Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.
Python aicode-complexitycode-healthcode-intelligencecode-qualitydead-codedeveloper-toolsdocumentationgit-analyticsmcpopen-sourcepythonrefactoringstatic-analysistechnical-debt
feat(generation): derive a concept outline deterministically, name it with an LLM (#1023)
Raghav Chamadiya5aac96ea71e6
diff --git a/packages/core/src/repowise/core/generation/concept_tree/__init__.py b/packages/core/src/repowise/core/generation/concept_tree/__init__.pynew file mode 100644index 000000000..ff716834b--- /dev/null+++ b/packages/core/src/repowise/core/generation/concept_tree/__init__.py@@ -0,0 +1,30 @@+"""Derive a repo-specific concept outline: deterministic grouping, LLM naming.++The split is not a style preference, it is what the probes measured. Feeding a+planner the whole directory inventory and asking it to both allocate files and+name the result gave 91.7% coverage on one run and 87.5% on the next from an+identical payload, and pushing harder on allocation dropped coverage to 88.2%+while raising fabricated paths ninefold. Membership decisions are the part an+LLM is unreliable at and a bounded-partition function is exact at; naming is+the part it is genuinely good at.++So :mod:`grouping` decides which files each page covers, with no model in the+loop, and :mod:`naming` decides what those pages are called. Coverage becomes+100% by construction rather than by luck, and the identity of a page stops+depending on a sampling temperature.+"""++from .grouping import ConceptGroup, GroupingParams, group_files+from .models import ConceptOutline, ConceptPage, ConceptSection, OutlineReport+from .validation import validate_outline++__all__ = [+ "ConceptGroup",+ "ConceptOutline",+ "ConceptPage",+ "ConceptSection",+ "GroupingParams",+ "OutlineReport",+ "group_files",+ "validate_outline",+]diff --git a/packages/core/src/repowise/core/generation/concept_tree/grouping.py b/packages/core/src/repowise/core/generation/concept_tree/grouping.pynew file mode 100644index 000000000..d6c0b967e--- /dev/null+++ b/packages/core/src/repowise/core/generation/concept_tree/grouping.py@@ -0,0 +1,445 @@+"""Bin a repository's production files into bounded, path-local groups.++One page per directory gives ``analysis/dead_code`` a page whether it earns one+and ``core/generation`` exactly one whether it needs three. One page per LLM+whim gives a different answer every run. This gives a third thing: a partition+of the directory tree into subtrees of bounded size, computed from the tree+itself.++The rule is a single bottom-up pass. A directory whose whole subtree fits under+the ceiling becomes one group and stops recursing, which is what keeps a page+about a place rather than about an arbitrary slice of one. A directory too big+to fit descends, and the pieces its children could not fill on their own are+accumulated in path order and flushed when they would overflow. Path locality+therefore falls out of the traversal rather than being asked for.++Three properties this has to hold, all of them load-bearing elsewhere:++* **Deterministic.** Children are visited in sorted order and every tie breaks+ on a path string, so the same file set always produces the same groups. Page+ identity hashes the member list (D2), so a grouping that drifted between runs+ would mint new page ids on every index and strand the old rows.+* **Total.** Every non-test production file lands in exactly one group. The+ validator asserts this rather than trusting it, but it is true by+ construction: the recursion partitions the tree and never drops a branch.+* **Local.** A group is a subtree or a run of adjacent siblings, never a+ scattering. That is what lets a group keep a real directory in+ ``target_path``, which the bench gold set matches on.++The layer signal steers which adjacent runs merge; it does not force a split.+A subtree small enough to fit stays one group even when it spans two layers,+because recursion stops as soon as a subtree fits. Splitting on layer wherever+layers disagree would fragment the outline and would inherit the layer data's+own noise, and an occasionally impure small page is the better trade.++The size ceiling comes from a coarse ladder keyed on repository size rather+than from a division, because a ceiling computed as ``total / target`` moves+whenever any file is added and would reshuffle the whole tree on a one-line+commit. The ladder only moves at a band edge.+"""++from __future__ import annotations++from collections.abc import Iterable, Sequence+from dataclasses import dataclass, field++from ..models import member_structural_key++#: Prefix for a concept group's structural key. Distinct from ``module`` and+#: ``scc`` so a key can never be mistaken for another page type's.+STRUCTURAL_KEY_PREFIX = "concept"+++@dataclass(frozen=True)+class GroupingParams:+ """Size bounds for the partition.++ ``max_files`` is the ceiling a single group may reach; ``min_files`` is the+ size below which a group prefers to be absorbed by a neighbour rather than+ stand alone. ``min_files`` is advisory — a repository can contain a single+ isolated directory smaller than it — while ``max_files`` is enforced except+ for one unavoidable case: a single directory holding more files than the+ ceiling cannot be split any further by a path-based rule, and splitting it+ on some other axis would break locality. Those are reported, not hidden.+ """++ min_files: int = 10+ max_files: int = 30+++# Repository size bands and the ceiling each one uses. Chosen so a repository+# of a few hundred files lands near 50 pages and one of a few thousand lands+# near 80, rather than growing without bound. Bands are wide on purpose: the+# ceiling must not move when a commit adds a file, because every group's+# identity would move with it.+_SIZE_LADDER: tuple[tuple[int, GroupingParams], ...] = (+ (600, GroupingParams(min_files=6, max_files=14)),+ (1200, GroupingParams(min_files=8, max_files=22)),+ (2400, GroupingParams(min_files=10, max_files=28)),+ (4800, GroupingParams(min_files=14, max_files=52)),+ (10_000, GroupingParams(min_files=20, max_files=95)),+)++_LADDER_TOP = GroupingParams(min_files=32, max_files=160)+++def params_for(file_count: int) -> GroupingParams:+ """The size bounds a repository of *file_count* production files uses."""+ for ceiling, params in _SIZE_LADDER:+ if file_count <= ceiling:+ return params+ return _LADDER_TOP+++@dataclass+class ConceptGroup:+ """The files one concept page will cover.++ ``target_path`` is a real directory and stays one. It is the page id+ (``compute_page_id`` is ``"{type}:{target_path}"``) and it is what the+ bench gold set matches by exact equality, so a group that merged several+ directories still has to name one of them. ``structural_key`` is where the+ abstract identity lives — a hash of the members, which is the only thing+ that actually says which page this is.+ """++ members: list[str]+ #: Directories fully contained in this group, sorted. A group is usually a+ #: subtree, so this is normally that subtree's directories.+ dirs: list[str]+ target_path: str+ dominant_layer: str = ""+ #: Set when the group is a single directory larger than the ceiling and+ #: therefore could not be split by any path-based rule.+ oversized: bool = False+ structural_key: str = field(default="", repr=False)++ def __post_init__(self) -> None:+ if not self.structural_key:+ self.structural_key = member_structural_key(+ self.members, prefix=STRUCTURAL_KEY_PREFIX+ )++ @property+ def file_count(self) -> int:+ return len(self.members)+++# ---------------------------------------------------------------------------+# Directory tree+# ---------------------------------------------------------------------------+++@dataclass+class _Node:+ path: str+ own: list[str] = field(default_factory=list)+ children: dict[str, _Node] = field(default_factory=dict)+ subtree: int = 0++ def child(self, name: str) -> _Node:+ node = self.children.get(name)+ if node is None:+ node = _Node(path=f"{self.path}/{name}" if self.path else name)+ self.children[name] = node+ return node+++def _build_tree(paths: Iterable[str]) -> _Node:+ root = _Node(path="")+ for path in paths:+ parts = path.split("/")+ node = root+ for part in parts[:-1]:+ node = node.child(part)+ node.own.append(path)+ _count(root)+ return root+++def _count(node: _Node) -> int:+ node.subtree = len(node.own) + sum(_count(c) for c in node.children.values())+ return node.subtree+++def _subtree_files(node: _Node) -> list[str]:+ out = list(node.own)+ for _name, child in sorted(node.children.items()):+ out.extend(_subtree_files(child))+ return out+++def _dirs_of(paths: Sequence[str]) -> list[str]:+ return sorted({p.rsplit("/", 1)[0] if "/" in p else "" for p in paths})+++# ---------------------------------------------------------------------------+# Partition+# ---------------------------------------------------------------------------+++def _dominant(paths: Sequence[str], layer_of_file: dict[str, str]) -> str:+ """The layer most of *paths* belong to, ties broken on the layer id.++ Matches ``page_tree._dominant_layer`` deliberately: a group placed under a+ layer in the tree and a group whose merge decision consulted a different+ notion of "its layer" would disagree about where it belongs.+ """+ counts: dict[str, int] = {}+ for path in paths:+ layer = layer_of_file.get(path)+ if layer:+ counts[layer] = counts.get(layer, 0) + 1+ if not counts:+ return ""+ return sorted(counts.items(), key=lambda kv: (-kv[1], kv[0]))[0][0]+++def _target_path(members: Sequence[str], dirs: Sequence[str]) -> str:+ """The directory that names this group.++ The shallowest directory containing every member, which for a subtree group+ is that subtree's root and for a run of siblings is their shared parent.+ Derived from the members rather than picked from a list, so it does not+ depend on which directory happened to sort first.++ **This value is the page id** (``compute_page_id`` is+ ``"{page_type}:{target_path}"``), so two groups sharing one would be two+ pages sharing a row. The partition splits a directory's own files from its+ subdirectories' files, so a "files directly in ``routers``" group and a+ "everything under ``routers/``" group both compute ``routers`` here.+ :func:`_assign_targets` resolves that; this function is the unqualified+ answer.+ """+ if not members:+ return dirs[0] if dirs else ""+ split = [m.split("/")[:-1] for m in members]+ common = split[0]+ for parts in split[1:]:+ limit = min(len(common), len(parts))+ cut = limit+ for i in range(limit):+ if common[i] != parts[i]:+ cut = i+ break+ common = common[:cut]+ return "/".join(common)+++class _Partitioner:+ def __init__(self, params: GroupingParams, layer_of_file: dict[str, str]):+ self.params = params+ self.layer_of_file = layer_of_file+ self.groups: list[ConceptGroup] = []++ def make(self, members: Sequence[str]) -> ConceptGroup:+ ordered = sorted(members)+ dirs = _dirs_of(ordered)+ # A group over the ceiling whose files all sit directly in one+ # directory has hit the floor of what a path-based rule can do: there+ # is no sub-path to split on. Chunking such a directory by filename+ # order would split it, but the chunk boundaries would move whenever a+ # file was added, and every chunk's identity hashes its members — so a+ # one-file commit would remint every page in the directory. Stability+ # is a correctness requirement here and page size is a quality target,+ # so the group stays whole and says that it is oversized.+ oversized = len(ordered) > self.params.max_files and len(dirs) == 1+ return ConceptGroup(+ members=ordered,+ dirs=dirs,+ target_path=_target_path(ordered, dirs),+ dominant_layer=_dominant(ordered, self.layer_of_file),+ oversized=oversized,+ )++ def _same_layer(self, left: Sequence[str], right: Sequence[str]) -> bool:+ """Whether two pending runs may be merged across a layer boundary.++ A merge that straddles two architectural layers produces a page about+ no single thing, which is the failure the layer hint exists to prevent.+ The exception is a run too thin to stand alone: a three-file group with+ a title is worse documentation than a slightly impure eight-file one.+ """+ if len(left) + len(right) < self.params.min_files:+ return True+ a = _dominant(left, self.layer_of_file)+ b = _dominant(right, self.layer_of_file)+ return not a or not b or a == b++ def visit(self, node: _Node) -> list[str]:+ """Emit groups for *node*'s subtree; return the files left for its parent.++ The returned files are ones that did not amount to a group on their+ own. The parent either merges them with an adjacent sibling's leftovers+ or, at the top, flushes them as their own group.+ """+ if node.subtree <= self.params.max_files:+ return _subtree_files(node)++ pending: list[str] = list(node.own)+ for _name, child in sorted(node.children.items()):+ left = self.visit(child)+ if not left:+ continue+ if len(pending) + len(left) <= self.params.max_files and self._same_layer(+ pending, left+ ):+ pending.extend(left)+ continue+ if pending:+ self.groups.append(self.make(pending))+ # A child only returns files when its whole subtree fits under the+ # ceiling, so this can never itself overflow. An over-ceiling+ # single directory is emitted by the recursion below, not here.+ pending = list(left)++ # A thin remainder is folded back into the group before it rather than+ # left as a two-file page, but only when they share a layer and the+ # result still fits.+ if pending and len(pending) < self.params.min_files and self.groups:+ last = self.groups[-1]+ if len(last.members) + len(pending) <= self.params.max_files and self._same_layer(+ last.members, pending+ ):+ merged = self.make(last.members + pending)+ self.groups[-1] = merged+ pending = []+ if pending:+ self.groups.append(self.make(pending))+ return []+++def _absorb_thin(+ groups: list[ConceptGroup], part: _Partitioner+) -> list[ConceptGroup]:+ """Fold groups too small to deserve a page into their closest neighbour.++ The recursion flushes a remainder wherever a subtree boundary falls, so a+ directory holding two files next to one holding twenty-nine can leave a+ two-file group standing on its own. A page about two files is not a+ concept, it is a directory listing with a title.++ Neighbours are considered in path order and the one sharing the longer+ prefix wins, so a thin group joins the thing it is actually near rather+ than whichever side happened to be checked first. A group with no legal+ neighbour — the merge would breach the ceiling, or cross a layer — stays,+ because a slightly thin page is better than a wrong one.+ """+ params = part.params+ working = sorted(groups, key=lambda g: g.target_path)+ changed = True+ while changed:+ changed = False+ for i, group in enumerate(working):+ if group.file_count >= params.min_files or len(working) == 1:+ continue+ best: int | None = None+ best_rank = (-1, -1)+ for j in (i - 1, i + 1):+ if j < 0 or j >= len(working):+ continue+ other = working[j]+ if other.file_count + group.file_count > params.max_files:+ continue+ # A same-layer neighbour is preferred, but a thin group takes+ # a cross-layer neighbour over standing alone. The layer gate+ # exists to stop a page spanning two subsystems; it should not+ # produce a one-file page, which is a worse outcome by every+ # measure the outline is judged on.+ same = part._same_layer(other.members, group.members)+ # Depth of the directory the two would share. ``"".split("/")``+ # and ``"src".split("/")`` are both length one, so measuring by+ # segment count rated "shares nothing" equal to "shares a+ # top-level directory" and let the tie fall to whichever side+ # was examined first.+ common = _target_path(group.members + other.members, [])+ shared = 0 if not common else common.count("/") + 1+ rank = (1 if same else 0, shared)+ if rank > best_rank:+ best_rank = rank+ best = j+ if best is None:+ continue+ merged = part.make(working[best].members + group.members)+ working[best] = merged+ del working[i]+ changed = True+ break+ return working+++def _assign_targets(groups: list[ConceptGroup]) -> None:+ """Give every group a distinct ``target_path`` that is still a real directory.++ A group whose shallowest common directory also physically holds some of its+ files keeps that directory: it is the one that owns it. A group covering+ only subdirectories of that directory takes its own first directory+ instead, which is real, is inside the group, and cannot be claimed by+ anyone else because the partition never splits one directory's files across+ two groups.++ That argument is sound but it is an argument, and the cost of it being+ wrong is two pages silently sharing a row, so the loop below checks the+ result and walks to the next candidate rather than trusting it.+ """+ taken: set[str] = set()+ # Largest first so the group with the strongest claim to a directory keeps+ # the shortest name; ties break on the key so the order never depends on+ # the traversal.+ for group in sorted(groups, key=lambda g: (-g.file_count, g.structural_key)):+ owned = _target_path(group.members, group.dirs)+ candidates = [owned] if owned in group.dirs else []+ candidates.extend(d for d in group.dirs if d and d not in candidates)+ if owned and owned not in candidates:+ candidates.append(owned)+ # ``None`` rather than "" for "nothing chosen yet": the repository root+ # is a real, legitimate directory whose path *is* the empty string, and+ # a falsy check cannot tell it apart from failure. It could not, once —+ # a group holding root-level files (``setup.py``, ``manage.py``,+ # ``index.js``) fell through to the hash fallback and was titled after+ # its own digest.+ chosen: str | None = None+ for candidate in candidates:+ if candidate not in taken:+ chosen = candidate+ break+ if chosen is None:+ # Every real directory is spoken for. Fall back to the members'+ # common prefix qualified by the group's identity, which is stable+ # and unique even though it is no longer only a directory.+ chosen = f"{owned}#{group.structural_key.rsplit('-', 1)[-1]}"+ group.target_path = chosen+ taken.add(chosen)+++def group_files(+ paths: Iterable[str],+ *,+ layer_of_file: dict[str, str] | None = None,+ params: GroupingParams | None = None,+) -> list[ConceptGroup]:+ """Partition *paths* into bounded, path-local concept groups.++ *paths* must already exclude test files (D8) — this function does not+ filter, so a caller that wants tests in gets them in and the count reflects+ it. *layer_of_file* maps a file to its curated layer id and only affects+ whether two adjacent runs may be merged; grouping works without it.++ Groups come back in path order, which is also the order their members were+ walked, so the sequence itself is stable.+ """+ files = sorted({p.replace("\\", "/") for p in paths if p})+ if not files:+ return []+ tree = _build_tree(files)+ resolved = params or params_for(len(files))+ part = _Partitioner(resolved, layer_of_file or {})+ leftover = part.visit(tree)+ if leftover:+ # The whole repository fits under the ceiling: one group, which is the+ # honest answer for a tiny repository.+ part.groups.append(part.make(leftover))+ final = _absorb_thin(part.groups, part)+ _assign_targets(final)+ final.sort(key=lambda g: (g.target_path, g.structural_key))+ return finaldiff --git a/packages/core/src/repowise/core/generation/concept_tree/models.py b/packages/core/src/repowise/core/generation/concept_tree/models.pynew file mode 100644index 000000000..aa2b0e61e--- /dev/null+++ b/packages/core/src/repowise/core/generation/concept_tree/models.py@@ -0,0 +1,164 @@+"""The outline itself, and the numbers that say whether it is any good.++An outline is a two-level thing — sections that group pages — but the tree it+becomes is not, because sections are not pages. A section exists to be a+heading; giving it a row would mean inventing pages nobody asked for and would+break the property that placement adds no pages. So a section is carried here+as a label plus a number, and the pages under it inherit the number.+"""++from __future__ import annotations++from dataclasses import dataclass, field++from .grouping import ConceptGroup+++@dataclass+class ConceptPage:+ """One leaf of the outline: a named group of files."""++ title: str+ #: One sentence saying what the page covers and what it deliberately does+ #: not. The "and does not" half is the part that stops adjacent pages+ #: describing each other.+ scope: str+ group: ConceptGroup+ #: Dotted number within the outline, e.g. ``"3.2"``. Assigned by the+ #: planner after ordering, never by the model.+ number: str = ""+ #: False when the title came from the deterministic fallback because the+ #: model skipped this group. Long inputs get skipped near the tail, so+ #: these are what the repair pass goes back for.+ named_by_model: bool = False++ @property+ def target_path(self) -> str:+ return self.group.target_path++ @property+ def structural_key(self) -> str:+ return self.group.structural_key++ @property+ def members(self) -> list[str]:+ return self.group.members+++@dataclass+class ConceptSection:+ """A heading over one or more pages."""++ title: str+ pages: list[ConceptPage] = field(default_factory=list)+ number: str = ""+++@dataclass+class ConceptOutline:+ """A complete, ordered outline over a repository's production files."""++ sections: list[ConceptSection] = field(default_factory=list)+ #: Terms lifted from the repository's own docs and bound to a real group.+ #: Present for provenance: a title that came from the project's vocabulary+ #: should be traceable to the sentence that supplied it.+ vocabulary: dict[str, str] = field(default_factory=dict)+ #: ``"llm"`` when a model named the groups, ``"deterministic"`` when the+ #: keyless fallback did. One axis, per D5: the map is free, the prose+ #: needs a key.+ naming_mode: str = "deterministic"++ @property+ def pages(self) -> list[ConceptPage]:+ return [p for s in self.sections for p in s.pages]++ def number_sections(self) -> None:+ """Assign dotted numbers by position. Sections are 1-based."""+ for si, section in enumerate(self.sections, start=1):+ section.number = str(si)+ for pi, page in enumerate(section.pages, start=1):+ page.number = f"{si}.{pi}"+++@dataclass+class OutlineReport:+ """What the validator measured.++ Deliberately numbers rather than a boolean. "The outline validated" says+ nothing a reader can act on; "two directories unclaimed, nine paths+ invented" says exactly where the planner went wrong, and the probes showed+ those two failures move independently.+ """++ total_files: int = 0+ covered_files: int = 0+ unclaimed_files: list[str] = field(default_factory=list)+ double_claimed_files: list[str] = field(default_factory=list)+ #: Paths the model emitted that are not in the index. The probes measured+ #: one on a good run and nine on a constrained one, so this fires.+ invented_paths: list[str] = field(default_factory=list)+ page_count: int = 0+ section_count: int = 0+ empty_pages: list[str] = field(default_factory=list)+ oversized_pages: list[str] = field(default_factory=list)+ single_child_sections: list[str] = field(default_factory=list)+ duplicate_titles: list[str] = field(default_factory=list)+ bad_length_titles: list[str] = field(default_factory=list)+ bare_directory_titles: list[str] = field(default_factory=list)+ duplicate_structural_keys: list[str] = field(default_factory=list)+ #: Two pages resolving to one ``target_path``. That is two pages sharing a+ #: page id, so one silently overwrites the other on persist.+ duplicate_target_paths: list[str] = field(default_factory=list)+ test_paths_included: list[str] = field(default_factory=list)+ title_word_avg: float = 0.0++ @property+ def coverage(self) -> float:+ return self.covered_files / self.total_files if self.total_files else 0.0++ @property+ def hard_failures(self) -> list[str]:+ """The failures that make an outline wrong rather than untidy.++ Coverage and fabrication are correctness; a title one word too long is+ taste. Only the first kind blocks.+ """+ out: list[str] = []+ if self.unclaimed_files:+ out.append(f"{len(self.unclaimed_files)} unclaimed files")+ if self.double_claimed_files:+ out.append(f"{len(self.double_claimed_files)} double-claimed files")+ if self.invented_paths:+ out.append(f"{len(self.invented_paths)} invented paths")+ if self.empty_pages:+ out.append(f"{len(self.empty_pages)} pages claiming no files")+ if self.duplicate_structural_keys:+ out.append(f"{len(self.duplicate_structural_keys)} duplicate structural keys")+ if self.duplicate_target_paths:+ out.append(f"{len(self.duplicate_target_paths)} duplicate target paths")+ if self.test_paths_included:+ out.append(f"{len(self.test_paths_included)} test files in the tree")+ return out++ @property+ def ok(self) -> bool:+ return not self.hard_failures++ def as_lines(self) -> list[str]:+ """The report as the generation log and the progress doc want it."""+ return [+ f"coverage {self.covered_files}/{self.total_files} = "+ f"{100 * self.coverage:.1f}%",+ f"pages {self.page_count} in {self.section_count} sections",+ f"unclaimed {len(self.unclaimed_files)}",+ f"double-claimed {len(self.double_claimed_files)}",+ f"invented paths {len(self.invented_paths)}",+ f"empty pages {len(self.empty_pages)}",+ f"oversized {len(self.oversized_pages)}",+ f"one-child sects {len(self.single_child_sections)}",+ f"dup titles {len(self.duplicate_titles)}",+ f"title words avg {self.title_word_avg:.1f}",+ f"dup struct keys {len(self.duplicate_structural_keys)}",+ f"dup target paths {len(self.duplicate_target_paths)}",+ f"test files in {len(self.test_paths_included)}",+ ]diff --git a/packages/core/src/repowise/core/generation/concept_tree/naming.py b/packages/core/src/repowise/core/generation/concept_tree/naming.pynew file mode 100644index 000000000..f5a4c8771--- /dev/null+++ b/packages/core/src/repowise/core/generation/concept_tree/naming.py@@ -0,0 +1,428 @@+"""Name and order the groups. The model never decides which files they hold.++This is the half of the planner an LLM is good at, and it is bounded so that+the half it is bad at cannot leak back in. Groups arrive with opaque ids and+the model returns a title, a scope sentence and a section for each id. It is+never shown a blank canvas and never asked which directory belongs where, so+the two failures the probes measured — dropping directories and inventing+plausible-looking paths — have nowhere to enter through. A group the model+forgets to name keeps a deterministic name; a group id it invents is discarded.+Coverage is therefore unchanged by anything the model does or fails to do.++The payload conveys breadth, never importance. Ranking the input by PageRank+and sampling the top files produced 12.7% coverage against 91.7% for the full+inventory, because hub files are infrastructure — ``models.py``, ``client.ts``,+``__init__.py`` — and an outline built around them describes the plumbing+rather than the product.++Repo docs do not come in here as prose. Nine thousand characters of README+injected alongside a 145-directory structural task collapsed the outline to+three pages. Vocabulary arrives already bound to a group by+:mod:`vocabulary`, as a short list of suggested names.+"""++from __future__ import annotations++import json+import re+from dataclasses import dataclass+from typing import Any++import structlog++from ..onboarding.grounding import check_grounding+from .grouping import ConceptGroup++logger = structlog.get_logger(__name__)++MAX_TITLE_WORDS = 7+MIN_TITLE_WORDS = 2++_FENCE = re.compile(r"^```[a-zA-Z]*\n|\n```$")+++def parse_json_object(content: str) -> dict[str, Any]:+ """Parse a JSON object out of a model response, tolerating decoration.++ Providers here are asked for JSON by instruction rather than by an API+ flag, because no provider in the set exposes a native structured-output+ mode. So the response can arrive fenced, prefaced, or trailed with prose,+ and the recovery ladder is the same one decision extraction already uses:+ strip fences, parse, and failing that pull the outermost braces out.+ Returns an empty dict rather than raising — an unparseable outline falls+ back to deterministic names, which is a worse wiki, not a broken run.+ """+ text = (content or "").strip()+ if text.startswith("```"):+ text = "\n".join(+ line for line in text.split("\n") if not line.strip().startswith("```")+ )+ try:+ parsed = json.loads(text)+ except (json.JSONDecodeError, ValueError):+ match = re.search(r"\{.*\}", text, re.DOTALL)+ if not match:+ return {}+ try:+ parsed = json.loads(match.group())+ except (json.JSONDecodeError, ValueError):+ return {}+ return parsed if isinstance(parsed, dict) else {}+++# ---------------------------------------------------------------------------+# Deterministic naming — the keyless path (D5)+# ---------------------------------------------------------------------------++_STOP_SEGMENTS = {"src", "lib", "app", "packages", "core", "internal", "pkg", "source"}+++def _humanise(segment: str) -> str:+ words = re.split(r"[-_.\s]+", segment)+ out: list[str] = []+ for word in words:+ if not word:+ continue+ # Split camelCase so ``pageGenerator`` reads as two words.+ parts = re.findall(r"[A-Z]+(?![a-z])|[A-Z][a-z]*|[a-z]+|\d+", word) or [word]+ out.extend(p.capitalize() if p.islower() else p for p in parts)+ return " ".join(out)+++def deterministic_title(group: ConceptGroup, layer_label: str = "") -> str:+ """A readable name from structure alone, for a run with no provider.++ Uses the last one or two meaningful path segments, prefixed by the layer+ when that is what makes the name unambiguous. Not as good as a named+ capability and not meant to be: per D5 the map is free and the prose needs+ a key, so a keyless wiki gets a real tree with plainer labels rather than a+ worse tree.+ """+ segments = [s for s in group.target_path.split("/") if s]+ meaningful = [s for s in segments if s.lower() not in _STOP_SEGMENTS] or segments+ tail = [_humanise(s) for s in meaningful[-2:]]+ title = " ".join(tail).strip()+ if layer_label and len(title.split()) < MIN_TITLE_WORDS:+ title = f"{layer_label} {title}".strip()+ if not title:+ # The repository root has no path segments to name it by. It is a real+ # place and a common one — top-level entry points live there — so it+ # gets a name of its own rather than one derived from an empty path.+ title = "Repository Root Files"+ words = title.split()+ if len(words) > MAX_TITLE_WORDS:+ title = " ".join(words[:MAX_TITLE_WORDS])+ if len(title.split()) < MIN_TITLE_WORDS:+ title = f"{title} Components".strip()+ return title+++def deterministic_scope(group: ConceptGroup) -> str:+ """A scope line stating coverage and its boundary, with no model."""+ # The root directory is the empty string, which is a real directory and+ # not an absent one; filtering it out left a group reading "0 directories".+ dirs = list(group.dirs)+ if len(dirs) == 1:+ where = dirs[0] or "the repository root"+ else:+ where = f"{len(dirs)} directories under {group.target_path or 'the repository root'}"+ return (+ f"Covers the {group.file_count} source files in {where}. "+ "Does not cover code outside those directories, which is documented on "+ "its own pages."+ )+++# ---------------------------------------------------------------------------+# LLM naming+# ---------------------------------------------------------------------------+++@dataclass+class NamedGroup:+ """What the namer decided about one group."""++ group: ConceptGroup+ title: str+ scope: str+ section: str+ order: int = 0+ #: True when this name came from the fallback rather than the model, either+ #: because there was no provider or because the model skipped the group.+ fallback: bool = False+++SYSTEM_PROMPT = (+ "You are an information architect naming the sections of a technical wiki "+ "about a specific codebase, read by senior engineers and AI coding agents. "+ "You name and order groups of files that have already been decided for you. "+ "You never decide which files belong to which group. Reply with JSON only."+)++NAMING_INSTRUCTIONS = """\+Below is the complete set of file groups for the `{repo}` codebase. The grouping \+is fixed: every group is a contiguous part of the directory tree, every source \+file is in exactly one group, and you must not change, merge, split or drop any \+group.++For EVERY group id, return a title, a one-sentence scope, and the section it \+belongs to. Then order the sections.++TITLES+- Name the CAPABILITY or SUBSYSTEM, not the directory and not the layer. \+Good: "Dependency Graph Construction", "Output Distillation", \+"Code Health Scoring". Bad: "core/ingestion", "Ingestion Layer", \+"Utilities and Helpers".+- {min_words} to {max_words} words. Every title must be unique.+- An enumerative title is right only when a group genuinely spans several small \+areas, e.g. "Dead Code, Security, and Change Risk".+- Where a suggested name is given for a group, prefer it: it is the project's \+own word for this, taken from its documentation.++SCOPE+- One sentence saying what the page covers AND what it deliberately does not, \+so that adjacent pages do not describe each other.+- Refer only to paths that appear in that group's own listing. Do not name a \+file or directory that is not in the input.++SECTIONS+- Group the pages into {min_sections} to {max_sections} sections. Order them as \+a narrative: orientation first, then the core engine, then interfaces in order \+of distance from the engine, then operations, then reference.+- No section may contain exactly one page.+- Section titles are 1 to 5 words.++Return ONLY this JSON:+{{"sections": [{{"title": "...", "groups": ["g01", "g07", ...]}}],+ "names": {{"g01": {{"title": "...", "scope": "..."}}, ...}}}}++GROUPS:+"""+++def _sample_names(members: list[str], limit: int) -> list[str]:+ """Filenames spread across the group's directories, not the first *limit*.++ Taking the alphabetically-first names from a group spanning two+ directories draws all of them from whichever directory sorts first, so the+ namer sees half the page and names it after that half. Round-robin gives+ every directory a turn before any gets a second.+ """+ by_dir: dict[str, list[str]] = {}+ for member in sorted(members):+ by_dir.setdefault(member.rsplit("/", 1)[0], []).append(+ member.rsplit("/", 1)[-1]+ )+ out: list[str] = []+ order = sorted(by_dir)+ depth = 0+ while len(out) < limit:+ added = False+ for directory in order:+ names = by_dir[directory]+ if depth < len(names):+ out.append(names[depth])+ added = True+ if len(out) >= limit:+ break+ if not added:+ break+ depth += 1+ return out+++def _common_prefix(dirs: list[str]) -> str:+ """The deepest directory every entry of *dirs* sits under."""+ if not dirs:+ return ""+ split = [d.split("/") for d in dirs]+ common = split[0]+ for parts in split[1:]:+ cut = min(len(common), len(parts))+ for i in range(cut):+ if common[i] != parts[i]:+ cut = i+ break+ common = common[:cut]+ return "/".join(common)+++def build_payload(+ groups: list[ConceptGroup],+ *,+ layer_labels: dict[str, str] | None = None,+ summaries: dict[str, str] | None = None,+ suggestions: dict[str, str] | None = None,+ entry_points: set[str] | None = None,+ max_filenames: int = 6,+) -> tuple[dict[str, Any], dict[str, ConceptGroup]]:+ """Build the namer's payload and the id-to-group index that decodes it.++ Ids are positional (``g01``) rather than derived from the path, so the+ model has nothing path-shaped to echo back and a fabricated id is+ immediately recognisable as one. Filenames are included because they carry+ what a directory name does not, but they are basenames only: a model given+ full paths will quote them, and a quoted path is a citation we would then+ have to verify.+ """+ labels = layer_labels or {}+ mod_summaries = summaries or {}+ hints = suggestions or {}+ entries = entry_points or set()++ index: dict[str, ConceptGroup] = {}+ entries_out: list[dict[str, Any]] = []+ for i, group in enumerate(groups, start=1):+ gid = f"g{i:02d}"+ index[gid] = group+ entry: dict[str, Any] = {+ "id": gid,+ "dir": group.target_path,+ "files": group.file_count,+ "names": _sample_names(group.members, max_filenames),+ }+ if len(group.dirs) > 1:+ # Relative to what the directories share, not to ``target_path``.+ # The target is one of the group's directories and is not always a+ # prefix of the others: a page covering ``ingestion/git_indexer``+ # and ``ingestion/graph`` is targeted at the first, and stripping+ # that prefix from the second yielded an empty string. Both+ # directories then arrived as ".", so the namer could not see that+ # the page covered two subsystems and named it after one.+ common = _common_prefix(group.dirs)+ cut = len(common) + 1 if common else 0+ entry["subdirs"] = [d[cut:] or d.rsplit("/", 1)[-1] for d in group.dirs[:8]]+ label = labels.get(group.dominant_layer)+ if label:+ entry["layer"] = label+ summary = mod_summaries.get(group.target_path)+ if summary:+ entry["summary"] = summary.replace("\n", " ")[:200]+ hint = hints.get(gid)+ if hint:+ entry["suggested_name"] = hint+ group_entries = sorted(e.rsplit("/", 1)[-1] for e in entries if e in set(group.members))+ if group_entries:+ entry["entry_points"] = group_entries[:3]+ entries_out.append(entry)++ return {"repo": "", "groups": entries_out}, index+++_MAX_TITLE_CHARS = 120+++def _clean_title(raw: Any) -> str:+ # Only a string is a title. A dict or a list stringifies into something+ # that looks like a title to every downstream check and reads as garbage+ # to a person.+ if not isinstance(raw, str):+ return ""+ title = raw.strip().strip('"').strip()+ title = re.sub(r"\s+", " ", title)+ # A model asked for a title sometimes returns "Section 3: Ingestion".+ title = re.sub(r"^\d+(\.\d+)*[.:)\-]\s*", "", title)+ # Length is bounded here rather than left to the validator, which treats+ # it as taste. An unbounded title is a rendering problem, not a taste one.+ if len(title) > _MAX_TITLE_CHARS:+ title = title[:_MAX_TITLE_CHARS].rsplit(" ", 1)[0]+ return title+++def decode_response(+ data: dict[str, Any],+ index: dict[str, ConceptGroup],+ *,+ layer_labels: dict[str, str] | None = None,+) -> tuple[list[NamedGroup], list[str]]:+ """Turn the model's JSON into named groups, keeping the grouping intact.++ Returns ``(named, invented_ids)``. Every group in *index* comes back+ exactly once whatever the model said: an id it skipped gets a+ deterministic name, an id it invented is dropped and reported. This is the+ property that makes coverage independent of the model.+ """+ labels = layer_labels or {}+ names = data.get("names")+ names = names if isinstance(names, dict) else {}+ sections = data.get("sections")+ sections = sections if isinstance(sections, list) else []++ section_of: dict[str, str] = {}+ order_of: dict[str, int] = {}+ section_rank: dict[str, int] = {}+ invented: list[str] = []+ counter = 0+ for rank, section in enumerate(sections):+ if not isinstance(section, dict):+ continue+ title = _clean_title(section.get("title"))+ if not title:+ continue+ section_rank.setdefault(title, rank)+ # A scalar here is not hypothetical: a model asked for a list of ids+ # returned a count instead, and ``or []`` passes a non-empty scalar+ # straight through to the loop. The whole point of this function is+ # that no model response can break the run, so the shape is checked+ # rather than assumed.+ raw_groups = section.get("groups")+ if not isinstance(raw_groups, (list, tuple)):+ raw_groups = []+ for gid in raw_groups:+ if not isinstance(gid, (str, int)):+ continue+ gid = str(gid)+ if gid not in index:+ invented.append(gid)+ continue+ if gid in section_of:+ # Claimed twice. The first section wins, deterministically.+ continue+ section_of[gid] = title+ order_of[gid] = counter+ counter += 1++ named: list[NamedGroup] = []+ for gid, group in index.items():+ entry = names.get(gid)+ entry = entry if isinstance(entry, dict) else {}+ title = _clean_title(entry.get("title"))+ scope = str(entry.get("scope") or "").strip()+ fallback = False+ if not title:+ title = deterministic_title(group, labels.get(group.dominant_layer, ""))+ fallback = True+ if not scope:+ scope = deterministic_scope(group)+ section = section_of.get(gid) or labels.get(group.dominant_layer) or "Reference"+ named.append(+ NamedGroup(+ group=group,+ title=title,+ scope=scope,+ section=section,+ order=order_of.get(gid, len(index) + len(named)),+ fallback=fallback,+ )+ )++ named.sort(key=lambda n: (section_rank.get(n.section, len(sections)), n.order,+ n.group.target_path))+ return named, sorted(set(invented))+++def ground_scopes(named: list[NamedGroup]) -> list[str]:+ """Strip citations from scope sentences that the group's files do not support.++ Reuses the onboarding grounding check rather than repeating its shape+ rules: a backticked token that looks like a path or a symbol and is absent+ from the group's own file list loses its backticks, so it reads as prose+ instead of as a verified reference. Returns every token demoted, which the+ validator counts as an invented path.+ """+ ungrounded: list[str] = []+ for entry in named:+ cleaned, bad = check_grounding(entry.scope, entry.group.members)+ entry.scope = cleaned+ ungrounded.extend(bad)+ return ungroundeddiff --git a/packages/core/src/repowise/core/generation/concept_tree/planner.py b/packages/core/src/repowise/core/generation/concept_tree/planner.pynew file mode 100644index 000000000..73e29e3ee--- /dev/null+++ b/packages/core/src/repowise/core/generation/concept_tree/planner.py@@ -0,0 +1,474 @@+"""Plan the outline: group, name, validate, repair, validate again.++Two passes rather than one, because a single call cannot hold coverage and+allocation at the same time — pushing on structure made a planner drop+directories and invent paths at nine times the rate. Here the two jobs are+separated at the source, so the second pass has almost nothing to do: grouping+already guarantees coverage, and repair only ever revisits names.++The repair pass sees only what failed. Handing back the whole tree invites the+model to rewrite the parts that were fine, which is how a "fix these four+titles" request turns into a different outline.+"""++from __future__ import annotations++import contextlib+import json+from collections.abc import Iterator+from dataclasses import dataclass, field+from pathlib import Path+from typing import Any++import structlog++from .grouping import ConceptGroup, GroupingParams, group_files, params_for+from .models import ConceptOutline, ConceptPage, ConceptSection, OutlineReport+from .naming import (+ NAMING_INSTRUCTIONS,+ SYSTEM_PROMPT,+ NamedGroup,+ _humanise,+ build_payload,+ decode_response,+ deterministic_scope,+ deterministic_title,+ ground_scopes,+ parse_json_object,+)+from .validation import validate_outline+from .vocabulary import bind_terms, extract_terms++logger = structlog.get_logger(__name__)++COST_OPERATION = "outline_planning"++MIN_SECTIONS = 5+MAX_SECTIONS = 11+++@dataclass+class PlannerInputs:+ """Everything the planner needs, already resolved by the caller.++ Deliberately plain data. The planner is then testable without a pipeline,+ a database or a provider, which matters because its interesting properties+ are about determinism and those need to be asserted cheaply and often.+ """++ repo_name: str+ #: Production source files, tests already excluded (D8).+ production_files: list[str]+ repo_root: Path | None = None+ #: File path -> curated layer id.+ layer_of_file: dict[str, str] = field(default_factory=dict)+ #: Layer id -> display name, for section fallbacks and payload hints.+ layer_labels: dict[str, str] = field(default_factory=dict)+ #: Directory -> module summary written from the code.+ summaries: dict[str, str] = field(default_factory=dict)+ entry_points: set[str] = field(default_factory=set)+ #: Test files, kept only so the validator can prove none leaked in.+ test_files: set[str] = field(default_factory=set)+++def _billed_as(provider: Any, operation: str) -> Iterator[None]:+ """Label this step's spend on the Costs page; no-op without a tracker."""+ tracker = getattr(provider, "_cost_tracker", None)+ if tracker is not None and hasattr(tracker, "record_as"):+ with tracker.record_as(operation):+ yield+ else:+ yield+++def _build_outline(named: list[NamedGroup]) -> ConceptOutline:+ """Assemble sections from named groups, preserving the namer's order."""+ outline = ConceptOutline()+ by_section: dict[str, ConceptSection] = {}+ for entry in named:+ section = by_section.get(entry.section)+ if section is None:+ section = ConceptSection(title=entry.section)+ by_section[entry.section] = section+ outline.sections.append(section)+ section.pages.append(+ ConceptPage(+ title=entry.title,+ scope=entry.scope,+ group=entry.group,+ named_by_model=not entry.fallback,+ )+ )+ outline.number_sections()+ return outline+++def _merge_thin_sections(outline: ConceptOutline) -> None:+ """Fold a section holding a single page into its neighbour.++ A heading over one page is not a grouping, it is a heading pretending to+ be one, and it makes a table of contents look padded. The page keeps its+ own title, so nothing is lost by moving it.+ """+ if len(outline.sections) <= 1:+ return+ changed = True+ while changed and len(outline.sections) > 1:+ changed = False+ for i, section in enumerate(outline.sections):+ if len(section.pages) != 1:+ continue+ target = i - 1 if i > 0 else i + 1+ outline.sections[target].pages.extend(section.pages)+ del outline.sections[i]+ changed = True+ break+ outline.number_sections()+++def _disambiguate_titles(named: list[NamedGroup]) -> None:+ """Make path-derived titles unique by qualifying them with their parent.++ Two groups carved out of one directory tree — one holding a directory's own+ files, one holding its subdirectories — produce the same name from the same+ path segments. Rather than numbering them, which tells a reader nothing,+ the later one borrows a segment from further up its own path.+ """+ seen: dict[str, int] = {}+ for entry in sorted(named, key=lambda n: (n.title.lower(), n.group.target_path)):+ key = entry.title.lower()+ if key not in seen:+ seen[key] = 1+ continue+ segments = [s for s in entry.group.target_path.split("/") if s]+ for extra in reversed(segments[:-1]):+ candidate = f"{deterministic_title(entry.group)} {_humanise(extra)}".strip()+ candidate = " ".join(dict.fromkeys(candidate.split()))+ if candidate.lower() not in seen and len(candidate.split()) <= 7:+ entry.title = candidate+ break+ else:+ seen[key] += 1+ entry.title = f"{entry.title} {seen[key]}"+ seen[entry.title.lower()] = 1+++def plan_deterministic(+ inputs: PlannerInputs, *, params: GroupingParams | None = None+) -> tuple[ConceptOutline, list[ConceptGroup]]:+ """The keyless outline: real structure, names derived from paths and layers.++ Per D5 this is the same tree the LLM path produces — identical grouping,+ identical identities — with plainer titles. Adding a key upgrades the+ prose, never the shape, so a user who adds one later does not get their+ wiki re-partitioned underneath them.++ That guarantee only holds if both paths partition with the same bounds, so+ *params* is threaded rather than re-derived. It was not, once: the caller's+ bounds were dropped here and the two paths produced different page counts+ for the same repository.+ """+ groups = group_files(+ inputs.production_files, layer_of_file=inputs.layer_of_file, params=params+ )+ named = [+ NamedGroup(+ group=g,+ title=deterministic_title(g, inputs.layer_labels.get(g.dominant_layer, "")),+ scope=deterministic_scope(g),+ section=inputs.layer_labels.get(g.dominant_layer) or "Reference",+ order=i,+ fallback=True,+ )+ for i, g in enumerate(groups)+ ]+ _disambiguate_titles(named)+ outline = _build_outline(named)+ _merge_thin_sections(outline)+ outline.naming_mode = "deterministic"+ return outline, groups+++_REPAIR_INSTRUCTIONS = """\+These page titles from the outline you just produced need fixing. Nothing else \+about the outline is changing — do not restructure it, and do not rename any \+page that is not listed here.++Return a corrected title AND a scope sentence for exactly these group ids.++The title is {min_words} to {max_words} words, names the capability rather than \+the directory or the layer, and is unique across the whole outline.++The scope is a full English sentence saying what the page covers and what it \+deliberately does not. It is never a path: do not echo the `dir` value back.++Titles already in use elsewhere in the outline (do not reuse any of these):+{taken}++GROUPS TO RENAME:+{failures}++Return ONLY this JSON:+{{"names": {{"g07": {{"title": "...", "scope": "..."}}, ...}}}}+"""+++def _ground_pages(outline: ConceptOutline) -> list[str]:+ """Strip unsupported citations from every page's scope. Returns the tokens."""+ from ..onboarding.grounding import check_grounding++ ungrounded: list[str] = []+ for page in outline.pages:+ cleaned, bad = check_grounding(page.scope, page.members)+ page.scope = cleaned+ ungrounded.extend(bad)+ return ungrounded+++def _usable_scope(scope: str, target_path: str) -> bool:+ """Whether a returned scope is a sentence rather than an echoed path.++ Asked for a title and a scope for a group described by its directory, a+ model will sometimes hand the directory back in the scope field. That+ reads as a broken page rather than a terse one, so it is rejected and the+ deterministic sentence stands.+ """+ text = scope.strip()+ if len(text.split()) < 4:+ return False+ if text.rstrip("/") == target_path.rstrip("/"):+ return False+ # A lone path has no spaces before its first separator run; a sentence does.+ return "/" not in text.split(" ", 1)[0] or " " in text.strip()+++def _repair_targets(outline: ConceptOutline, report: OutlineReport) -> set[str]:+ """The page titles worth a second call. Everything else is left alone.++ Three kinds qualify: titles the model repeated, titles that are just the+ directory respelled, and groups the model never named at all. The last is+ the common one on a large repository — a request to name seventy groups+ reliably loses a few near the end of the list — and it is also the+ cheapest to fix, because the second call carries only the stragglers.+ """+ bad = set(report.duplicate_titles)+ targets = {p.title for p in outline.pages if p.title.lower() in bad}+ targets |= set(report.bare_directory_titles)+ targets |= {t.rsplit(" (", 1)[0] for t in report.bad_length_titles}+ targets |= {p.title for p in outline.pages if not p.named_by_model}+ return targets+++async def plan_outline(+ inputs: PlannerInputs,+ *,+ provider: Any | None = None,+ deterministic: bool = False,+ params: GroupingParams | None = None,+ repair: bool = True,+) -> tuple[ConceptOutline, OutlineReport]:+ """Produce a validated outline for *inputs*.++ Falls back to :func:`plan_deterministic` whenever there is no provider, the+ run is in deterministic mode, or the model's response cannot be used. A+ failed naming call costs the wiki its titles, never its structure.+ """+ all_files = set(inputs.production_files)+ resolved = params or params_for(len(all_files))++ if deterministic or provider is None:+ outline, _groups = plan_deterministic(inputs, params=params)+ report = validate_outline(+ outline,+ all_files=all_files,+ test_files=inputs.test_files,+ max_files_per_page=resolved.max_files,+ )+ return outline, report++ groups = group_files(+ inputs.production_files, layer_of_file=inputs.layer_of_file, params=params+ )+ payload, index = build_payload(+ groups,+ layer_labels=inputs.layer_labels,+ summaries=inputs.summaries,+ entry_points=inputs.entry_points,+ )+ payload["repo"] = inputs.repo_name++ terms: list[str] = []+ bound: dict[str, str] = {}+ if inputs.repo_root is not None:+ terms = extract_terms(inputs.repo_root)+ bound = bind_terms(terms, index)+ for gid, term in bound.items():+ for entry in payload["groups"]:+ if entry["id"] == gid:+ entry["suggested_name"] = term+ break+ logger.info(+ "concept_outline_vocabulary",+ terms_found=len(terms),+ terms_bound=len(bound),+ )++ sections_lo = min(MIN_SECTIONS, max(2, len(groups) // 8))+ sections_hi = max(sections_lo + 1, min(MAX_SECTIONS, len(groups) // 3 or 2))+ instructions = NAMING_INSTRUCTIONS.format(+ repo=inputs.repo_name,+ min_words=2,+ max_words=7,+ min_sections=sections_lo,+ max_sections=sections_hi,+ )+ body = json.dumps(payload, separators=(",", ":"))++ data: dict[str, Any] = {}+ try:+ with _billed_as(provider, COST_OPERATION):+ response = await provider.generate(+ system_prompt=SYSTEM_PROMPT,+ user_prompt=instructions + body,+ max_tokens=16000,+ temperature=0.2,+ )+ data = parse_json_object(getattr(response, "content", "") or "")+ except Exception as exc:+ logger.warning("concept_outline_naming_failed", error=str(exc))++ # Decoding sits inside the guard too. It used to sit outside, which meant a+ # response that parsed as JSON but held the wrong shape reached the decoder+ # and could take the run down — the one failure mode this design exists to+ # rule out. Decoding is defensive in its own right; this is the second line.+ try:+ named, invented_ids = decode_response(+ data, index, layer_labels=inputs.layer_labels+ )+ except Exception as exc:+ logger.warning("concept_outline_decode_failed", error=str(exc))+ named, invented_ids = decode_response({}, index, layer_labels=inputs.layer_labels)+ ungrounded = ground_scopes(named)+ if invented_ids:+ logger.warning("concept_outline_invented_group_ids", ids=invented_ids[:10])++ outline = _build_outline(named)+ _merge_thin_sections(outline)+ outline.naming_mode = "llm" if any(not n.fallback for n in named) else "deterministic"+ outline.vocabulary = {bound[gid]: index[gid].target_path for gid in bound}++ report = validate_outline(+ outline,+ all_files=all_files,+ test_files=inputs.test_files,+ max_files_per_page=resolved.max_files,+ )+ report.invented_paths = sorted(set(report.invented_paths) | set(ungrounded))++ if repair:+ targets = _repair_targets(outline, report)+ if targets:+ await _repair_titles(outline, index, targets, provider=provider)+ # Repair writes new prose, so it has to face the same citation+ # check the first pass did. Grounding after the last write rather+ # than after the first is the difference between checking the page+ # and checking a draft of it.+ ungrounded.extend(_ground_pages(outline))+ report = validate_outline(+ outline,+ all_files=all_files,+ test_files=inputs.test_files,+ max_files_per_page=resolved.max_files,+ )+ report.invented_paths = sorted(set(report.invented_paths) | set(ungrounded))++ logger.info(+ "concept_outline_planned",+ pages=report.page_count,+ sections=report.section_count,+ coverage=round(report.coverage, 4),+ invented=len(report.invented_paths),+ naming_mode=outline.naming_mode,+ )+ return outline, report+++async def _repair_titles(+ outline: ConceptOutline,+ index: dict[str, ConceptGroup],+ targets: set[str],+ *,+ provider: Any,+) -> None:+ """Re-ask for just the failing titles, then apply only what improved.++ A repair that made things worse is discarded: the replacement has to be+ non-empty, correctly sized and not already in use, or the original stands.+ That keeps a bad second call from being worse than no second call.+ """+ gid_of = {g.structural_key: gid for gid, g in index.items()}+ failing = [p for p in outline.pages if p.title in targets]+ if not failing:+ return+ taken = sorted({p.title for p in outline.pages if p.title not in targets})++ lines = []+ for page in failing:+ gid = gid_of.get(page.structural_key, "")+ names = sorted(m.rsplit("/", 1)[-1] for m in page.members)[:6]+ lines.append(+ json.dumps(+ {+ "id": gid,+ "current_title": page.title,+ "dir": page.target_path,+ "files": len(page.members),+ "names": names,+ },+ separators=(",", ":"),+ )+ )++ prompt = _REPAIR_INSTRUCTIONS.format(+ min_words=2,+ max_words=7,+ taken="\n".join(f"- {t}" for t in taken) or "(none)",+ failures="\n".join(lines),+ )+ try:+ with _billed_as(provider, COST_OPERATION):+ response = await provider.generate(+ system_prompt=SYSTEM_PROMPT,+ user_prompt=prompt,+ max_tokens=2000,+ temperature=0.2,+ )+ data = parse_json_object(getattr(response, "content", "") or "")+ except Exception as exc:+ logger.warning("concept_outline_repair_failed", error=str(exc))+ return++ names = data.get("names")+ if not isinstance(names, dict):+ return+ in_use = {p.title.lower() for p in outline.pages}+ fixed = 0+ for page in failing:+ gid = gid_of.get(page.structural_key, "")+ entry = names.get(gid)+ if not isinstance(entry, dict):+ continue+ title = str(entry.get("title") or "").strip()+ words = len(title.split())+ if not title or not (2 <= words <= 7) or title.lower() in in_use:+ continue+ in_use.discard(page.title.lower())+ in_use.add(title.lower())+ page.title = title+ page.named_by_model = True+ scope = str(entry.get("scope") or "").strip()+ if _usable_scope(scope, page.target_path):+ page.scope = scope+ fixed += 1+ logger.info("concept_outline_repaired", requested=len(failing), applied=fixed)diff --git a/packages/core/src/repowise/core/generation/concept_tree/validation.py b/packages/core/src/repowise/core/generation/concept_tree/validation.pynew file mode 100644index 000000000..d95b50ae1--- /dev/null+++ b/packages/core/src/repowise/core/generation/concept_tree/validation.py@@ -0,0 +1,112 @@+"""Check the outline against the index, and count what is wrong.++Not a theoretical gap. Measured on this codebase, a planner run invented one+directory and a more heavily constrained one invented nine, among them+``core/generation/cost_tracker.py`` and ``web/src/lib/workspace-mode.ts`` —+paths that read as if they belong here and do not exist. An unchecked citation+like that reaches the reader as an authoritative reference, which is worse than+saying nothing.++Grouping is deterministic, so coverage and double-claiming ought to be true by+construction. They are still asserted here, because "ought to be" is how the+last two placement bugs got in, and because the repair pass can hand back a+tree the model touched.+"""++from __future__ import annotations++from collections import Counter++from ..models import compute_page_id # noqa: F401 (documents the id contract)+from .models import ConceptOutline, OutlineReport++MIN_TITLE_WORDS = 2+MAX_TITLE_WORDS = 7+++def _is_bare_directory(title: str, dirs: list[str]) -> bool:+ """Whether the title is just the directory it covers, respelled.++ ``core/ingestion`` and ``Ingestion`` are both failures of the same kind:+ they tell a reader where the code is, which they can already see, instead+ of what it does.+ """+ flat = title.strip().lower().replace("-", " ").replace("_", " ").replace("/", " ")+ for d in dirs:+ segs = [s for s in d.replace("-", " ").replace("_", " ").split("/") if s]+ if not segs:+ continue+ if flat == " ".join(segs).lower() or flat == segs[-1].lower():+ return True+ return False+++def validate_outline(+ outline: ConceptOutline,+ *,+ all_files: set[str],+ test_files: set[str] | None = None,+ max_files_per_page: int,+) -> OutlineReport:+ """Measure *outline* against the file set it claims to cover.++ *all_files* is the production file set the grouper was given — the ground+ truth for both coverage and fabrication. A member the outline names that is+ not in it is an invented path, and the distinction between "invented" and+ "unclaimed" matters: they have different causes and the probes moved them+ in opposite directions.+ """+ report = OutlineReport()+ pages = outline.pages+ report.page_count = len(pages)+ report.section_count = len(outline.sections)+ report.total_files = len(all_files)++ claimed: Counter[str] = Counter()+ for page in pages:+ if not page.members:+ report.empty_pages.append(page.title or page.target_path)+ for member in page.members:+ claimed[member] += 1++ known = set(claimed)+ report.invented_paths = sorted(known - all_files)+ report.covered_files = len(known & all_files)+ report.unclaimed_files = sorted(all_files - known)+ report.double_claimed_files = sorted(p for p, n in claimed.items() if n > 1)++ if test_files:+ report.test_paths_included = sorted(known & test_files)++ for page in pages:+ if len(page.members) > max_files_per_page and not page.group.oversized:+ report.oversized_pages.append(f"{page.title} ({len(page.members)})")++ for section in outline.sections:+ if len(section.pages) == 1:+ report.single_child_sections.append(section.title)++ titles = [p.title.strip() for p in pages]+ report.duplicate_titles = sorted(+ t for t, n in Counter(t.lower() for t in titles).items() if n > 1+ )+ for page in pages:+ words = len(page.title.split())+ if words < MIN_TITLE_WORDS or words > MAX_TITLE_WORDS:+ report.bad_length_titles.append(f"{page.title} ({words}w)")+ if _is_bare_directory(page.title, page.group.dirs):+ report.bare_directory_titles.append(page.title)+ report.title_word_avg = (+ sum(len(t.split()) for t in titles) / len(titles) if titles else 0.0+ )++ keys = Counter(p.structural_key for p in pages)+ report.duplicate_structural_keys = sorted(k for k, n in keys.items() if n > 1)++ # The page id is derived from the target path alone, so a repeat here is a+ # row collision rather than a cosmetic one: the second page would overwrite+ # the first on persist and the wiki would quietly lose a page.+ targets = Counter(p.target_path for p in pages)+ report.duplicate_target_paths = sorted(t for t, n in targets.items() if n > 1)++ return reportdiff --git a/packages/core/src/repowise/core/generation/concept_tree/vocabulary.py b/packages/core/src/repowise/core/generation/concept_tree/vocabulary.pynew file mode 100644index 000000000..71f748a3f--- /dev/null+++ b/packages/core/src/repowise/core/generation/concept_tree/vocabulary.py@@ -0,0 +1,245 @@+"""The project's own words for its own subsystems, bound to real groups.++A wiki that reuses the repository's terminology reads as one document instead+of as a pile of pages. The naive way to get that is to hand the planner the+README, and it is measured to be a bad idea: nine thousand characters of+positioning prose alongside a structural task collapsed the outline from 23+pages covering 87.5% of the code to three pages covering 5.4%, and it invented+a docs path that matched the project's marketing language rather than its+directory tree.++So terminology enters through a keyhole. Terms are lifted from the repository's+docs here, and a term only survives if it can be **bound to a group that+structure already produced**. A feature that is marketed but not built has no+group to bind to and therefore gets no page. Structure leads and vocabulary+decorates, which is the only arrangement where a thin, stale or absent README —+the common case in the wild — costs nothing structural.++Extraction is deterministic rather than another model call. It has to run on+the keyless path, it has to give the same answer twice, and the job is small+enough that a model adds cost and variance without adding much.+"""++from __future__ import annotations++import re+from pathlib import Path++from .grouping import ConceptGroup++#: Documents worth mining, relative to the repository root. A doc that names a+#: subsystem in a heading is a much stronger signal than one that mentions it+#: in a sentence, so headings are what this reads.+#:+#: These are root-anchored on purpose. Glob patterns containing ``/`` match a+#: path *tail*, so a pattern like ``docs/*.md`` also matches+#: ``some/vendored/thing/docs/notes.md`` — which on this repository pulled+#: planning notes and marketing copy into the glossary and bound terms like+#: "Headline facts updated" to source directories.+DOC_FILES: tuple[str, ...] = (+ "README.md",+ "README.rst",+ "ARCHITECTURE.md",+ "CONTRIBUTING.md",+)++#: Directories, relative to the root, whose top level is scanned for docs.+DOC_DIRS: tuple[str, ...] = ("docs", "doc", "adr", "docs/adr", "docs/decisions")++_MAX_DOC_BYTES = 200_000+_MAX_DOCS = 60++# A markdown or rst heading. The term is the heading text.+_HEADING = re.compile(r"^\s{0,3}#{1,4}\s+(.+?)\s*#*\s*$", re.MULTILINE)+# Bolded lead-ins are how glossaries are usually written: "**Blast radius** — ".+_BOLD_TERM = re.compile(r"\*\*([A-Z][^*\n]{2,40})\*\*")++_STOPWORDS = frozenset(+ {+ "the", "a", "an", "and", "or", "of", "for", "to", "in", "on", "with",+ "how", "what", "why", "when", "is", "are", "it", "this", "that", "you",+ "your", "we", "our", "using", "use", "usage", "getting", "started",+ "installation", "install", "quickstart", "quick", "start", "license",+ "contributing", "changelog", "table", "contents", "overview",+ "introduction", "intro", "example", "examples", "faq", "notes",+ "requirements", "setup", "configuration", "config", "options", "api",+ "reference", "guide", "tutorial", "docs", "documentation", "features",+ "roadmap", "credits", "acknowledgements", "badges", "support",+ }+)++_MIN_TERM_WORDS = 1+_MAX_TERM_WORDS = 4++# A heading that is a sentence is describing something, not naming it.+# "Local paths are masked" and "Headline facts updated 2026-06-22" are both+# real headings from this repository's own docs, and neither is a subsystem.+_SENTENCE_VERBS = frozenset(+ {+ "is", "are", "was", "were", "has", "have", "had", "do", "does", "did",+ "can", "will", "would", "should", "must", "may", "updated", "added",+ "removed", "fixed", "works", "means", "needs", "gets", "goes", "runs",+ "returns", "uses", "supports", "requires", "shows", "makes", "keeps",+ }+)+++def _normalise(term: str) -> str:+ term = re.sub(r"`([^`]*)`", r"\1", term)+ term = re.sub(r"\[([^\]]*)\]\([^)]*\)", r"\1", term) # markdown links+ term = re.sub(r"[^\w\s/&+-]", " ", term)+ term = re.sub(r"\s+", " ", term).strip()+ return term+++def _is_useful(term: str) -> bool:+ words = term.split()+ if not (_MIN_TERM_WORDS <= len(words) <= _MAX_TERM_WORDS):+ return False+ if all(w.lower() in _STOPWORDS for w in words):+ return False+ # A heading that is entirely boilerplate ("Getting Started") says nothing+ # about this repository in particular.+ meaningful = [w for w in words if w.lower() not in _STOPWORDS]+ if not meaningful:+ return False+ if not any(len(w) > 2 for w in meaningful):+ return False+ # Sentences and dated notes are prose, not names.+ if any(w.lower() in _SENTENCE_VERBS for w in words):+ return False+ if any(any(ch.isdigit() for ch in w) for w in words):+ return False+ # A name starts like a name. All-lowercase multi-word headings are almost+ # always prose fragments.+ return len(words) == 1 or words[0][:1].isupper()+++def extract_terms(repo_root: Path, *, max_terms: int = 200) -> list[str]:+ """Lift candidate subsystem names from the repository's own documentation.++ Returns terms in document order with duplicates removed. Never raises: a+ repository with no docs returns an empty list, which is a supported and+ common outcome, not a degraded one.+ """+ seen: set[str] = set()+ terms: list[str] = []+ files: list[Path] = []+ try:+ for name in DOC_FILES:+ candidate = repo_root / name+ if candidate.is_file():+ files.append(candidate)+ for rel_dir in DOC_DIRS:+ directory = repo_root / rel_dir+ if not directory.is_dir():+ continue+ for candidate in sorted(directory.glob("*.md")):+ if candidate.is_file() and candidate not in files:+ files.append(candidate)+ if len(files) >= _MAX_DOCS:+ break+ if len(files) >= _MAX_DOCS:+ break+ except OSError:+ return []++ for path in files:+ try:+ text = path.read_text(encoding="utf-8", errors="replace")[:_MAX_DOC_BYTES]+ except OSError:+ continue+ for match in list(_HEADING.finditer(text)) + list(_BOLD_TERM.finditer(text)):+ term = _normalise(match.group(1))+ if not _is_useful(term):+ continue+ key = term.lower()+ if key in seen:+ continue+ seen.add(key)+ terms.append(term)+ if len(terms) >= max_terms:+ return terms+ return terms+++# ---------------------------------------------------------------------------+# Binding+# ---------------------------------------------------------------------------+++def _tokens(text: str) -> set[str]:+ parts = re.split(r"[^A-Za-z0-9]+", text)+ out: set[str] = set()+ for part in parts:+ if not part:+ continue+ for piece in re.findall(r"[A-Z]+(?![a-z])|[A-Z][a-z]*|[a-z]+|\d+", part) or [part]:+ low = piece.lower()+ if len(low) > 2 and low not in _STOPWORDS:+ out.add(low)+ # A trailing plural should still match its directory.+ if low.endswith("s"):+ out.add(low[:-1])+ return out+++def _group_tokens(group: ConceptGroup) -> tuple[set[str], set[str]]:+ """The group's words, split into structural ones and incidental ones.++ Directory names are what the codebase calls a *place*; filenames are what+ it calls the things in that place. A term matching only a filename is+ usually a coincidence — every group contains a file with "client" or+ "store" in its name — so the two are scored differently.+ """+ structural = _tokens(group.target_path)+ for d in group.dirs:+ structural |= _tokens(d)+ incidental: set[str] = set()+ for member in group.members:+ incidental |= _tokens(member.rsplit("/", 1)[-1].rsplit(".", 1)[0])+ return structural, incidental - structural+++def bind_terms(+ terms: list[str],+ groups: dict[str, ConceptGroup],+) -> dict[str, str]:+ """Bind each term to at most one group, and each group to at most one term.++ A term binds only when the codebase spells **all** of it and at least part+ of it in a directory name. Both halves matter. Requiring every word keeps+ "Change Risk" off a group that merely contains the word "change";+ requiring a directory hit keeps a term from attaching to whichever group+ happens to hold a file with a matching name. A marketed feature with no+ code cluster satisfies neither and therefore never becomes a page title,+ which is the entire point of binding rather than injecting.++ Returns ``{group_id: term}``. Ties break on how much of the match was+ structural, then the term's document order, then the group id, so two runs+ over an unchanged repository bind identically.+ """+ token_cache = {gid: _group_tokens(g) for gid, g in groups.items()}+ scored: list[tuple[int, int, int, str, str]] = []+ for order, term in enumerate(terms):+ wanted = _tokens(term)+ if not wanted:+ continue+ for gid, (structural, incidental) in token_cache.items():+ in_struct = wanted & structural+ if not in_struct:+ continue+ if wanted - structural - incidental:+ # Some word of the term is absent from this group entirely.+ continue+ scored.append((-len(in_struct), -len(wanted), order, gid, term))++ scored.sort()+ bound: dict[str, str] = {}+ used_terms: set[str] = set()+ for _s, _w, _order, gid, term in scored:+ if gid in bound or term.lower() in used_terms:+ continue+ bound[gid] = term+ used_terms.add(term.lower())+ return bounddiff --git a/packages/core/src/repowise/core/generation/page_generator/orchestrate.py b/packages/core/src/repowise/core/generation/page_generator/orchestrate.pyindex 6f15e2af0..dfff264c3 100644--- a/packages/core/src/repowise/core/generation/page_generator/orchestrate.py+++ b/packages/core/src/repowise/core/generation/page_generator/orchestrate.py@@ -750,10 +750,20 @@ def _stamp_structural_keys(pages: list[GeneratedPage]) -> None:A page keyed on a real file path has no identity separate from that path,so it is left unset rather than given a copy of the path.++ A key already set by whatever produced the page is kept. The concept-tree+ planner computes its groups' identities as part of deciding what the+ groups *are*, and hashes exactly the same member list this would; letting+ the stamp recompute it would give two places that must agree about page+ identity, which is the arrangement D2 exists to avoid. It also lets the+ key say which algorithm produced the page, so a wiki holding both the old+ per-directory modules and the new concept groups can tell them apart."""for page in pages:if page.page_type not in STRUCTURALLY_KEYED_PAGE_TYPES:continue+ if page.structural_key:+ continueprefix = _MEMBER_KEYED_PREFIX.get(page.page_type)members = page.metadata.get("file_paths") or []members = [m for m in members if isinstance(m, str)]diff --git a/packages/core/src/repowise/core/pipeline/persist.py b/packages/core/src/repowise/core/pipeline/persist.pyindex bb2bb4ab7..af79c25f3 100644--- a/packages/core/src/repowise/core/pipeline/persist.py+++ b/packages/core/src/repowise/core/pipeline/persist.py@@ -598,6 +598,100 @@ async def _sweep_stale_generated_pages(return swept+async def sweep_superseded_generated_pages(+ session: Any,+ repo_id: str,+ generated_pages: list[Any] | None,+) -> list[str]:+ """Retire structurally-keyed rows whose coverage a scoped run just took over.++ :func:`_sweep_stale_generated_pages` cannot be used on the update path. It+ deletes every row of a page type the run did not reproduce, which is right+ for a full index and catastrophic for a scoped one: regenerating a single+ page would wipe the other fifty.++ That left a real gap. A structurally-keyed page's id is its+ ``target_path``, and a page that groups files can legitimately change which+ directory names it when its membership changes. ``repowise update`` goes+ through ``scoped_generation`` and never swept, so the regenerated page was+ written under its new id and the old row stayed behind as a duplicate until+ somebody ran a full index. Harmless while module and cycle pages were never+ regenerated incrementally, and not harmless once concept pages are.++ The rule here is narrower and does not need to know what the caller asked+ for: a prior row is superseded when **every file it covered is now covered+ by pages this run produced**, of the same type. That is exactly the+ condition under which the old row documents nothing that is not documented+ elsewhere. A page that failed to generate keeps files no produced page+ claims, so it survives — which matters, because deleting a page because its+ generation errored would be a worse bug than the one this fixes.++ Returns the swept page ids so the caller can drop them from FTS after the+ session closes.+ """+ from sqlalchemy import delete, select++ from repowise.core.persistence.models import Page, PageVersion++ produced_ids: dict[str, set[str]] = {}+ covered: dict[str, set[str]] = {}+ for page in generated_pages or []:+ page_type = page.page_type+ if page_type not in _SWEPT_GENERATED_PAGE_TYPES:+ continue+ produced_ids.setdefault(page_type, set()).add(page.page_id)+ metadata = getattr(page, "metadata", None) or {}+ members = metadata.get("file_paths") or metadata.get("files") or []+ covered.setdefault(page_type, set()).update(+ m for m in members if isinstance(m, str)+ )++ swept: list[str] = []+ for page_type, current in produced_ids.items():+ union = covered.get(page_type) or set()+ if not union:+ # Nothing to compare against: a run that recorded no membership+ # cannot prove it superseded anything, so it retires nothing.+ continue+ rows = (+ await session.execute(+ select(Page.id, Page.metadata_json).where(+ Page.repository_id == repo_id, Page.page_type == page_type+ )+ )+ ).all()+ stale: list[str] = []+ for page_id, raw in rows:+ if page_id in current:+ continue+ try:+ metadata = json.loads(raw) if isinstance(raw, str) else (raw or {})+ except (TypeError, ValueError):+ continue+ if not isinstance(metadata, dict):+ continue+ members = metadata.get("file_paths") or metadata.get("files") or []+ members = {m for m in members if isinstance(m, str)}+ # No recorded membership means no proof of supersession. Those rows+ # are the pre-membership ones (backlog B20) and they are left for a+ # full index to retire.+ if members and members <= union:+ stale.append(page_id)+ for i in range(0, len(stale), _PRUNE_CHUNK):+ batch = stale[i : i + _PRUNE_CHUNK]+ await session.execute(delete(PageVersion).where(PageVersion.page_id.in_(batch)))+ await session.execute(+ delete(Page).where(Page.repository_id == repo_id, Page.id.in_(batch))+ )+ swept.extend(stale)++ if swept:+ logger.info(+ "superseded_generated_pages_swept", repo_id=repo_id, count=len(swept)+ )+ return swept++async def persist_ingestion(result: Any, session: Any, repo_id: str) -> int:"""Persist ingestion-phase outputs: graph nodes/edges, external systems,symbols, and the per-file security scan.diff --git a/packages/core/src/repowise/core/pipeline/scoped_generation.py b/packages/core/src/repowise/core/pipeline/scoped_generation.pyindex 118be7bff..232fe859b 100644--- a/packages/core/src/repowise/core/pipeline/scoped_generation.py+++ b/packages/core/src/repowise/core/pipeline/scoped_generation.py@@ -26,7 +26,7 @@ code, so hosted reuses it as-is.from __future__ import annotationsimport asyncio-from dataclasses import dataclass+from dataclasses import dataclass, fieldfrom pathlib import Pathfrom typing import Any@@ -67,6 +67,10 @@ class ScopedGenerationResult:generated_pages: list[Any]marked_stale: int+ #: Rows retired because pages this run produced now cover everything they+ #: covered. Reported so an update can say it removed a duplicate rather+ #: than doing it silently.+ swept_page_ids: list[str] = field(default_factory=list)def load_kg_context(repo_path: Path) -> Any:@@ -202,7 +206,10 @@ async def execute_scoped_generation(from repowise.core.persistence import get_session, upsert_pages_from_generatedfrom repowise.core.persistence.crud import backfill_related_pagesfrom repowise.core.pipeline import run_generation- from repowise.core.pipeline.persist import mark_page_ids_stale+ from repowise.core.pipeline.persist import (+ mark_page_ids_stale,+ sweep_superseded_generated_pages,+ )# Bill this run's LLM calls to the caller's tracker. run_generation reads the# passed tracker; some providers additionally consult ``_cost_tracker``.@@ -229,11 +236,22 @@ async def execute_scoped_generation(await cost_tracker.flush()marked_stale = 0+ swept_page_ids: list[str] = []async with get_session(session_factory) as session:await upsert_pages_from_generated(session, generated_pages, repo_id)+ # A page whose identity is its members can change which directory names+ # it when those members change, and the new row is written under a new+ # id. Without this the old row survives as a duplicate until somebody+ # runs a full index, because the full sweep cannot run here — it would+ # delete every page of a type this run did not reproduce, which on a+ # scoped run is nearly all of them.+ swept_page_ids = await sweep_superseded_generated_pages(+ session, repo_id, generated_pages+ )# A scoped run holds only the pages it was asked for, so it cannot# work out where they sit. Placement comes from the store, which has# the whole set; without this the pages it touched lose their place.+ # After the sweep, so a retired row cannot be given a place.from .page_tree_sync import rebuild_page_treeawait rebuild_page_tree(session, repo_id)@@ -260,8 +278,22 @@ async def execute_scoped_generation(except Exception as exc:logger.debug("related_pages_backfill_skipped", error=str(exc))+ # Same ordering rule the full-index path follows: the vector delete+ # goes before the SQL commit, because the store is a separate engine+ # and an interrupted run then self-heals rather than leaving an+ # embedding for a row that no longer exists.+ if swept_page_ids and vector_store is not None:+ try:+ await vector_store.delete_many(swept_page_ids)+ except Exception as exc:+ logger.debug("vector_sweep_skipped", error=str(exc))++ # FTS lives in the same SQLite file as the session, so it is only safe to+ # touch after that session has closed.if fts is not None:try:+ if swept_page_ids:+ await fts.delete_many(swept_page_ids)for page in generated_pages:await fts.index(page.page_id, page.title, page.content)except Exception as exc:@@ -270,4 +302,5 @@ async def execute_scoped_generation(return ScopedGenerationResult(generated_pages=generated_pages,marked_stale=marked_stale,+ swept_page_ids=swept_page_ids,)diff --git a/tests/unit/generation/test_concept_grouping.py b/tests/unit/generation/test_concept_grouping.pynew file mode 100644index 000000000..1720496e8--- /dev/null+++ b/tests/unit/generation/test_concept_grouping.py@@ -0,0 +1,347 @@+"""The deterministic partition: totality, stability, and bounded page size.++Grouping is the half of the outline planner that must not vary, because page+identity hashes the member list. A grouping that reshuffled between two runs of+the same commit would remint every page id, strand the vectors behind them, and+leave the old rows as duplicates. So the properties asserted here are+correctness properties, not tidiness ones.++Every fixture that depends on ordering is built so the structure disagrees with+the alphabet. A fixture whose spine and alphabet agree cannot tell a test that+reads structure from one that sorts.+"""++from __future__ import annotations++import os++from repowise.core.generation.concept_tree.grouping import (+ GroupingParams,+ group_files,+ params_for,+)++# Two sibling subsystems plus a deep one. Named so alphabetical order+# (alpha, beta, zulu) is NOT the order the tree nests them in.+TINY = GroupingParams(min_files=3, max_files=6)+++def _repo(spec: dict[str, int]) -> list[str]:+ """Build a file list: ``{directory: count}``. ``""`` is the repo root."""+ out: list[str] = []+ for directory, count in spec.items():+ prefix = f"{directory}/" if directory else ""+ out.extend(f"{prefix}f{i:02d}.py" for i in range(count))+ return out+++SPEC = {+ # Root-level files are the common case, not an edge case: setup.py,+ # manage.py, index.js, vite.config.ts all live here. A fixture without them+ # cannot see that the root's directory path is the empty string, which is+ # a real directory and was once mistaken for a missing one.+ "": 3,+ "src/zulu": 5,+ "src/alpha": 5,+ "src/beta/deep": 5,+ "src/beta/other": 4,+}+++class TestTotality:+ def test_every_file_lands_in_exactly_one_group(self):+ files = _repo(SPEC)+ groups = group_files(files, params=TINY)+ claimed = [m for g in groups for m in g.members]+ assert sorted(claimed) == sorted(files)+ assert len(claimed) == len(set(claimed))++ def test_no_group_is_empty(self):+ """An empty group is a page about nothing, and a hard validator failure."""+ groups = group_files(_repo(SPEC), params=TINY)+ assert groups+ assert all(g.members for g in groups)++ def test_empty_input_is_harmless(self):+ assert group_files([]) == []++ def test_a_repo_smaller_than_the_ceiling_is_one_group(self):+ groups = group_files(_repo({"src": 4}), params=TINY)+ assert len(groups) == 1+ assert groups[0].file_count == 4+++class TestStability:+ def test_two_processes_produce_identical_structural_keys(self):+ """Determinism that survives a fresh interpreter, not just a fresh call.++ Calling a pure function twice in one process cannot detect the failure+ that actually matters here. Python randomises string hashing per+ process, so a grouping that leaked set or dict iteration order would+ agree with itself all day and disagree between two indexing runs —+ reminting every page id. These two subprocesses are given different+ hash seeds on purpose.+ """+ import json+ import subprocess+ import sys+ import textwrap++ script = textwrap.dedent(+ """+ import json+ from repowise.core.generation.concept_tree.grouping import (+ GroupingParams, group_files,+ )+ spec = json.loads(input())+ files = []+ for d, n in spec.items():+ p = f"{d}/" if d else ""+ files.extend(f"{p}f{i:02d}.py" for i in range(n))+ groups = group_files(files, params=GroupingParams(3, 6))+ print(json.dumps([[g.target_path, g.structural_key] for g in groups]))+ """+ )++ def run(seed: str) -> list[list[str]]:+ out = subprocess.run(+ [sys.executable, "-c", script],+ input=json.dumps(SPEC),+ capture_output=True,+ text=True,+ env={**os.environ, "PYTHONHASHSEED": seed},+ check=True,+ )+ return json.loads(out.stdout.strip().splitlines()[-1])++ assert run("0") == run("12345")++ def test_input_order_does_not_change_the_grouping(self):+ files = _repo(SPEC)+ forward = [g.structural_key for g in group_files(files, params=TINY)]+ reverse = [+ g.structural_key for g in group_files(list(reversed(files)), params=TINY)+ ]+ assert forward == reverse++ def test_one_added_file_does_not_reshuffle_the_tree(self):+ """The property the whole design exists for.++ A commit that adds one file must change the identity of the page that+ gained it and of nothing else. If a global parameter were derived from+ the file count, every key would move on every commit.+ """+ files = _repo(SPEC)+ before = {g.structural_key for g in group_files(files, params=TINY)}+ after = {+ g.structural_key+ for g in group_files([*files, "src/alpha/new_thing.py"], params=TINY)+ }+ assert len(before - after) == 1, "more than one group changed identity"+ assert len(after - before) == 1++ def test_the_size_ladder_is_flat_across_a_whole_band(self):+ """No parameter may be computed from the file count.++ A ceiling of ``total / target`` would move on almost every commit, and+ because the ceiling decides the partition, every page in the wiki+ would be reminted. Asserting two adjacent counts is not enough to catch+ that — integer division agrees with itself most of the time — so this+ asserts the whole band is flat.+ """+ band = [params_for(n) for n in range(1250, 2350, 7)]+ assert len(set(band)) == 1, "the ceiling varies with the file count"++ def test_the_ladder_does_step_between_bands(self):+ """The flatness above must not be flatness everywhere."""+ assert params_for(500) != params_for(5000)++ def test_an_unrelated_group_keeps_its_identity_when_another_changes(self):+ files = _repo(SPEC)+ base = {g.target_path: g.structural_key for g in group_files(files, params=TINY)}+ after = {+ g.target_path: g.structural_key+ for g in group_files([*files, "src/alpha/new_thing.py"], params=TINY)+ }+ untouched = [t for t in base if "alpha" not in t and t in after]+ assert untouched, "fixture proves nothing if no group is untouched"+ for target in untouched:+ assert base[target] == after[target]+++class TestIdentity:+ def test_target_paths_are_unique(self):+ """Two groups sharing a target path would share a page id."""+ groups = group_files(_repo(SPEC), params=TINY)+ targets = [g.target_path for g in groups]+ assert len(targets) == len(set(targets))++ def test_two_runs_of_siblings_under_one_parent_do_not_collide(self):+ """The case that actually produced a page-id collision.++ Four sibling directories partition into two groups of two. Neither+ group holds a file sitting directly in the shared parent, so both+ compute that parent as their shallowest common directory — and the+ page id is the target path, so the second page would have overwritten+ the first on persist. Verified to fail when target assignment is+ removed.+ """+ params = GroupingParams(min_files=3, max_files=9)+ files = _repo({"src/x/a": 4, "src/x/b": 4, "src/x/c": 4, "src/x/d": 4})+ groups = group_files(files, params=params)+ assert len(groups) > 1, "fixture must produce more than one group"+ targets = [g.target_path for g in groups]+ assert len(targets) == len(set(targets)), targets++ def test_target_path_stays_a_real_directory(self):+ files = _repo(SPEC)+ directories = {f.rsplit("/", 1)[0] if "/" in f else "" for f in files}+ for group in group_files(files, params=TINY):+ assert group.target_path in directories, (+ f"{group.target_path!r} is not a directory in this repository"+ )++ def test_a_group_of_root_level_files_is_targeted_at_the_root(self):+ """The repository root is a directory whose path is the empty string.++ Treating that as "no directory found" sent the group to the+ identity-hash fallback, which then became its page id, its title+ ("D 73789 F 001 Bd") and its scope sentence.+ """+ files = _repo({"": 3, "src/pkg": 10})+ groups = group_files(files, params=GroupingParams(min_files=2, max_files=4))+ root = next(g for g in groups if "setup" in " ".join(g.members) or "f00.py" in g.members)+ assert root.target_path == ""+ assert "#" not in root.target_path+ assert all("#" not in g.target_path for g in groups)++ def test_structural_key_follows_members_not_target(self):+ a = group_files(_repo({"src/one": 4}), params=TINY)[0]+ b = group_files(_repo({"src/two": 4}), params=TINY)[0]+ # Different paths, same shape: the key must differ because the members+ # differ, not because the directory name does.+ assert a.structural_key != b.structural_key++ def test_reordering_members_does_not_change_the_key(self):+ files = _repo({"src/one": 4})+ a = group_files(files, params=TINY)[0]+ b = group_files(list(reversed(files)), params=TINY)[0]+ assert a.structural_key == b.structural_key+++class TestSize:+ def test_no_group_exceeds_the_ceiling_unless_it_is_one_directory(self):+ """The fixture must actually contain an over-ceiling group.++ It did not, once: every group in ``SPEC`` fits under the ceiling, so+ the loop body never ran and the test passed whatever the code did.+ """+ spec = {**SPEC, "src/huge": TINY.max_files * 3}+ groups = group_files(_repo(spec), params=TINY)+ over = [g for g in groups if g.file_count > TINY.max_files]+ assert over, "fixture produced no over-ceiling group, so this proves nothing"+ for group in over:+ assert group.oversized+ assert len(group.dirs) == 1++ def test_a_flat_directory_over_the_ceiling_is_flagged_not_split(self):+ """No path rule can split it, so it stays whole and says so."""+ groups = group_files(_repo({"src/big": 20}), params=TINY)+ assert len(groups) == 1+ assert groups[0].oversized+ assert groups[0].file_count == 20++ def test_absorption_considers_the_left_neighbour_too(self):+ """A unit test of the absorption pass, not of a whole partition.++ The walk merges forward only: a thin remainder is offered to the group+ it was flushed after. Absorption is what gives a thin group its other+ neighbour, and the case is reachable but fiddly to provoke end to end,+ so it is asserted directly. Here the right neighbour is full and only+ the left one has room.+ """+ from repowise.core.generation.concept_tree.grouping import (+ _absorb_thin,+ _Partitioner,+ )++ params = GroupingParams(min_files=4, max_files=8)+ part = _Partitioner(params, {})+ groups = [+ part.make(_repo({"src/a": 5})),+ part.make(_repo({"src/b": 1})),+ part.make(_repo({"src/c": 8})),+ ]+ result = _absorb_thin(groups, part)+ assert [g.file_count for g in result] == [6, 8]+ assert not any(g.file_count < params.min_files for g in result)++ def test_grouping_actually_runs_the_absorption_pass(self, monkeypatch):+ """Wiring, asserted separately from behaviour.++ The pass above can be correct and simply not called. The end-to-end+ fixtures cannot catch that, because the forward merge in the walk+ already absorbs most thin remainders, so removing the call leaves+ their output unchanged.+ """+ from repowise.core.generation.concept_tree import grouping as mod++ called: list[int] = []++ def spy(groups, part):+ called.append(len(groups))+ return groups++ monkeypatch.setattr(mod, "_absorb_thin", spy)+ mod.group_files(_repo(SPEC), params=TINY)+ assert called, "group_files did not run the absorption pass"+++class TestLayers:+ def test_the_layer_boundary_changes_which_siblings_merge(self):+ """The layer signal steers a split that is already happening.++ Without it the walk merges the first two siblings that fit; with it,+ the first sibling is flushed alone because its layer differs, and the+ two that share a layer merge instead. Asserting composition rather+ than group count is what makes this discriminate: both answers have+ two groups.+ """+ params = GroupingParams(min_files=3, max_files=12)+ files = _repo({"src/aaa": 5, "src/bbb": 5, "src/ccc": 5})+ layers = {f: ("layer:ui" if "/aaa/" in f else "layer:core") for f in files}++ blind = {g.target_path: g.file_count for g in group_files(files, params=params)}+ aware = {+ g.target_path: g.file_count+ for g in group_files(files, layer_of_file=layers, params=params)+ }+ # Blind: aaa and bbb merge, ccc is flushed on its own.+ assert blind == {"src/aaa": 10, "src/ccc": 5}+ # Aware: aaa is flushed alone at the layer boundary, bbb and ccc merge.+ assert aware == {"src/aaa": 5, "src/bbb": 10}+ assert len(blind) == len(aware), "counts match, so composition is the signal"++ def test_a_subtree_within_the_ceiling_is_not_split_by_layer(self):+ """A documented limitation, asserted so it cannot change silently.++ Recursion stops as soon as a subtree fits, so the layer signal is+ never consulted for one that does — a small directory spanning two+ layers stays one page. Splitting every fitting subtree on layer would+ fragment the outline and would inherit the layer data's own noise,+ which is worse than an occasionally impure small page.+ """+ params = GroupingParams(min_files=3, max_files=12)+ files = _repo({"src/aaa": 5, "src/bbb": 5})+ layers = {f: ("layer:ui" if "/aaa/" in f else "layer:core") for f in files}+ groups = group_files(files, layer_of_file=layers, params=params)+ assert len(groups) == 1++ def test_dominant_layer_ties_break_deterministically(self):+ files = _repo({"src/x": 4})+ layers = {f: ("layer:a" if i % 2 else "layer:b") for i, f in enumerate(files)}+ first = group_files(files, layer_of_file=layers, params=TINY)[0]+ second = group_files(+ list(reversed(files)), layer_of_file=layers, params=TINY+ )[0]+ assert first.dominant_layer == second.dominant_layerdiff --git a/tests/unit/generation/test_concept_outline.py b/tests/unit/generation/test_concept_outline.pynew file mode 100644index 000000000..35ad33e9e--- /dev/null+++ b/tests/unit/generation/test_concept_outline.py@@ -0,0 +1,346 @@+"""The planner: what the model can and cannot break, and what the validator sees.++The design claim this file exists to hold is narrow and testable: **naming is+the only thing the model decides**. Coverage, membership and page identity come+from the deterministic partition, so a model that skips half the groups, echoes+back ids that were never in the input, or returns nothing at all still leaves a+complete and correctly-identified outline behind.+"""++from __future__ import annotations++import pytest++from repowise.core.generation.concept_tree.grouping import (+ GroupingParams,+ group_files,+)+from repowise.core.generation.concept_tree.naming import (+ build_payload,+ decode_response,+ deterministic_title,+)+from repowise.core.generation.concept_tree.planner import PlannerInputs, plan_outline+from repowise.core.generation.concept_tree.validation import validate_outline++TINY = GroupingParams(min_files=3, max_files=6)++FILES = [+ "src/ingest/reader.py",+ "src/ingest/parser.py",+ "src/ingest/walker.py",+ "src/ingest/cache.py",+ "src/render/html.py",+ "src/render/markdown.py",+ "src/render/theme.py",+ "src/render/layout.py",+]+TESTS = {"tests/test_reader.py"}+++def _inputs() -> PlannerInputs:+ return PlannerInputs(+ repo_name="demo",+ production_files=list(FILES),+ layer_labels={"layer:core": "Core"},+ test_files=set(TESTS),+ )+++class FakeProvider:+ """A provider whose reply is whatever the test needs it to be."""++ def __init__(self, content: str):+ self.content = content+ self.calls: list[str] = []++ async def generate(self, *, system_prompt, user_prompt, **kwargs):+ self.calls.append(user_prompt)++ class R:+ content = self.content++ return R()+++class TestModelCannotBreakStructure:+ @pytest.mark.asyncio+ async def test_a_model_that_returns_nothing_still_covers_every_file(self):+ _outline, report = await plan_outline(+ _inputs(), provider=FakeProvider(""), params=TINY, repair=False+ )+ assert report.coverage == 1.0+ assert report.unclaimed_files == []+ assert report.ok++ @pytest.mark.asyncio+ async def test_a_model_that_invents_group_ids_does_not_add_pages(self):+ groups = group_files(FILES, params=TINY)+ _payload, index = build_payload(groups)+ data = {+ "sections": [+ {"title": "Made Up", "groups": ["g99", "gzz"]},+ {"title": "Real", "groups": list(index)},+ ],+ "names": {"g99": {"title": "Ghost Page", "scope": "x"}},+ }+ named, invented = decode_response(data, index)+ assert invented == ["g99", "gzz"]+ assert len(named) == len(index)+ assert "Ghost Page" not in [n.title for n in named]++ @pytest.mark.asyncio+ async def test_a_model_that_skips_groups_leaves_them_named_anyway(self):+ groups = group_files(FILES, params=TINY)+ _payload, index = build_payload(groups)+ first = next(iter(index))+ data = {"sections": [], "names": {first: {"title": "Real Title", "scope": "s"}}}+ named, _ = decode_response(data, index)+ assert len(named) == len(index)+ skipped = [n for n in named if n.fallback]+ assert skipped, "fixture must leave at least one group unnamed"+ assert all(n.title for n in named)++ def test_decode_survives_a_non_list_group_field(self):+ """A model asked for a list of ids returned a count instead.++ ``section.get("groups") or []`` passes a non-empty scalar straight+ through to the loop, which raised. Asserted against ``decode_response``+ directly, because the planner also wraps this call and a test that only+ went through the planner would pass on the safety net alone.+ """+ groups = group_files(FILES, params=TINY)+ _payload, index = build_payload(groups)+ named, invented = decode_response(+ {"sections": [{"title": "Core", "groups": 3}], "names": {}}, index+ )+ assert len(named) == len(index)+ assert invented == []++ def test_decode_ignores_group_ids_that_are_not_scalars(self):+ groups = group_files(FILES, params=TINY)+ _payload, index = build_payload(groups)+ named, _ = decode_response(+ {"sections": [{"title": "Core", "groups": [{"a": 1}, ["b"], None]}],+ "names": {}},+ index,+ )+ assert len(named) == len(index)++ @pytest.mark.asyncio+ async def test_a_non_list_group_field_does_not_crash_the_run(self):+ """The same malformed response, end to end through the planner."""+ _outline, report = await plan_outline(+ _inputs(),+ provider=FakeProvider('{"sections":[{"title":"Core","groups":3}],"names":{}}'),+ params=TINY,+ repair=False,+ )+ assert report.coverage == 1.0+ assert report.ok++ @pytest.mark.asyncio+ async def test_junk_types_in_the_response_are_ignored(self):+ content = (+ '{"sections": {"not": "a list"},'+ ' "names": {"g01": ["not", "a", "dict"], "g02": 7}}'+ )+ outline, report = await plan_outline(+ _inputs(), provider=FakeProvider(content), params=TINY, repair=False+ )+ assert report.coverage == 1.0+ assert report.ok+ assert all(p.title for p in outline.pages)++ @pytest.mark.asyncio+ async def test_an_enormous_title_is_bounded(self):+ groups = group_files(FILES, params=TINY)+ _payload, index = build_payload(groups)+ gid = next(iter(index))+ data = {"sections": [], "names": {gid: {"title": "Wildly " * 400, "scope": "s"}}}+ named, _ = decode_response(data, index)+ assert all(len(n.title) <= 120 for n in named)++ @pytest.mark.asyncio+ async def test_a_model_claiming_one_group_twice_places_it_once(self):+ groups = group_files(FILES, params=TINY)+ _payload, index = build_payload(groups)+ gid = next(iter(index))+ data = {+ "sections": [+ {"title": "First", "groups": [gid]},+ {"title": "Second", "groups": [gid]},+ ],+ "names": {},+ }+ named, _ = decode_response(data, index)+ assert sum(1 for n in named if n.group is index[gid]) == 1+++class TestDeterministicPath:+ @pytest.mark.asyncio+ async def test_no_provider_still_produces_a_valid_outline(self):+ outline, report = await plan_outline(_inputs(), provider=None, params=TINY)+ assert outline.naming_mode == "deterministic"+ assert report.coverage == 1.0+ assert report.ok++ @pytest.mark.asyncio+ async def test_deterministic_mode_ignores_a_provider_that_is_present(self):+ provider = FakeProvider('{"sections":[],"names":{}}')+ outline, _ = await plan_outline(+ _inputs(), provider=provider, deterministic=True, params=TINY+ )+ assert provider.calls == []+ assert outline.naming_mode == "deterministic"++ @pytest.mark.asyncio+ async def test_the_keyless_tree_has_the_same_shape_as_the_named_one(self):+ """D5: adding a key changes the prose, never the partition."""+ keyless, _ = await plan_outline(_inputs(), provider=None, params=TINY)+ named, _ = await plan_outline(+ _inputs(), provider=FakeProvider(""), params=TINY, repair=False+ )+ assert sorted(p.structural_key for p in keyless.pages) == sorted(+ p.structural_key for p in named.pages+ )++ def test_deterministic_titles_are_not_bare_directory_names(self):+ """The name of this test is what it must actually assert.++ A word-count bound alone would pass on "src ingest", which is the+ exact failure the validator's bare-directory check exists to catch.+ """+ from repowise.core.generation.concept_tree.validation import (+ _is_bare_directory,+ )++ for group in group_files(FILES, params=TINY):+ title = deterministic_title(group)+ assert 2 <= len(title.split()) <= 7+ assert not _is_bare_directory(title, group.dirs), title+++class TestValidator:+ def _outline(self):+ from repowise.core.generation.concept_tree.models import (+ ConceptOutline,+ ConceptPage,+ ConceptSection,+ )++ groups = group_files(FILES, params=TINY)+ outline = ConceptOutline()+ outline.sections.append(+ ConceptSection(+ title="All",+ pages=[+ ConceptPage(title=f"Page {i}", scope="s", group=g)+ for i, g in enumerate(groups)+ ],+ )+ )+ outline.number_sections()+ return outline++ def test_a_clean_outline_has_no_hard_failures(self):+ report = validate_outline(+ self._outline(),+ all_files=set(FILES),+ test_files=TESTS,+ max_files_per_page=TINY.max_files,+ )+ assert report.hard_failures == []++ def test_an_invented_member_is_a_hard_failure(self):+ outline = self._outline()+ outline.pages[0].group.members.append("src/ghost/nowhere.py")+ report = validate_outline(+ outline,+ all_files=set(FILES),+ test_files=TESTS,+ max_files_per_page=TINY.max_files,+ )+ assert report.invented_paths == ["src/ghost/nowhere.py"]+ assert not report.ok++ def test_an_unclaimed_file_is_a_hard_failure(self):+ report = validate_outline(+ self._outline(),+ all_files=set(FILES) | {"src/orphan.py"},+ test_files=TESTS,+ max_files_per_page=TINY.max_files,+ )+ assert report.unclaimed_files == ["src/orphan.py"]+ assert not report.ok++ def test_two_pages_on_one_target_path_is_a_hard_failure(self):+ outline = self._outline()+ outline.pages[1].group.target_path = outline.pages[0].group.target_path+ report = validate_outline(+ outline,+ all_files=set(FILES),+ test_files=TESTS,+ max_files_per_page=TINY.max_files,+ )+ assert report.duplicate_target_paths+ assert not report.ok++ def test_a_test_file_in_the_tree_is_a_hard_failure(self):+ outline = self._outline()+ outline.pages[0].group.members.append("tests/test_reader.py")+ report = validate_outline(+ outline,+ all_files=set(FILES) | TESTS,+ test_files=TESTS,+ max_files_per_page=TINY.max_files,+ )+ assert report.test_paths_included == ["tests/test_reader.py"]+ assert not report.ok++ def test_a_long_title_is_reported_but_does_not_block(self):+ outline = self._outline()+ outline.pages[0].title = "A Very Long Title That Runs On And On Forever Indeed"+ report = validate_outline(+ outline,+ all_files=set(FILES),+ test_files=TESTS,+ max_files_per_page=TINY.max_files,+ )+ assert report.bad_length_titles+ assert report.ok, "title length is taste, not correctness"+++class TestPayload:+ def test_group_ids_carry_no_path_information(self):+ """A fabricated id must be recognisable as fabricated."""+ groups = group_files(FILES, params=TINY)+ _payload, index = build_payload(groups)+ for gid in index:+ assert gid.startswith("g") and gid[1:].isdigit()++ def test_filenames_are_sampled_across_every_directory(self):+ """A merged group must not be described by only one of its halves."""+ from repowise.core.generation.concept_tree.grouping import ConceptGroup++ group = ConceptGroup(+ members=[f"src/aaa/a{i}.py" for i in range(6)]+ + [f"src/zzz/z{i}.py" for i in range(6)],+ dirs=["src/aaa", "src/zzz"],+ target_path="src/aaa",+ )+ payload, _ = build_payload([group], max_filenames=4)+ names = payload["groups"][0]["names"]+ assert any(n.startswith("a") for n in names)+ assert any(n.startswith("z") for n in names), names++ def test_sibling_directories_are_shown_relative_to_what_they_share(self):+ from repowise.core.generation.concept_tree.grouping import ConceptGroup++ group = ConceptGroup(+ members=["src/git_indexer/a.py", "src/graph/b.py"],+ dirs=["src/git_indexer", "src/graph"],+ target_path="src/git_indexer",+ )+ payload, _ = build_payload([group])+ assert sorted(payload["groups"][0]["subdirs"]) == ["git_indexer", "graph"]diff --git a/tests/unit/persistence/test_page_tree_wiring.py b/tests/unit/persistence/test_page_tree_wiring.pyindex 4dfe29d4b..2317def52 100644--- a/tests/unit/persistence/test_page_tree_wiring.py+++ b/tests/unit/persistence/test_page_tree_wiring.py@@ -46,3 +46,29 @@ def test_persist_path_rebuilds_the_tree(module_name: str, description: str):def test_the_writer_list_is_not_silently_empty():"""Guards the parametrisation itself."""assert len(_PERSIST_PATHS) >= 6+++def test_the_scoped_path_sweeps_superseded_rows():+ """Backlog B19, wired the same way and asserted the same way.++ A page whose members change is rewritten under a new id, and until this+ call existed the old row stayed behind as a duplicate on every+ ``repowise update``. That the sweep is *correct* is tested elsewhere; this+ asserts it is reached, because a function that is right and never called+ looks exactly like a function that was not needed.+ """+ module = __import__("repowise.core.pipeline.scoped_generation", fromlist=["*"])+ source = inspect.getsource(module)+ assert "sweep_superseded_generated_pages(\n" in source, (+ "a scoped regeneration writes pages without retiring the rows they "+ "supersede, so a membership change strands the old page as a duplicate"+ )+++def test_the_scoped_sweep_runs_before_the_tree_is_rebuilt():+ """Order matters: a retired row must not be handed a place in the tree."""+ module = __import__("repowise.core.pipeline.scoped_generation", fromlist=["*"])+ source = inspect.getsource(module)+ sweep = source.index("sweep_superseded_generated_pages(\n")+ rebuild = source.index("await rebuild_page_tree(")+ assert sweep < rebuilddiff --git a/tests/unit/persistence/test_superseded_sweep.py b/tests/unit/persistence/test_superseded_sweep.pynew file mode 100644index 000000000..2b92e3c06--- /dev/null+++ b/tests/unit/persistence/test_superseded_sweep.py@@ -0,0 +1,192 @@+"""The update-path sweep (backlog B19).++A structurally-keyed page's id is its ``target_path``, and a page defined by the+files it covers can legitimately change which directory names it when its+membership changes. On a full index the existing sweep retires the old row. On+``repowise update`` nothing did, because the full sweep deletes every row of a+page type the run did not reproduce — correct for a full index, and on a scoped+run that regenerates one page out of fifty it would delete the other forty-nine.++So the rule here is different and the tests that matter are the ones proving it+is *narrow*: a page this run did not touch must survive, and so must a page+whose generation failed.+"""++from __future__ import annotations++import json+import uuid+from datetime import UTC, datetime++import pytest+from sqlalchemy import select++from repowise.core.persistence.models import Page, Repository+from repowise.core.pipeline.persist import sweep_superseded_generated_pages+++async def session(async_session):+ return async_session+++async def _new_repo(session, name: str) -> str:+ repo_id = uuid.uuid4().hex[:32]+ session.add(+ Repository(id=repo_id, name=name, local_path=f"/tmp/{name}", url="")+ )+ await session.flush()+ return repo_id+++async def repo_id(session):+ return await _new_repo(session, "primary")+++async def other_repo_id(session):+ return await _new_repo(session, "other")+++_NOW = datetime.now(UTC)+++class _Generated:+ def __init__(self, page_id: str, page_type: str, members: list[str]):+ self.page_id = page_id+ self.page_type = page_type+ self.metadata = {"file_paths": list(members)}+++async def _add(session, repo_id: str, page_id: str, page_type: str, members):+ session.add(+ Page(+ id=page_id,+ repository_id=repo_id,+ page_type=page_type,+ target_path=page_id.split(":", 1)[1],+ title=page_id,+ content="body",+ source_hash="h" * 64,+ model_name="m",+ provider_name="template",+ metadata_json=json.dumps({"file_paths": list(members)}),+ created_at=_NOW,+ updated_at=_NOW,+ )+ )+ await session.flush()+++async def _ids(session, repo_id: str) -> set[str]:+ rows = await session.execute(select(Page.id).where(Page.repository_id == repo_id))+ return set(rows.scalars())+++async def test_a_row_whose_coverage_moved_to_a_new_page_is_retired(+ session, repo_id+):+ """The bug: the same page under a new id, with the old row left behind."""+ members = ["src/a.py", "src/b.py"]+ await _add(session, repo_id, "module_page:src/old", "module_page", members)+ swept = await sweep_superseded_generated_pages(+ session,+ repo_id,+ [_Generated("module_page:src/new", "module_page", members)],+ )+ assert swept == ["module_page:src/old"]+ assert "module_page:src/old" not in await _ids(session, repo_id)+++async def test_an_untouched_page_survives(session, repo_id):+ """The reason the full sweep cannot be reused here."""+ await _add(session, repo_id, "module_page:src/keep", "module_page", ["src/z.py"])+ await _add(session, repo_id, "module_page:src/old", "module_page", ["src/a.py"])+ swept = await sweep_superseded_generated_pages(+ session,+ repo_id,+ [_Generated("module_page:src/new", "module_page", ["src/a.py"])],+ )+ assert swept == ["module_page:src/old"]+ assert "module_page:src/keep" in await _ids(session, repo_id)+++async def test_a_page_whose_generation_failed_is_not_deleted(session, repo_id):+ """The failure mode worse than the bug being fixed.++ A page the run meant to regenerate but could not still covers files no+ produced page claims, so it is left alone. Deleting a page because its+ generation errored would lose content rather than a duplicate.+ """+ await _add(session, repo_id, "module_page:src/failed", "module_page", ["src/f.py"])+ swept = await sweep_superseded_generated_pages(+ session,+ repo_id,+ [_Generated("module_page:src/ok", "module_page", ["src/a.py"])],+ )+ assert swept == []+ assert "module_page:src/failed" in await _ids(session, repo_id)+++async def test_partial_overlap_does_not_retire_a_row(session, repo_id):+ """Supersession means *all* of it, not some of it."""+ await _add(+ session, repo_id, "module_page:src/old", "module_page", ["src/a.py", "src/b.py"]+ )+ swept = await sweep_superseded_generated_pages(+ session,+ repo_id,+ [_Generated("module_page:src/new", "module_page", ["src/a.py"])],+ )+ assert swept == []+++async def test_a_row_with_no_recorded_membership_is_left_alone(session, repo_id):+ """Pre-membership rows (backlog B20) cannot be proven superseded."""+ await _add(session, repo_id, "module_page:src/old", "module_page", [])+ swept = await sweep_superseded_generated_pages(+ session,+ repo_id,+ [_Generated("module_page:src/new", "module_page", ["src/a.py"])],+ )+ assert swept == []+ assert "module_page:src/old" in await _ids(session, repo_id)+++async def test_only_structurally_keyed_types_are_swept(session, repo_id):+ """A file page's id is its path; it has no identity that can move."""+ await _add(session, repo_id, "file_page:src/a.py", "file_page", ["src/a.py"])+ swept = await sweep_superseded_generated_pages(+ session,+ repo_id,+ [_Generated("file_page:src/b.py", "file_page", ["src/a.py"])],+ )+ assert swept == []+ assert "file_page:src/a.py" in await _ids(session, repo_id)+++async def test_a_run_that_produced_nothing_retires_nothing(session, repo_id):+ await _add(session, repo_id, "module_page:src/old", "module_page", ["src/a.py"])+ assert await sweep_superseded_generated_pages(session, repo_id, []) == []+ assert await sweep_superseded_generated_pages(session, repo_id, None) == []+ assert "module_page:src/old" in await _ids(session, repo_id)+++async def test_a_different_repository_is_untouched(session, repo_id, other_repo_id):+ await _add(session, other_repo_id, "module_page:src/old", "module_page", ["src/a.py"])+ swept = await sweep_superseded_generated_pages(+ session,+ repo_id,+ [_Generated("module_page:src/new", "module_page", ["src/a.py"])],+ )+ assert swept == []+ assert "module_page:src/old" in await _ids(session, other_repo_id)