Skip to content

Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.

fix(generation): package roots are walls in the concept tree (#2705)

Swati Ahuja8b59505a67c1
diff --git a/packages/core/src/repowise/core/generation/concept_tree/grouping.py b/packages/core/src/repowise/core/generation/concept_tree/grouping.py
index 4f54a9a07..4c0384ba3 100644
--- a/packages/core/src/repowise/core/generation/concept_tree/grouping.py
+++ b/packages/core/src/repowise/core/generation/concept_tree/grouping.py
@@ -40,9 +40,11 @@ commit. The ladder only moves at a band edge.
from __future__ import annotations
+from collections import Counter
from collections.abc import Iterable, Sequence
from dataclasses import dataclass, field
+from ...ingestion.package_roots import module_for
from ..models import member_structural_key
#: Prefix for a concept group's structural key. Distinct from ``module`` and
@@ -66,6 +68,16 @@ class GroupingParams:
min_files: int = 10
max_files: int = 30
+ @property
+ def thin_package_files(self) -> int:
+ """Below this a package rolls up with its thin siblings instead of taking a page.
+
+ A third of the ceiling on the larger bands: a package that cannot fill a
+ third of a page is a table row, and per-package pages there would grow a
+ wide monorepo's page count by a quarter.
+ """
+ return max(self.min_files, self.max_files // 3)
+
# Repository size bands and the ceiling each one uses. Chosen so a repository
# of a few hundred files lands near 50 pages and one of a few thousand lands
@@ -113,6 +125,9 @@ class ConceptGroup:
#: therefore could not be split by any path-based rule.
oversized: bool = False
structural_key: str = field(default="", repr=False)
+ #: Set only on a roll-up of whole thin sibling packages: their roots, sorted.
+ #: Every other group sits inside one package, so this is empty.
+ packages: tuple[str, ...] = ()
def __post_init__(self) -> None:
if not self.structural_key:
@@ -243,13 +258,71 @@ def _is_local_merge(left: Sequence[str], right: Sequence[str]) -> bool:
return all("/" not in m for m in left) and all("/" not in m for m in right)
+def _package_of(path: str, roots: set[str]) -> str:
+ """The deepest package root holding *path*, or ``""`` outside every package."""
+ owner = module_for(path, roots)
+ return owner if owner in roots else ""
+
+
+def _roots_holding(path: str, roots: set[str]) -> list[str]:
+ """Every root in *roots* that *path* sits under, outermost first."""
+ parts = path.split("/")
+ return [d for d in ("/".join(parts[:i]) for i in range(1, len(parts))) if d in roots]
+
+
+def package_file_counts(members: Sequence[str], packages: Sequence[str]) -> list[tuple[str, int]]:
+ """``(package root, files)`` for each package a roll-up covers, in path order."""
+ return [(p, sum(1 for m in members if m.startswith(p + "/"))) for p in sorted(packages)]
+
+
class _Partitioner:
- def __init__(self, params: GroupingParams, layer_of_file: dict[str, str]):
+ def __init__(
+ self,
+ params: GroupingParams,
+ layer_of_file: dict[str, str],
+ package_of: dict[str, str] | None = None,
+ ):
self.params = params
self.layer_of_file = layer_of_file
+ self.package_of = package_of or {}
self.groups: list[ConceptGroup] = []
- def make(self, members: Sequence[str]) -> ConceptGroup:
+ def _one_package(self, paths: Iterable[str]) -> bool:
+ return len({self.package_of.get(p, "") for p in paths}) <= 1
+
+ def may_merge(self, left: Sequence[str], right: Sequence[str]) -> bool:
+ """Local, and never across a package root: two packages never share a page."""
+ return _is_local_merge(left, right) and self._one_package([*left, *right])
+
+ def roll_up_thin_packages(self, files: Sequence[str]) -> list[str]:
+ """Group thin sibling packages whole; return the files left to partition.
+
+ A package too small for a page of its own cannot merge with a
+ neighbour, so sibling thin packages (same parent directory) become one
+ roll-up whose page lists them one row each. A thin package with no thin
+ sibling stays in the partition as its own small page.
+ """
+ # Sized by whole subtree: a root whose code sits under a nested manifest
+ # owns few files directly but is not thin.
+ owners = {o for o in self.package_of.values() if o}
+ sizes = Counter(r for f in files for r in _roots_holding(f, owners))
+ thin = {r for r, n in sizes.items() if n < self.params.thin_package_files}
+ by_parent: dict[str, list[str]] = {}
+ for root in sorted(thin):
+ # A thin root inside another thin root rides along with its ancestor.
+ if any(root.startswith(t + "/") for t in thin):
+ continue
+ by_parent.setdefault("/".join(root.split("/")[:-1]), []).append(root)
+ runs = [tuple(roots) for _parent, roots in sorted(by_parent.items()) if len(roots) > 1]
+ rolled: set[str] = set()
+ for run in runs:
+ held = set(run)
+ members = [f for f in files if _roots_holding(f, held)]
+ self.groups.append(self.make(members, packages=run))
+ rolled.update(members)
+ return [f for f in files if f not in rolled]
+
+ def make(self, members: Sequence[str], packages: tuple[str, ...] = ()) -> ConceptGroup:
ordered = sorted(members)
dirs = _dirs_of(ordered)
# A group over the ceiling whose files all sit directly in one
@@ -267,6 +340,7 @@ class _Partitioner:
target_path=_target_path(ordered, dirs),
dominant_layer=_dominant(ordered, self.layer_of_file),
oversized=oversized,
+ packages=packages,
)
def _same_layer(self, left: Sequence[str], right: Sequence[str]) -> bool:
@@ -291,7 +365,11 @@ class _Partitioner:
or, at the top, flushes them as their own group.
"""
if node.subtree <= self.params.max_files:
- return _subtree_files(node)
+ files = _subtree_files(node)
+ # A subtree that fits but holds two packages descends instead, so
+ # each package is grouped on its own side of the wall.
+ if self._one_package(files):
+ return files
pending: list[str] = list(node.own)
for _name, child in sorted(node.children.items()):
@@ -301,7 +379,7 @@ class _Partitioner:
if (
len(pending) + len(left) <= self.params.max_files
and self._same_layer(pending, left)
- and _is_local_merge(pending, left)
+ and self.may_merge(pending, left)
):
pending.extend(left)
continue
@@ -320,7 +398,7 @@ class _Partitioner:
if (
len(last.members) + len(pending) <= self.params.max_files
and self._same_layer(last.members, pending)
- and _is_local_merge(last.members, pending)
+ and self.may_merge(last.members, pending)
):
merged = self.make(last.members + pending)
self.groups[-1] = merged
@@ -360,7 +438,7 @@ def _absorb_thin(groups: list[ConceptGroup], part: _Partitioner) -> list[Concept
other = working[j]
if other.file_count + group.file_count > params.max_files:
continue
- if not _is_local_merge(other.members, group.members):
+ if not part.may_merge(other.members, group.members):
continue
# A same-layer neighbour is preferred, but a thin group takes
# a cross-layer neighbour over standing alone. The layer gate
@@ -419,16 +497,19 @@ def _assign_targets(groups: list[ConceptGroup]) -> None:
# Largest first so the group with the strongest claim to a directory keeps
# the shortest name; ties break on the key so the order never depends on
# the traversal.
- for group in sorted(groups, key=lambda g: (-g.file_count, g.structural_key)):
+ # Roll-ups last: a directory's own files have the stronger claim to its name.
+ for group in sorted(groups, key=lambda g: (bool(g.packages), -g.file_count, g.structural_key)):
owned = _target_path(group.members, group.dirs)
- candidates = [owned] if owned in group.dirs else []
+ # A package roll-up is named for the parent its packages share.
+ candidates = [owned] if group.packages or owned in group.dirs else []
# A whole repository under the ceiling is one group spanning several
# top-level directories — the guard cannot refuse it, there is nothing
# to merge. Naming it after a member would put the repository under a
# name describing one corner of it, so it takes the root.
if not owned and "" not in candidates and len({m.split("/")[0] for m in group.members}) > 1:
candidates.append("")
- candidates.extend(d for d in group.dirs if d and d not in candidates)
+ if not group.packages:
+ candidates.extend(d for d in group.dirs if d and d not in candidates)
if owned and owned not in candidates:
candidates.append(owned)
# The root is substituted before the uniqueness check, never after.
@@ -463,6 +544,7 @@ def group_files(
*,
layer_of_file: dict[str, str] | None = None,
params: GroupingParams | None = None,
+ package_roots: Iterable[str] = (),
) -> list[ConceptGroup]:
"""Partition *paths* into bounded, path-local concept groups.
@@ -471,16 +553,21 @@ def group_files(
it. *layer_of_file* maps a file to its curated layer id and only affects
whether two adjacent runs may be merged; grouping works without it.
+ *package_roots* are hard walls: no group holds files of two packages,
+ except a roll-up of whole thin sibling packages, which records them in
+ ``packages``.
+
Groups come back in path order, which is also the order their members were
walked, so the sequence itself is stable.
"""
files = sorted({p.replace("\\", "/") for p in paths if p})
if not files:
return []
- tree = _build_tree(files)
resolved = params or params_for(len(files))
- part = _Partitioner(resolved, layer_of_file or {})
- leftover = part.visit(tree)
+ roots = set(package_roots)
+ package_of = {f: _package_of(f, roots) for f in files} if roots else {}
+ part = _Partitioner(resolved, layer_of_file or {}, package_of)
+ leftover = part.visit(_build_tree(part.roll_up_thin_packages(files)))
if leftover:
# The whole repository fits under the ceiling: one group, which is the
# honest answer for a tiny repository.
diff --git a/packages/core/src/repowise/core/generation/concept_tree/naming.py b/packages/core/src/repowise/core/generation/concept_tree/naming.py
index efd1ace9b..69496c41d 100644
--- a/packages/core/src/repowise/core/generation/concept_tree/naming.py
+++ b/packages/core/src/repowise/core/generation/concept_tree/naming.py
@@ -113,6 +113,8 @@ def deterministic_title(group: ConceptGroup, layer_label: str = "") -> str:
a key, so a keyless wiki gets a real tree with plainer labels rather than a
worse tree.
"""
+ if group.packages:
+ return _package_rollup_title(group.packages)
segments = [s for s in group.target_path.split("/") if s]
meaningful = [s for s in segments if s.lower() not in _STOP_SEGMENTS] or segments
tail = [_humanise(s) for s in meaningful[-2:]]
@@ -343,8 +345,30 @@ def disambiguate_titles(
return [t for t, _ in out]
+def _package_rollup_title(packages: Sequence[str]) -> str:
+ """The shared parent plus a member list, never one member's name alone."""
+ parent = _common_prefix(list(packages))
+ lead = _humanise(_chapter_subject(parent)) or "Root"
+ if not lead.lower().endswith("packages"):
+ lead = f"{lead} Packages"
+ names = [_humanise(p.rsplit("/", 1)[-1]) for p in sorted(packages)]
+ more = f" and {len(names) - 2} More" if len(names) > 2 else ""
+ title = f"{lead}: {names[0]}, {names[1]}{more}"
+ if len(title.split()) <= MAX_TITLE_WORDS:
+ return title
+ return f"{lead} ({len(names)})"
+
+
def deterministic_scope(group: ConceptGroup) -> str:
"""A scope line stating coverage and its boundary, with no model."""
+ if group.packages:
+ names = [p.rsplit("/", 1)[-1] for p in group.packages]
+ listed = ", ".join(names[:8]) + (f" and {len(names) - 8} more" if len(names) > 8 else "")
+ return (
+ f"Covers {len(names)} small sibling packages ({listed}), "
+ f"{group.file_count} source files in all. Each is its own package; "
+ "code outside them is documented on its own pages."
+ )
# The root directory is the empty string, which is a real directory and
# not an absent one; filtering it out left a group reading "0 directories".
dirs = list(group.dirs)
@@ -524,6 +548,9 @@ def build_payload(
common = _common_prefix(group.dirs)
cut = len(common) + 1 if common else 0
entry["subdirs"] = [d[cut:] or d.rsplit("/", 1)[-1] for d in group.dirs[:8]]
+ if group.packages:
+ # Sections only: the title stays the deterministic parent + members.
+ entry["packages"] = [p.rsplit("/", 1)[-1] for p in group.packages]
label = labels.get(group.dominant_layer)
if label:
entry["layer"] = label
@@ -626,6 +653,9 @@ def decode_response(
title = _clean_title(entry.get("title"))
scope = str(entry.get("scope") or "").strip()
fallback = False
+ if group.packages:
+ # A model shown several packages names the page after one of them.
+ title, scope = "", ""
if not title:
title = deterministic_title(group, labels.get(group.dominant_layer, ""))
fallback = True
diff --git a/packages/core/src/repowise/core/generation/concept_tree/planner.py b/packages/core/src/repowise/core/generation/concept_tree/planner.py
index 7098106d1..f7133168a 100644
--- a/packages/core/src/repowise/core/generation/concept_tree/planner.py
+++ b/packages/core/src/repowise/core/generation/concept_tree/planner.py
@@ -70,6 +70,8 @@ class PlannerInputs:
entry_points: set[str] = field(default_factory=set)
#: Test files, kept only so the validator can prove none leaked in.
test_files: set[str] = field(default_factory=set)
+ #: Package roots that wall the grouping, as selection computes them.
+ package_roots: set[str] = field(default_factory=set)
def _reasoning_kwargs(reasoning: str | None) -> dict[str, str]:
@@ -193,6 +195,16 @@ def name_deterministically(groups: list[ConceptGroup], inputs: PlannerInputs) ->
return outline
+def _group(inputs: PlannerInputs, params: GroupingParams | None) -> list[ConceptGroup]:
+ """The one grouping call both planner entry points share."""
+ return group_files(
+ inputs.production_files,
+ layer_of_file=inputs.layer_of_file,
+ params=params,
+ package_roots=inputs.package_roots,
+ )
+
+
def plan_deterministic(
inputs: PlannerInputs, *, params: GroupingParams | None = None
) -> tuple[ConceptOutline, list[ConceptGroup]]:
@@ -208,7 +220,7 @@ def plan_deterministic(
bounds were dropped here and the two paths produced different page counts
for the same repository.
"""
- groups = group_files(inputs.production_files, layer_of_file=inputs.layer_of_file, params=params)
+ groups = _group(inputs, params)
return name_deterministically(groups, inputs), groups
@@ -293,7 +305,8 @@ def _repair_targets(outline: ConceptOutline, report: OutlineReport) -> set[str]:
targets |= set(report.bare_directory_titles)
targets |= {t.rsplit(" (", 1)[0] for t in report.bad_length_titles}
targets |= {p.title for p in outline.pages if not p.named_by_model}
- return targets
+ # A package roll-up keeps its deterministic title; see ``decode_response``.
+ return targets - {p.title for p in outline.pages if p.group.packages}
async def plan_outline(
@@ -306,7 +319,7 @@ async def plan_outline(
reasoning: str | None = None,
) -> tuple[ConceptOutline, OutlineReport]:
"""Group *inputs* and produce a validated outline over that grouping."""
- groups = group_files(inputs.production_files, layer_of_file=inputs.layer_of_file, params=params)
+ groups = _group(inputs, params)
return await name_groups(
groups,
inputs,
diff --git a/packages/core/src/repowise/core/generation/context/assembler.py b/packages/core/src/repowise/core/generation/context/assembler.py
index de3db2cef..4c3323b11 100644
--- a/packages/core/src/repowise/core/generation/context/assembler.py
+++ b/packages/core/src/repowise/core/generation/context/assembler.py
@@ -505,6 +505,7 @@ class ContextAssembler:
scope: str = "",
is_rollup: bool = False,
child_pages: list[dict] | None = None,
+ packages: list[dict] | None = None,
) -> ModulePageContext:
"""Assemble context for the module_page template."""
total_symbols = sum(len(fc.symbols) for fc in file_contexts)
@@ -647,6 +648,7 @@ class ContextAssembler:
scope=scope,
is_rollup=is_rollup,
child_pages=child_pages or [],
+ packages=packages or [],
hotspot_count=hotspot_count,
stable_count=stable_count,
single_owner_files=single_owner_files,
diff --git a/packages/core/src/repowise/core/generation/context/contexts.py b/packages/core/src/repowise/core/generation/context/contexts.py
index 438adefe5..1554a3a64 100644
--- a/packages/core/src/repowise/core/generation/context/contexts.py
+++ b/packages/core/src/repowise/core/generation/context/contexts.py
@@ -120,6 +120,8 @@ class ModulePageContext:
is_rollup: bool = False
# Child concept pages this rollup sits above: [{"title", "path", "summary"}].
child_pages: list[dict] = field(default_factory=list)
+ # Sibling packages a package roll-up covers: [{"path", "files"}].
+ packages: list[dict] = field(default_factory=list)
# Git-derived subsystem health, aggregated over the page's member files.
# All degrade to zero/empty when no git metadata is available, so the
# template renders nothing rather than a wrong number.
diff --git a/packages/core/src/repowise/core/generation/page_generator/levels.py b/packages/core/src/repowise/core/generation/page_generator/levels.py
index 24c35b950..39aa68f77 100644
--- a/packages/core/src/repowise/core/generation/page_generator/levels.py
+++ b/packages/core/src/repowise/core/generation/page_generator/levels.py
@@ -17,6 +17,7 @@ import structlog
from repowise.core.ids import is_external
from .. import onboarding as _onboarding
+from ..concept_tree.grouping import package_file_counts
from ..context_assembler import FilePageContext
from ..models import compute_page_id
from .helpers import _is_infra_file, decisions_for_files, rank_decisions
@@ -314,6 +315,10 @@ def build_level4_coros(run: _GenerationRun) -> list[tuple[str, Any]]:
# shape implies: a chapter that is also a leaf directory
# heads its children *and* documents its own loose files.
owns_files=bool(mg.file_paths),
+ packages=[
+ {"path": path, "files": count}
+ for path, count in package_file_counts(mg.file_paths, mg.packages)
+ ],
),
)
)
diff --git a/packages/core/src/repowise/core/generation/page_generator/pertype.py b/packages/core/src/repowise/core/generation/page_generator/pertype.py
index 96ef364f2..212ee8865 100644
--- a/packages/core/src/repowise/core/generation/page_generator/pertype.py
+++ b/packages/core/src/repowise/core/generation/page_generator/pertype.py
@@ -209,6 +209,7 @@ class PerTypeGenerationMixin:
is_rollup: bool = False,
child_pages: list[dict] | None = None,
owns_files: bool = True,
+ packages: list[dict] | None = None,
) -> GeneratedPage:
ctx = self._assembler.assemble_module_page(
title,
@@ -225,6 +226,7 @@ class PerTypeGenerationMixin:
scope=scope,
is_rollup=is_rollup,
child_pages=child_pages,
+ packages=packages,
)
module_git_summary = None
if git_meta_map:
@@ -320,13 +322,21 @@ class PerTypeGenerationMixin:
After ``_build_generated_page`` too, so the page summary is still
drawn from the model's opening rather than from a table row.
"""
+ tables = []
+ if ctx.packages:
+ # One row per package of a roll-up, for the same reason.
+ tables.append(
+ self._render("_package_table.j2", style_prefix=False, rows=ctx.packages)
+ )
rows, omitted = build_concept_index(file_contexts)
- if not rows:
- return page
- table = self._render(
- "_concept_index_table.j2", style_prefix=False, rows=rows, omitted=omitted
- )
- page.content = f"{(page.content or '').rstrip()}\n\n{table.strip()}\n"
+ if rows:
+ tables.append(
+ self._render(
+ "_concept_index_table.j2", style_prefix=False, rows=rows, omitted=omitted
+ )
+ )
+ for table in tables:
+ page.content = f"{(page.content or '').rstrip()}\n\n{table.strip()}\n"
return page
if self._config.deterministic:
diff --git a/packages/core/src/repowise/core/generation/selection/selector.py b/packages/core/src/repowise/core/generation/selection/selector.py
index 1ff156db0..b1a4d6e3a 100644
--- a/packages/core/src/repowise/core/generation/selection/selector.py
+++ b/packages/core/src/repowise/core/generation/selection/selector.py
@@ -16,6 +16,7 @@ from typing import Any
import structlog
from repowise.core.ingestion.languages.registry import REGISTRY as _LANG_REGISTRY
+from repowise.core.ingestion.package_roots import package_roots_from_paths
from ..concept_tree.grouping import ConceptGroup, group_files
from ..concept_tree.naming import (
@@ -105,6 +106,8 @@ class ModuleGroup:
#: silently broke all three: a rollup out-claimed its own leaves for every
#: file in the subtree.
context_paths: tuple[str, ...] = ()
+ #: Package roots of a roll-up of thin sibling packages; empty otherwise.
+ packages: tuple[str, ...] = ()
@dataclass
@@ -494,7 +497,12 @@ def _build_module_groups(inputs: SelectionInputs) -> ConceptCandidates:
return ConceptCandidates()
layer_of_file, layer_labels = _layer_map_from_kg(inputs)
- groups = group_files(files, layer_of_file=layer_of_file)
+ # Read off the parsed file list, not the disk, so the estimate, the run and
+ # the update cascade all see the same walls and mint the same page ids.
+ # Ceiling: a manifest the traverser does not emit (go.mod, pom.xml) is no
+ # wall; threading the disk scan through every SelectionInputs lifts it.
+ package_roots = package_roots_from_paths({p.file_info.path for p in inputs.parsed_files})
+ groups = group_files(files, layer_of_file=layer_of_file, package_roots=package_roots)
lang_of = {p.file_info.path: p.file_info.language for p in inputs.parsed_files}
# Names are decided over the whole set, not per group: two packages that
@@ -517,7 +525,8 @@ def _build_module_groups(inputs: SelectionInputs) -> ConceptCandidates:
langs = Counter(lang_of.get(m, "") for m in group.members)
langs.pop("", None)
score = sum(inputs.pagerank.get(m, 0.0) for m in group.members)
- subsystem = chapters.get(group.target_path)
+ # A package roll-up keeps its own name and material, never a chapter's.
+ subsystem = None if group.packages else chapters.get(group.target_path)
scored.append(
(
score,
@@ -543,6 +552,7 @@ def _build_module_groups(inputs: SelectionInputs) -> ConceptCandidates:
),
is_rollup=subsystem is not None,
context_paths=tuple(subsystem or ()),
+ packages=group.packages,
),
)
)
diff --git a/packages/core/src/repowise/core/generation/templates/_package_table.j2 b/packages/core/src/repowise/core/generation/templates/_package_table.j2
new file mode 100644
index 000000000..4e05a953c
--- /dev/null
+++ b/packages/core/src/repowise/core/generation/templates/_package_table.j2
@@ -0,0 +1,12 @@
+{#- One row per package a package roll-up covers, rendered from the grouping and
+ appended after the model has written the page, so no response can drop or
+ merge a package. -#}
+## Packages
+
+This page covers {{ rows | length }} sibling packages, each with its own manifest.
+
+| Package | Files |
+| --- | --- |
+{% for row in rows -%}
+| `{{ row.path }}` | {{ row.files }} |
+{% endfor -%}
diff --git a/packages/core/src/repowise/core/generation/templates/module_page.j2 b/packages/core/src/repowise/core/generation/templates/module_page.j2
index 754c1a47c..bd2260951 100644
--- a/packages/core/src/repowise/core/generation/templates/module_page.j2
+++ b/packages/core/src/repowise/core/generation/templates/module_page.j2
@@ -3,6 +3,9 @@ You are documenting **{{ ctx.title }}**, an OVERVIEW of the `{{ ctx.directories[
{% else %}
You are documenting **{{ ctx.title }}** (language: {{ ctx.language }}), a group of files that belong together, not a single directory. Write about what the group does as a whole.
{% endif %}
+{% if ctx.packages %}
+This page covers {{ ctx.packages | length }} sibling packages: {% for p in ctx.packages %}`{{ p.path }}`{% if not loop.last %}, {% endif %}{% endfor %}. Each is a separate package with its own manifest. Say what they share and how they differ; never describe the group as one of them or name the page after one. A table listing every package is appended after you write the page, so do not reproduce it.
+{% endif %}
{% if ctx.scope %}
SCOPE (use this to open by situating the page against its siblings — say what it covers and what it deliberately leaves to other pages; do NOT quote this line verbatim):
diff --git a/tests/unit/generation/test_concept_package_walls.py b/tests/unit/generation/test_concept_package_walls.py
new file mode 100644
index 000000000..70d174c37
--- /dev/null
+++ b/tests/unit/generation/test_concept_package_walls.py
@@ -0,0 +1,276 @@
+"""Package roots are walls in the concept tree.
+
+A page that merged several small packages used to take one member's directory
+as its target and its title, then describe every member as that one. Now no
+group holds files of two packages, except a roll-up of whole thin sibling
+packages, which is named for their shared parent plus a member list and lists
+each package in a deterministic table.
+"""
+
+from __future__ import annotations
+
+import networkx as nx
+
+from repowise.core.generation.concept_tree.grouping import (
+ GroupingParams,
+ group_files,
+ package_file_counts,
+)
+from repowise.core.generation.concept_tree.naming import (
+ build_payload,
+ decode_response,
+ deterministic_title,
+)
+from repowise.core.generation.concept_tree.planner import PlannerInputs, plan_deterministic
+from repowise.core.generation.context_assembler import ContextAssembler, FilePageContext
+from repowise.core.generation.models import GenerationConfig
+from repowise.core.generation.page_generator import PageGenerator
+from repowise.core.generation.selection import SelectionInputs
+from repowise.core.generation.selection.selector import _build_module_groups
+from repowise.core.providers.llm.mock import MockProvider
+from tests.unit.generation.test_selection_contract import (
+ FakeFileInfo,
+ FakeParsedFile,
+ FakeSymbol,
+)
+
+PARAMS = GroupingParams(min_files=4, max_files=10)
+
+THIN = ["alpha", "bravo", "charlie", "delta", "echo", "foxtrot"]
+
+
+def _monorepo() -> tuple[list[str], set[str]]:
+ """Six thin packages and one fat one under ``packages/``, plus root files."""
+ files = [f"packages/{name}/src/f{i}.ts" for name in THIN for i in range(2)]
+ files += [
+ f"packages/kernel/src/{sub}/f{i}.ts" for sub in ("io", "model", "run") for i in range(6)
+ ]
+ files += ["build.ts", "index.ts"]
+ roots = {f"packages/{name}" for name in [*THIN, "kernel"]}
+ return files, roots
+
+
+def _package_of(path: str, roots: set[str]) -> str:
+ hits = [r for r in roots if path.startswith(r + "/")]
+ return max(hits, key=len) if hits else ""
+
+
+def _groups():
+ files, roots = _monorepo()
+ return group_files(files, params=PARAMS, package_roots=roots), roots
+
+
+def test_no_leaf_group_spans_two_packages():
+ groups, roots = _groups()
+ for g in groups:
+ owners = {_package_of(m, roots) for m in g.members}
+ if g.packages:
+ continue
+ assert len(owners) == 1, (g.target_path, owners)
+
+
+def test_thin_siblings_roll_up_whole_under_their_parent():
+ groups, _roots = _groups()
+ rollups = [g for g in groups if g.packages]
+ assert len(rollups) == 1
+ rollup = rollups[0]
+ assert rollup.packages == tuple(f"packages/{n}" for n in THIN)
+ # Whole packages only: every file of each member is on this page.
+ for pkg in rollup.packages:
+ assert {m for m in _monorepo()[0] if m.startswith(pkg + "/")} <= set(rollup.members)
+ assert rollup.target_path == "packages"
+ assert package_file_counts(rollup.members, rollup.packages) == [
+ (f"packages/{n}", 2) for n in THIN
+ ]
+
+
+def test_no_title_or_target_names_a_single_member():
+ groups, _roots = _groups()
+ for g in groups:
+ if not g.packages:
+ continue
+ assert not any(g.target_path == p or g.target_path.startswith(p + "/") for p in g.packages)
+ title = deterministic_title(g)
+ assert title.startswith("Packages:"), title
+ assert "Alpha" in title and "Bravo" in title and "More" in title
+
+
+def test_a_subtree_that_fits_still_splits_at_a_package_root():
+ """Two packages small enough to share one subtree group stay two groups."""
+ files = [f"libs/one/f{i}.py" for i in range(4)] + [f"libs/two/f{i}.py" for i in range(4)]
+ groups = group_files(files, params=PARAMS, package_roots={"libs/one", "libs/two"})
+ assert sorted(g.target_path for g in groups) == ["libs/one", "libs/two"]
+ assert not any(g.packages for g in groups)
+
+
+def test_a_lone_thin_package_does_not_absorb_its_neighbour():
+ files = [f"src/app/f{i}.py" for i in range(8)] + ["src/plugin/a.py", "src/plugin/b.py"]
+ groups = group_files(files, params=PARAMS, package_roots={"src/plugin"})
+ by_target = {g.target_path: sorted(g.members) for g in groups}
+ assert by_target["src/plugin"] == ["src/plugin/a.py", "src/plugin/b.py"]
+
+
+def test_without_package_roots_the_partition_is_unchanged():
+ files, _roots = _monorepo()
+ plain = group_files(files, params=PARAMS)
+ walled_nothing = group_files(files, params=PARAMS, package_roots=())
+ assert [g.members for g in plain] == [g.members for g in walled_nothing]
+
+
+def test_the_model_cannot_rename_a_rollup_after_one_member():
+ groups, _roots = _groups()
+ payload, index = build_payload(groups)
+ gid = next(gid for gid, g in index.items() if g.packages)
+ assert next(e for e in payload["groups"] if e["id"] == gid)["packages"] == THIN
+ named, _ = decode_response(
+ {"names": {gid: {"title": "Alpha Provider Package", "scope": "The alpha provider."}}},
+ index,
+ )
+ entry = next(n for n in named if n.group.packages)
+ assert entry.title == deterministic_title(entry.group)
+ assert "6 small sibling packages" in entry.scope
+
+
+def _selection_inputs() -> SelectionInputs:
+ files, roots = _monorepo()
+ parsed = [
+ FakeParsedFile(
+ file_info=FakeFileInfo(path=p, language="typescript"), symbols=[FakeSymbol(name="fn")]
+ )
+ for p in files
+ ]
+ parsed += [
+ FakeParsedFile(
+ file_info=FakeFileInfo(path=f"{r}/package.json", language="json"), symbols=[]
+ )
+ for r in sorted(roots)
+ ]
+ paths = [p.file_info.path for p in parsed]
+ return SelectionInputs(
+ parsed_files=parsed,
+ pagerank={p: 0.1 for p in paths},
+ betweenness={p: 0.0 for p in paths},
+ community={p: 0 for p in paths},
+ community_info=None,
+ sccs=[],
+ git_meta_map=None,
+ config=GenerationConfig(coverage_pct=0.20),
+ )
+
+
+def test_selection_reads_the_walls_off_the_parsed_manifests():
+ groups = [mg for _, mg in _build_module_groups(_selection_inputs()).scored]
+ rollups = [mg for mg in groups if mg.packages]
+ assert len(rollups) == 1
+ assert rollups[0].key == "packages"
+ assert rollups[0].display.startswith("Packages:")
+ kernel = [mg for mg in groups if any(f.startswith("packages/kernel/") for f in mg.file_paths)]
+ assert all(all(f.startswith("packages/kernel/") for f in mg.file_paths) for mg in kernel)
+
+
+class _RecordingProvider(MockProvider):
+ async def generate(self, *args, **kwargs):
+ response = await super().generate(*args, **kwargs)
+ response.content = "# Small packages\n\nThese packages each wrap one provider.\n"
+ return response
+
+
+def _context(path: str) -> FilePageContext:
+ return FilePageContext(
+ file_path=path,
+ language="typescript",
+ docstring=None,
+ symbols=[],
+ imports=[],
+ exports=[],
+ pagerank_score=0.1,
+ betweenness_score=0.0,
+ community_id=0,
+ dependents=[],
+ dependencies=[],
+ is_api_contract=False,
+ is_entry_point=False,
+ is_test=False,
+ parse_errors=[],
+ estimated_tokens=10,
+ )
+
+
+async def test_rollup_prompt_names_every_package_and_page_lists_them(sample_config):
+ provider = _RecordingProvider()
+ gen = PageGenerator(provider, ContextAssembler(sample_config), sample_config)
+ members = [f"packages/{n}/src/f0.ts" for n in ("alpha", "bravo", "charlie")]
+ page = await gen.generate_module_page(
+ "Packages: Alpha, Bravo and 1 More",
+ "typescript",
+ [_context(m) for m in members],
+ nx.DiGraph(),
+ target_path="packages",
+ structural_key="k",
+ members=members,
+ packages=[{"path": f"packages/{n}", "files": 1} for n in ("alpha", "bravo", "charlie")],
+ )
+ prompt = provider._calls[-1]["user_prompt"]
+ assert (
+ "covers 3 sibling packages: `packages/alpha`, `packages/bravo`, `packages/charlie`"
+ in prompt
+ )
+ assert "## Packages" in page.content
+ for n in ("alpha", "bravo", "charlie"):
+ assert f"| `packages/{n}` | 1 |" in page.content
+
+
+def test_a_rollup_at_a_chapter_directory_keeps_its_name_and_material():
+ """Two fat siblings make ``packages`` chapter-eligible; the roll-up stays a roll-up."""
+ files, roots = _monorepo()
+ files += [f"packages/{n}/f{i}.ts" for n in ("golf", "hotel") for i in range(7)]
+ files += [f"apps/web/{d}/f{i}.ts" for d in ("a", "b", "c", "d", "e", "f") for i in range(8)]
+ roots |= {"packages/golf", "packages/hotel"}
+ parsed = [FakeParsedFile(file_info=FakeFileInfo(path=p, language="typescript")) for p in files]
+ parsed += [
+ FakeParsedFile(file_info=FakeFileInfo(path=f"{r}/package.json", language="json"))
+ for r in sorted(roots)
+ ]
+ inputs = _selection_inputs()
+ inputs.parsed_files = parsed
+ rollup = next(mg for _, mg in _build_module_groups(inputs).scored if mg.packages)
+ assert not rollup.is_rollup
+ assert rollup.display.startswith("Packages:")
+ assert rollup.context_paths == ()
+
+
+def test_the_planner_walls_packages_the_way_selection_does():
+ files, roots = _monorepo()
+ walled = group_files(files, params=PARAMS, package_roots=roots)
+ _outline, groups = plan_deterministic(
+ PlannerInputs(repo_name="r", production_files=files, package_roots=roots), params=PARAMS
+ )
+ assert [g.members for g in groups] == [g.members for g in walled]
+ assert [g.packages for g in groups if g.packages] == [tuple(f"packages/{n}" for n in THIN)]
+
+
+def test_top_level_thin_packages_roll_up_at_the_root():
+ files = [f"{n}/f{i}.py" for n in ("alpha", "bravo") for i in range(2)]
+ files += [f"core/{d}/f{i}.py" for d in ("x", "y") for i in range(6)]
+ groups = group_files(files, params=PARAMS, package_roots={"alpha", "bravo"})
+ rollup = next(g for g in groups if g.packages)
+ assert rollup.target_path == "root"
+ assert deterministic_title(rollup) == "Root Packages: Alpha, Bravo"
+
+
+def test_a_package_with_a_nested_manifest_is_sized_by_its_subtree():
+ """``pkg/package.json`` over ``pkg/src/package.json``: the outer root owns one
+ file directly but its code sits under the inner root, so it is not thin."""
+ files = [f"packages/{n}/src/f{i}.ts" for n in ("alpha", "bravo") for i in range(2)]
+ files += ["packages/vue/index.ts"]
+ files += [f"packages/vue/src/{d}/f{i}.ts" for d in ("a", "b") for i in range(4)]
+ roots = {"packages/alpha", "packages/bravo", "packages/vue", "packages/vue/src"}
+ groups = group_files(files, params=PARAMS, package_roots=roots)
+ rollup = next(g for g in groups if g.packages)
+ assert rollup.packages == ("packages/alpha", "packages/bravo")
+ assert not any(m.startswith("packages/vue/") for m in rollup.members)
+ assert package_file_counts(files, ["packages/vue", *rollup.packages]) == [
+ ("packages/alpha", 2),
+ ("packages/bravo", 2),
+ ("packages/vue", 9),
+ ]