Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.
Python aicode-complexitycode-healthcode-intelligencecode-qualitydead-codedeveloper-toolsdocumentationgit-analyticsmcpopen-sourcepythonrefactoringstatic-analysistechnical-debt
refactor(core): move KG module derivation and validation out of kg_curation.py (#2592)
Raghav Chamadiya8e499d2c2572
diff --git a/packages/core/src/repowise/core/analysis/kg_curation.py b/packages/core/src/repowise/core/analysis/kg_curation.pyindex a6fea743a..502ddc3c2 100644--- a/packages/core/src/repowise/core/analysis/kg_curation.py+++ b/packages/core/src/repowise/core/analysis/kg_curation.py@@ -11,23 +11,18 @@ summaries.**Hard invariant.** Curation reads the NetworkX graph, communities, andcentrality, but it *only ever writes the returned* :class:`KnowledgeGraphResult`.It never mutates ``graph_builder``'s graph, ``graph_edges``, centrality caches,-community detection, or any DB table. There is a regression test that asserts the-graph's node/edge counts are identical before and after this pass.+community detection, or any DB table.Curation is feature-flagged (``REPOWISE_KG_CURATION``) and defaults **on**;-the 38-repo cross-language validation matrix is the acceptance gate that-flipped it. Setting the flag to ``0``/``false``/``no``/``off`` makes-:func:`curate_knowledge_graph` a no-op that returns its input unchanged-(the raw uncurated export).+see :func:`curation_enabled`."""from __future__ import annotationsimport loggingimport os-import re-from collections import Counter, defaultdict-from dataclasses import dataclass, field+from collections import defaultdict+from collections.abc import Callablefrom pathlib import PurePosixPathfrom typing import Any@@ -38,11 +33,24 @@ from repowise.core.analysis.kg_inputs import (_file_nodes,_is_barrel,)+from repowise.core.analysis.kg_modules import ( # noqa: F401 (re-exported)+ GENERIC_ORG_SEGMENTS,+ _common_dir_prefix,+ derive_modules,+ dominant_segments,+)from repowise.core.analysis.kg_tour import ( # noqa: F401 (re-exported)_anchor_fanout_rank,_curate_tour,_import_pairs_excluding_fanout,)+from repowise.core.analysis.kg_validation import (+ _MAX_ENTRY_POINTS,+ _MAX_LAYERS,+ KGValidation,+ build_portable_kg,+ validate_kg,+)from repowise.core.analysis.knowledge_graph import KnowledgeGraphResult, _slugifyfrom repowise.core.entry_candidacy import (conventional_entry_stems,@@ -55,10 +63,7 @@ from repowise.core.generation.layers import (infer_layer,layer_order_basis,)-from repowise.core.generation.tour import (- DEFAULT_MAX_STOPS,- score_entry_points,-)+from repowise.core.generation.tour import score_entry_pointsfrom repowise.core.generation.well_known_files import well_known_rolefrom repowise.core.ingestion.languages.registry import REGISTRY as _LANG_REGISTRYfrom repowise.core.support_paths import is_support_path@@ -81,27 +86,16 @@ _FLAG_ENV = "REPOWISE_KG_CURATION"# A primary layer larger than this many files, or spanning more than this many# distinct sub-directories, is given a two-level structure (primary → named# sub-groups) so a mega-layer like core/* or ui/* stays drill-down legible-# instead of becoming one opaque bucket (plan §Phase 1, edge case B).+# instead of becoming one opaque bucket._SUBSPLIT_FILE_THRESHOLD = 60_SUBSPLIT_DIR_THRESHOLD = 8-# Hard bound on the curated primary-layer count. The spine is bounded ≤~11 by-# construction; if a future change ever blows past this we degrade to the-# uncurated layers rather than ship an unreadable list.-_MAX_LAYERS = 15--# Entry points surfaced in ``project.entry_points``; the full ranked list is-# kept as ``entry_candidates``.-_MAX_ENTRY_POINTS = 8-def curation_enabled() -> bool:"""Whether KG curation is enabled via the ``REPOWISE_KG_CURATION`` env flag.- Defaults to **on** — the cross-language validation matrix (38 pinned- repos, enforced density/orphan/catch-all thresholds, honest degradation- modes) is the acceptance gate that flipped it. Set ``0``/``false``/``no``/- ``off`` (case-insensitive) to fall back to the raw uncurated export.+ Defaults to **on**. Set ``0``/``false``/``no``/``off`` (case-insensitive)+ to fall back to the raw uncurated export.Resolved at the call site so :func:`curate_knowledge_graph` itself stayspure and trivially testable with an explicit ``enabled=``."""@@ -123,8 +117,7 @@ def curate_knowledge_graph(Pure with respect to the AST graph: reads ``graph_builder`` /``community_info`` but writes only the returned result. When ``enabled`` is- ``False`` this is a strict no-op returning ``kg`` unchanged (the default, so- the exported KG is unaffected until the flag flips).+ ``False`` this is a strict no-op returning ``kg`` unchanged.``defer_summary_floor`` skips the never-empty summary floor here so it canrun *after* the wiki-page backfill in generate mode (where richer summaries@@ -136,88 +129,83 @@ def curate_knowledge_graph(if not enabled:return kg- # Each step mutates only ``kg`` (the presentation result) and is guarded so- # a failure degrades to the prior, uncurated field rather than aborting the- # export. Steps are layered in by subsequent phases:- # _curate_layers -> _curate_entry_points -> _curate_tour- # -> _curate_node_types -> _curate_summaries- layers_curated = False- try:- curated = _curate_layers(kg, graph_builder)- if curated is not None:- kg.layers = curated- layers_curated = True- except Exception: # pragma: no cover - defensive; keep uncurated layers- logger.exception("kg_curation._curate_layers failed; keeping community layers")-- # Wiki modules are a *sibling* artifact of the curated layers (same- # splitting machinery, module-sized granularity). Only derived when the- # spine landed — community layers would make the dir-split meaningless,- # and downstream consumers fall back to community grouping when this- # stays empty (the fallback matrix's "degraded" row).- if layers_curated:- try:- modules = _curate_modules(kg)- if modules is not None:- kg.modules = modules- except Exception: # pragma: no cover - defensive; ship no modules- logger.exception("kg_curation._curate_modules failed; exporting no modules")+ layers = _guarded(+ "kg_curation._curate_layers failed; keeping community layers",+ _curate_layers,+ kg,+ graph_builder,+ )+ if layers is not None:+ kg.layers = layers+ # Modules need the curated spine; over community layers the dir split+ # is meaningless, and consumers fall back to community grouping.+ modules = _guarded(+ "kg_curation._curate_modules failed; exporting no modules", _curate_modules, kg+ )+ if modules is not None:+ kg.modules = modules++ _guarded(+ "kg_curation._curate_entry_points failed; keeping raw entry points",+ _curate_entry_points,+ kg,+ parsed_files,+ graph_builder,+ )- try:- _curate_entry_points(kg, parsed_files, graph_builder)- except Exception: # pragma: no cover - defensive; keep skeleton entry points- logger.exception("kg_curation._curate_entry_points failed; keeping raw entry points")-- # Genuine churn hotspots (a constantly-edited file off the hot import path)- # earn a tour stop on this signal alone. Only flagged hotspots with recent- # commits qualify; repos without git history pass an empty map and the tour- # is unchanged.- hotspot_commits = {- path: int(meta.get("commit_count_90d", 0) or 0)- for path, meta in (git_meta_map or {}).items()- if meta.get("is_hotspot")- }- try:- tour = _curate_tour(kg, parsed_files, graph_builder, hotspot_commits=hotspot_commits)- if tour is not None:- kg.tour = tour- except Exception: # pragma: no cover - defensive; keep skeleton/LLM tour- logger.exception("kg_curation._curate_tour failed; keeping existing tour")+ tour = _guarded(+ "kg_curation._curate_tour failed; keeping existing tour",+ _curate_tour,+ kg,+ parsed_files,+ graph_builder,+ hotspot_commits=_hotspot_commits(git_meta_map),+ )+ if tour is not None:+ kg.tour = tour- try:- _curate_node_types(kg)- except Exception: # pragma: no cover - defensive; keep skeleton types- logger.exception("kg_curation._curate_node_types failed; keeping coarse types")+ _guarded("kg_curation._curate_node_types failed; keeping coarse types", _curate_node_types, kg)if not defer_summary_floor:- try:- apply_summary_floor(kg, parsed_files)- except Exception: # pragma: no cover - defensive; leave summaries as-is- logger.exception("kg_curation summary floor failed; leaving summaries empty")+ _guarded(+ "kg_curation summary floor failed; leaving summaries empty",+ apply_summary_floor,+ kg,+ parsed_files,+ )return kg+def _guarded(failure: str, step: Callable[..., Any], *args: Any, **kwargs: Any) -> Any:+ """Run one curation *step*; on error log *failure* and return ``None``."""+ try:+ return step(*args, **kwargs)+ except Exception: # pragma: no cover - defensive; keep the uncurated field+ logger.exception(failure)+ return None+++def _hotspot_commits(git_meta_map: dict[str, dict] | None) -> dict[str, int]:+ """Recent commit counts of the flagged churn hotspots.++ Genuine churn hotspots (a constantly-edited file off the hot import path)+ earn a tour stop on this signal alone. Only flagged hotspots with recent+ commits qualify; repos without git history pass an empty map and the tour+ is unchanged.+ """+ return {+ path: int(meta.get("commit_count_90d", 0) or 0)+ for path, meta in (git_meta_map or {}).items()+ if meta.get("is_hotspot")+ }++# ---------------------------------------------------------------------------# Phase 1 — curated layers (replace raw-community layers with the spine)# ----------------------------------------------------------------------------def _common_dir_prefix(seg_lists: list[tuple[str, ...]]) -> tuple[str, ...]:- """Longest common leading directory-segment prefix across *seg_lists*."""- if not seg_lists:- return ()- common = list(seg_lists[0])- for segs in seg_lists[1:]:- i = 0- while i < len(common) and i < len(segs) and common[i] == segs[i]:- i += 1- del common[i:]- if not common:- break- return tuple(common)--def _sub_split(layer_id: str, node_ids: list[str], id_to_path: dict[str, str]) -> list[dict] | None:"""Two-level sub-groups for an oversized/wide primary layer, else ``None``.@@ -269,371 +257,62 @@ def _curate_layers(kg: KnowledgeGraphResult, graph_builder: Any) -> list[dict] |}import_edges = _file_import_edges(graph_builder)order = compute_layer_order(file_layers, import_edges)- # Honesty label (additive export field): "imports" when inter-layer edges- # informed the order, "canonical" when it is pure convention — consumers- # must not claim "X sits above Y" for a canonical order.+ # "imports" when inter-layer edges informed the order, "canonical" when it+ # is pure convention: consumers must not claim "X sits above Y" for the latter.order_basis = layer_order_basis(file_layers, import_edges)by_layer: dict[str, list[str]] = defaultdict(list)for n in file_nodes:by_layer[file_layers[n["filePath"]]].append(n["id"])- layers: list[dict] = []- for display_order, layer_name in enumerate(order):- node_ids = by_layer[layer_name]- layer_id = f"layer:{_slugify(layer_name)}"- layer: dict[str, Any] = {- "id": layer_id,- "name": layer_name,- "description": "",- "nodeIds": node_ids,- "display_order": display_order,- "order_basis": order_basis,- }- sub_groups = _sub_split(layer_id, node_ids, id_to_path)- if sub_groups:- layer["subGroups"] = sub_groups- layers.append(layer)-- # Degrade rather than ship a broken artifact: enforce bound + partition.+ layers = [+ _layer_record(layer_name, display_order, by_layer[layer_name], order_basis, id_to_path)+ for display_order, layer_name in enumerate(order)+ ]+ return layers if _layers_hold_invariants(layers, len(file_nodes)) else None+++def _layer_record(+ layer_name: str,+ display_order: int,+ node_ids: list[str],+ order_basis: str,+ id_to_path: dict[str, str],+) -> dict[str, Any]:+ layer_id = f"layer:{_slugify(layer_name)}"+ layer: dict[str, Any] = {+ "id": layer_id,+ "name": layer_name,+ "description": "",+ "nodeIds": node_ids,+ "display_order": display_order,+ "order_basis": order_basis,+ }+ sub_groups = _sub_split(layer_id, node_ids, id_to_path)+ if sub_groups:+ layer["subGroups"] = sub_groups+ return layer+++def _layers_hold_invariants(layers: list[dict], file_count: int) -> bool:+ """Degrade rather than ship a broken artifact: enforce bound + partition."""total = sum(len(layer["nodeIds"]) for layer in layers)- if not layers or len(layers) > _MAX_LAYERS or total != len(file_nodes):+ if not layers or len(layers) > _MAX_LAYERS or total != file_count:logger.warning("kg_curation: curated layers failed invariant ""(count=%d, partition=%d/%d); keeping community layers",len(layers),total,- len(file_nodes),+ file_count,)- return None- return layers+ return False+ return True# ---------------------------------------------------------------------------# Wiki modules — right-sized directory groups derived from the curated layers# ----------------------------------------------------------------------------# Granularity window for derived wiki modules. Sub-groups verbatim are NOT-# module-sized (a 452-file ``core`` sub-group would make one vague mush of a-# doc; a 1-file ``examples`` group would mint a confetti page), so the layer-# node sets are split *recursively* by directory until every group fits the-# window — bottoming out honestly on flat directories. ``target_max`` keeps-# the 10 key-file template slots representative; ``target_min`` is the-# merge-up floor below which a group folds into its nearest sibling.-_MODULE_TARGET_MIN = 8-_MODULE_TARGET_MAX = 120-# A layer smaller than this yields no module at all (matches the selection-# layer's ``min_module_size`` floor that kills singleton pages).-_MODULE_MIN_FILES = 3-# A directory segment present in more than this fraction of all repo paths is-# *generic* (namespace dirs: ``src``, ``packages``, the repo's own name) and-# never appears in a module name. Data-driven — no hardcoded segment list.-_GENERIC_SEGMENT_FRACTION = 0.60-# The legacy community labels' size-suffix dedupe ("ingestion (32)") is the-# exact failure mode module names must never reproduce.-_SIZE_SUFFIX_RE = re.compile(r"\(\d+\)\s*$")---# Universal organizational directory names — containers, not domain labels.-# Shared with community labeling; the data-driven ``dominant_segments`` set-# complements this with per-repo namespace noise (the repo's own name).-GENERIC_ORG_SEGMENTS = frozenset({- "src", "lib", "core", "common", "shared", "internal", "pkg",- "main", "app", "utils", "helpers", "index", "mod",- # Monorepo organisational directories- "packages", "modules", "workspace", "workspaces", "libs",- "projects", "services", "apps",-})---def dominant_segments(paths: list[str]) -> set[str]:- """Directory segments appearing in > 60% of *paths* (namespace noise).-- Shared with community labeling (``analysis/communities.py``) so both- vocabularies strip the same namespace dirs (``src``, ``packages``, the- repo's own name) without depending on a hardcoded list.- """- n = len(paths)- if not n:- return set()- counts: Counter[str] = Counter()- for p in paths:- for seg in set(PurePosixPath(p).parts[:-1]):- counts[seg] += 1- return {s for s, c in counts.items() if c / n > _GENERIC_SEGMENT_FRACTION}---def _split_to_granularity(- node_ids: list[str], id_to_path: dict[str, str], target_max: int-) -> list[tuple[tuple[str, ...], list[str]]]:- """Recursively split *node_ids* by directory until groups fit *target_max*.-- Returns ``[(dir_segments, sorted_node_ids), ...]``. Reuses ``_sub_split``'s- prefix logic (group by the first segment that distinguishes members after- the common directory prefix) but, unlike sub-groups, recurses into any- group still above ``target_max``. Recursion bottoms out when a directory- has no distinguishing subdirs — a 200-file flat dir stays one module- (honest), never an artificial split.- """- dir_segs = {nid: PurePosixPath(id_to_path[nid]).parts[:-1] for nid in node_ids}-- def rec(ids: list[str]) -> list[tuple[tuple[str, ...], list[str]]]:- common = _common_dir_prefix([dir_segs[i] for i in ids])- if len(ids) <= target_max:- return [(common, ids)]- groups: dict[str, list[str]] = defaultdict(list)- for nid in ids:- segs = dir_segs[nid]- key = segs[len(common)] if len(segs) > len(common) else ""- groups[key].append(nid)- if len(groups) < 2:- return [(common, ids)] # flat directory — no honest split exists- out: list[tuple[tuple[str, ...], list[str]]] = []- for key in sorted(groups):- if key == "":- # Files sitting directly in the common dir (the "(root)"- # group). Usually below target_min → folded by merge-up.- out.append((common, groups[key]))- else:- out.extend(rec(groups[key]))- return out-- return [(d, sorted(ids)) for d, ids in rec(sorted(node_ids))]---def _merge_small_groups(- groups: list[tuple[tuple[str, ...], list[str]]], target_min: int-) -> list[tuple[tuple[str, ...], list[str]]]:- """Fold groups below *target_min* into their nearest sibling.-- "Nearest" = the group sharing the longest directory prefix (the parent- subtree), largest first as the tie-break — so a 2-file "(root)" remnant- folds into its own subtree's biggest module, and an isolated small dir- folds into the layer's dominant module rather than minting a confetti- page. Never merges across layers (callers pass one layer at a time). A- layer that is itself below ``target_min`` stays one whole group.-- A pre-pass fuses *small sibling* groups into one group at their common- parent when that collection is itself module-sized — ninety tiny locale- dirs become one ``conf/locale`` module instead of folding into whichever- sibling sorts first and misnaming it. The fold-in loop then never renames- a survivor: a healthy ``core/providers`` absorbing a 2-file sibling keeps- its identity.- """- merged = [(d, list(ids)) for d, ids in groups]-- by_parent: dict[tuple[str, ...], list[tuple[tuple[str, ...], list[str]]]] = {}- for g in merged:- if len(g[1]) < target_min and len(g[0]) > 0:- by_parent.setdefault(g[0][:-1], []).append(g)- for parent, sibs in sorted(by_parent.items()):- if len(sibs) < 2 or sum(len(g[1]) for g in sibs) < target_min:- continue- fused = sorted(nid for g in sibs for nid in g[1])- for g in sibs:- merged.remove(g)- existing = next((g for g in merged if g[0] == parent), None)- if existing is not None:- existing[1].extend(fused)- existing[1].sort()- else:- merged.append((parent, fused))- merged.sort(key=lambda g: g[0])-- def shared(a: tuple[str, ...], b: tuple[str, ...]) -> int:- return len(_common_dir_prefix([a, b]))-- while len(merged) > 1:- small = min(- (g for g in merged if len(g[1]) < target_min),- key=lambda g: (len(g[1]), g[0]),- default=None,- )- if small is None:- break- merged.remove(small)- target = min(- merged,- key=lambda g: (-shared(g[0], small[0]), -len(g[1]), g[0]),- )- target[1].extend(small[1])- target[1].sort()- return [(d, ids) for d, ids in merged]---def _name_modules(mods: list[dict], generic: set[str]) -> None:- """Assign unique, human module names in place.-- Initial name = the last one or two *informative* directory segments- (generic namespace segments stripped; when stripping consumes every- segment, the raw tail is used instead). Collisions extend leftward by- one more parent segment — NEVER a size suffix. Single-module layers- take the layer's name; the root group (empty dir) becomes- "<Layer> (top-level)". The absolute fallback (identical informative- paths across layers) appends the layer name, which is unique by- construction.- """- per_layer: Counter[str] = Counter(m["layerId"] for m in mods)- info_by: dict[int, list[str]] = {}- used: dict[int, int | None] = {} # informative segments consumed; None = fixed- for m in mods:- # Data-driven stripping can consume EVERY segment on fixture-dominated- # repos (aeson: tests/JSONTestSuite/test_parsing is >60% of all- # paths). The raw dir tail is still the honest name there —- # "(top-level)" would mislabel a real directory and collide across- # sibling groups (which trips the export degradation guard and ships- # no modules). Universal organizational dirs (pkg, src, packages…)- # stay excluded even in the fallback: "(top-level)" reads better than- # a container name, so it remains the name for true root groups.- info = [s for s in m["_dir"] if s not in generic] or [- s for s in m["_dir"] if s.lower() not in GENERIC_ORG_SEGMENTS- ]- info_by[id(m)] = info- if per_layer[m["layerId"]] == 1:- m["name"] = m["_layerName"]- used[id(m)] = None- elif not info:- m["name"] = f"{m['_layerName']} (top-level)"- used[id(m)] = None- else:- k = min(2, len(info))- m["name"] = "/".join(info[-k:])- used[id(m)] = k-- for _ in range(16): # bounded: each round consumes ≥1 segment somewhere- names = Counter(m["name"] for m in mods)- colliding = [m for m in mods if names[m["name"]] > 1]- if not colliding:- return- progressed = False- for m in colliding:- k = used.get(id(m))- info = info_by[id(m)]- if k is not None and k < len(info):- used[id(m)] = k + 1- m["name"] = "/".join(info[-(k + 1) :])- progressed = True- if not progressed:- break-- # Two all-organizational groups in one layer (a root remnant plus a- # "packages"-style container) would both read "<Layer> (top-level)" —- # the container's raw tail is the honest tiebreak.- names = Counter(m["name"] for m in mods)- for m in mods:- if names[m["name"]] > 1 and not info_by[id(m)] and m["_dir"]:- m["name"] = "/".join(m["_dir"][-min(2, len(m["_dir"])) :])-- # Same informative dir in two layers (or no segments left): the layer- # name disambiguates — (dir, layer) is unique by construction.- names = Counter(m["name"] for m in mods)- for m in mods:- if names[m["name"]] > 1:- m["name"] = f"{m['name']} ({m['_layerName']})"-- # Absolute backstop (two all-org dirs in one layer sharing a tail): the- # full dir path is unique per layer.- names = Counter(m["name"] for m in mods)- for m in mods:- if names[m["name"]] > 1 and m["_dir"]:- m["name"] = "/".join(m["_dir"])---def derive_modules(- layers: list[dict],- id_to_path: dict[str, str],- *,- target_min: int = _MODULE_TARGET_MIN,- target_max: int = _MODULE_TARGET_MAX,- min_module_size: int = _MODULE_MIN_FILES,- lang_by_id: dict[str, str] | None = None,-) -> list[dict]:- """Derive right-sized, stably-identified wiki modules from curated layers.-- ``Module = {"id": "module:<dir-slug>", "name": <human>, "path": <dir or "">,- "layerId": ..., "nodeIds": [...], "language": ...}``-- Properties (each one an edge case from the research pass):-- - **Partition per layer**: every node of every layer ≥ ``min_module_size``- lands in exactly one module; layers below the floor yield none. Never- merges across layers.- - **Granularity**: recursive directory splitting to the- [``target_min``, ``target_max``] window; flat dirs stay one honest- module; sub-``target_min`` remnants merge up into their subtree.- - **Names**: informative path segments only (data-driven generic-segment- stripping kills ``src``/``packages``/repo-name automatically); collision- resolution extends the path leftward — never a size suffix.- - **Ids**: ``module:`` + slug of the real directory path — stable across- runs and under file adds/renames inside the dir; changes only when the- directory itself moves. ``path`` is the actual dir (not the slug) so- path-prefix child lookups (``target_path LIKE 'dir/%'``) work.- - **Files only**: operates on ids present in ``id_to_path`` — external- nodes never pollute a module.- - **Determinism**: sorted iteration throughout; same inputs → same bytes.- """- generic = dominant_segments(sorted(set(id_to_path.values())))-- mods: list[dict] = []- for layer in layers:- node_ids = [nid for nid in layer.get("nodeIds", []) if nid in id_to_path]- if len(node_ids) < min_module_size:- continue- groups = _merge_small_groups(- _split_to_granularity(node_ids, id_to_path, target_max), target_min- )- for dir_parts, ids in sorted(groups):- mods.append(- {- "_dir": dir_parts,- "_layerName": layer.get("name", ""),- "path": "/".join(dir_parts),- "layerId": layer.get("id", ""),- "nodeIds": sorted(ids),- }- )-- _name_modules(mods, generic)-- # Ids: path-derived slugs; the bigger module keeps the plain id on the- # rare cross-layer dir collision (a dir whose files split across layers).- used_ids: set[str] = set()- for m in sorted(mods, key=lambda m: (-len(m["nodeIds"]), m["path"], m["layerId"])):- base = "module:" + _slugify(m["path"] or m["_layerName"])- mid = base- n = 1- while mid in used_ids:- mid = f"{base}--{_slugify(m['_layerName'])}" + ("" if n == 1 else f"-{n}")- n += 1- used_ids.add(mid)- m["id"] = mid-- # A single-module layer is 1:1 with its layer page — mark it so page- # generation can skip the duplicate doc (the module stays in the- # artifact: canvas containers and the coverage invariant need it).- per_layer_count: Counter[str] = Counter(m["layerId"] for m in mods)-- out: list[dict] = []- for m in mods:- module = {- "id": m["id"],- "name": m["name"],- "path": m["path"],- "layerId": m["layerId"],- "nodeIds": m["nodeIds"],- }- if per_layer_count[m["layerId"]] == 1:- module["wholeLayer"] = True- if lang_by_id is not None:- langs = Counter(- lang for nid in m["nodeIds"] if (lang := lang_by_id.get(nid, ""))- )- module["language"] = (- min(langs, key=lambda tag: (-langs[tag], tag)) if langs else ""- )- out.append(module)- return out-def _curate_modules(kg: KnowledgeGraphResult) -> list[dict] | None:"""Derive wiki modules from the curated layers, or ``None`` on degradation.@@ -649,9 +328,12 @@ def _curate_modules(kg: KnowledgeGraphResult) -> list[dict] | None:lang_by_id = {n["id"]: (n.get("language") or "").lower() for n in file_nodes}modules = derive_modules(kg.layers, id_to_path, lang_by_id=lang_by_id)- if not modules:+ if not modules or not _modules_hold_invariants(modules, id_to_path):return None+ return modules++def _modules_hold_invariants(modules: list[dict], id_to_path: dict[str, str]) -> bool:seen: set[str] = set()for m in modules:for nid in m["nodeIds"]:@@ -660,7 +342,7 @@ def _curate_modules(kg: KnowledgeGraphResult) -> list[dict] | None:"kg_curation: derived modules failed partition invariant; ""exporting no modules")- return None+ return Falseseen.add(nid)names = [m["name"] for m in modules]ids = [m["id"] for m in modules]@@ -668,8 +350,8 @@ def _curate_modules(kg: KnowledgeGraphResult) -> list[dict] | None:logger.warning("kg_curation: derived module names/ids not unique; exporting no modules")- return None- return modules+ return False+ return True# ---------------------------------------------------------------------------@@ -698,30 +380,31 @@ def _curate_entry_points(pf_by_path = {pf.file_info.path: pf for pf in parsed_files if getattr(pf, "file_info", None)}lang_by_path = {n["filePath"]: (n.get("language") or "").lower() for n in _file_nodes(kg)}pagerank = graph_builder.pagerank() or {}- try:- betweenness = graph_builder.betweenness_centrality() or {}- except Exception: # pragma: no cover - defensive- betweenness = {}-- candidates = [- (path, pagerank.get(path, 0.0), betweenness.get(path, 0.0))- for path in _flagged_entry_paths(kg, pf_by_path)- ]- if not candidates:- # No ingestion-flagged entries (or all were barrels): fall back to the- # strong filename scorers the tour seeds from (score >= 3 means an- # entry-style name or flag, never just shallow/high-PageRank).- candidates = [- (path, pagerank.get(path, 0.0), betweenness.get(path, 0.0))+ betweenness = _betweenness(graph_builder)++ paths = _flagged_entry_paths(kg, pf_by_path)+ if not paths:+ # No flagged entries survived: score >= 3 means an entry-style name or+ # flag, never just a shallow or high-PageRank file.+ paths = [+ pathfor s, path in score_entry_points(parsed_files, pagerank)if s >= 3.0 and _is_entry_candidate(path, lang_by_path.get(path, ""), pf_by_path)]+ candidates = [(path, pagerank.get(path, 0.0), betweenness.get(path, 0.0)) for path in paths]ranked = rank_entry_points(candidates, conventional_entry_stems())kg.project["entry_points"] = ranked[:_MAX_ENTRY_POINTS]kg.project["entry_candidates"] = ranked+def _betweenness(graph_builder: Any) -> dict[str, float]:+ try:+ return graph_builder.betweenness_centrality() or {}+ except Exception: # pragma: no cover - defensive+ return {}++def _flagged_entry_paths(kg: KnowledgeGraphResult, pf_by_path: dict[str, Any]) -> list[str]:"""Ingestion-flagged entry files that survive candidacy, in node order.@@ -730,29 +413,39 @@ def _flagged_entry_paths(kg: KnowledgeGraphResult, pf_by_path: dict[str, Any]) -"""paths: list[str] = []for node in kg.nodes:- nid = node.get("id", "")- if not (isinstance(nid, str) and nid.startswith("file:")):- continue- tags = node.get("tags") or []- if "entry_point" not in tags:+ if not _is_flagged_file_node(node):continuepath = node.get("filePath", "")language = (node.get("language") or "").lower()if _off_the_entry_path(path, language):continue- pf = pf_by_path.get(path)- if pf is not None and _is_barrel(pf):- new_tags = [t for t in tags if t != "entry_point"]- if "barrel" not in new_tags:- new_tags.append("barrel")- node["tags"] = new_tags+ if _is_barrel_path(path, pf_by_path):+ _retag_as_barrel(node)continue- if not_an_execution_start(path, language):- continue- paths.append(path)+ if not not_an_execution_start(path, language):+ paths.append(path)return paths+def _is_flagged_file_node(node: dict) -> bool:+ nid = node.get("id", "")+ if not (isinstance(nid, str) and nid.startswith("file:")):+ return False+ return "entry_point" in (node.get("tags") or [])+++def _retag_as_barrel(node: dict) -> None:+ new_tags = [t for t in (node.get("tags") or []) if t != "entry_point"]+ if "barrel" not in new_tags:+ new_tags.append("barrel")+ node["tags"] = new_tags+++def _is_barrel_path(path: str, pf_by_path: dict[str, Any]) -> bool:+ pf = pf_by_path.get(path)+ return pf is not None and _is_barrel(pf)++def _off_the_entry_path(path: str, language: str) -> bool:# Test fixtures (a wsgi.py inside tests/) and sample programs# (examples/*/main.go) may carry the ingestion flag, but they are@@ -762,10 +455,7 @@ def _off_the_entry_path(path: str, language: str) -> bool:def _is_entry_candidate(path: str, language: str, pf_by_path: dict[str, Any]) -> bool:"""Whether a scored (unflagged) file may stand as an entry point."""- if _off_the_entry_path(path, language):- return False- pf = pf_by_path.get(path)- if pf is not None and _is_barrel(pf):+ if _off_the_entry_path(path, language) or _is_barrel_path(path, pf_by_path):return Falsereturn not not_an_execution_start(path, language)@@ -793,10 +483,8 @@ _DATA_SUFFIXES = (".sql", ".prisma")# Source-code extensions. A code file is never CI/infra config however its# name or directory reads — ``languages/specs/dockerfile.py`` *parses*-# Dockerfiles, it isn't one. Registry-derived: every is_code,-# non-infra language's extensions are protected — .dart/.hs/.clj included;-# shell/terraform stay promotable (they ARE infra); the historical orphan-# ``.pl`` (no perl spec) is gone.+# Dockerfiles, it isn't one. Registry-derived from every non-infra code+# language; shell/terraform stay promotable because they are infra._CODE_SUFFIXES = _LANG_REGISTRY.non_infra_code_extensions()@@ -812,22 +500,25 @@ def _enrich_type(path: str, current_type: str) -> tuple[str, str | None]:suffix = PurePosixPath(p).suffixis_code = suffix in _CODE_SUFFIXES- if not is_code and (any(m in p for m in _CI_PATH_MARKERS) or name == "jenkinsfile"):+ if not is_code and _is_ci_path(p, name):return "pipeline", "ci"- if (- not is_code- and (- name.startswith("dockerfile")- or any(m in name for m in _INFRA_NAME_MARKERS)- or any(m in p for m in _INFRA_PATH_MARKERS)- )- ) or suffix in _INFRA_SUFFIXES:+ if (not is_code and _is_infra_path(p, name)) or suffix in _INFRA_SUFFIXES:return "service", "infra"if any(m in p for m in _DATA_PATH_MARKERS) or suffix in _DATA_SUFFIXES:return "schema", "data"return current_type, None+def _is_ci_path(lowered_path: str, name: str) -> bool:+ return any(m in lowered_path for m in _CI_PATH_MARKERS) or name == "jenkinsfile"+++def _is_infra_path(lowered_path: str, name: str) -> bool:+ if name.startswith("dockerfile") or any(m in name for m in _INFRA_NAME_MARKERS):+ return True+ return any(m in lowered_path for m in _INFRA_PATH_MARKERS)++def _curate_node_types(kg: KnowledgeGraphResult) -> None:"""Promote infra/CI/data file nodes to first-class presentation types."""for node in _file_nodes(kg):@@ -850,49 +541,53 @@ def _infer_test_target(path: str) -> str:return stem+# Support-file summary templates in precedence order: the first row whose+# presentation type or tag matches the node names it.+_SUPPORT_TEMPLATES: tuple[tuple[str, str | None, str], ...] = (+ ("pipeline", "ci", "CI / pipeline definition: {name}."),+ ("service", "infra", "Infrastructure definition: {name}."),+ ("schema", "data", "Data / schema definition: {name}."),+ ("config", "config", "Configuration file: {name}."),+ ("document", None, "Documentation: {name}."),+)++def _cheap_summary(node: dict, parsed_file: Any | None) -> str:"""A deterministic, honest fallback summary (zero LLM cost)."""path = node["filePath"]- stem = PurePosixPath(path).stem- parent = PurePosixPath(path).parent.name or "root"- node_type = node.get("type", "file")tags = node.get("tags") or []- layer = infer_layer(path, (node.get("language") or "").lower())if "barrel" in tags:- return f"Re-export barrel for {parent}/."-- name = PurePosixPath(path).name- if node_type in {"pipeline", "service", "schema", "config", "document"} or (- tags and ({"ci", "infra", "data", "config"} & set(tags))- ):- # Recognised scaffolding earns a real role instead of a bare name- # restatement; only genuinely opaque support files fall back to the- # type template below.- role = well_known_role(path)- if role is not None:- return role- if node_type == "pipeline" or "ci" in tags:- return f"CI / pipeline definition: {name}."- if node_type == "service" or "infra" in tags:- return f"Infrastructure definition: {name}."- if node_type == "schema" or "data" in tags:- return f"Data / schema definition: {name}."- if node_type == "config" or "config" in tags:- return f"Configuration file: {name}."- if node_type == "document":- return f"Documentation: {name}."+ return f"Re-export barrel for {PurePosixPath(path).parent.name or 'root'}/."+ support = _support_summary(path, node.get("type", "file"), tags)+ if support is not None:+ return supportif "test" in tags:return f"Tests for {_infer_test_target(path)}."+ return _code_summary(node, parsed_file)+- # Code file: name the layer and its most prominent symbols.- symbol_names: list[str] = []- if parsed_file is not None:- symbol_names = [- getattr(s, "name", "")- for s in (getattr(parsed_file, "symbols", []) or [])- if getattr(s, "kind", "") in _SUBSTANTIVE_KINDS and getattr(s, "name", "")- ][:3]+def _support_summary(path: str, node_type: str, tags: list[str]) -> str | None:+ template = next(+ (text for kind, tag, text in _SUPPORT_TEMPLATES if node_type == kind or tag in tags),+ None,+ )+ if template is None:+ return None+ # Recognised scaffolding earns a real role; only opaque support files fall+ # back to the type template.+ role = well_known_role(path)+ if role is not None:+ return role+ return template.format(name=PurePosixPath(path).name)+++def _code_summary(node: dict, parsed_file: Any | None) -> str:+ """Name the code file's layer and its most prominent symbols."""+ path = node["filePath"]+ stem = PurePosixPath(path).stem+ layer = infer_layer(path, (node.get("language") or "").lower())+ symbol_names = _top_symbol_names(parsed_file)if symbol_names:return f"{layer} module {stem} defining {', '.join(symbol_names)}."count = node.get("symbolCount", 0)@@ -901,6 +596,16 @@ def _cheap_summary(node: dict, parsed_file: Any | None) -> str:return f"{layer} module {stem}."+def _top_symbol_names(parsed_file: Any | None) -> list[str]:+ if parsed_file is None:+ return []+ return [+ getattr(s, "name", "")+ for s in (getattr(parsed_file, "symbols", []) or [])+ if getattr(s, "kind", "") in _SUBSTANTIVE_KINDS and getattr(s, "name", "")+ ][:3]++def apply_summary_floor(kg: KnowledgeGraphResult, parsed_files: list[Any] | None = None) -> None:"""Ensure every file node carries a summary (cheap deterministic floor).@@ -916,234 +621,3 @@ def apply_summary_floor(kg: KnowledgeGraphResult, parsed_files: list[Any] | Noneif node.get("summary"):continuenode["summary"] = _cheap_summary(node, pf_by_path.get(node["filePath"]))---# ----------------------------------------------------------------------------# Phase 7 — invariant validation (shared by tests and the portable writer)-# -----------------------------------------------------------------------------# Quality thresholds. The lower layer bound and coverage targets are *soft*-# (warnings) because they depend on repo size/shape; the partition, hard count-# bound, capped entry set, never-empty summaries, and tour budget are *hard*.-_MIN_LAYERS = 6-_MAX_LAYER_FRACTION = 0.35-_MAX_CATCHALL_FRACTION = 0.20-_MAX_SINGLETON_FRACTION = 0.10-_MIN_TOUR_COVERAGE = 0.90---@dataclass-class KGValidation:- """Outcome of :func:`validate_kg` — hard errors, soft warnings, metrics."""-- ok: bool- errors: list[str] = field(default_factory=list)- warnings: list[str] = field(default_factory=list)- metrics: dict[str, Any] = field(default_factory=dict)-- def as_dict(self) -> dict[str, Any]:- return {- "ok": self.ok,- "errors": self.errors,- "warnings": self.warnings,- "metrics": self.metrics,- }---def validate_kg(kg: KnowledgeGraphResult) -> KGValidation:- """Validate a curated KG against the intuitiveness invariants (plan §5/§7).-- Pure and side-effect free. Hard violations set ``ok=False`` and populate- ``errors``; size/shape-dependent shortfalls go to ``warnings``. The- ``metrics`` block is the per-repo intuitiveness scorecard.- """- errors: list[str] = []- warnings: list[str] = []-- file_nodes = _file_nodes(kg)- file_count = len(file_nodes)- file_ids = {n["id"] for n in file_nodes}- tags_by_path = {n["filePath"]: (n.get("tags") or []) for n in file_nodes}- summary_by_id = {n["id"]: n.get("summary") for n in file_nodes}-- layers = kg.layers or []- shape = _check_layers(layers, file_ids, file_count, errors, warnings)-- entry_points = kg.project.get("entry_points", []) if isinstance(kg.project, dict) else []- _check_entry_points(entry_points, tags_by_path, errors)-- tour = kg.tour or []- tour_coverage = _check_tour(tour, layers, errors, warnings)-- # Modules are only checked when the curated artifact carries them.- modules = getattr(kg, "modules", None) or []- module_covered = _check_modules(modules, file_ids, errors, warnings) if modules else set()-- # -- Summaries ---------------------------------------------------------- empty_summaries = [nid for nid, s in summary_by_id.items() if not s]- if empty_summaries:- errors.append(f"{len(empty_summaries)} file nodes have an empty summary")- summary_completeness = 1.0 - len(empty_summaries) / file_count if file_count else 1.0-- metrics = {- "file_count": file_count,- "layer_count": len(layers),- "module_count": len(modules),- "module_coverage_pct": round(- (len(module_covered) / file_count * 100) if (modules and file_count) else 0.0, 1- ),- "singleton_layer_pct": round(shape.singleton_frac * 100, 1),- "largest_layer_pct": round(shape.largest_frac * 100, 1),- "application_pct": round(shape.catchall_frac * 100, 1),- "entry_point_count": len(entry_points),- "tour_steps": len(tour),- "tour_coverage_pct": round(tour_coverage * 100, 1),- "summary_completeness_pct": round(summary_completeness * 100, 1),- }-- return KGValidation(ok=not errors, errors=errors, warnings=warnings, metrics=metrics)---@dataclass(frozen=True)-class _LayerShape:- singleton_frac: float- largest_frac: float- catchall_frac: float---def _check_layers(- layers: list[dict],- file_ids: set[str],- file_count: int,- errors: list[str],- warnings: list[str],-) -> _LayerShape:- """Layer count, partition, singleton spam and mega-layer balance."""- n_layers = len(layers)-- # -- Layer count -------------------------------------------------------- if n_layers == 0:- errors.append("no layers")- elif n_layers > _MAX_LAYERS:- errors.append(f"too many layers: {n_layers} > {_MAX_LAYERS}")- elif n_layers < _MIN_LAYERS:- warnings.append(f"few layers: {n_layers} < {_MIN_LAYERS} (small/flat repo?)")-- # -- Partition ---------------------------------------------------------- layered: list[str] = [nid for layer in layers for nid in layer.get("nodeIds", [])]- layered_set = set(layered)- if len(layered) != len(layered_set):- errors.append("partition: a file appears in more than one layer")- if file_count and layered_set != file_ids:- missing = len(file_ids - layered_set)- extra = len(layered_set - file_ids)- errors.append(f"partition: {missing} unlayered, {extra} unknown ids")-- # -- Singleton spam & mega-layer balance -------------------------------- sizes = [len(layer.get("nodeIds", [])) for layer in layers]- singleton_frac = (sum(1 for s in sizes if s == 1) / n_layers) if n_layers else 0.0- if singleton_frac >= _MAX_SINGLETON_FRACTION:- warnings.append(f"singleton layers {singleton_frac:.0%} ≥ {_MAX_SINGLETON_FRACTION:.0%}")-- largest_frac = (max(sizes) / file_count) if (sizes and file_count) else 0.0- if largest_frac > _MAX_LAYER_FRACTION:- warnings.append(f"largest layer {largest_frac:.0%} > {_MAX_LAYER_FRACTION:.0%}")-- catchall = next((layer for layer in layers if layer.get("name") == "Application"), None)- catchall_frac = (- (len(catchall.get("nodeIds", [])) / file_count) if (catchall and file_count) else 0.0- )- if catchall_frac > _MAX_CATCHALL_FRACTION:- warnings.append(f"Application catch-all {catchall_frac:.0%} > {_MAX_CATCHALL_FRACTION:.0%}")- return _LayerShape(singleton_frac, largest_frac, catchall_frac)---def _check_entry_points(- entry_points: list[str], tags_by_path: dict[str, list[str]], errors: list[str]-) -> None:- if len(entry_points) > _MAX_ENTRY_POINTS:- errors.append(f"too many entry points: {len(entry_points)} > {_MAX_ENTRY_POINTS}")- barrels_surfaced = [p for p in entry_points if "barrel" in tags_by_path.get(p, [])]- if barrels_surfaced:- errors.append(f"barrels surfaced as entry points: {barrels_surfaced}")---def _check_tour(- tour: list[dict], layers: list[dict], errors: list[str], warnings: list[str]-) -> float:- """Tour budget and opening, returning the fraction of layers it visits."""- if not tour:- return 0.0- if len(tour) > DEFAULT_MAX_STOPS:- errors.append(f"tour too long: {len(tour)} > {DEFAULT_MAX_STOPS}")- if tour[0].get("kind") != "overview":- errors.append("tour does not open with an overview/README step")- layer_ids = {layer.get("id") for layer in layers}- covered = {- s.get("layer_id")- for s in tour- if s.get("kind") != "overview" and s.get("layer_id") in layer_ids- }- tour_coverage = (len(covered) / len(layer_ids)) if layer_ids else 0.0- if tour_coverage < _MIN_TOUR_COVERAGE:- warnings.append(f"tour covers {tour_coverage:.0%} of layers < {_MIN_TOUR_COVERAGE:.0%}")- return tour_coverage---def _check_modules(- modules: list[dict], file_ids: set[str], errors: list[str], warnings: list[str]-) -> set[str]:- """Module partition, naming and size, returning the file ids modules cover."""- module_member_lists = [m.get("nodeIds", []) for m in modules]- flat = [nid for ids in module_member_lists for nid in ids]- module_covered = set(flat)- if len(flat) != len(module_covered):- errors.append("modules: a file appears in more than one module")- if not module_covered <= file_ids:- errors.append(- f"modules: {len(module_covered - file_ids)} unknown ids in modules"- )- module_names = [m.get("name", "") for m in modules]- if len(set(module_names)) != len(module_names):- errors.append("modules: names not unique")- size_suffixed = [n for n in module_names if _SIZE_SUFFIX_RE.search(n)]- if size_suffixed:- errors.append(f"modules: size-suffixed names: {size_suffixed}")- oversized = sum(- 1 for ids in module_member_lists if len(ids) > _MODULE_TARGET_MAX- )- if oversized:- # Flat dirs may honestly exceed the window — soft signal only.- warnings.append(f"{oversized} modules above target_max (flat dirs?)")- return module_covered---# ----------------------------------------------------------------------------# Phase 6 — portable, self-validated export artifact-# ------------------------------------------------------------------------------def build_portable_kg(kg: KnowledgeGraphResult) -> tuple[dict, KGValidation]:- """Assemble a self-contained, self-validated ``knowledge-graph.json`` dict.-- Kept separate from :meth:`KnowledgeGraphResult.to_dict` so the *default*- export stays byte-identical (curation flag-off contract); the portable- artifact adds a ``meta`` block (counts, fingerprint) and an embedded- ``validation`` report so an external consumer can trust it without a server.- Returns ``(data, validation)`` so the writer can decide on hard violations.- """- data = kg.to_dict()- validation = validate_kg(kg)- data["meta"] = {- # The integer the loader gates on, not the "1.0.0" display label.- "schema_version": data.get("schema_version", 1),- "generator": "repowise-kg-curation",- "fingerprint": getattr(kg, "fingerprint", ""),- "file_count": validation.metrics.get("file_count", 0),- "layer_count": validation.metrics.get("layer_count", 0),- "module_count": validation.metrics.get("module_count", 0),- "entry_point_count": validation.metrics.get("entry_point_count", 0),- "tour_steps": validation.metrics.get("tour_steps", 0),- "validation": validation.as_dict(),- }- return data, validationdiff --git a/packages/core/src/repowise/core/analysis/kg_modules.py b/packages/core/src/repowise/core/analysis/kg_modules.pynew file mode 100644index 000000000..cea69237c--- /dev/null+++ b/packages/core/src/repowise/core/analysis/kg_modules.py@@ -0,0 +1,382 @@+"""Wiki modules: right-sized directory groups derived from the curated KG layers."""++from __future__ import annotations++import re+from collections import Counter, defaultdict+from collections.abc import Callable+from pathlib import PurePosixPath++from repowise.core.analysis.kg_inputs import _dominant_language+from repowise.core.analysis.knowledge_graph import _slugify++# A directory's segments and the (sorted) node ids grouped under it.+_DirGroup = tuple[tuple[str, ...], list[str]]+++def _common_dir_prefix(seg_lists: list[tuple[str, ...]]) -> tuple[str, ...]:+ """Longest common leading directory-segment prefix across *seg_lists*."""+ if not seg_lists:+ return ()+ common = list(seg_lists[0])+ for segs in seg_lists[1:]:+ i = 0+ while i < len(common) and i < len(segs) and common[i] == segs[i]:+ i += 1+ del common[i:]+ if not common:+ break+ return tuple(common)+++# Granularity window for derived wiki modules. Layer node sets are split+# recursively by directory until every group fits, stopping at flat+# directories. ``target_max`` keeps the 10 key-file template slots+# representative; below ``target_min`` a group folds into its nearest sibling.+_MODULE_TARGET_MIN = 8+_MODULE_TARGET_MAX = 120+# A layer smaller than this yields no module at all (matches the selection+# layer's ``min_module_size`` floor that kills singleton pages).+_MODULE_MIN_FILES = 3+# A directory segment present in more than this fraction of all repo paths is+# *generic* (namespace dirs: ``src``, ``packages``, the repo's own name) and+# never appears in a module name. Data-driven — no hardcoded segment list.+_GENERIC_SEGMENT_FRACTION = 0.60+# Module names never carry a size suffix such as "ingestion (32)".+_SIZE_SUFFIX_RE = re.compile(r"\(\d+\)\s*$")+++# Universal organizational directory names — containers, not domain labels.+# Shared with community labeling; the data-driven ``dominant_segments`` set+# complements this with per-repo namespace noise (the repo's own name).+GENERIC_ORG_SEGMENTS = frozenset({+ "src", "lib", "core", "common", "shared", "internal", "pkg",+ "main", "app", "utils", "helpers", "index", "mod",+ # Monorepo organisational directories+ "packages", "modules", "workspace", "workspaces", "libs",+ "projects", "services", "apps",+})+++def dominant_segments(paths: list[str]) -> set[str]:+ """Directory segments appearing in > 60% of *paths* (namespace noise).++ Shared with community labeling (``analysis/communities.py``) so both+ vocabularies strip the same namespace dirs (``src``, ``packages``, the+ repo's own name) without depending on a hardcoded list.+ """+ n = len(paths)+ if not n:+ return set()+ counts: Counter[str] = Counter()+ for p in paths:+ for seg in set(PurePosixPath(p).parts[:-1]):+ counts[seg] += 1+ return {s for s, c in counts.items() if c / n > _GENERIC_SEGMENT_FRACTION}+++def _split_to_granularity(+ node_ids: list[str], id_to_path: dict[str, str], target_max: int+) -> list[tuple[tuple[str, ...], list[str]]]:+ """Recursively split *node_ids* by directory until groups fit *target_max*.++ Returns ``[(dir_segments, sorted_node_ids), ...]``. Reuses ``_sub_split``'s+ prefix logic (group by the first segment that distinguishes members after+ the common directory prefix) but, unlike sub-groups, recurses into any+ group still above ``target_max``. Recursion bottoms out when a directory+ has no distinguishing subdirs — a 200-file flat dir stays one module+ (honest), never an artificial split.+ """+ dir_segs = {nid: PurePosixPath(id_to_path[nid]).parts[:-1] for nid in node_ids}++ def rec(ids: list[str]) -> list[tuple[tuple[str, ...], list[str]]]:+ common = _common_dir_prefix([dir_segs[i] for i in ids])+ if len(ids) <= target_max:+ return [(common, ids)]+ groups: dict[str, list[str]] = defaultdict(list)+ for nid in ids:+ segs = dir_segs[nid]+ key = segs[len(common)] if len(segs) > len(common) else ""+ groups[key].append(nid)+ if len(groups) < 2:+ return [(common, ids)] # flat directory — no honest split exists+ out: list[tuple[tuple[str, ...], list[str]]] = []+ for key in sorted(groups):+ if key == "":+ # Files sitting directly in the common dir (the "(root)"+ # group). Usually below target_min → folded by merge-up.+ out.append((common, groups[key]))+ else:+ out.extend(rec(groups[key]))+ return out++ return [(d, sorted(ids)) for d, ids in rec(sorted(node_ids))]+++def _merge_small_groups(+ groups: list[tuple[tuple[str, ...], list[str]]], target_min: int+) -> list[tuple[tuple[str, ...], list[str]]]:+ """Fold groups below *target_min* into their nearest sibling.++ "Nearest" = the group sharing the longest directory prefix (the parent+ subtree), largest first as the tie-break — so a 2-file "(root)" remnant+ folds into its own subtree's biggest module, and an isolated small dir+ folds into the layer's dominant module rather than minting a confetti+ page. Never merges across layers (callers pass one layer at a time). A+ layer that is itself below ``target_min`` stays one whole group.++ A pre-pass fuses *small sibling* groups into one group at their common+ parent when that collection is itself module-sized — ninety tiny locale+ dirs become one ``conf/locale`` module instead of folding into whichever+ sibling sorts first and misnaming it. The fold-in loop then never renames+ a survivor: a healthy ``core/providers`` absorbing a 2-file sibling keeps+ its identity.+ """+ merged = [(d, list(ids)) for d, ids in groups]+ _fuse_small_siblings(merged, target_min)+ merged.sort(key=lambda g: g[0])+ _fold_small_groups(merged, target_min)+ return [(d, ids) for d, ids in merged]+++def _fuse_small_siblings(merged: list[_DirGroup], target_min: int) -> None:+ for parent, sibs in sorted(_small_groups_by_parent(merged, target_min).items()):+ if len(sibs) < 2 or sum(len(g[1]) for g in sibs) < target_min:+ continue+ fused = sorted(nid for g in sibs for nid in g[1])+ for g in sibs:+ merged.remove(g)+ existing = next((g for g in merged if g[0] == parent), None)+ if existing is not None:+ existing[1].extend(fused)+ existing[1].sort()+ else:+ merged.append((parent, fused))+++def _small_groups_by_parent(+ merged: list[_DirGroup], target_min: int+) -> dict[tuple[str, ...], list[_DirGroup]]:+ by_parent: dict[tuple[str, ...], list[_DirGroup]] = {}+ for g in merged:+ if len(g[1]) < target_min and len(g[0]) > 0:+ by_parent.setdefault(g[0][:-1], []).append(g)+ return by_parent+++def _fold_small_groups(merged: list[_DirGroup], target_min: int) -> None:+ while len(merged) > 1:+ small = min(+ (g for g in merged if len(g[1]) < target_min),+ key=lambda g: (len(g[1]), g[0]),+ default=None,+ )+ if small is None:+ break+ merged.remove(small)+ target = min(+ merged,+ key=lambda g: (-_shared_depth(g[0], small[0]), -len(g[1]), g[0]),+ )+ target[1].extend(small[1])+ target[1].sort()+++def _shared_depth(a: tuple[str, ...], b: tuple[str, ...]) -> int:+ return len(_common_dir_prefix([a, b]))+++def _name_modules(mods: list[dict], generic: set[str]) -> None:+ """Assign unique, human module names in place.++ Initial name = the last one or two *informative* directory segments+ (generic namespace segments stripped; when stripping consumes every+ segment, the raw tail is used instead). Collisions extend leftward by+ one more parent segment — NEVER a size suffix. Single-module layers+ take the layer's name; the root group (empty dir) becomes+ "<Layer> (top-level)". The absolute fallback (identical informative+ paths across layers) appends the layer name, which is unique by+ construction.+ """+ info_by, used = _initial_module_names(mods, generic)+ _extend_colliding_names(mods, info_by, used)++ # A root remnant and an organizational container in one layer would both+ # read "<Layer> (top-level)"; the container's raw tail breaks the tie.+ _rename_collisions(mods, lambda m: None if info_by[id(m)] else _dir_tail(m))++ # Same informative dir in two layers: (dir, layer) is unique by construction.+ _rename_collisions(mods, lambda m: f"{m['name']} ({m['_layerName']})")++ # Backstop: the full dir path is unique per layer.+ _rename_collisions(mods, lambda m: "/".join(m["_dir"]) or None)+++def _initial_module_names(+ mods: list[dict], generic: set[str]+) -> tuple[dict[int, list[str]], dict[int, int | None]]:+ """Name each module; return its informative segments and how many it uses."""+ per_layer: Counter[str] = Counter(m["layerId"] for m in mods)+ info_by: dict[int, list[str]] = {}+ used: dict[int, int | None] = {} # informative segments consumed; None = fixed+ for m in mods:+ # Generic stripping can consume every segment on fixture-dominated+ # repos; the raw tail minus organizational dirs is still the honest+ # name, and "(top-level)" stays reserved for true root groups.+ info = [s for s in m["_dir"] if s not in generic] or [+ s for s in m["_dir"] if s.lower() not in GENERIC_ORG_SEGMENTS+ ]+ info_by[id(m)] = info+ if per_layer[m["layerId"]] == 1:+ m["name"] = m["_layerName"]+ used[id(m)] = None+ elif not info:+ m["name"] = f"{m['_layerName']} (top-level)"+ used[id(m)] = None+ else:+ k = min(2, len(info))+ m["name"] = "/".join(info[-k:])+ used[id(m)] = k+ return info_by, used+++def _extend_colliding_names(+ mods: list[dict], info_by: dict[int, list[str]], used: dict[int, int | None]+) -> None:+ """Extend colliding names leftward, one informative segment per round."""+ for _ in range(16): # bounded: each round consumes ≥1 segment somewhere+ names = Counter(m["name"] for m in mods)+ colliding = [m for m in mods if names[m["name"]] > 1]+ if not colliding:+ return+ progressed = False+ for m in colliding:+ k = used.get(id(m))+ info = info_by[id(m)]+ if k is not None and k < len(info):+ used[id(m)] = k + 1+ m["name"] = "/".join(info[-(k + 1) :])+ progressed = True+ if not progressed:+ return+++def _rename_collisions(mods: list[dict], rename: Callable[[dict], str | None]) -> None:+ """Rename every module whose name is still shared, where *rename* has one."""+ names = Counter(m["name"] for m in mods)+ for m in mods:+ if names[m["name"]] > 1:+ new_name = rename(m)+ if new_name is not None:+ m["name"] = new_name+++def _dir_tail(module: dict) -> str | None:+ return "/".join(module["_dir"][-2:]) or None+++def derive_modules(+ layers: list[dict],+ id_to_path: dict[str, str],+ *,+ target_min: int = _MODULE_TARGET_MIN,+ target_max: int = _MODULE_TARGET_MAX,+ min_module_size: int = _MODULE_MIN_FILES,+ lang_by_id: dict[str, str] | None = None,+) -> list[dict]:+ """Derive right-sized, stably-identified wiki modules from curated layers.++ ``Module = {"id": "module:<dir-slug>", "name": <human>, "path": <dir or "">,+ "layerId": ..., "nodeIds": [...], "language": ...}``++ Properties:++ - **Partition per layer**: every node of every layer ≥ ``min_module_size``+ lands in exactly one module; layers below the floor yield none. Never+ merges across layers.+ - **Granularity**: recursive directory splitting to the+ [``target_min``, ``target_max``] window; flat dirs stay one honest+ module; sub-``target_min`` remnants merge up into their subtree.+ - **Names**: informative path segments only (data-driven generic-segment+ stripping kills ``src``/``packages``/repo-name automatically); collision+ resolution extends the path leftward — never a size suffix.+ - **Ids**: ``module:`` + slug of the real directory path — stable across+ runs and under file adds/renames inside the dir; changes only when the+ directory itself moves. ``path`` is the actual dir (not the slug) so+ path-prefix child lookups (``target_path LIKE 'dir/%'``) work.+ - **Files only**: operates on ids present in ``id_to_path`` — external+ nodes never pollute a module.+ - **Determinism**: sorted iteration throughout; same inputs → same bytes.+ """+ generic = dominant_segments(sorted(set(id_to_path.values())))+ mods = [+ module+ for layer in layers+ for module in _layer_modules(layer, id_to_path, target_min, target_max, min_module_size)+ ]+ _name_modules(mods, generic)+ _assign_module_ids(mods)++ # A single-module layer duplicates its layer page; the flag lets page+ # generation skip it while the module stays in the artifact.+ per_layer_count: Counter[str] = Counter(m["layerId"] for m in mods)+ return [+ _export_module(m, per_layer_count[m["layerId"]] == 1, lang_by_id) for m in mods+ ]+++def _layer_modules(+ layer: dict, id_to_path: dict[str, str], target_min: int, target_max: int, min_size: int+) -> list[dict]:+ """One layer's unnamed module groups, or none below *min_size* files."""+ node_ids = [nid for nid in layer.get("nodeIds", []) if nid in id_to_path]+ if len(node_ids) < min_size:+ return []+ groups = _merge_small_groups(+ _split_to_granularity(node_ids, id_to_path, target_max), target_min+ )+ return [+ {+ "_dir": dir_parts,+ "_layerName": layer.get("name", ""),+ "path": "/".join(dir_parts),+ "layerId": layer.get("id", ""),+ "nodeIds": sorted(ids),+ }+ for dir_parts, ids in sorted(groups)+ ]+++def _assign_module_ids(mods: list[dict]) -> None:+ """Give each module a path-derived slug id.++ The bigger module keeps the plain id on the rare cross-layer dir collision+ (a dir whose files split across layers).+ """+ used_ids: set[str] = set()+ for m in sorted(mods, key=lambda m: (-len(m["nodeIds"]), m["path"], m["layerId"])):+ base = "module:" + _slugify(m["path"] or m["_layerName"])+ mid = base+ n = 1+ while mid in used_ids:+ mid = f"{base}--{_slugify(m['_layerName'])}" + ("" if n == 1 else f"-{n}")+ n += 1+ used_ids.add(mid)+ m["id"] = mid+++def _export_module(m: dict, whole_layer: bool, lang_by_id: dict[str, str] | None) -> dict:+ module = {+ "id": m["id"],+ "name": m["name"],+ "path": m["path"],+ "layerId": m["layerId"],+ "nodeIds": m["nodeIds"],+ }+ if whole_layer:+ module["wholeLayer"] = True+ if lang_by_id is not None:+ module["language"] = _dominant_language(+ [lang for nid in m["nodeIds"] if (lang := lang_by_id.get(nid, ""))]+ )+ return modulediff --git a/packages/core/src/repowise/core/analysis/kg_validation.py b/packages/core/src/repowise/core/analysis/kg_validation.pynew file mode 100644index 000000000..3399dad62--- /dev/null+++ b/packages/core/src/repowise/core/analysis/kg_validation.py@@ -0,0 +1,260 @@+"""Invariant validation of a curated KG, and the portable self-validated export."""++from __future__ import annotations++from dataclasses import dataclass, field+from typing import Any++from repowise.core.analysis.kg_inputs import _file_nodes+from repowise.core.analysis.kg_modules import _MODULE_TARGET_MAX, _SIZE_SUFFIX_RE+from repowise.core.analysis.knowledge_graph import KnowledgeGraphResult+from repowise.core.generation.tour import DEFAULT_MAX_STOPS++# Hard bound on the curated primary-layer count. The spine stays well below it+# by construction; past it curation keeps the uncurated layers.+_MAX_LAYERS = 15++# Entry points surfaced in ``project.entry_points``; the full ranked list is+# kept as ``entry_candidates``.+_MAX_ENTRY_POINTS = 8++# Quality thresholds. The lower layer bound and coverage targets are *soft*+# (warnings) because they depend on repo size/shape; the partition, hard count+# bound, capped entry set, never-empty summaries, and tour budget are *hard*.+_MIN_LAYERS = 6+_MAX_LAYER_FRACTION = 0.35+_MAX_CATCHALL_FRACTION = 0.20+_MAX_SINGLETON_FRACTION = 0.10+_MIN_TOUR_COVERAGE = 0.90+++@dataclass+class KGValidation:+ """Outcome of :func:`validate_kg` — hard errors, soft warnings, metrics."""++ ok: bool+ errors: list[str] = field(default_factory=list)+ warnings: list[str] = field(default_factory=list)+ metrics: dict[str, Any] = field(default_factory=dict)++ def as_dict(self) -> dict[str, Any]:+ return {+ "ok": self.ok,+ "errors": self.errors,+ "warnings": self.warnings,+ "metrics": self.metrics,+ }+++def validate_kg(kg: KnowledgeGraphResult) -> KGValidation:+ """Validate a curated KG against the intuitiveness invariants.++ Pure and side-effect free. Hard violations set ``ok=False`` and populate+ ``errors``; size/shape-dependent shortfalls go to ``warnings``. The+ ``metrics`` block is the per-repo intuitiveness scorecard.+ """+ errors: list[str] = []+ warnings: list[str] = []++ file_nodes = _file_nodes(kg)+ file_count = len(file_nodes)+ file_ids = {n["id"] for n in file_nodes}+ tags_by_path = {n["filePath"]: (n.get("tags") or []) for n in file_nodes}++ layers = kg.layers or []+ shape = _check_layers(layers, file_ids, file_count, errors, warnings)++ entry_points = kg.project.get("entry_points", []) if isinstance(kg.project, dict) else []+ _check_entry_points(entry_points, tags_by_path, errors)++ tour = kg.tour or []+ tour_coverage = _check_tour(tour, layers, errors, warnings)++ # Modules are only checked when the curated artifact carries them.+ modules = getattr(kg, "modules", None) or []+ module_covered = _check_modules(modules, file_ids, errors, warnings) if modules else set()++ summary_completeness = _check_summaries(file_nodes, errors)++ metrics = {+ "file_count": file_count,+ "layer_count": len(layers),+ "module_count": len(modules),+ "module_coverage_pct": (+ round(len(module_covered) / file_count * 100, 1) if file_count else 0.0+ ),+ "singleton_layer_pct": round(shape.singleton_frac * 100, 1),+ "largest_layer_pct": round(shape.largest_frac * 100, 1),+ "application_pct": round(shape.catchall_frac * 100, 1),+ "entry_point_count": len(entry_points),+ "tour_steps": len(tour),+ "tour_coverage_pct": round(tour_coverage * 100, 1),+ "summary_completeness_pct": round(summary_completeness * 100, 1),+ }++ return KGValidation(ok=not errors, errors=errors, warnings=warnings, metrics=metrics)+++@dataclass(frozen=True)+class _LayerShape:+ singleton_frac: float+ largest_frac: float+ catchall_frac: float+++def _check_layers(+ layers: list[dict],+ file_ids: set[str],+ file_count: int,+ errors: list[str],+ warnings: list[str],+) -> _LayerShape:+ """Layer count, partition, singleton spam and mega-layer balance."""+ _check_layer_count(len(layers), errors, warnings)+ _check_layer_partition(layers, file_ids, file_count, errors)+ shape = _layer_shape(layers, file_count)+ if shape.singleton_frac >= _MAX_SINGLETON_FRACTION:+ warnings.append(+ f"singleton layers {shape.singleton_frac:.0%} ≥ {_MAX_SINGLETON_FRACTION:.0%}"+ )+ if shape.largest_frac > _MAX_LAYER_FRACTION:+ warnings.append(f"largest layer {shape.largest_frac:.0%} > {_MAX_LAYER_FRACTION:.0%}")+ if shape.catchall_frac > _MAX_CATCHALL_FRACTION:+ warnings.append(+ f"Application catch-all {shape.catchall_frac:.0%} > {_MAX_CATCHALL_FRACTION:.0%}"+ )+ return shape+++def _check_layer_count(n_layers: int, errors: list[str], warnings: list[str]) -> None:+ if n_layers == 0:+ errors.append("no layers")+ elif n_layers > _MAX_LAYERS:+ errors.append(f"too many layers: {n_layers} > {_MAX_LAYERS}")+ elif n_layers < _MIN_LAYERS:+ warnings.append(f"few layers: {n_layers} < {_MIN_LAYERS} (small/flat repo?)")+++def _check_layer_partition(+ layers: list[dict], file_ids: set[str], file_count: int, errors: list[str]+) -> None:+ layered: list[str] = [nid for layer in layers for nid in layer.get("nodeIds", [])]+ layered_set = set(layered)+ if len(layered) != len(layered_set):+ errors.append("partition: a file appears in more than one layer")+ if file_count and layered_set != file_ids:+ missing = len(file_ids - layered_set)+ extra = len(layered_set - file_ids)+ errors.append(f"partition: {missing} unlayered, {extra} unknown ids")+++def _layer_shape(layers: list[dict], file_count: int) -> _LayerShape:+ """Singleton-layer, largest-layer and ``Application`` catch-all fractions."""+ sizes = [len(layer.get("nodeIds", [])) for layer in layers]+ singleton_frac = (sum(1 for s in sizes if s == 1) / len(layers)) if layers else 0.0+ largest_frac = (max(sizes) / file_count) if (sizes and file_count) else 0.0+ catchall = next((layer for layer in layers if layer.get("name") == "Application"), None)+ catchall_frac = (+ (len(catchall.get("nodeIds", [])) / file_count) if (catchall and file_count) else 0.0+ )+ return _LayerShape(singleton_frac, largest_frac, catchall_frac)+++def _check_summaries(file_nodes: list[dict], errors: list[str]) -> float:+ """Never-empty summaries, returning the fraction of file nodes that have one."""+ summary_by_id = {n["id"]: n.get("summary") for n in file_nodes}+ empty_summaries = [nid for nid, s in summary_by_id.items() if not s]+ if empty_summaries:+ errors.append(f"{len(empty_summaries)} file nodes have an empty summary")+ return 1.0 - len(empty_summaries) / len(file_nodes) if file_nodes else 1.0+++def _check_entry_points(+ entry_points: list[str], tags_by_path: dict[str, list[str]], errors: list[str]+) -> None:+ if len(entry_points) > _MAX_ENTRY_POINTS:+ errors.append(f"too many entry points: {len(entry_points)} > {_MAX_ENTRY_POINTS}")+ barrels_surfaced = [p for p in entry_points if "barrel" in tags_by_path.get(p, [])]+ if barrels_surfaced:+ errors.append(f"barrels surfaced as entry points: {barrels_surfaced}")+++def _check_tour(+ tour: list[dict], layers: list[dict], errors: list[str], warnings: list[str]+) -> float:+ """Tour budget and opening, returning the fraction of layers it visits."""+ if not tour:+ return 0.0+ if len(tour) > DEFAULT_MAX_STOPS:+ errors.append(f"tour too long: {len(tour)} > {DEFAULT_MAX_STOPS}")+ if tour[0].get("kind") != "overview":+ errors.append("tour does not open with an overview/README step")+ layer_ids = {layer.get("id") for layer in layers}+ covered = {+ s.get("layer_id")+ for s in tour+ if s.get("kind") != "overview" and s.get("layer_id") in layer_ids+ }+ tour_coverage = (len(covered) / len(layer_ids)) if layer_ids else 0.0+ if tour_coverage < _MIN_TOUR_COVERAGE:+ warnings.append(f"tour covers {tour_coverage:.0%} of layers < {_MIN_TOUR_COVERAGE:.0%}")+ return tour_coverage+++def _check_modules(+ modules: list[dict], file_ids: set[str], errors: list[str], warnings: list[str]+) -> set[str]:+ """Module partition, naming and size, returning the file ids modules cover."""+ module_member_lists = [m.get("nodeIds", []) for m in modules]+ flat = [nid for ids in module_member_lists for nid in ids]+ module_covered = set(flat)+ if len(flat) != len(module_covered):+ errors.append("modules: a file appears in more than one module")+ if not module_covered <= file_ids:+ errors.append(+ f"modules: {len(module_covered - file_ids)} unknown ids in modules"+ )+ module_names = [m.get("name", "") for m in modules]+ if len(set(module_names)) != len(module_names):+ errors.append("modules: names not unique")+ size_suffixed = [n for n in module_names if _SIZE_SUFFIX_RE.search(n)]+ if size_suffixed:+ errors.append(f"modules: size-suffixed names: {size_suffixed}")+ oversized = sum(+ 1 for ids in module_member_lists if len(ids) > _MODULE_TARGET_MAX+ )+ if oversized:+ # Flat dirs may honestly exceed the window — soft signal only.+ warnings.append(f"{oversized} modules above target_max (flat dirs?)")+ return module_covered+++# ---------------------------------------------------------------------------+# Phase 6 — portable, self-validated export artifact+# ---------------------------------------------------------------------------+++def build_portable_kg(kg: KnowledgeGraphResult) -> tuple[dict, KGValidation]:+ """Assemble a self-contained, self-validated ``knowledge-graph.json`` dict.++ Kept separate from :meth:`KnowledgeGraphResult.to_dict` so the *default*+ export stays byte-identical (curation flag-off contract); the portable+ artifact adds a ``meta`` block (counts, fingerprint) and an embedded+ ``validation`` report so an external consumer can trust it without a server.+ Returns ``(data, validation)`` so the writer can decide on hard violations.+ """+ data = kg.to_dict()+ validation = validate_kg(kg)+ data["meta"] = {+ # The integer the loader gates on, not the "1.0.0" display label.+ "schema_version": data.get("schema_version", 1),+ "generator": "repowise-kg-curation",+ "fingerprint": getattr(kg, "fingerprint", ""),+ "file_count": validation.metrics.get("file_count", 0),+ "layer_count": validation.metrics.get("layer_count", 0),+ "module_count": validation.metrics.get("module_count", 0),+ "entry_point_count": validation.metrics.get("entry_point_count", 0),+ "tour_steps": validation.metrics.get("tour_steps", 0),+ "validation": validation.as_dict(),+ }+ return data, validation