Skip to content

Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.

refactor(core): move KG module derivation and validation out of kg_curation.py (#2592)

Raghav Chamadiya8e499d2c2572
diff --git a/packages/core/src/repowise/core/analysis/kg_curation.py b/packages/core/src/repowise/core/analysis/kg_curation.py
index a6fea743a..502ddc3c2 100644
--- a/packages/core/src/repowise/core/analysis/kg_curation.py
+++ b/packages/core/src/repowise/core/analysis/kg_curation.py
@@ -11,23 +11,18 @@ summaries.
**Hard invariant.** Curation reads the NetworkX graph, communities, and
centrality, but it *only ever writes the returned* :class:`KnowledgeGraphResult`.
It never mutates ``graph_builder``'s graph, ``graph_edges``, centrality caches,
-community detection, or any DB table. There is a regression test that asserts the
-graph's node/edge counts are identical before and after this pass.
+community detection, or any DB table.
Curation is feature-flagged (``REPOWISE_KG_CURATION``) and defaults **on**;
-the 38-repo cross-language validation matrix is the acceptance gate that
-flipped it. Setting the flag to ``0``/``false``/``no``/``off`` makes
-:func:`curate_knowledge_graph` a no-op that returns its input unchanged
-(the raw uncurated export).
+see :func:`curation_enabled`.
"""
from __future__ import annotations
import logging
import os
-import re
-from collections import Counter, defaultdict
-from dataclasses import dataclass, field
+from collections import defaultdict
+from collections.abc import Callable
from pathlib import PurePosixPath
from typing import Any
@@ -38,11 +33,24 @@ from repowise.core.analysis.kg_inputs import (
_file_nodes,
_is_barrel,
)
+from repowise.core.analysis.kg_modules import ( # noqa: F401 (re-exported)
+ GENERIC_ORG_SEGMENTS,
+ _common_dir_prefix,
+ derive_modules,
+ dominant_segments,
+)
from repowise.core.analysis.kg_tour import ( # noqa: F401 (re-exported)
_anchor_fanout_rank,
_curate_tour,
_import_pairs_excluding_fanout,
)
+from repowise.core.analysis.kg_validation import (
+ _MAX_ENTRY_POINTS,
+ _MAX_LAYERS,
+ KGValidation,
+ build_portable_kg,
+ validate_kg,
+)
from repowise.core.analysis.knowledge_graph import KnowledgeGraphResult, _slugify
from repowise.core.entry_candidacy import (
conventional_entry_stems,
@@ -55,10 +63,7 @@ from repowise.core.generation.layers import (
infer_layer,
layer_order_basis,
)
-from repowise.core.generation.tour import (
- DEFAULT_MAX_STOPS,
- score_entry_points,
-)
+from repowise.core.generation.tour import score_entry_points
from repowise.core.generation.well_known_files import well_known_role
from repowise.core.ingestion.languages.registry import REGISTRY as _LANG_REGISTRY
from repowise.core.support_paths import is_support_path
@@ -81,27 +86,16 @@ _FLAG_ENV = "REPOWISE_KG_CURATION"
# A primary layer larger than this many files, or spanning more than this many
# distinct sub-directories, is given a two-level structure (primary → named
# sub-groups) so a mega-layer like core/* or ui/* stays drill-down legible
-# instead of becoming one opaque bucket (plan §Phase 1, edge case B).
+# instead of becoming one opaque bucket.
_SUBSPLIT_FILE_THRESHOLD = 60
_SUBSPLIT_DIR_THRESHOLD = 8
-# Hard bound on the curated primary-layer count. The spine is bounded ≤~11 by
-# construction; if a future change ever blows past this we degrade to the
-# uncurated layers rather than ship an unreadable list.
-_MAX_LAYERS = 15
-
-# Entry points surfaced in ``project.entry_points``; the full ranked list is
-# kept as ``entry_candidates``.
-_MAX_ENTRY_POINTS = 8
-
def curation_enabled() -> bool:
"""Whether KG curation is enabled via the ``REPOWISE_KG_CURATION`` env flag.
- Defaults to **on** — the cross-language validation matrix (38 pinned
- repos, enforced density/orphan/catch-all thresholds, honest degradation
- modes) is the acceptance gate that flipped it. Set ``0``/``false``/``no``/
- ``off`` (case-insensitive) to fall back to the raw uncurated export.
+ Defaults to **on**. Set ``0``/``false``/``no``/``off`` (case-insensitive)
+ to fall back to the raw uncurated export.
Resolved at the call site so :func:`curate_knowledge_graph` itself stays
pure and trivially testable with an explicit ``enabled=``.
"""
@@ -123,8 +117,7 @@ def curate_knowledge_graph(
Pure with respect to the AST graph: reads ``graph_builder`` /
``community_info`` but writes only the returned result. When ``enabled`` is
- ``False`` this is a strict no-op returning ``kg`` unchanged (the default, so
- the exported KG is unaffected until the flag flips).
+ ``False`` this is a strict no-op returning ``kg`` unchanged.
``defer_summary_floor`` skips the never-empty summary floor here so it can
run *after* the wiki-page backfill in generate mode (where richer summaries
@@ -136,88 +129,83 @@ def curate_knowledge_graph(
if not enabled:
return kg
- # Each step mutates only ``kg`` (the presentation result) and is guarded so
- # a failure degrades to the prior, uncurated field rather than aborting the
- # export. Steps are layered in by subsequent phases:
- # _curate_layers -> _curate_entry_points -> _curate_tour
- # -> _curate_node_types -> _curate_summaries
- layers_curated = False
- try:
- curated = _curate_layers(kg, graph_builder)
- if curated is not None:
- kg.layers = curated
- layers_curated = True
- except Exception: # pragma: no cover - defensive; keep uncurated layers
- logger.exception("kg_curation._curate_layers failed; keeping community layers")
-
- # Wiki modules are a *sibling* artifact of the curated layers (same
- # splitting machinery, module-sized granularity). Only derived when the
- # spine landed — community layers would make the dir-split meaningless,
- # and downstream consumers fall back to community grouping when this
- # stays empty (the fallback matrix's "degraded" row).
- if layers_curated:
- try:
- modules = _curate_modules(kg)
- if modules is not None:
- kg.modules = modules
- except Exception: # pragma: no cover - defensive; ship no modules
- logger.exception("kg_curation._curate_modules failed; exporting no modules")
+ layers = _guarded(
+ "kg_curation._curate_layers failed; keeping community layers",
+ _curate_layers,
+ kg,
+ graph_builder,
+ )
+ if layers is not None:
+ kg.layers = layers
+ # Modules need the curated spine; over community layers the dir split
+ # is meaningless, and consumers fall back to community grouping.
+ modules = _guarded(
+ "kg_curation._curate_modules failed; exporting no modules", _curate_modules, kg
+ )
+ if modules is not None:
+ kg.modules = modules
+
+ _guarded(
+ "kg_curation._curate_entry_points failed; keeping raw entry points",
+ _curate_entry_points,
+ kg,
+ parsed_files,
+ graph_builder,
+ )
- try:
- _curate_entry_points(kg, parsed_files, graph_builder)
- except Exception: # pragma: no cover - defensive; keep skeleton entry points
- logger.exception("kg_curation._curate_entry_points failed; keeping raw entry points")
-
- # Genuine churn hotspots (a constantly-edited file off the hot import path)
- # earn a tour stop on this signal alone. Only flagged hotspots with recent
- # commits qualify; repos without git history pass an empty map and the tour
- # is unchanged.
- hotspot_commits = {
- path: int(meta.get("commit_count_90d", 0) or 0)
- for path, meta in (git_meta_map or {}).items()
- if meta.get("is_hotspot")
- }
- try:
- tour = _curate_tour(kg, parsed_files, graph_builder, hotspot_commits=hotspot_commits)
- if tour is not None:
- kg.tour = tour
- except Exception: # pragma: no cover - defensive; keep skeleton/LLM tour
- logger.exception("kg_curation._curate_tour failed; keeping existing tour")
+ tour = _guarded(
+ "kg_curation._curate_tour failed; keeping existing tour",
+ _curate_tour,
+ kg,
+ parsed_files,
+ graph_builder,
+ hotspot_commits=_hotspot_commits(git_meta_map),
+ )
+ if tour is not None:
+ kg.tour = tour
- try:
- _curate_node_types(kg)
- except Exception: # pragma: no cover - defensive; keep skeleton types
- logger.exception("kg_curation._curate_node_types failed; keeping coarse types")
+ _guarded("kg_curation._curate_node_types failed; keeping coarse types", _curate_node_types, kg)
if not defer_summary_floor:
- try:
- apply_summary_floor(kg, parsed_files)
- except Exception: # pragma: no cover - defensive; leave summaries as-is
- logger.exception("kg_curation summary floor failed; leaving summaries empty")
+ _guarded(
+ "kg_curation summary floor failed; leaving summaries empty",
+ apply_summary_floor,
+ kg,
+ parsed_files,
+ )
return kg
+def _guarded(failure: str, step: Callable[..., Any], *args: Any, **kwargs: Any) -> Any:
+ """Run one curation *step*; on error log *failure* and return ``None``."""
+ try:
+ return step(*args, **kwargs)
+ except Exception: # pragma: no cover - defensive; keep the uncurated field
+ logger.exception(failure)
+ return None
+
+
+def _hotspot_commits(git_meta_map: dict[str, dict] | None) -> dict[str, int]:
+ """Recent commit counts of the flagged churn hotspots.
+
+ Genuine churn hotspots (a constantly-edited file off the hot import path)
+ earn a tour stop on this signal alone. Only flagged hotspots with recent
+ commits qualify; repos without git history pass an empty map and the tour
+ is unchanged.
+ """
+ return {
+ path: int(meta.get("commit_count_90d", 0) or 0)
+ for path, meta in (git_meta_map or {}).items()
+ if meta.get("is_hotspot")
+ }
+
+
# ---------------------------------------------------------------------------
# Phase 1 — curated layers (replace raw-community layers with the spine)
# ---------------------------------------------------------------------------
-def _common_dir_prefix(seg_lists: list[tuple[str, ...]]) -> tuple[str, ...]:
- """Longest common leading directory-segment prefix across *seg_lists*."""
- if not seg_lists:
- return ()
- common = list(seg_lists[0])
- for segs in seg_lists[1:]:
- i = 0
- while i < len(common) and i < len(segs) and common[i] == segs[i]:
- i += 1
- del common[i:]
- if not common:
- break
- return tuple(common)
-
-
def _sub_split(layer_id: str, node_ids: list[str], id_to_path: dict[str, str]) -> list[dict] | None:
"""Two-level sub-groups for an oversized/wide primary layer, else ``None``.
@@ -269,371 +257,62 @@ def _curate_layers(kg: KnowledgeGraphResult, graph_builder: Any) -> list[dict] |
}
import_edges = _file_import_edges(graph_builder)
order = compute_layer_order(file_layers, import_edges)
- # Honesty label (additive export field): "imports" when inter-layer edges
- # informed the order, "canonical" when it is pure convention — consumers
- # must not claim "X sits above Y" for a canonical order.
+ # "imports" when inter-layer edges informed the order, "canonical" when it
+ # is pure convention: consumers must not claim "X sits above Y" for the latter.
order_basis = layer_order_basis(file_layers, import_edges)
by_layer: dict[str, list[str]] = defaultdict(list)
for n in file_nodes:
by_layer[file_layers[n["filePath"]]].append(n["id"])
- layers: list[dict] = []
- for display_order, layer_name in enumerate(order):
- node_ids = by_layer[layer_name]
- layer_id = f"layer:{_slugify(layer_name)}"
- layer: dict[str, Any] = {
- "id": layer_id,
- "name": layer_name,
- "description": "",
- "nodeIds": node_ids,
- "display_order": display_order,
- "order_basis": order_basis,
- }
- sub_groups = _sub_split(layer_id, node_ids, id_to_path)
- if sub_groups:
- layer["subGroups"] = sub_groups
- layers.append(layer)
-
- # Degrade rather than ship a broken artifact: enforce bound + partition.
+ layers = [
+ _layer_record(layer_name, display_order, by_layer[layer_name], order_basis, id_to_path)
+ for display_order, layer_name in enumerate(order)
+ ]
+ return layers if _layers_hold_invariants(layers, len(file_nodes)) else None
+
+
+def _layer_record(
+ layer_name: str,
+ display_order: int,
+ node_ids: list[str],
+ order_basis: str,
+ id_to_path: dict[str, str],
+) -> dict[str, Any]:
+ layer_id = f"layer:{_slugify(layer_name)}"
+ layer: dict[str, Any] = {
+ "id": layer_id,
+ "name": layer_name,
+ "description": "",
+ "nodeIds": node_ids,
+ "display_order": display_order,
+ "order_basis": order_basis,
+ }
+ sub_groups = _sub_split(layer_id, node_ids, id_to_path)
+ if sub_groups:
+ layer["subGroups"] = sub_groups
+ return layer
+
+
+def _layers_hold_invariants(layers: list[dict], file_count: int) -> bool:
+ """Degrade rather than ship a broken artifact: enforce bound + partition."""
total = sum(len(layer["nodeIds"]) for layer in layers)
- if not layers or len(layers) > _MAX_LAYERS or total != len(file_nodes):
+ if not layers or len(layers) > _MAX_LAYERS or total != file_count:
logger.warning(
"kg_curation: curated layers failed invariant "
"(count=%d, partition=%d/%d); keeping community layers",
len(layers),
total,
- len(file_nodes),
+ file_count,
)
- return None
- return layers
+ return False
+ return True
# ---------------------------------------------------------------------------
# Wiki modules — right-sized directory groups derived from the curated layers
# ---------------------------------------------------------------------------
-# Granularity window for derived wiki modules. Sub-groups verbatim are NOT
-# module-sized (a 452-file ``core`` sub-group would make one vague mush of a
-# doc; a 1-file ``examples`` group would mint a confetti page), so the layer
-# node sets are split *recursively* by directory until every group fits the
-# window — bottoming out honestly on flat directories. ``target_max`` keeps
-# the 10 key-file template slots representative; ``target_min`` is the
-# merge-up floor below which a group folds into its nearest sibling.
-_MODULE_TARGET_MIN = 8
-_MODULE_TARGET_MAX = 120
-# A layer smaller than this yields no module at all (matches the selection
-# layer's ``min_module_size`` floor that kills singleton pages).
-_MODULE_MIN_FILES = 3
-# A directory segment present in more than this fraction of all repo paths is
-# *generic* (namespace dirs: ``src``, ``packages``, the repo's own name) and
-# never appears in a module name. Data-driven — no hardcoded segment list.
-_GENERIC_SEGMENT_FRACTION = 0.60
-# The legacy community labels' size-suffix dedupe ("ingestion (32)") is the
-# exact failure mode module names must never reproduce.
-_SIZE_SUFFIX_RE = re.compile(r"\(\d+\)\s*$")
-
-
-# Universal organizational directory names — containers, not domain labels.
-# Shared with community labeling; the data-driven ``dominant_segments`` set
-# complements this with per-repo namespace noise (the repo's own name).
-GENERIC_ORG_SEGMENTS = frozenset({
- "src", "lib", "core", "common", "shared", "internal", "pkg",
- "main", "app", "utils", "helpers", "index", "mod",
- # Monorepo organisational directories
- "packages", "modules", "workspace", "workspaces", "libs",
- "projects", "services", "apps",
-})
-
-
-def dominant_segments(paths: list[str]) -> set[str]:
- """Directory segments appearing in > 60% of *paths* (namespace noise).
-
- Shared with community labeling (``analysis/communities.py``) so both
- vocabularies strip the same namespace dirs (``src``, ``packages``, the
- repo's own name) without depending on a hardcoded list.
- """
- n = len(paths)
- if not n:
- return set()
- counts: Counter[str] = Counter()
- for p in paths:
- for seg in set(PurePosixPath(p).parts[:-1]):
- counts[seg] += 1
- return {s for s, c in counts.items() if c / n > _GENERIC_SEGMENT_FRACTION}
-
-
-def _split_to_granularity(
- node_ids: list[str], id_to_path: dict[str, str], target_max: int
-) -> list[tuple[tuple[str, ...], list[str]]]:
- """Recursively split *node_ids* by directory until groups fit *target_max*.
-
- Returns ``[(dir_segments, sorted_node_ids), ...]``. Reuses ``_sub_split``'s
- prefix logic (group by the first segment that distinguishes members after
- the common directory prefix) but, unlike sub-groups, recurses into any
- group still above ``target_max``. Recursion bottoms out when a directory
- has no distinguishing subdirs — a 200-file flat dir stays one module
- (honest), never an artificial split.
- """
- dir_segs = {nid: PurePosixPath(id_to_path[nid]).parts[:-1] for nid in node_ids}
-
- def rec(ids: list[str]) -> list[tuple[tuple[str, ...], list[str]]]:
- common = _common_dir_prefix([dir_segs[i] for i in ids])
- if len(ids) <= target_max:
- return [(common, ids)]
- groups: dict[str, list[str]] = defaultdict(list)
- for nid in ids:
- segs = dir_segs[nid]
- key = segs[len(common)] if len(segs) > len(common) else ""
- groups[key].append(nid)
- if len(groups) < 2:
- return [(common, ids)] # flat directory — no honest split exists
- out: list[tuple[tuple[str, ...], list[str]]] = []
- for key in sorted(groups):
- if key == "":
- # Files sitting directly in the common dir (the "(root)"
- # group). Usually below target_min → folded by merge-up.
- out.append((common, groups[key]))
- else:
- out.extend(rec(groups[key]))
- return out
-
- return [(d, sorted(ids)) for d, ids in rec(sorted(node_ids))]
-
-
-def _merge_small_groups(
- groups: list[tuple[tuple[str, ...], list[str]]], target_min: int
-) -> list[tuple[tuple[str, ...], list[str]]]:
- """Fold groups below *target_min* into their nearest sibling.
-
- "Nearest" = the group sharing the longest directory prefix (the parent
- subtree), largest first as the tie-break — so a 2-file "(root)" remnant
- folds into its own subtree's biggest module, and an isolated small dir
- folds into the layer's dominant module rather than minting a confetti
- page. Never merges across layers (callers pass one layer at a time). A
- layer that is itself below ``target_min`` stays one whole group.
-
- A pre-pass fuses *small sibling* groups into one group at their common
- parent when that collection is itself module-sized — ninety tiny locale
- dirs become one ``conf/locale`` module instead of folding into whichever
- sibling sorts first and misnaming it. The fold-in loop then never renames
- a survivor: a healthy ``core/providers`` absorbing a 2-file sibling keeps
- its identity.
- """
- merged = [(d, list(ids)) for d, ids in groups]
-
- by_parent: dict[tuple[str, ...], list[tuple[tuple[str, ...], list[str]]]] = {}
- for g in merged:
- if len(g[1]) < target_min and len(g[0]) > 0:
- by_parent.setdefault(g[0][:-1], []).append(g)
- for parent, sibs in sorted(by_parent.items()):
- if len(sibs) < 2 or sum(len(g[1]) for g in sibs) < target_min:
- continue
- fused = sorted(nid for g in sibs for nid in g[1])
- for g in sibs:
- merged.remove(g)
- existing = next((g for g in merged if g[0] == parent), None)
- if existing is not None:
- existing[1].extend(fused)
- existing[1].sort()
- else:
- merged.append((parent, fused))
- merged.sort(key=lambda g: g[0])
-
- def shared(a: tuple[str, ...], b: tuple[str, ...]) -> int:
- return len(_common_dir_prefix([a, b]))
-
- while len(merged) > 1:
- small = min(
- (g for g in merged if len(g[1]) < target_min),
- key=lambda g: (len(g[1]), g[0]),
- default=None,
- )
- if small is None:
- break
- merged.remove(small)
- target = min(
- merged,
- key=lambda g: (-shared(g[0], small[0]), -len(g[1]), g[0]),
- )
- target[1].extend(small[1])
- target[1].sort()
- return [(d, ids) for d, ids in merged]
-
-
-def _name_modules(mods: list[dict], generic: set[str]) -> None:
- """Assign unique, human module names in place.
-
- Initial name = the last one or two *informative* directory segments
- (generic namespace segments stripped; when stripping consumes every
- segment, the raw tail is used instead). Collisions extend leftward by
- one more parent segment — NEVER a size suffix. Single-module layers
- take the layer's name; the root group (empty dir) becomes
- "<Layer> (top-level)". The absolute fallback (identical informative
- paths across layers) appends the layer name, which is unique by
- construction.
- """
- per_layer: Counter[str] = Counter(m["layerId"] for m in mods)
- info_by: dict[int, list[str]] = {}
- used: dict[int, int | None] = {} # informative segments consumed; None = fixed
- for m in mods:
- # Data-driven stripping can consume EVERY segment on fixture-dominated
- # repos (aeson: tests/JSONTestSuite/test_parsing is >60% of all
- # paths). The raw dir tail is still the honest name there —
- # "(top-level)" would mislabel a real directory and collide across
- # sibling groups (which trips the export degradation guard and ships
- # no modules). Universal organizational dirs (pkg, src, packages…)
- # stay excluded even in the fallback: "(top-level)" reads better than
- # a container name, so it remains the name for true root groups.
- info = [s for s in m["_dir"] if s not in generic] or [
- s for s in m["_dir"] if s.lower() not in GENERIC_ORG_SEGMENTS
- ]
- info_by[id(m)] = info
- if per_layer[m["layerId"]] == 1:
- m["name"] = m["_layerName"]
- used[id(m)] = None
- elif not info:
- m["name"] = f"{m['_layerName']} (top-level)"
- used[id(m)] = None
- else:
- k = min(2, len(info))
- m["name"] = "/".join(info[-k:])
- used[id(m)] = k
-
- for _ in range(16): # bounded: each round consumes ≥1 segment somewhere
- names = Counter(m["name"] for m in mods)
- colliding = [m for m in mods if names[m["name"]] > 1]
- if not colliding:
- return
- progressed = False
- for m in colliding:
- k = used.get(id(m))
- info = info_by[id(m)]
- if k is not None and k < len(info):
- used[id(m)] = k + 1
- m["name"] = "/".join(info[-(k + 1) :])
- progressed = True
- if not progressed:
- break
-
- # Two all-organizational groups in one layer (a root remnant plus a
- # "packages"-style container) would both read "<Layer> (top-level)" —
- # the container's raw tail is the honest tiebreak.
- names = Counter(m["name"] for m in mods)
- for m in mods:
- if names[m["name"]] > 1 and not info_by[id(m)] and m["_dir"]:
- m["name"] = "/".join(m["_dir"][-min(2, len(m["_dir"])) :])
-
- # Same informative dir in two layers (or no segments left): the layer
- # name disambiguates — (dir, layer) is unique by construction.
- names = Counter(m["name"] for m in mods)
- for m in mods:
- if names[m["name"]] > 1:
- m["name"] = f"{m['name']} ({m['_layerName']})"
-
- # Absolute backstop (two all-org dirs in one layer sharing a tail): the
- # full dir path is unique per layer.
- names = Counter(m["name"] for m in mods)
- for m in mods:
- if names[m["name"]] > 1 and m["_dir"]:
- m["name"] = "/".join(m["_dir"])
-
-
-def derive_modules(
- layers: list[dict],
- id_to_path: dict[str, str],
- *,
- target_min: int = _MODULE_TARGET_MIN,
- target_max: int = _MODULE_TARGET_MAX,
- min_module_size: int = _MODULE_MIN_FILES,
- lang_by_id: dict[str, str] | None = None,
-) -> list[dict]:
- """Derive right-sized, stably-identified wiki modules from curated layers.
-
- ``Module = {"id": "module:<dir-slug>", "name": <human>, "path": <dir or "">,
- "layerId": ..., "nodeIds": [...], "language": ...}``
-
- Properties (each one an edge case from the research pass):
-
- - **Partition per layer**: every node of every layer ≥ ``min_module_size``
- lands in exactly one module; layers below the floor yield none. Never
- merges across layers.
- - **Granularity**: recursive directory splitting to the
- [``target_min``, ``target_max``] window; flat dirs stay one honest
- module; sub-``target_min`` remnants merge up into their subtree.
- - **Names**: informative path segments only (data-driven generic-segment
- stripping kills ``src``/``packages``/repo-name automatically); collision
- resolution extends the path leftward — never a size suffix.
- - **Ids**: ``module:`` + slug of the real directory path — stable across
- runs and under file adds/renames inside the dir; changes only when the
- directory itself moves. ``path`` is the actual dir (not the slug) so
- path-prefix child lookups (``target_path LIKE 'dir/%'``) work.
- - **Files only**: operates on ids present in ``id_to_path`` — external
- nodes never pollute a module.
- - **Determinism**: sorted iteration throughout; same inputs → same bytes.
- """
- generic = dominant_segments(sorted(set(id_to_path.values())))
-
- mods: list[dict] = []
- for layer in layers:
- node_ids = [nid for nid in layer.get("nodeIds", []) if nid in id_to_path]
- if len(node_ids) < min_module_size:
- continue
- groups = _merge_small_groups(
- _split_to_granularity(node_ids, id_to_path, target_max), target_min
- )
- for dir_parts, ids in sorted(groups):
- mods.append(
- {
- "_dir": dir_parts,
- "_layerName": layer.get("name", ""),
- "path": "/".join(dir_parts),
- "layerId": layer.get("id", ""),
- "nodeIds": sorted(ids),
- }
- )
-
- _name_modules(mods, generic)
-
- # Ids: path-derived slugs; the bigger module keeps the plain id on the
- # rare cross-layer dir collision (a dir whose files split across layers).
- used_ids: set[str] = set()
- for m in sorted(mods, key=lambda m: (-len(m["nodeIds"]), m["path"], m["layerId"])):
- base = "module:" + _slugify(m["path"] or m["_layerName"])
- mid = base
- n = 1
- while mid in used_ids:
- mid = f"{base}--{_slugify(m['_layerName'])}" + ("" if n == 1 else f"-{n}")
- n += 1
- used_ids.add(mid)
- m["id"] = mid
-
- # A single-module layer is 1:1 with its layer page — mark it so page
- # generation can skip the duplicate doc (the module stays in the
- # artifact: canvas containers and the coverage invariant need it).
- per_layer_count: Counter[str] = Counter(m["layerId"] for m in mods)
-
- out: list[dict] = []
- for m in mods:
- module = {
- "id": m["id"],
- "name": m["name"],
- "path": m["path"],
- "layerId": m["layerId"],
- "nodeIds": m["nodeIds"],
- }
- if per_layer_count[m["layerId"]] == 1:
- module["wholeLayer"] = True
- if lang_by_id is not None:
- langs = Counter(
- lang for nid in m["nodeIds"] if (lang := lang_by_id.get(nid, ""))
- )
- module["language"] = (
- min(langs, key=lambda tag: (-langs[tag], tag)) if langs else ""
- )
- out.append(module)
- return out
-
def _curate_modules(kg: KnowledgeGraphResult) -> list[dict] | None:
"""Derive wiki modules from the curated layers, or ``None`` on degradation.
@@ -649,9 +328,12 @@ def _curate_modules(kg: KnowledgeGraphResult) -> list[dict] | None:
lang_by_id = {n["id"]: (n.get("language") or "").lower() for n in file_nodes}
modules = derive_modules(kg.layers, id_to_path, lang_by_id=lang_by_id)
- if not modules:
+ if not modules or not _modules_hold_invariants(modules, id_to_path):
return None
+ return modules
+
+def _modules_hold_invariants(modules: list[dict], id_to_path: dict[str, str]) -> bool:
seen: set[str] = set()
for m in modules:
for nid in m["nodeIds"]:
@@ -660,7 +342,7 @@ def _curate_modules(kg: KnowledgeGraphResult) -> list[dict] | None:
"kg_curation: derived modules failed partition invariant; "
"exporting no modules"
)
- return None
+ return False
seen.add(nid)
names = [m["name"] for m in modules]
ids = [m["id"] for m in modules]
@@ -668,8 +350,8 @@ def _curate_modules(kg: KnowledgeGraphResult) -> list[dict] | None:
logger.warning(
"kg_curation: derived module names/ids not unique; exporting no modules"
)
- return None
- return modules
+ return False
+ return True
# ---------------------------------------------------------------------------
@@ -698,30 +380,31 @@ def _curate_entry_points(
pf_by_path = {pf.file_info.path: pf for pf in parsed_files if getattr(pf, "file_info", None)}
lang_by_path = {n["filePath"]: (n.get("language") or "").lower() for n in _file_nodes(kg)}
pagerank = graph_builder.pagerank() or {}
- try:
- betweenness = graph_builder.betweenness_centrality() or {}
- except Exception: # pragma: no cover - defensive
- betweenness = {}
-
- candidates = [
- (path, pagerank.get(path, 0.0), betweenness.get(path, 0.0))
- for path in _flagged_entry_paths(kg, pf_by_path)
- ]
- if not candidates:
- # No ingestion-flagged entries (or all were barrels): fall back to the
- # strong filename scorers the tour seeds from (score >= 3 means an
- # entry-style name or flag, never just shallow/high-PageRank).
- candidates = [
- (path, pagerank.get(path, 0.0), betweenness.get(path, 0.0))
+ betweenness = _betweenness(graph_builder)
+
+ paths = _flagged_entry_paths(kg, pf_by_path)
+ if not paths:
+ # No flagged entries survived: score >= 3 means an entry-style name or
+ # flag, never just a shallow or high-PageRank file.
+ paths = [
+ path
for s, path in score_entry_points(parsed_files, pagerank)
if s >= 3.0 and _is_entry_candidate(path, lang_by_path.get(path, ""), pf_by_path)
]
+ candidates = [(path, pagerank.get(path, 0.0), betweenness.get(path, 0.0)) for path in paths]
ranked = rank_entry_points(candidates, conventional_entry_stems())
kg.project["entry_points"] = ranked[:_MAX_ENTRY_POINTS]
kg.project["entry_candidates"] = ranked
+def _betweenness(graph_builder: Any) -> dict[str, float]:
+ try:
+ return graph_builder.betweenness_centrality() or {}
+ except Exception: # pragma: no cover - defensive
+ return {}
+
+
def _flagged_entry_paths(kg: KnowledgeGraphResult, pf_by_path: dict[str, Any]) -> list[str]:
"""Ingestion-flagged entry files that survive candidacy, in node order.
@@ -730,29 +413,39 @@ def _flagged_entry_paths(kg: KnowledgeGraphResult, pf_by_path: dict[str, Any]) -
"""
paths: list[str] = []
for node in kg.nodes:
- nid = node.get("id", "")
- if not (isinstance(nid, str) and nid.startswith("file:")):
- continue
- tags = node.get("tags") or []
- if "entry_point" not in tags:
+ if not _is_flagged_file_node(node):
continue
path = node.get("filePath", "")
language = (node.get("language") or "").lower()
if _off_the_entry_path(path, language):
continue
- pf = pf_by_path.get(path)
- if pf is not None and _is_barrel(pf):
- new_tags = [t for t in tags if t != "entry_point"]
- if "barrel" not in new_tags:
- new_tags.append("barrel")
- node["tags"] = new_tags
+ if _is_barrel_path(path, pf_by_path):
+ _retag_as_barrel(node)
continue
- if not_an_execution_start(path, language):
- continue
- paths.append(path)
+ if not not_an_execution_start(path, language):
+ paths.append(path)
return paths
+def _is_flagged_file_node(node: dict) -> bool:
+ nid = node.get("id", "")
+ if not (isinstance(nid, str) and nid.startswith("file:")):
+ return False
+ return "entry_point" in (node.get("tags") or [])
+
+
+def _retag_as_barrel(node: dict) -> None:
+ new_tags = [t for t in (node.get("tags") or []) if t != "entry_point"]
+ if "barrel" not in new_tags:
+ new_tags.append("barrel")
+ node["tags"] = new_tags
+
+
+def _is_barrel_path(path: str, pf_by_path: dict[str, Any]) -> bool:
+ pf = pf_by_path.get(path)
+ return pf is not None and _is_barrel(pf)
+
+
def _off_the_entry_path(path: str, language: str) -> bool:
# Test fixtures (a wsgi.py inside tests/) and sample programs
# (examples/*/main.go) may carry the ingestion flag, but they are
@@ -762,10 +455,7 @@ def _off_the_entry_path(path: str, language: str) -> bool:
def _is_entry_candidate(path: str, language: str, pf_by_path: dict[str, Any]) -> bool:
"""Whether a scored (unflagged) file may stand as an entry point."""
- if _off_the_entry_path(path, language):
- return False
- pf = pf_by_path.get(path)
- if pf is not None and _is_barrel(pf):
+ if _off_the_entry_path(path, language) or _is_barrel_path(path, pf_by_path):
return False
return not not_an_execution_start(path, language)
@@ -793,10 +483,8 @@ _DATA_SUFFIXES = (".sql", ".prisma")
# Source-code extensions. A code file is never CI/infra config however its
# name or directory reads — ``languages/specs/dockerfile.py`` *parses*
-# Dockerfiles, it isn't one. Registry-derived: every is_code,
-# non-infra language's extensions are protected — .dart/.hs/.clj included;
-# shell/terraform stay promotable (they ARE infra); the historical orphan
-# ``.pl`` (no perl spec) is gone.
+# Dockerfiles, it isn't one. Registry-derived from every non-infra code
+# language; shell/terraform stay promotable because they are infra.
_CODE_SUFFIXES = _LANG_REGISTRY.non_infra_code_extensions()
@@ -812,22 +500,25 @@ def _enrich_type(path: str, current_type: str) -> tuple[str, str | None]:
suffix = PurePosixPath(p).suffix
is_code = suffix in _CODE_SUFFIXES
- if not is_code and (any(m in p for m in _CI_PATH_MARKERS) or name == "jenkinsfile"):
+ if not is_code and _is_ci_path(p, name):
return "pipeline", "ci"
- if (
- not is_code
- and (
- name.startswith("dockerfile")
- or any(m in name for m in _INFRA_NAME_MARKERS)
- or any(m in p for m in _INFRA_PATH_MARKERS)
- )
- ) or suffix in _INFRA_SUFFIXES:
+ if (not is_code and _is_infra_path(p, name)) or suffix in _INFRA_SUFFIXES:
return "service", "infra"
if any(m in p for m in _DATA_PATH_MARKERS) or suffix in _DATA_SUFFIXES:
return "schema", "data"
return current_type, None
+def _is_ci_path(lowered_path: str, name: str) -> bool:
+ return any(m in lowered_path for m in _CI_PATH_MARKERS) or name == "jenkinsfile"
+
+
+def _is_infra_path(lowered_path: str, name: str) -> bool:
+ if name.startswith("dockerfile") or any(m in name for m in _INFRA_NAME_MARKERS):
+ return True
+ return any(m in lowered_path for m in _INFRA_PATH_MARKERS)
+
+
def _curate_node_types(kg: KnowledgeGraphResult) -> None:
"""Promote infra/CI/data file nodes to first-class presentation types."""
for node in _file_nodes(kg):
@@ -850,49 +541,53 @@ def _infer_test_target(path: str) -> str:
return stem
+# Support-file summary templates in precedence order: the first row whose
+# presentation type or tag matches the node names it.
+_SUPPORT_TEMPLATES: tuple[tuple[str, str | None, str], ...] = (
+ ("pipeline", "ci", "CI / pipeline definition: {name}."),
+ ("service", "infra", "Infrastructure definition: {name}."),
+ ("schema", "data", "Data / schema definition: {name}."),
+ ("config", "config", "Configuration file: {name}."),
+ ("document", None, "Documentation: {name}."),
+)
+
+
def _cheap_summary(node: dict, parsed_file: Any | None) -> str:
"""A deterministic, honest fallback summary (zero LLM cost)."""
path = node["filePath"]
- stem = PurePosixPath(path).stem
- parent = PurePosixPath(path).parent.name or "root"
- node_type = node.get("type", "file")
tags = node.get("tags") or []
- layer = infer_layer(path, (node.get("language") or "").lower())
if "barrel" in tags:
- return f"Re-export barrel for {parent}/."
-
- name = PurePosixPath(path).name
- if node_type in {"pipeline", "service", "schema", "config", "document"} or (
- tags and ({"ci", "infra", "data", "config"} & set(tags))
- ):
- # Recognised scaffolding earns a real role instead of a bare name
- # restatement; only genuinely opaque support files fall back to the
- # type template below.
- role = well_known_role(path)
- if role is not None:
- return role
- if node_type == "pipeline" or "ci" in tags:
- return f"CI / pipeline definition: {name}."
- if node_type == "service" or "infra" in tags:
- return f"Infrastructure definition: {name}."
- if node_type == "schema" or "data" in tags:
- return f"Data / schema definition: {name}."
- if node_type == "config" or "config" in tags:
- return f"Configuration file: {name}."
- if node_type == "document":
- return f"Documentation: {name}."
+ return f"Re-export barrel for {PurePosixPath(path).parent.name or 'root'}/."
+ support = _support_summary(path, node.get("type", "file"), tags)
+ if support is not None:
+ return support
if "test" in tags:
return f"Tests for {_infer_test_target(path)}."
+ return _code_summary(node, parsed_file)
+
- # Code file: name the layer and its most prominent symbols.
- symbol_names: list[str] = []
- if parsed_file is not None:
- symbol_names = [
- getattr(s, "name", "")
- for s in (getattr(parsed_file, "symbols", []) or [])
- if getattr(s, "kind", "") in _SUBSTANTIVE_KINDS and getattr(s, "name", "")
- ][:3]
+def _support_summary(path: str, node_type: str, tags: list[str]) -> str | None:
+ template = next(
+ (text for kind, tag, text in _SUPPORT_TEMPLATES if node_type == kind or tag in tags),
+ None,
+ )
+ if template is None:
+ return None
+ # Recognised scaffolding earns a real role; only opaque support files fall
+ # back to the type template.
+ role = well_known_role(path)
+ if role is not None:
+ return role
+ return template.format(name=PurePosixPath(path).name)
+
+
+def _code_summary(node: dict, parsed_file: Any | None) -> str:
+ """Name the code file's layer and its most prominent symbols."""
+ path = node["filePath"]
+ stem = PurePosixPath(path).stem
+ layer = infer_layer(path, (node.get("language") or "").lower())
+ symbol_names = _top_symbol_names(parsed_file)
if symbol_names:
return f"{layer} module {stem} defining {', '.join(symbol_names)}."
count = node.get("symbolCount", 0)
@@ -901,6 +596,16 @@ def _cheap_summary(node: dict, parsed_file: Any | None) -> str:
return f"{layer} module {stem}."
+def _top_symbol_names(parsed_file: Any | None) -> list[str]:
+ if parsed_file is None:
+ return []
+ return [
+ getattr(s, "name", "")
+ for s in (getattr(parsed_file, "symbols", []) or [])
+ if getattr(s, "kind", "") in _SUBSTANTIVE_KINDS and getattr(s, "name", "")
+ ][:3]
+
+
def apply_summary_floor(kg: KnowledgeGraphResult, parsed_files: list[Any] | None = None) -> None:
"""Ensure every file node carries a summary (cheap deterministic floor).
@@ -916,234 +621,3 @@ def apply_summary_floor(kg: KnowledgeGraphResult, parsed_files: list[Any] | None
if node.get("summary"):
continue
node["summary"] = _cheap_summary(node, pf_by_path.get(node["filePath"]))
-
-
-# ---------------------------------------------------------------------------
-# Phase 7 — invariant validation (shared by tests and the portable writer)
-# ---------------------------------------------------------------------------
-
-# Quality thresholds. The lower layer bound and coverage targets are *soft*
-# (warnings) because they depend on repo size/shape; the partition, hard count
-# bound, capped entry set, never-empty summaries, and tour budget are *hard*.
-_MIN_LAYERS = 6
-_MAX_LAYER_FRACTION = 0.35
-_MAX_CATCHALL_FRACTION = 0.20
-_MAX_SINGLETON_FRACTION = 0.10
-_MIN_TOUR_COVERAGE = 0.90
-
-
-@dataclass
-class KGValidation:
- """Outcome of :func:`validate_kg` — hard errors, soft warnings, metrics."""
-
- ok: bool
- errors: list[str] = field(default_factory=list)
- warnings: list[str] = field(default_factory=list)
- metrics: dict[str, Any] = field(default_factory=dict)
-
- def as_dict(self) -> dict[str, Any]:
- return {
- "ok": self.ok,
- "errors": self.errors,
- "warnings": self.warnings,
- "metrics": self.metrics,
- }
-
-
-def validate_kg(kg: KnowledgeGraphResult) -> KGValidation:
- """Validate a curated KG against the intuitiveness invariants (plan §5/§7).
-
- Pure and side-effect free. Hard violations set ``ok=False`` and populate
- ``errors``; size/shape-dependent shortfalls go to ``warnings``. The
- ``metrics`` block is the per-repo intuitiveness scorecard.
- """
- errors: list[str] = []
- warnings: list[str] = []
-
- file_nodes = _file_nodes(kg)
- file_count = len(file_nodes)
- file_ids = {n["id"] for n in file_nodes}
- tags_by_path = {n["filePath"]: (n.get("tags") or []) for n in file_nodes}
- summary_by_id = {n["id"]: n.get("summary") for n in file_nodes}
-
- layers = kg.layers or []
- shape = _check_layers(layers, file_ids, file_count, errors, warnings)
-
- entry_points = kg.project.get("entry_points", []) if isinstance(kg.project, dict) else []
- _check_entry_points(entry_points, tags_by_path, errors)
-
- tour = kg.tour or []
- tour_coverage = _check_tour(tour, layers, errors, warnings)
-
- # Modules are only checked when the curated artifact carries them.
- modules = getattr(kg, "modules", None) or []
- module_covered = _check_modules(modules, file_ids, errors, warnings) if modules else set()
-
- # -- Summaries ---------------------------------------------------------
- empty_summaries = [nid for nid, s in summary_by_id.items() if not s]
- if empty_summaries:
- errors.append(f"{len(empty_summaries)} file nodes have an empty summary")
- summary_completeness = 1.0 - len(empty_summaries) / file_count if file_count else 1.0
-
- metrics = {
- "file_count": file_count,
- "layer_count": len(layers),
- "module_count": len(modules),
- "module_coverage_pct": round(
- (len(module_covered) / file_count * 100) if (modules and file_count) else 0.0, 1
- ),
- "singleton_layer_pct": round(shape.singleton_frac * 100, 1),
- "largest_layer_pct": round(shape.largest_frac * 100, 1),
- "application_pct": round(shape.catchall_frac * 100, 1),
- "entry_point_count": len(entry_points),
- "tour_steps": len(tour),
- "tour_coverage_pct": round(tour_coverage * 100, 1),
- "summary_completeness_pct": round(summary_completeness * 100, 1),
- }
-
- return KGValidation(ok=not errors, errors=errors, warnings=warnings, metrics=metrics)
-
-
-@dataclass(frozen=True)
-class _LayerShape:
- singleton_frac: float
- largest_frac: float
- catchall_frac: float
-
-
-def _check_layers(
- layers: list[dict],
- file_ids: set[str],
- file_count: int,
- errors: list[str],
- warnings: list[str],
-) -> _LayerShape:
- """Layer count, partition, singleton spam and mega-layer balance."""
- n_layers = len(layers)
-
- # -- Layer count -------------------------------------------------------
- if n_layers == 0:
- errors.append("no layers")
- elif n_layers > _MAX_LAYERS:
- errors.append(f"too many layers: {n_layers} > {_MAX_LAYERS}")
- elif n_layers < _MIN_LAYERS:
- warnings.append(f"few layers: {n_layers} < {_MIN_LAYERS} (small/flat repo?)")
-
- # -- Partition ---------------------------------------------------------
- layered: list[str] = [nid for layer in layers for nid in layer.get("nodeIds", [])]
- layered_set = set(layered)
- if len(layered) != len(layered_set):
- errors.append("partition: a file appears in more than one layer")
- if file_count and layered_set != file_ids:
- missing = len(file_ids - layered_set)
- extra = len(layered_set - file_ids)
- errors.append(f"partition: {missing} unlayered, {extra} unknown ids")
-
- # -- Singleton spam & mega-layer balance -------------------------------
- sizes = [len(layer.get("nodeIds", [])) for layer in layers]
- singleton_frac = (sum(1 for s in sizes if s == 1) / n_layers) if n_layers else 0.0
- if singleton_frac >= _MAX_SINGLETON_FRACTION:
- warnings.append(f"singleton layers {singleton_frac:.0%} ≥ {_MAX_SINGLETON_FRACTION:.0%}")
-
- largest_frac = (max(sizes) / file_count) if (sizes and file_count) else 0.0
- if largest_frac > _MAX_LAYER_FRACTION:
- warnings.append(f"largest layer {largest_frac:.0%} > {_MAX_LAYER_FRACTION:.0%}")
-
- catchall = next((layer for layer in layers if layer.get("name") == "Application"), None)
- catchall_frac = (
- (len(catchall.get("nodeIds", [])) / file_count) if (catchall and file_count) else 0.0
- )
- if catchall_frac > _MAX_CATCHALL_FRACTION:
- warnings.append(f"Application catch-all {catchall_frac:.0%} > {_MAX_CATCHALL_FRACTION:.0%}")
- return _LayerShape(singleton_frac, largest_frac, catchall_frac)
-
-
-def _check_entry_points(
- entry_points: list[str], tags_by_path: dict[str, list[str]], errors: list[str]
-) -> None:
- if len(entry_points) > _MAX_ENTRY_POINTS:
- errors.append(f"too many entry points: {len(entry_points)} > {_MAX_ENTRY_POINTS}")
- barrels_surfaced = [p for p in entry_points if "barrel" in tags_by_path.get(p, [])]
- if barrels_surfaced:
- errors.append(f"barrels surfaced as entry points: {barrels_surfaced}")
-
-
-def _check_tour(
- tour: list[dict], layers: list[dict], errors: list[str], warnings: list[str]
-) -> float:
- """Tour budget and opening, returning the fraction of layers it visits."""
- if not tour:
- return 0.0
- if len(tour) > DEFAULT_MAX_STOPS:
- errors.append(f"tour too long: {len(tour)} > {DEFAULT_MAX_STOPS}")
- if tour[0].get("kind") != "overview":
- errors.append("tour does not open with an overview/README step")
- layer_ids = {layer.get("id") for layer in layers}
- covered = {
- s.get("layer_id")
- for s in tour
- if s.get("kind") != "overview" and s.get("layer_id") in layer_ids
- }
- tour_coverage = (len(covered) / len(layer_ids)) if layer_ids else 0.0
- if tour_coverage < _MIN_TOUR_COVERAGE:
- warnings.append(f"tour covers {tour_coverage:.0%} of layers < {_MIN_TOUR_COVERAGE:.0%}")
- return tour_coverage
-
-
-def _check_modules(
- modules: list[dict], file_ids: set[str], errors: list[str], warnings: list[str]
-) -> set[str]:
- """Module partition, naming and size, returning the file ids modules cover."""
- module_member_lists = [m.get("nodeIds", []) for m in modules]
- flat = [nid for ids in module_member_lists for nid in ids]
- module_covered = set(flat)
- if len(flat) != len(module_covered):
- errors.append("modules: a file appears in more than one module")
- if not module_covered <= file_ids:
- errors.append(
- f"modules: {len(module_covered - file_ids)} unknown ids in modules"
- )
- module_names = [m.get("name", "") for m in modules]
- if len(set(module_names)) != len(module_names):
- errors.append("modules: names not unique")
- size_suffixed = [n for n in module_names if _SIZE_SUFFIX_RE.search(n)]
- if size_suffixed:
- errors.append(f"modules: size-suffixed names: {size_suffixed}")
- oversized = sum(
- 1 for ids in module_member_lists if len(ids) > _MODULE_TARGET_MAX
- )
- if oversized:
- # Flat dirs may honestly exceed the window — soft signal only.
- warnings.append(f"{oversized} modules above target_max (flat dirs?)")
- return module_covered
-
-
-# ---------------------------------------------------------------------------
-# Phase 6 — portable, self-validated export artifact
-# ---------------------------------------------------------------------------
-
-
-def build_portable_kg(kg: KnowledgeGraphResult) -> tuple[dict, KGValidation]:
- """Assemble a self-contained, self-validated ``knowledge-graph.json`` dict.
-
- Kept separate from :meth:`KnowledgeGraphResult.to_dict` so the *default*
- export stays byte-identical (curation flag-off contract); the portable
- artifact adds a ``meta`` block (counts, fingerprint) and an embedded
- ``validation`` report so an external consumer can trust it without a server.
- Returns ``(data, validation)`` so the writer can decide on hard violations.
- """
- data = kg.to_dict()
- validation = validate_kg(kg)
- data["meta"] = {
- # The integer the loader gates on, not the "1.0.0" display label.
- "schema_version": data.get("schema_version", 1),
- "generator": "repowise-kg-curation",
- "fingerprint": getattr(kg, "fingerprint", ""),
- "file_count": validation.metrics.get("file_count", 0),
- "layer_count": validation.metrics.get("layer_count", 0),
- "module_count": validation.metrics.get("module_count", 0),
- "entry_point_count": validation.metrics.get("entry_point_count", 0),
- "tour_steps": validation.metrics.get("tour_steps", 0),
- "validation": validation.as_dict(),
- }
- return data, validation
diff --git a/packages/core/src/repowise/core/analysis/kg_modules.py b/packages/core/src/repowise/core/analysis/kg_modules.py
new file mode 100644
index 000000000..cea69237c
--- /dev/null
+++ b/packages/core/src/repowise/core/analysis/kg_modules.py
@@ -0,0 +1,382 @@
+"""Wiki modules: right-sized directory groups derived from the curated KG layers."""
+
+from __future__ import annotations
+
+import re
+from collections import Counter, defaultdict
+from collections.abc import Callable
+from pathlib import PurePosixPath
+
+from repowise.core.analysis.kg_inputs import _dominant_language
+from repowise.core.analysis.knowledge_graph import _slugify
+
+# A directory's segments and the (sorted) node ids grouped under it.
+_DirGroup = tuple[tuple[str, ...], list[str]]
+
+
+def _common_dir_prefix(seg_lists: list[tuple[str, ...]]) -> tuple[str, ...]:
+ """Longest common leading directory-segment prefix across *seg_lists*."""
+ if not seg_lists:
+ return ()
+ common = list(seg_lists[0])
+ for segs in seg_lists[1:]:
+ i = 0
+ while i < len(common) and i < len(segs) and common[i] == segs[i]:
+ i += 1
+ del common[i:]
+ if not common:
+ break
+ return tuple(common)
+
+
+# Granularity window for derived wiki modules. Layer node sets are split
+# recursively by directory until every group fits, stopping at flat
+# directories. ``target_max`` keeps the 10 key-file template slots
+# representative; below ``target_min`` a group folds into its nearest sibling.
+_MODULE_TARGET_MIN = 8
+_MODULE_TARGET_MAX = 120
+# A layer smaller than this yields no module at all (matches the selection
+# layer's ``min_module_size`` floor that kills singleton pages).
+_MODULE_MIN_FILES = 3
+# A directory segment present in more than this fraction of all repo paths is
+# *generic* (namespace dirs: ``src``, ``packages``, the repo's own name) and
+# never appears in a module name. Data-driven — no hardcoded segment list.
+_GENERIC_SEGMENT_FRACTION = 0.60
+# Module names never carry a size suffix such as "ingestion (32)".
+_SIZE_SUFFIX_RE = re.compile(r"\(\d+\)\s*$")
+
+
+# Universal organizational directory names — containers, not domain labels.
+# Shared with community labeling; the data-driven ``dominant_segments`` set
+# complements this with per-repo namespace noise (the repo's own name).
+GENERIC_ORG_SEGMENTS = frozenset({
+ "src", "lib", "core", "common", "shared", "internal", "pkg",
+ "main", "app", "utils", "helpers", "index", "mod",
+ # Monorepo organisational directories
+ "packages", "modules", "workspace", "workspaces", "libs",
+ "projects", "services", "apps",
+})
+
+
+def dominant_segments(paths: list[str]) -> set[str]:
+ """Directory segments appearing in > 60% of *paths* (namespace noise).
+
+ Shared with community labeling (``analysis/communities.py``) so both
+ vocabularies strip the same namespace dirs (``src``, ``packages``, the
+ repo's own name) without depending on a hardcoded list.
+ """
+ n = len(paths)
+ if not n:
+ return set()
+ counts: Counter[str] = Counter()
+ for p in paths:
+ for seg in set(PurePosixPath(p).parts[:-1]):
+ counts[seg] += 1
+ return {s for s, c in counts.items() if c / n > _GENERIC_SEGMENT_FRACTION}
+
+
+def _split_to_granularity(
+ node_ids: list[str], id_to_path: dict[str, str], target_max: int
+) -> list[tuple[tuple[str, ...], list[str]]]:
+ """Recursively split *node_ids* by directory until groups fit *target_max*.
+
+ Returns ``[(dir_segments, sorted_node_ids), ...]``. Reuses ``_sub_split``'s
+ prefix logic (group by the first segment that distinguishes members after
+ the common directory prefix) but, unlike sub-groups, recurses into any
+ group still above ``target_max``. Recursion bottoms out when a directory
+ has no distinguishing subdirs — a 200-file flat dir stays one module
+ (honest), never an artificial split.
+ """
+ dir_segs = {nid: PurePosixPath(id_to_path[nid]).parts[:-1] for nid in node_ids}
+
+ def rec(ids: list[str]) -> list[tuple[tuple[str, ...], list[str]]]:
+ common = _common_dir_prefix([dir_segs[i] for i in ids])
+ if len(ids) <= target_max:
+ return [(common, ids)]
+ groups: dict[str, list[str]] = defaultdict(list)
+ for nid in ids:
+ segs = dir_segs[nid]
+ key = segs[len(common)] if len(segs) > len(common) else ""
+ groups[key].append(nid)
+ if len(groups) < 2:
+ return [(common, ids)] # flat directory — no honest split exists
+ out: list[tuple[tuple[str, ...], list[str]]] = []
+ for key in sorted(groups):
+ if key == "":
+ # Files sitting directly in the common dir (the "(root)"
+ # group). Usually below target_min → folded by merge-up.
+ out.append((common, groups[key]))
+ else:
+ out.extend(rec(groups[key]))
+ return out
+
+ return [(d, sorted(ids)) for d, ids in rec(sorted(node_ids))]
+
+
+def _merge_small_groups(
+ groups: list[tuple[tuple[str, ...], list[str]]], target_min: int
+) -> list[tuple[tuple[str, ...], list[str]]]:
+ """Fold groups below *target_min* into their nearest sibling.
+
+ "Nearest" = the group sharing the longest directory prefix (the parent
+ subtree), largest first as the tie-break — so a 2-file "(root)" remnant
+ folds into its own subtree's biggest module, and an isolated small dir
+ folds into the layer's dominant module rather than minting a confetti
+ page. Never merges across layers (callers pass one layer at a time). A
+ layer that is itself below ``target_min`` stays one whole group.
+
+ A pre-pass fuses *small sibling* groups into one group at their common
+ parent when that collection is itself module-sized — ninety tiny locale
+ dirs become one ``conf/locale`` module instead of folding into whichever
+ sibling sorts first and misnaming it. The fold-in loop then never renames
+ a survivor: a healthy ``core/providers`` absorbing a 2-file sibling keeps
+ its identity.
+ """
+ merged = [(d, list(ids)) for d, ids in groups]
+ _fuse_small_siblings(merged, target_min)
+ merged.sort(key=lambda g: g[0])
+ _fold_small_groups(merged, target_min)
+ return [(d, ids) for d, ids in merged]
+
+
+def _fuse_small_siblings(merged: list[_DirGroup], target_min: int) -> None:
+ for parent, sibs in sorted(_small_groups_by_parent(merged, target_min).items()):
+ if len(sibs) < 2 or sum(len(g[1]) for g in sibs) < target_min:
+ continue
+ fused = sorted(nid for g in sibs for nid in g[1])
+ for g in sibs:
+ merged.remove(g)
+ existing = next((g for g in merged if g[0] == parent), None)
+ if existing is not None:
+ existing[1].extend(fused)
+ existing[1].sort()
+ else:
+ merged.append((parent, fused))
+
+
+def _small_groups_by_parent(
+ merged: list[_DirGroup], target_min: int
+) -> dict[tuple[str, ...], list[_DirGroup]]:
+ by_parent: dict[tuple[str, ...], list[_DirGroup]] = {}
+ for g in merged:
+ if len(g[1]) < target_min and len(g[0]) > 0:
+ by_parent.setdefault(g[0][:-1], []).append(g)
+ return by_parent
+
+
+def _fold_small_groups(merged: list[_DirGroup], target_min: int) -> None:
+ while len(merged) > 1:
+ small = min(
+ (g for g in merged if len(g[1]) < target_min),
+ key=lambda g: (len(g[1]), g[0]),
+ default=None,
+ )
+ if small is None:
+ break
+ merged.remove(small)
+ target = min(
+ merged,
+ key=lambda g: (-_shared_depth(g[0], small[0]), -len(g[1]), g[0]),
+ )
+ target[1].extend(small[1])
+ target[1].sort()
+
+
+def _shared_depth(a: tuple[str, ...], b: tuple[str, ...]) -> int:
+ return len(_common_dir_prefix([a, b]))
+
+
+def _name_modules(mods: list[dict], generic: set[str]) -> None:
+ """Assign unique, human module names in place.
+
+ Initial name = the last one or two *informative* directory segments
+ (generic namespace segments stripped; when stripping consumes every
+ segment, the raw tail is used instead). Collisions extend leftward by
+ one more parent segment — NEVER a size suffix. Single-module layers
+ take the layer's name; the root group (empty dir) becomes
+ "<Layer> (top-level)". The absolute fallback (identical informative
+ paths across layers) appends the layer name, which is unique by
+ construction.
+ """
+ info_by, used = _initial_module_names(mods, generic)
+ _extend_colliding_names(mods, info_by, used)
+
+ # A root remnant and an organizational container in one layer would both
+ # read "<Layer> (top-level)"; the container's raw tail breaks the tie.
+ _rename_collisions(mods, lambda m: None if info_by[id(m)] else _dir_tail(m))
+
+ # Same informative dir in two layers: (dir, layer) is unique by construction.
+ _rename_collisions(mods, lambda m: f"{m['name']} ({m['_layerName']})")
+
+ # Backstop: the full dir path is unique per layer.
+ _rename_collisions(mods, lambda m: "/".join(m["_dir"]) or None)
+
+
+def _initial_module_names(
+ mods: list[dict], generic: set[str]
+) -> tuple[dict[int, list[str]], dict[int, int | None]]:
+ """Name each module; return its informative segments and how many it uses."""
+ per_layer: Counter[str] = Counter(m["layerId"] for m in mods)
+ info_by: dict[int, list[str]] = {}
+ used: dict[int, int | None] = {} # informative segments consumed; None = fixed
+ for m in mods:
+ # Generic stripping can consume every segment on fixture-dominated
+ # repos; the raw tail minus organizational dirs is still the honest
+ # name, and "(top-level)" stays reserved for true root groups.
+ info = [s for s in m["_dir"] if s not in generic] or [
+ s for s in m["_dir"] if s.lower() not in GENERIC_ORG_SEGMENTS
+ ]
+ info_by[id(m)] = info
+ if per_layer[m["layerId"]] == 1:
+ m["name"] = m["_layerName"]
+ used[id(m)] = None
+ elif not info:
+ m["name"] = f"{m['_layerName']} (top-level)"
+ used[id(m)] = None
+ else:
+ k = min(2, len(info))
+ m["name"] = "/".join(info[-k:])
+ used[id(m)] = k
+ return info_by, used
+
+
+def _extend_colliding_names(
+ mods: list[dict], info_by: dict[int, list[str]], used: dict[int, int | None]
+) -> None:
+ """Extend colliding names leftward, one informative segment per round."""
+ for _ in range(16): # bounded: each round consumes ≥1 segment somewhere
+ names = Counter(m["name"] for m in mods)
+ colliding = [m for m in mods if names[m["name"]] > 1]
+ if not colliding:
+ return
+ progressed = False
+ for m in colliding:
+ k = used.get(id(m))
+ info = info_by[id(m)]
+ if k is not None and k < len(info):
+ used[id(m)] = k + 1
+ m["name"] = "/".join(info[-(k + 1) :])
+ progressed = True
+ if not progressed:
+ return
+
+
+def _rename_collisions(mods: list[dict], rename: Callable[[dict], str | None]) -> None:
+ """Rename every module whose name is still shared, where *rename* has one."""
+ names = Counter(m["name"] for m in mods)
+ for m in mods:
+ if names[m["name"]] > 1:
+ new_name = rename(m)
+ if new_name is not None:
+ m["name"] = new_name
+
+
+def _dir_tail(module: dict) -> str | None:
+ return "/".join(module["_dir"][-2:]) or None
+
+
+def derive_modules(
+ layers: list[dict],
+ id_to_path: dict[str, str],
+ *,
+ target_min: int = _MODULE_TARGET_MIN,
+ target_max: int = _MODULE_TARGET_MAX,
+ min_module_size: int = _MODULE_MIN_FILES,
+ lang_by_id: dict[str, str] | None = None,
+) -> list[dict]:
+ """Derive right-sized, stably-identified wiki modules from curated layers.
+
+ ``Module = {"id": "module:<dir-slug>", "name": <human>, "path": <dir or "">,
+ "layerId": ..., "nodeIds": [...], "language": ...}``
+
+ Properties:
+
+ - **Partition per layer**: every node of every layer ≥ ``min_module_size``
+ lands in exactly one module; layers below the floor yield none. Never
+ merges across layers.
+ - **Granularity**: recursive directory splitting to the
+ [``target_min``, ``target_max``] window; flat dirs stay one honest
+ module; sub-``target_min`` remnants merge up into their subtree.
+ - **Names**: informative path segments only (data-driven generic-segment
+ stripping kills ``src``/``packages``/repo-name automatically); collision
+ resolution extends the path leftward — never a size suffix.
+ - **Ids**: ``module:`` + slug of the real directory path — stable across
+ runs and under file adds/renames inside the dir; changes only when the
+ directory itself moves. ``path`` is the actual dir (not the slug) so
+ path-prefix child lookups (``target_path LIKE 'dir/%'``) work.
+ - **Files only**: operates on ids present in ``id_to_path`` — external
+ nodes never pollute a module.
+ - **Determinism**: sorted iteration throughout; same inputs → same bytes.
+ """
+ generic = dominant_segments(sorted(set(id_to_path.values())))
+ mods = [
+ module
+ for layer in layers
+ for module in _layer_modules(layer, id_to_path, target_min, target_max, min_module_size)
+ ]
+ _name_modules(mods, generic)
+ _assign_module_ids(mods)
+
+ # A single-module layer duplicates its layer page; the flag lets page
+ # generation skip it while the module stays in the artifact.
+ per_layer_count: Counter[str] = Counter(m["layerId"] for m in mods)
+ return [
+ _export_module(m, per_layer_count[m["layerId"]] == 1, lang_by_id) for m in mods
+ ]
+
+
+def _layer_modules(
+ layer: dict, id_to_path: dict[str, str], target_min: int, target_max: int, min_size: int
+) -> list[dict]:
+ """One layer's unnamed module groups, or none below *min_size* files."""
+ node_ids = [nid for nid in layer.get("nodeIds", []) if nid in id_to_path]
+ if len(node_ids) < min_size:
+ return []
+ groups = _merge_small_groups(
+ _split_to_granularity(node_ids, id_to_path, target_max), target_min
+ )
+ return [
+ {
+ "_dir": dir_parts,
+ "_layerName": layer.get("name", ""),
+ "path": "/".join(dir_parts),
+ "layerId": layer.get("id", ""),
+ "nodeIds": sorted(ids),
+ }
+ for dir_parts, ids in sorted(groups)
+ ]
+
+
+def _assign_module_ids(mods: list[dict]) -> None:
+ """Give each module a path-derived slug id.
+
+ The bigger module keeps the plain id on the rare cross-layer dir collision
+ (a dir whose files split across layers).
+ """
+ used_ids: set[str] = set()
+ for m in sorted(mods, key=lambda m: (-len(m["nodeIds"]), m["path"], m["layerId"])):
+ base = "module:" + _slugify(m["path"] or m["_layerName"])
+ mid = base
+ n = 1
+ while mid in used_ids:
+ mid = f"{base}--{_slugify(m['_layerName'])}" + ("" if n == 1 else f"-{n}")
+ n += 1
+ used_ids.add(mid)
+ m["id"] = mid
+
+
+def _export_module(m: dict, whole_layer: bool, lang_by_id: dict[str, str] | None) -> dict:
+ module = {
+ "id": m["id"],
+ "name": m["name"],
+ "path": m["path"],
+ "layerId": m["layerId"],
+ "nodeIds": m["nodeIds"],
+ }
+ if whole_layer:
+ module["wholeLayer"] = True
+ if lang_by_id is not None:
+ module["language"] = _dominant_language(
+ [lang for nid in m["nodeIds"] if (lang := lang_by_id.get(nid, ""))]
+ )
+ return module
diff --git a/packages/core/src/repowise/core/analysis/kg_validation.py b/packages/core/src/repowise/core/analysis/kg_validation.py
new file mode 100644
index 000000000..3399dad62
--- /dev/null
+++ b/packages/core/src/repowise/core/analysis/kg_validation.py
@@ -0,0 +1,260 @@
+"""Invariant validation of a curated KG, and the portable self-validated export."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any
+
+from repowise.core.analysis.kg_inputs import _file_nodes
+from repowise.core.analysis.kg_modules import _MODULE_TARGET_MAX, _SIZE_SUFFIX_RE
+from repowise.core.analysis.knowledge_graph import KnowledgeGraphResult
+from repowise.core.generation.tour import DEFAULT_MAX_STOPS
+
+# Hard bound on the curated primary-layer count. The spine stays well below it
+# by construction; past it curation keeps the uncurated layers.
+_MAX_LAYERS = 15
+
+# Entry points surfaced in ``project.entry_points``; the full ranked list is
+# kept as ``entry_candidates``.
+_MAX_ENTRY_POINTS = 8
+
+# Quality thresholds. The lower layer bound and coverage targets are *soft*
+# (warnings) because they depend on repo size/shape; the partition, hard count
+# bound, capped entry set, never-empty summaries, and tour budget are *hard*.
+_MIN_LAYERS = 6
+_MAX_LAYER_FRACTION = 0.35
+_MAX_CATCHALL_FRACTION = 0.20
+_MAX_SINGLETON_FRACTION = 0.10
+_MIN_TOUR_COVERAGE = 0.90
+
+
+@dataclass
+class KGValidation:
+ """Outcome of :func:`validate_kg` — hard errors, soft warnings, metrics."""
+
+ ok: bool
+ errors: list[str] = field(default_factory=list)
+ warnings: list[str] = field(default_factory=list)
+ metrics: dict[str, Any] = field(default_factory=dict)
+
+ def as_dict(self) -> dict[str, Any]:
+ return {
+ "ok": self.ok,
+ "errors": self.errors,
+ "warnings": self.warnings,
+ "metrics": self.metrics,
+ }
+
+
+def validate_kg(kg: KnowledgeGraphResult) -> KGValidation:
+ """Validate a curated KG against the intuitiveness invariants.
+
+ Pure and side-effect free. Hard violations set ``ok=False`` and populate
+ ``errors``; size/shape-dependent shortfalls go to ``warnings``. The
+ ``metrics`` block is the per-repo intuitiveness scorecard.
+ """
+ errors: list[str] = []
+ warnings: list[str] = []
+
+ file_nodes = _file_nodes(kg)
+ file_count = len(file_nodes)
+ file_ids = {n["id"] for n in file_nodes}
+ tags_by_path = {n["filePath"]: (n.get("tags") or []) for n in file_nodes}
+
+ layers = kg.layers or []
+ shape = _check_layers(layers, file_ids, file_count, errors, warnings)
+
+ entry_points = kg.project.get("entry_points", []) if isinstance(kg.project, dict) else []
+ _check_entry_points(entry_points, tags_by_path, errors)
+
+ tour = kg.tour or []
+ tour_coverage = _check_tour(tour, layers, errors, warnings)
+
+ # Modules are only checked when the curated artifact carries them.
+ modules = getattr(kg, "modules", None) or []
+ module_covered = _check_modules(modules, file_ids, errors, warnings) if modules else set()
+
+ summary_completeness = _check_summaries(file_nodes, errors)
+
+ metrics = {
+ "file_count": file_count,
+ "layer_count": len(layers),
+ "module_count": len(modules),
+ "module_coverage_pct": (
+ round(len(module_covered) / file_count * 100, 1) if file_count else 0.0
+ ),
+ "singleton_layer_pct": round(shape.singleton_frac * 100, 1),
+ "largest_layer_pct": round(shape.largest_frac * 100, 1),
+ "application_pct": round(shape.catchall_frac * 100, 1),
+ "entry_point_count": len(entry_points),
+ "tour_steps": len(tour),
+ "tour_coverage_pct": round(tour_coverage * 100, 1),
+ "summary_completeness_pct": round(summary_completeness * 100, 1),
+ }
+
+ return KGValidation(ok=not errors, errors=errors, warnings=warnings, metrics=metrics)
+
+
+@dataclass(frozen=True)
+class _LayerShape:
+ singleton_frac: float
+ largest_frac: float
+ catchall_frac: float
+
+
+def _check_layers(
+ layers: list[dict],
+ file_ids: set[str],
+ file_count: int,
+ errors: list[str],
+ warnings: list[str],
+) -> _LayerShape:
+ """Layer count, partition, singleton spam and mega-layer balance."""
+ _check_layer_count(len(layers), errors, warnings)
+ _check_layer_partition(layers, file_ids, file_count, errors)
+ shape = _layer_shape(layers, file_count)
+ if shape.singleton_frac >= _MAX_SINGLETON_FRACTION:
+ warnings.append(
+ f"singleton layers {shape.singleton_frac:.0%} ≥ {_MAX_SINGLETON_FRACTION:.0%}"
+ )
+ if shape.largest_frac > _MAX_LAYER_FRACTION:
+ warnings.append(f"largest layer {shape.largest_frac:.0%} > {_MAX_LAYER_FRACTION:.0%}")
+ if shape.catchall_frac > _MAX_CATCHALL_FRACTION:
+ warnings.append(
+ f"Application catch-all {shape.catchall_frac:.0%} > {_MAX_CATCHALL_FRACTION:.0%}"
+ )
+ return shape
+
+
+def _check_layer_count(n_layers: int, errors: list[str], warnings: list[str]) -> None:
+ if n_layers == 0:
+ errors.append("no layers")
+ elif n_layers > _MAX_LAYERS:
+ errors.append(f"too many layers: {n_layers} > {_MAX_LAYERS}")
+ elif n_layers < _MIN_LAYERS:
+ warnings.append(f"few layers: {n_layers} < {_MIN_LAYERS} (small/flat repo?)")
+
+
+def _check_layer_partition(
+ layers: list[dict], file_ids: set[str], file_count: int, errors: list[str]
+) -> None:
+ layered: list[str] = [nid for layer in layers for nid in layer.get("nodeIds", [])]
+ layered_set = set(layered)
+ if len(layered) != len(layered_set):
+ errors.append("partition: a file appears in more than one layer")
+ if file_count and layered_set != file_ids:
+ missing = len(file_ids - layered_set)
+ extra = len(layered_set - file_ids)
+ errors.append(f"partition: {missing} unlayered, {extra} unknown ids")
+
+
+def _layer_shape(layers: list[dict], file_count: int) -> _LayerShape:
+ """Singleton-layer, largest-layer and ``Application`` catch-all fractions."""
+ sizes = [len(layer.get("nodeIds", [])) for layer in layers]
+ singleton_frac = (sum(1 for s in sizes if s == 1) / len(layers)) if layers else 0.0
+ largest_frac = (max(sizes) / file_count) if (sizes and file_count) else 0.0
+ catchall = next((layer for layer in layers if layer.get("name") == "Application"), None)
+ catchall_frac = (
+ (len(catchall.get("nodeIds", [])) / file_count) if (catchall and file_count) else 0.0
+ )
+ return _LayerShape(singleton_frac, largest_frac, catchall_frac)
+
+
+def _check_summaries(file_nodes: list[dict], errors: list[str]) -> float:
+ """Never-empty summaries, returning the fraction of file nodes that have one."""
+ summary_by_id = {n["id"]: n.get("summary") for n in file_nodes}
+ empty_summaries = [nid for nid, s in summary_by_id.items() if not s]
+ if empty_summaries:
+ errors.append(f"{len(empty_summaries)} file nodes have an empty summary")
+ return 1.0 - len(empty_summaries) / len(file_nodes) if file_nodes else 1.0
+
+
+def _check_entry_points(
+ entry_points: list[str], tags_by_path: dict[str, list[str]], errors: list[str]
+) -> None:
+ if len(entry_points) > _MAX_ENTRY_POINTS:
+ errors.append(f"too many entry points: {len(entry_points)} > {_MAX_ENTRY_POINTS}")
+ barrels_surfaced = [p for p in entry_points if "barrel" in tags_by_path.get(p, [])]
+ if barrels_surfaced:
+ errors.append(f"barrels surfaced as entry points: {barrels_surfaced}")
+
+
+def _check_tour(
+ tour: list[dict], layers: list[dict], errors: list[str], warnings: list[str]
+) -> float:
+ """Tour budget and opening, returning the fraction of layers it visits."""
+ if not tour:
+ return 0.0
+ if len(tour) > DEFAULT_MAX_STOPS:
+ errors.append(f"tour too long: {len(tour)} > {DEFAULT_MAX_STOPS}")
+ if tour[0].get("kind") != "overview":
+ errors.append("tour does not open with an overview/README step")
+ layer_ids = {layer.get("id") for layer in layers}
+ covered = {
+ s.get("layer_id")
+ for s in tour
+ if s.get("kind") != "overview" and s.get("layer_id") in layer_ids
+ }
+ tour_coverage = (len(covered) / len(layer_ids)) if layer_ids else 0.0
+ if tour_coverage < _MIN_TOUR_COVERAGE:
+ warnings.append(f"tour covers {tour_coverage:.0%} of layers < {_MIN_TOUR_COVERAGE:.0%}")
+ return tour_coverage
+
+
+def _check_modules(
+ modules: list[dict], file_ids: set[str], errors: list[str], warnings: list[str]
+) -> set[str]:
+ """Module partition, naming and size, returning the file ids modules cover."""
+ module_member_lists = [m.get("nodeIds", []) for m in modules]
+ flat = [nid for ids in module_member_lists for nid in ids]
+ module_covered = set(flat)
+ if len(flat) != len(module_covered):
+ errors.append("modules: a file appears in more than one module")
+ if not module_covered <= file_ids:
+ errors.append(
+ f"modules: {len(module_covered - file_ids)} unknown ids in modules"
+ )
+ module_names = [m.get("name", "") for m in modules]
+ if len(set(module_names)) != len(module_names):
+ errors.append("modules: names not unique")
+ size_suffixed = [n for n in module_names if _SIZE_SUFFIX_RE.search(n)]
+ if size_suffixed:
+ errors.append(f"modules: size-suffixed names: {size_suffixed}")
+ oversized = sum(
+ 1 for ids in module_member_lists if len(ids) > _MODULE_TARGET_MAX
+ )
+ if oversized:
+ # Flat dirs may honestly exceed the window — soft signal only.
+ warnings.append(f"{oversized} modules above target_max (flat dirs?)")
+ return module_covered
+
+
+# ---------------------------------------------------------------------------
+# Phase 6 — portable, self-validated export artifact
+# ---------------------------------------------------------------------------
+
+
+def build_portable_kg(kg: KnowledgeGraphResult) -> tuple[dict, KGValidation]:
+ """Assemble a self-contained, self-validated ``knowledge-graph.json`` dict.
+
+ Kept separate from :meth:`KnowledgeGraphResult.to_dict` so the *default*
+ export stays byte-identical (curation flag-off contract); the portable
+ artifact adds a ``meta`` block (counts, fingerprint) and an embedded
+ ``validation`` report so an external consumer can trust it without a server.
+ Returns ``(data, validation)`` so the writer can decide on hard violations.
+ """
+ data = kg.to_dict()
+ validation = validate_kg(kg)
+ data["meta"] = {
+ # The integer the loader gates on, not the "1.0.0" display label.
+ "schema_version": data.get("schema_version", 1),
+ "generator": "repowise-kg-curation",
+ "fingerprint": getattr(kg, "fingerprint", ""),
+ "file_count": validation.metrics.get("file_count", 0),
+ "layer_count": validation.metrics.get("layer_count", 0),
+ "module_count": validation.metrics.get("module_count", 0),
+ "entry_point_count": validation.metrics.get("entry_point_count", 0),
+ "tour_steps": validation.metrics.get("tour_steps", 0),
+ "validation": validation.as_dict(),
+ }
+ return data, validation