Skip to content

Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.

feat(core): stop example trees shaping community detection (#1966)

Raghav Chamadiya0847cbff32c0
diff --git a/packages/core/src/repowise/core/analysis/communities.py b/packages/core/src/repowise/core/analysis/communities.py
index d0143f394..06a92e512 100644
--- a/packages/core/src/repowise/core/analysis/communities.py
+++ b/packages/core/src/repowise/core/analysis/communities.py
@@ -22,6 +22,7 @@ import structlog
from repowise.core.analysis.kg_curation import GENERIC_ORG_SEGMENTS, dominant_segments
from repowise.core.ingestion.models import FILE_DEPENDENCY_EDGE_TYPES, SYMBOL_USE_EDGE_TYPES
+from repowise.core.support_paths import is_example_path
from repowise.core.test_paths import is_test_related_path
log = structlog.get_logger(__name__)
@@ -402,14 +403,27 @@ def _is_test_node(node_id: str, data: dict) -> bool:
return is_test_related_path(node_id, data.get("language"))
+def _is_example_node(node_id: str, data: dict) -> bool:
+ """Whether a graph file node is example, demo, or benchmark code.
+
+ Derived from the path rather than a stored flag. Unlike ``is_test`` there
+ is no ``is_example`` on the node, and the path is the whole signal, so this
+ does not justify a new ``FileInfo`` field and ``GraphNode`` column.
+ """
+ return is_example_path(node_id)
+
+
def _assign_tests_to_communities(
test_nodes: list[str],
prod_assignment: dict[str, int],
graph: nx.DiGraph,
) -> dict[str, int]:
- """Assign each test file to the community of a production file it links to.
+ """Assign each non-core file to the community of a file it links to.
- Falls back to a catch-all "tests" community when no import link exists.
+ Falls back to a shared catch-all community when no import link exists.
+ Used for tests and for example/benchmark code alike: both are real files a
+ reader may open, but neither should shape what the production communities
+ are or what they are called.
"""
result: dict[str, int] = {}
next_cid = max(prod_assignment.values(), default=-1) + 1
@@ -457,13 +471,21 @@ def detect_file_communities(
if not file_nodes:
return {}, {}, "none"
- # Separate test files from production files. Test files are clustered
- # separately then assigned to the community of their most-imported
- # production file, preventing test directories from dominating labels
- # and mixing unrelated production modules.
- is_test = {n: _is_test_node(n, graph.nodes[n]) for n in file_nodes}
- prod_nodes = [n for n in file_nodes if not is_test[n]]
- test_nodes = [n for n in file_nodes if is_test[n]]
+ # Separate test and example files from production files. Both are clustered
+ # separately then assigned to the community of a production file they link
+ # to, preventing those directories from dominating labels and mixing
+ # unrelated production modules.
+ #
+ # Examples join tests here rather than being dropped: demos import across
+ # the whole system, so left in the partition they merge otherwise-distinct
+ # subsystems into one community and name it after the demo. They keep their
+ # nodes and their assignment, so nothing becomes unreachable.
+ non_core = {
+ n: _is_test_node(n, graph.nodes[n]) or _is_example_node(n, graph.nodes[n])
+ for n in file_nodes
+ }
+ prod_nodes = [n for n in file_nodes if not non_core[n]]
+ test_nodes = [n for n in file_nodes if non_core[n]]
# Build undirected subgraph from production files + relevant edges
undirected = nx.Graph()
diff --git a/packages/core/src/repowise/core/analysis/kg_curation.py b/packages/core/src/repowise/core/analysis/kg_curation.py
index 5b296d952..de2742cbc 100644
--- a/packages/core/src/repowise/core/analysis/kg_curation.py
+++ b/packages/core/src/repowise/core/analysis/kg_curation.py
@@ -41,7 +41,6 @@ from repowise.core.generation.layers import (
ADJACENT_LAYERS,
compute_layer_order,
infer_layer,
- is_support_path,
layer_order_basis,
)
from repowise.core.generation.tour import (
@@ -53,6 +52,7 @@ from repowise.core.generation.tour import (
from repowise.core.generation.well_known_files import well_known_role
from repowise.core.ids import is_external
from repowise.core.ingestion.languages.registry import REGISTRY as _LANG_REGISTRY
+from repowise.core.support_paths import is_support_path
# Closing-stop anchors (conftest, spec_helper, test_helper, …) and
# declaration descriptors (module-info.java) — both registry-declared.
diff --git a/packages/core/src/repowise/core/generation/layers.py b/packages/core/src/repowise/core/generation/layers.py
index 30af5afcd..5563df713 100644
--- a/packages/core/src/repowise/core/generation/layers.py
+++ b/packages/core/src/repowise/core/generation/layers.py
@@ -27,6 +27,7 @@ from pathlib import PurePosixPath
from repowise.core.ids import is_external
from repowise.core.ingestion.languages.registry import REGISTRY as _LANG_REGISTRY
+from repowise.core.support_paths import DOC_DIR_TOKENS, EXAMPLE_DIR_TOKENS
from repowise.core.test_paths import is_test_related_path
# ---------------------------------------------------------------------------
@@ -98,37 +99,11 @@ for _tag, _hints in _LANG_REGISTRY.layer_dir_hints_by_language().items():
_LANG_ROOT_HINTS[_tag] = _roots
-# Example/demo/benchmark directories: documentation-by-code and support
-# harnesses, not the system itself. Their files carry entry-style names
-# (main.go, index.js, decode.exs) by convention, so without demotion they
-# flood entry points and the tour on any repo that ships samples (express,
-# chi, …) or benchmarks (cargo's benches/, jason's bench/).
-_EXAMPLE_DIR_TOKENS = frozenset(
- {
- "examples", "_examples", "example", "samples", "sample", "demo", "demos",
- "bench", "benches", "benchmarks",
- }
-)
-
-
-# Documentation directories: sphinx/docusaurus/vitepress sites and runnable
-# doc snippets (libuv's docs/code/*/main.c, docfx template assets). Like the
-# example dirs above, their files carry entry-style names by convention but
-# document the system rather than being it.
-_DOC_DIR_TOKENS = frozenset({"docs", "doc", "website"})
-
-
-def is_support_path(path: str) -> bool:
- """Whether *path* is support material (examples/benchmarks/docs sites).
-
- Support files never seed or anchor a tour and never surface as entry
- points — a reader orienting in the repo must land in the system itself,
- not in its documentation or sample harnesses.
- """
- return any(
- s.lower() in _EXAMPLE_DIR_TOKENS or s.lower() in _DOC_DIR_TOKENS
- for s in PurePosixPath(path).parts[:-1]
- )
+# The directory vocabularies and `is_support_path` live in `core.support_paths`
+# so `analysis` can ask the same question without importing `generation`. Only
+# the layer-inference use of the token sets stays here.
+_EXAMPLE_DIR_TOKENS = EXAMPLE_DIR_TOKENS
+_DOC_DIR_TOKENS = DOC_DIR_TOKENS
# Build / CI / extension tooling directories: scripts, container definitions,
diff --git a/packages/core/src/repowise/core/generation/templates/stub/_footer.j2 b/packages/core/src/repowise/core/generation/templates/stub/_footer.j2
index 36f1471bf..f4f3d6437 100644
--- a/packages/core/src/repowise/core/generation/templates/stub/_footer.j2
+++ b/packages/core/src/repowise/core/generation/templates/stub/_footer.j2
@@ -1,4 +1,7 @@
---
+{#- No CLI command here: this page is read on the web as often as in a terminal,
+ and the CLI offers the prose upgrade in its own surfaces. The opening
+ sentence is matched by tests/integration/test_deterministic_generation.py. -#}
*Built from the code's structure. It states what is there, not why it is that
-way. Add an API key and run `repowise generate` to have that written.*
+way. The explanatory prose is a separate, model-written layer.*
diff --git a/packages/core/src/repowise/core/generation/tour.py b/packages/core/src/repowise/core/generation/tour.py
index 36468eb5e..1d593aa2b 100644
--- a/packages/core/src/repowise/core/generation/tour.py
+++ b/packages/core/src/repowise/core/generation/tour.py
@@ -36,9 +36,10 @@ from pathlib import PurePosixPath
from typing import Any
from repowise.core.entry_candidacy import non_code_entry_languages, not_an_execution_start
-from repowise.core.generation.layers import ADJACENT_LAYERS, infer_layer, is_support_path
+from repowise.core.generation.layers import ADJACENT_LAYERS, infer_layer
from repowise.core.ids import is_external
from repowise.core.ingestion.languages.registry import REGISTRY as _LANG_REGISTRY
+from repowise.core.support_paths import is_support_path
# Filenames that conventionally mark an executable / wiring entry point.
# Derived from the language registry (generic stems + per-language stems);
diff --git a/packages/core/src/repowise/core/support_paths.py b/packages/core/src/repowise/core/support_paths.py
new file mode 100644
index 000000000..ffd0514b2
--- /dev/null
+++ b/packages/core/src/repowise/core/support_paths.py
@@ -0,0 +1,72 @@
+"""Which paths are support material rather than the system itself.
+
+Sibling of :mod:`repowise.core.test_paths`, and there for the same reason: one
+traversal, so callers asking the same question cannot get different answers.
+
+Two predicates, deliberately not one. :func:`is_example_path` covers code that
+demonstrates or measures the system; :func:`is_support_path` adds documentation
+sites. A docs tree is not a module of the system the way an example crate is,
+so callers meaning "written to illustrate the subject" want the former.
+"""
+
+from __future__ import annotations
+
+from pathlib import PurePosixPath
+
+__all__ = ["is_example_path", "is_support_path"]
+
+
+# Example/demo/benchmark directories: documentation-by-code and support
+# harnesses, not the system itself. Their files carry entry-style names
+# (main.go, index.js) by convention, so without demotion they flood entry
+# points and the tour on any repo that ships samples or benchmarks.
+EXAMPLE_DIR_TOKENS = frozenset(
+ {
+ "examples",
+ "_examples",
+ "example",
+ "samples",
+ "sample",
+ "demo",
+ "demos",
+ "bench",
+ "benches",
+ "benchmarks",
+ }
+)
+
+
+# Documentation directories: static-site trees and runnable doc snippets. Like
+# the example dirs above, their files carry entry-style names by convention
+# but document the system rather than being it.
+DOC_DIR_TOKENS = frozenset({"docs", "doc", "website"})
+
+
+def _has_dir_token(path: str, tokens: frozenset[str]) -> bool:
+ """Whether any directory segment of *path* is one of *tokens*.
+
+ Matched at any depth, not just the repo root, since a workspace member
+ keeps its own ``examples/``. The basename is excluded so a file named
+ ``examples.py`` stays production code.
+ """
+ return any(s.lower() in tokens for s in PurePosixPath(path).parts[:-1])
+
+
+def is_example_path(path: str) -> bool:
+ """Whether *path* is example, demo, or benchmark code.
+
+ It ships in the repo and is worth reading, but it is not a subsystem:
+ grouped with the code it imports it inflates that module, and a large
+ example tree can dominate a community's label.
+ """
+ return _has_dir_token(path, EXAMPLE_DIR_TOKENS)
+
+
+def is_support_path(path: str) -> bool:
+ """Whether *path* is support material (examples/benchmarks/docs sites).
+
+ Support files never seed or anchor a tour and never surface as entry
+ points: a reader orienting in the repo must land in the system itself,
+ not in its documentation or sample harnesses.
+ """
+ return _has_dir_token(path, EXAMPLE_DIR_TOKENS | DOC_DIR_TOKENS)
diff --git a/tests/unit/analysis/test_community_labels.py b/tests/unit/analysis/test_community_labels.py
index 1eeff6943..1bc5d1752 100644
--- a/tests/unit/analysis/test_community_labels.py
+++ b/tests/unit/analysis/test_community_labels.py
@@ -113,3 +113,38 @@ class TestDetectFileCommunitiesLabels:
# The informative segments survive somewhere in the labels.
joined = " ".join(labels)
assert "ingestion" in joined and "web" in joined
+
+
+class TestExampleSeparation:
+ """Example trees are demoted alongside tests, not partitioned as production."""
+
+ def test_examples_are_demoted_with_tests_not_partitioned(self):
+ # An example and a test, neither importing production code. Partitioned
+ # as production they would each be an isolate with its own community;
+ # demoted they share the non-core catch-all.
+ paths = ["src/a1.py", "src/a2.py", "examples/demo.py", "tests/test_a.py"]
+ g = nx.DiGraph()
+ for path in paths:
+ g.add_node(
+ path,
+ node_type="file",
+ language="python",
+ is_test=path.startswith("tests/"),
+ )
+ g.add_edge("src/a1.py", "src/a2.py", edge_type="imports")
+
+ assignment, _info, _algo = detect_file_communities(g)
+
+ assert assignment["examples/demo.py"] == assignment["tests/test_a.py"]
+ assert assignment["examples/demo.py"] != assignment["src/a1.py"]
+
+ def test_example_files_keep_an_assignment(self):
+ # Demoted, not dropped: every file still lands in a community, so
+ # nothing becomes unreachable.
+ paths = ["src/a.py", "src/b.py", "examples/demo.py", "benches/bench.py"]
+ edges = [("src/a.py", "src/b.py"), ("examples/demo.py", "src/a.py")]
+ assignment, _info, _algo = detect_file_communities(_graph(paths, edges))
+
+ assert set(assignment) == set(paths)
+ # An example joins the community of the production file it imports.
+ assert assignment["examples/demo.py"] == assignment["src/a.py"]
diff --git a/tests/unit/generation/test_layers.py b/tests/unit/generation/test_layers.py
index 3e78c80ce..c827e4840 100644
--- a/tests/unit/generation/test_layers.py
+++ b/tests/unit/generation/test_layers.py
@@ -421,7 +421,7 @@ def test_infer_layer_c_cpp_include_is_api():
def test_is_support_path_covers_doc_dirs():
- from repowise.core.generation.layers import is_support_path
+ from repowise.core.support_paths import is_support_path
# Doc sites and runnable doc snippets are support material (libuv's
# docs/code/*/main.c, docfx templates, vitepress sites).
diff --git a/tests/unit/test_support_paths.py b/tests/unit/test_support_paths.py
new file mode 100644
index 000000000..745c98be8
--- /dev/null
+++ b/tests/unit/test_support_paths.py
@@ -0,0 +1,62 @@
+"""Support-path classification: examples and docs versus the system itself."""
+
+from __future__ import annotations
+
+import pytest
+
+from repowise.core.support_paths import is_example_path, is_support_path
+
+
+ "path",
+ [
+ "examples/demo.rs",
+ "example/main.go",
+ "_examples/basic/index.js",
+ "samples/sample.py",
+ "demos/app/main.ts",
+ "benches/parser.rs",
+ "benchmarks/run.py",
+ # Workspace members keep their own example trees, so the match is not
+ # anchored to the repo root.
+ "crates/render/examples/window.rs",
+ "packages/core/benches/walk.py",
+ ],
+)
+def test_example_dirs_are_examples(path: str):
+ assert is_example_path(path)
+ assert is_support_path(path)
+
+
+ "path",
+ [
+ "src/parser.rs",
+ "src/render/layout.py",
+ # Basename only: a module *about* examples is production code.
+ "src/examples.py",
+ "src/benchmark_runner.go",
+ # Not a whole segment.
+ "src/examples_helper/util.py",
+ ],
+)
+def test_production_paths_are_not_examples(path: str):
+ assert not is_example_path(path)
+ assert not is_support_path(path)
+
+
[email protected]("path", ["docs/guide/main.c", "doc/build.py", "website/theme/index.ts"])
+def test_doc_dirs_are_support_but_not_examples(path: str):
+ """The distinction the two predicates exist for.
+
+ A docs tree is not the system, so it is support material. But it is not
+ example code either: it is prose infrastructure, and callers grouping code
+ into modules should not treat it as a demo of the subject.
+ """
+ assert is_support_path(path)
+ assert not is_example_path(path)
+
+
+def test_matching_is_case_insensitive():
+ assert is_example_path("Examples/Demo.cs")
+ assert is_support_path("Docs/Guide.md")