Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.
Python aicode-complexitycode-healthcode-intelligencecode-qualitydead-codedeveloper-toolsdocumentationgit-analyticsmcpopen-sourcepythonrefactoringstatic-analysistechnical-debt
feat(core): stop example trees shaping community detection (#1966)
Raghav Chamadiya0847cbff32c0
diff --git a/packages/core/src/repowise/core/analysis/communities.py b/packages/core/src/repowise/core/analysis/communities.pyindex d0143f394..06a92e512 100644--- a/packages/core/src/repowise/core/analysis/communities.py+++ b/packages/core/src/repowise/core/analysis/communities.py@@ -22,6 +22,7 @@ import structlogfrom repowise.core.analysis.kg_curation import GENERIC_ORG_SEGMENTS, dominant_segmentsfrom repowise.core.ingestion.models import FILE_DEPENDENCY_EDGE_TYPES, SYMBOL_USE_EDGE_TYPES+from repowise.core.support_paths import is_example_pathfrom repowise.core.test_paths import is_test_related_pathlog = structlog.get_logger(__name__)@@ -402,14 +403,27 @@ def _is_test_node(node_id: str, data: dict) -> bool:return is_test_related_path(node_id, data.get("language"))+def _is_example_node(node_id: str, data: dict) -> bool:+ """Whether a graph file node is example, demo, or benchmark code.++ Derived from the path rather than a stored flag. Unlike ``is_test`` there+ is no ``is_example`` on the node, and the path is the whole signal, so this+ does not justify a new ``FileInfo`` field and ``GraphNode`` column.+ """+ return is_example_path(node_id)++def _assign_tests_to_communities(test_nodes: list[str],prod_assignment: dict[str, int],graph: nx.DiGraph,) -> dict[str, int]:- """Assign each test file to the community of a production file it links to.+ """Assign each non-core file to the community of a file it links to.- Falls back to a catch-all "tests" community when no import link exists.+ Falls back to a shared catch-all community when no import link exists.+ Used for tests and for example/benchmark code alike: both are real files a+ reader may open, but neither should shape what the production communities+ are or what they are called."""result: dict[str, int] = {}next_cid = max(prod_assignment.values(), default=-1) + 1@@ -457,13 +471,21 @@ def detect_file_communities(if not file_nodes:return {}, {}, "none"- # Separate test files from production files. Test files are clustered- # separately then assigned to the community of their most-imported- # production file, preventing test directories from dominating labels- # and mixing unrelated production modules.- is_test = {n: _is_test_node(n, graph.nodes[n]) for n in file_nodes}- prod_nodes = [n for n in file_nodes if not is_test[n]]- test_nodes = [n for n in file_nodes if is_test[n]]+ # Separate test and example files from production files. Both are clustered+ # separately then assigned to the community of a production file they link+ # to, preventing those directories from dominating labels and mixing+ # unrelated production modules.+ #+ # Examples join tests here rather than being dropped: demos import across+ # the whole system, so left in the partition they merge otherwise-distinct+ # subsystems into one community and name it after the demo. They keep their+ # nodes and their assignment, so nothing becomes unreachable.+ non_core = {+ n: _is_test_node(n, graph.nodes[n]) or _is_example_node(n, graph.nodes[n])+ for n in file_nodes+ }+ prod_nodes = [n for n in file_nodes if not non_core[n]]+ test_nodes = [n for n in file_nodes if non_core[n]]# Build undirected subgraph from production files + relevant edgesundirected = nx.Graph()diff --git a/packages/core/src/repowise/core/analysis/kg_curation.py b/packages/core/src/repowise/core/analysis/kg_curation.pyindex 5b296d952..de2742cbc 100644--- a/packages/core/src/repowise/core/analysis/kg_curation.py+++ b/packages/core/src/repowise/core/analysis/kg_curation.py@@ -41,7 +41,6 @@ from repowise.core.generation.layers import (ADJACENT_LAYERS,compute_layer_order,infer_layer,- is_support_path,layer_order_basis,)from repowise.core.generation.tour import (@@ -53,6 +52,7 @@ from repowise.core.generation.tour import (from repowise.core.generation.well_known_files import well_known_rolefrom repowise.core.ids import is_externalfrom repowise.core.ingestion.languages.registry import REGISTRY as _LANG_REGISTRY+from repowise.core.support_paths import is_support_path# Closing-stop anchors (conftest, spec_helper, test_helper, …) and# declaration descriptors (module-info.java) — both registry-declared.diff --git a/packages/core/src/repowise/core/generation/layers.py b/packages/core/src/repowise/core/generation/layers.pyindex 30af5afcd..5563df713 100644--- a/packages/core/src/repowise/core/generation/layers.py+++ b/packages/core/src/repowise/core/generation/layers.py@@ -27,6 +27,7 @@ from pathlib import PurePosixPathfrom repowise.core.ids import is_externalfrom repowise.core.ingestion.languages.registry import REGISTRY as _LANG_REGISTRY+from repowise.core.support_paths import DOC_DIR_TOKENS, EXAMPLE_DIR_TOKENSfrom repowise.core.test_paths import is_test_related_path# ---------------------------------------------------------------------------@@ -98,37 +99,11 @@ for _tag, _hints in _LANG_REGISTRY.layer_dir_hints_by_language().items():_LANG_ROOT_HINTS[_tag] = _roots-# Example/demo/benchmark directories: documentation-by-code and support-# harnesses, not the system itself. Their files carry entry-style names-# (main.go, index.js, decode.exs) by convention, so without demotion they-# flood entry points and the tour on any repo that ships samples (express,-# chi, …) or benchmarks (cargo's benches/, jason's bench/).-_EXAMPLE_DIR_TOKENS = frozenset(- {- "examples", "_examples", "example", "samples", "sample", "demo", "demos",- "bench", "benches", "benchmarks",- }-)---# Documentation directories: sphinx/docusaurus/vitepress sites and runnable-# doc snippets (libuv's docs/code/*/main.c, docfx template assets). Like the-# example dirs above, their files carry entry-style names by convention but-# document the system rather than being it.-_DOC_DIR_TOKENS = frozenset({"docs", "doc", "website"})---def is_support_path(path: str) -> bool:- """Whether *path* is support material (examples/benchmarks/docs sites).-- Support files never seed or anchor a tour and never surface as entry- points — a reader orienting in the repo must land in the system itself,- not in its documentation or sample harnesses.- """- return any(- s.lower() in _EXAMPLE_DIR_TOKENS or s.lower() in _DOC_DIR_TOKENS- for s in PurePosixPath(path).parts[:-1]- )+# The directory vocabularies and `is_support_path` live in `core.support_paths`+# so `analysis` can ask the same question without importing `generation`. Only+# the layer-inference use of the token sets stays here.+_EXAMPLE_DIR_TOKENS = EXAMPLE_DIR_TOKENS+_DOC_DIR_TOKENS = DOC_DIR_TOKENS# Build / CI / extension tooling directories: scripts, container definitions,diff --git a/packages/core/src/repowise/core/generation/templates/stub/_footer.j2 b/packages/core/src/repowise/core/generation/templates/stub/_footer.j2index 36f1471bf..f4f3d6437 100644--- a/packages/core/src/repowise/core/generation/templates/stub/_footer.j2+++ b/packages/core/src/repowise/core/generation/templates/stub/_footer.j2@@ -1,4 +1,7 @@---+{#- No CLI command here: this page is read on the web as often as in a terminal,+ and the CLI offers the prose upgrade in its own surfaces. The opening+ sentence is matched by tests/integration/test_deterministic_generation.py. -#}*Built from the code's structure. It states what is there, not why it is that-way. Add an API key and run `repowise generate` to have that written.*+way. The explanatory prose is a separate, model-written layer.*diff --git a/packages/core/src/repowise/core/generation/tour.py b/packages/core/src/repowise/core/generation/tour.pyindex 36468eb5e..1d593aa2b 100644--- a/packages/core/src/repowise/core/generation/tour.py+++ b/packages/core/src/repowise/core/generation/tour.py@@ -36,9 +36,10 @@ from pathlib import PurePosixPathfrom typing import Anyfrom repowise.core.entry_candidacy import non_code_entry_languages, not_an_execution_start-from repowise.core.generation.layers import ADJACENT_LAYERS, infer_layer, is_support_path+from repowise.core.generation.layers import ADJACENT_LAYERS, infer_layerfrom repowise.core.ids import is_externalfrom repowise.core.ingestion.languages.registry import REGISTRY as _LANG_REGISTRY+from repowise.core.support_paths import is_support_path# Filenames that conventionally mark an executable / wiring entry point.# Derived from the language registry (generic stems + per-language stems);diff --git a/packages/core/src/repowise/core/support_paths.py b/packages/core/src/repowise/core/support_paths.pynew file mode 100644index 000000000..ffd0514b2--- /dev/null+++ b/packages/core/src/repowise/core/support_paths.py@@ -0,0 +1,72 @@+"""Which paths are support material rather than the system itself.++Sibling of :mod:`repowise.core.test_paths`, and there for the same reason: one+traversal, so callers asking the same question cannot get different answers.++Two predicates, deliberately not one. :func:`is_example_path` covers code that+demonstrates or measures the system; :func:`is_support_path` adds documentation+sites. A docs tree is not a module of the system the way an example crate is,+so callers meaning "written to illustrate the subject" want the former.+"""++from __future__ import annotations++from pathlib import PurePosixPath++__all__ = ["is_example_path", "is_support_path"]+++# Example/demo/benchmark directories: documentation-by-code and support+# harnesses, not the system itself. Their files carry entry-style names+# (main.go, index.js) by convention, so without demotion they flood entry+# points and the tour on any repo that ships samples or benchmarks.+EXAMPLE_DIR_TOKENS = frozenset(+ {+ "examples",+ "_examples",+ "example",+ "samples",+ "sample",+ "demo",+ "demos",+ "bench",+ "benches",+ "benchmarks",+ }+)+++# Documentation directories: static-site trees and runnable doc snippets. Like+# the example dirs above, their files carry entry-style names by convention+# but document the system rather than being it.+DOC_DIR_TOKENS = frozenset({"docs", "doc", "website"})+++def _has_dir_token(path: str, tokens: frozenset[str]) -> bool:+ """Whether any directory segment of *path* is one of *tokens*.++ Matched at any depth, not just the repo root, since a workspace member+ keeps its own ``examples/``. The basename is excluded so a file named+ ``examples.py`` stays production code.+ """+ return any(s.lower() in tokens for s in PurePosixPath(path).parts[:-1])+++def is_example_path(path: str) -> bool:+ """Whether *path* is example, demo, or benchmark code.++ It ships in the repo and is worth reading, but it is not a subsystem:+ grouped with the code it imports it inflates that module, and a large+ example tree can dominate a community's label.+ """+ return _has_dir_token(path, EXAMPLE_DIR_TOKENS)+++def is_support_path(path: str) -> bool:+ """Whether *path* is support material (examples/benchmarks/docs sites).++ Support files never seed or anchor a tour and never surface as entry+ points: a reader orienting in the repo must land in the system itself,+ not in its documentation or sample harnesses.+ """+ return _has_dir_token(path, EXAMPLE_DIR_TOKENS | DOC_DIR_TOKENS)diff --git a/tests/unit/analysis/test_community_labels.py b/tests/unit/analysis/test_community_labels.pyindex 1eeff6943..1bc5d1752 100644--- a/tests/unit/analysis/test_community_labels.py+++ b/tests/unit/analysis/test_community_labels.py@@ -113,3 +113,38 @@ class TestDetectFileCommunitiesLabels:# The informative segments survive somewhere in the labels.joined = " ".join(labels)assert "ingestion" in joined and "web" in joined+++class TestExampleSeparation:+ """Example trees are demoted alongside tests, not partitioned as production."""++ def test_examples_are_demoted_with_tests_not_partitioned(self):+ # An example and a test, neither importing production code. Partitioned+ # as production they would each be an isolate with its own community;+ # demoted they share the non-core catch-all.+ paths = ["src/a1.py", "src/a2.py", "examples/demo.py", "tests/test_a.py"]+ g = nx.DiGraph()+ for path in paths:+ g.add_node(+ path,+ node_type="file",+ language="python",+ is_test=path.startswith("tests/"),+ )+ g.add_edge("src/a1.py", "src/a2.py", edge_type="imports")++ assignment, _info, _algo = detect_file_communities(g)++ assert assignment["examples/demo.py"] == assignment["tests/test_a.py"]+ assert assignment["examples/demo.py"] != assignment["src/a1.py"]++ def test_example_files_keep_an_assignment(self):+ # Demoted, not dropped: every file still lands in a community, so+ # nothing becomes unreachable.+ paths = ["src/a.py", "src/b.py", "examples/demo.py", "benches/bench.py"]+ edges = [("src/a.py", "src/b.py"), ("examples/demo.py", "src/a.py")]+ assignment, _info, _algo = detect_file_communities(_graph(paths, edges))++ assert set(assignment) == set(paths)+ # An example joins the community of the production file it imports.+ assert assignment["examples/demo.py"] == assignment["src/a.py"]diff --git a/tests/unit/generation/test_layers.py b/tests/unit/generation/test_layers.pyindex 3e78c80ce..c827e4840 100644--- a/tests/unit/generation/test_layers.py+++ b/tests/unit/generation/test_layers.py@@ -421,7 +421,7 @@ def test_infer_layer_c_cpp_include_is_api():def test_is_support_path_covers_doc_dirs():- from repowise.core.generation.layers import is_support_path+ from repowise.core.support_paths import is_support_path# Doc sites and runnable doc snippets are support material (libuv's# docs/code/*/main.c, docfx templates, vitepress sites).diff --git a/tests/unit/test_support_paths.py b/tests/unit/test_support_paths.pynew file mode 100644index 000000000..745c98be8--- /dev/null+++ b/tests/unit/test_support_paths.py@@ -0,0 +1,62 @@+"""Support-path classification: examples and docs versus the system itself."""++from __future__ import annotations++import pytest++from repowise.core.support_paths import is_example_path, is_support_path+++ "path",+ [+ "examples/demo.rs",+ "example/main.go",+ "_examples/basic/index.js",+ "samples/sample.py",+ "demos/app/main.ts",+ "benches/parser.rs",+ "benchmarks/run.py",+ # Workspace members keep their own example trees, so the match is not+ # anchored to the repo root.+ "crates/render/examples/window.rs",+ "packages/core/benches/walk.py",+ ],+)+def test_example_dirs_are_examples(path: str):+ assert is_example_path(path)+ assert is_support_path(path)+++ "path",+ [+ "src/parser.rs",+ "src/render/layout.py",+ # Basename only: a module *about* examples is production code.+ "src/examples.py",+ "src/benchmark_runner.go",+ # Not a whole segment.+ "src/examples_helper/util.py",+ ],+)+def test_production_paths_are_not_examples(path: str):+ assert not is_example_path(path)+ assert not is_support_path(path)++[email protected]("path", ["docs/guide/main.c", "doc/build.py", "website/theme/index.ts"])+def test_doc_dirs_are_support_but_not_examples(path: str):+ """The distinction the two predicates exist for.++ A docs tree is not the system, so it is support material. But it is not+ example code either: it is prose infrastructure, and callers grouping code+ into modules should not treat it as a demo of the subject.+ """+ assert is_support_path(path)+ assert not is_example_path(path)+++def test_matching_is_case_insensitive():+ assert is_example_path("Examples/Demo.cs")+ assert is_support_path("Docs/Guide.md")