Skip to content

Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.

fix(c4): no test or example containers, code-only language, manifest names (#2697)

Swati Ahuja17693a571ea8
diff --git a/packages/core/src/repowise/core/ingestion/traverser.py b/packages/core/src/repowise/core/ingestion/traverser.py
index cf5e741ea..de775f628 100644
--- a/packages/core/src/repowise/core/ingestion/traverser.py
+++ b/packages/core/src/repowise/core/ingestion/traverser.py
@@ -827,7 +827,7 @@ class FileTraverser:
declared member (a template or fixture shipped with that package).
Undeclared roots of kinds no declaration covers are kept.
"""
- from ..support_paths import is_example_path
+ from ..support_paths import is_test_or_example_path
from .workspace_members import declared_workspace_members, manifest_package_name
packages: list[PackageInfo] = []
@@ -839,8 +839,7 @@ class FileTraverser:
for rel_pkg, found in sorted(self.package_manifests().items()):
pkg_dir = self.repo_root / rel_pkg
manifests = sorted(found)
- probe = f"{rel_pkg}/{manifests[0]}"
- if is_test_related_path(probe) or is_example_path(probe):
+ if is_test_or_example_path(f"{rel_pkg}/{manifests[0]}"):
continue
declaring = [m for m in manifests if rel_pkg in members.get(m, ())]
if not declaring and (
diff --git a/packages/core/src/repowise/core/support_paths.py b/packages/core/src/repowise/core/support_paths.py
index 8c8f773dc..eadae58b0 100644
--- a/packages/core/src/repowise/core/support_paths.py
+++ b/packages/core/src/repowise/core/support_paths.py
@@ -14,15 +14,19 @@ from __future__ import annotations
from pathlib import PurePosixPath
from typing import Literal
+from .test_paths import is_test_related_path
+
__all__ = [
"CONFIG_EXTENSIONS",
"DOC_EXTENSIONS",
"FilePopulation",
"classification_token",
"file_population",
+ "is_config_path",
"is_doc_or_config_path",
"is_example_path",
"is_support_path",
+ "is_test_or_example_path",
]
@@ -72,6 +76,15 @@ def is_example_path(path: str) -> bool:
return _has_dir_token(path, EXAMPLE_DIR_TOKENS)
+def is_test_or_example_path(path: str) -> bool:
+ """Whether *path* sits in test material or example/benchmark code.
+
+ A manifest here marks a fixture or sample project, not one of the repo's
+ packages.
+ """
+ return is_test_related_path(path) or is_example_path(path)
+
+
# Shared with the knowledge graph's node classifier.
CONFIG_EXTENSIONS = frozenset(
{
@@ -96,6 +109,11 @@ def classification_token(path: str) -> str:
return (parsed.suffix or parsed.name).lower()
+def is_config_path(path: str) -> bool:
+ """Whether *path* is a configuration file."""
+ return classification_token(path) in CONFIG_EXTENSIONS
+
+
def is_doc_or_config_path(path: str) -> bool:
"""Whether *path* is documentation or configuration rather than code."""
token = classification_token(path)
diff --git a/packages/server/src/repowise/server/routers/c4.py b/packages/server/src/repowise/server/routers/c4.py
index 3b006744c..eaefb342e 100644
--- a/packages/server/src/repowise/server/routers/c4.py
+++ b/packages/server/src/repowise/server/routers/c4.py
@@ -79,9 +79,12 @@ async def get_c4_l1(
@router.get("/{repo_id}/c4/l2", response_model=C4L2Response)
async def get_c4_l2(
repo_id: str,
+ co_changes: bool = Query(
+ False, description="Overlay co-change relations on the dependency edges"
+ ),
session: AsyncSession = Depends(get_db_session),
) -> C4L2Response:
- view = await c4_builder.build_l2(session, repo_id)
+ view = await c4_builder.build_l2(session, repo_id, include_co_changes=co_changes)
return C4L2Response(
containers=[_container(c) for c in view.containers],
external_systems=[external_system_response(e) for e in view.external_systems],
diff --git a/packages/server/src/repowise/server/services/c4_builder/__init__.py b/packages/server/src/repowise/server/services/c4_builder/__init__.py
index 748317861..4cb6f2d4c 100644
--- a/packages/server/src/repowise/server/services/c4_builder/__init__.py
+++ b/packages/server/src/repowise/server/services/c4_builder/__init__.py
@@ -173,11 +173,12 @@ async def build_l1(session: AsyncSession, repo_id: str) -> C4L1:
)
-async def build_l2(session: AsyncSession, repo_id: str) -> C4L2:
+async def build_l2(
+ session: AsyncSession, repo_id: str, *, include_co_changes: bool = False
+) -> C4L2:
+ """Container view; ``include_co_changes`` overlays co-change relations."""
repo = await load_repo(session, repo_id)
- containers = await detect_containers(
- session, repo_id, root_name=repo.name if repo else None
- )
+ containers = await _detect_containers(session, repo_id, repo)
externals, _ = await _external_views(session, repo_id)
file_to_container = await _file_to_container_map(session, repo_id, containers)
@@ -192,6 +193,7 @@ async def build_l2(session: AsyncSession, repo_id: str) -> C4L2:
repo_id,
file_to_box=file_to_container,
file_to_external=file_to_external,
+ include_co_changes=include_co_changes,
)
# Only surface externals actually depended on by at least one container.
@@ -203,9 +205,7 @@ async def build_l2(session: AsyncSession, repo_id: str) -> C4L2:
async def build_l3(session: AsyncSession, repo_id: str, container_id_value: str) -> C4L3 | None:
"""Return L3 view for one container, or ``None`` if it doesn't exist."""
repo = await load_repo(session, repo_id)
- containers = await detect_containers(
- session, repo_id, root_name=repo.name if repo else None
- )
+ containers = await _detect_containers(session, repo_id, repo)
container = next((c for c in containers if c.id == container_id_value), None)
if container is None:
return None
@@ -280,7 +280,7 @@ async def build_model(
"""
repo = await load_repo(session, repo_id)
system = _system_for(repo, repo_id)
- containers = await detect_containers(session, repo_id, root_name=repo.name if repo else None)
+ containers = await _detect_containers(session, repo_id, repo)
externals_all, _ = await _external_views(session, repo_id)
people, actor_relations = await _actors_for(session, repo_id, system)
@@ -433,6 +433,17 @@ async def _tour_steps(session: AsyncSession, repo_id: str) -> list[TourStep]:
# ---------------------------------------------------------------------------
+async def _detect_containers(
+ session: AsyncSession, repo_id: str, repo: Repository | None
+) -> list[Container]:
+ return await detect_containers(
+ session,
+ repo_id,
+ root_name=repo.name if repo else None,
+ local_path=repo.local_path if repo else None,
+ )
+
+
async def _curated_entry_points(session: AsyncSession, repo_id: str) -> list[str]:
"""Return the curated entry-point paths persisted at index time.
diff --git a/packages/server/src/repowise/server/services/c4_builder/containers.py b/packages/server/src/repowise/server/services/c4_builder/containers.py
index 15a55c515..c3db217ee 100644
--- a/packages/server/src/repowise/server/services/c4_builder/containers.py
+++ b/packages/server/src/repowise/server/services/c4_builder/containers.py
@@ -7,24 +7,34 @@ Containers are derived from the persisted ``external_systems`` table — every
directory is a container root. This works for both monorepos (many manifests)
and single-package repos (one root manifest).
+Manifests under test or example trees are fixtures and samples, not the
+system's containers, so they are dropped and their files fold into the nearest
+remaining ancestor container (or into none, when no ancestor exists).
+
If no manifests were found (rare — exotic stacks), we fall back to the set
-of top-level directories that hold indexed files.
+of top-level directories that hold indexed production files.
Each container's ``language``, ``file_count``, and ``symbol_count`` are
aggregated from ``graph_nodes`` rows whose ``node_id`` (file path) lives
-inside the container directory.
+inside the container directory; ``language`` counts code files only. The name
+is the manifest's declared package name when the checkout is readable.
"""
from __future__ import annotations
-from collections import Counter, defaultdict
+from collections import defaultdict
from collections.abc import Iterable
+from pathlib import Path
from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession
+from repowise.core.analysis.kg_inputs import _dominant_language as _most_common_language
from repowise.core.ids import ContainerId, render
+from repowise.core.ingestion.languages.registry import REGISTRY
+from repowise.core.ingestion.workspace_members import manifest_package_name
from repowise.core.persistence import ExternalSystem, GraphNode
+from repowise.core.support_paths import is_test_or_example_path
from .models import Container
@@ -48,6 +58,8 @@ _MANIFEST_BASENAMES = frozenset(
)
_MANIFEST_SUFFIXES = (".csproj",)
+_CODE_LANGUAGES = REGISTRY.code_languages()
+
def _is_manifest(node_id: str) -> bool:
base = node_id.rsplit("/", 1)[-1].lower()
@@ -59,26 +71,32 @@ async def detect_containers(
repository_id: str,
*,
root_name: str | None = None,
+ local_path: str | None = None,
) -> list[Container]:
"""Return the list of containers detected for ``repository_id``.
- ``root_name`` (the repository name) labels the root container of a
- single-manifest / root-manifest repo, which would otherwise serialize as
- the uninformative ``"."``.
+ A container is named by its manifest's package name, read from
+ ``local_path`` when that checkout exists. Otherwise ``root_name`` (the
+ repository name) labels the root container, which would serialize as the
+ uninformative ``"."``, and any other container is named by its path.
Stable ordering by ``path`` so the C4 diagram doesn't reshuffle between
requests.
"""
- container_roots = await _container_roots_from_manifests(session, repository_id)
+ manifests = await _declared_manifests(session, repository_id)
file_nodes = await _file_nodes(session, repository_id)
# Also treat any directory holding a package manifest as a container root,
# even when that manifest declared no external deps (catches e.g. a core
# package that would otherwise dissolve into the repo-root catch-all).
- container_roots |= _manifest_roots(node.node_id for node in file_nodes)
+ manifests |= {n.node_id for n in file_nodes if _is_manifest(n.node_id)}
+ roots_to_manifests = _manifest_roots(manifests)
+ container_roots = set(roots_to_manifests)
if not container_roots:
- container_roots = _top_level_dirs(node.node_id for node in file_nodes)
+ container_roots = _top_level_dirs(
+ n.node_id for n in file_nodes if not is_test_or_example_path(n.node_id)
+ )
# Sort longest path first so nested manifests win over their parents.
sorted_roots = sorted(container_roots, key=len, reverse=True)
@@ -94,11 +112,11 @@ async def detect_containers(
for root in sorted(grouped):
nodes = grouped[root]
lang = _dominant_language(nodes)
- name = root.split("/")[-1] if root else fallback_root_name
+ name = _manifest_name(local_path, roots_to_manifests.get(root, ()))
containers.append(
Container(
id=container_id(root),
- name=name or fallback_root_name,
+ name=name or root or fallback_root_name,
path=root,
language=lang,
file_count=len(nodes),
@@ -113,22 +131,14 @@ def container_id(path: str) -> str:
return render(ContainerId(path or "."))
-async def _container_roots_from_manifests(
- session: AsyncSession, repository_id: str
-) -> set[str]:
- """Return the unique parent directories of every declared manifest."""
+async def _declared_manifests(session: AsyncSession, repository_id: str) -> set[str]:
+ """Every manifest path an external dependency was declared in."""
result = await session.execute(
select(ExternalSystem.declared_in).where(
ExternalSystem.repository_id == repository_id
)
)
- roots: set[str] = set()
- for (declared_in,) in result.all():
- if not declared_in:
- continue
- parent = declared_in.rsplit("/", 1)[0] if "/" in declared_in else ""
- roots.add(parent)
- return roots
+ return {declared_in for (declared_in,) in result.all() if declared_in}
async def _file_nodes(session: AsyncSession, repository_id: str) -> list[GraphNode]:
@@ -145,15 +155,27 @@ async def _file_nodes(session: AsyncSession, repository_id: str) -> list[GraphNo
return list(result.scalars())
-def _manifest_roots(paths: Iterable[str]) -> set[str]:
- """Parent directories of every package-manifest file node."""
- roots: set[str] = set()
- for path in paths:
- if _is_manifest(path):
- roots.add(path.rsplit("/", 1)[0] if "/" in path else "")
+def _manifest_roots(manifests: Iterable[str]) -> dict[str, list[str]]:
+ """``{container root: its manifests}``, fixture and sample manifests dropped."""
+ roots: dict[str, list[str]] = defaultdict(list)
+ for path in sorted(manifests):
+ if not is_test_or_example_path(path):
+ roots[path.rsplit("/", 1)[0] if "/" in path else ""].append(path)
return roots
+def _manifest_name(local_path: str | None, manifests: Iterable[str]) -> str | None:
+ """The first package name the root's manifests declare, read from the checkout."""
+ # Ceiling: needs the checkout on disk; persisting names at index time lifts it.
+ if not local_path or not Path(local_path).is_dir():
+ return None
+ for manifest in manifests:
+ name = manifest_package_name(Path(local_path) / manifest)
+ if name:
+ return name
+ return None
+
+
def _top_level_dirs(paths: Iterable[str]) -> set[str]:
out: set[str] = set()
for path in paths:
@@ -175,7 +197,6 @@ def _match_container(file_path: str, sorted_roots: list[str]) -> str | None:
def _dominant_language(nodes: list[GraphNode]) -> str:
- counter: Counter[str] = Counter(
- n.language for n in nodes if n.language and n.language != "unknown"
- )
- return counter.most_common(1)[0][0] if counter else "unknown"
+ """Most common code language; markdown and config never name a container."""
+ langs = [n.language for n in nodes if n.language in _CODE_LANGUAGES]
+ return _most_common_language(langs) or "unknown"
diff --git a/packages/server/src/repowise/server/services/c4_builder/relations.py b/packages/server/src/repowise/server/services/c4_builder/relations.py
index 778629fc5..85f8ca449 100644
--- a/packages/server/src/repowise/server/services/c4_builder/relations.py
+++ b/packages/server/src/repowise/server/services/c4_builder/relations.py
@@ -11,6 +11,11 @@ L2, between components at L3. This module does that aggregation:
4. Drop self-loops — an edge from a container to itself is not useful
to a viewer.
+Co-change edges are history, not a dependency, so they are left out unless a
+caller opts in to them as an overlay. Edges touching a configuration file
+(``tsconfig.json``, ``pyproject.toml``) are dropped: reading config is not a
+dependency between boxes.
+
External-system edges are produced from file→``external:*`` edges where
the target's ``external_system_id`` resolved to a row in the
``external_systems`` table.
@@ -25,6 +30,7 @@ from sqlalchemy.ext.asyncio import AsyncSession
from repowise.core.ids import ExternalSystemId, render
from repowise.core.persistence import ExternalSystem, GraphEdge, GraphNode
+from repowise.core.support_paths import is_config_path
from .labels import coupling_strength, relation_label
from .models import Relation
@@ -54,6 +60,7 @@ async def aggregate_relations(
*,
file_to_external: dict[str, str] | None = None,
edges: list[tuple[str, str, str]] | None = None,
+ include_co_changes: bool = False,
) -> list[Relation]:
"""Roll file→file edges up to box→box edges.
@@ -67,23 +74,23 @@ async def aggregate_relations(
emitted (as box → external).
edges:
Pre-loaded rows from :func:`load_edges`. Omit to read them here.
+ include_co_changes:
+ Also roll up ``co_changes`` edges, as an overlay on the dependencies.
"""
file_to_external = file_to_external or {}
if edges is None:
edges = await load_edges(session, repository_id)
- # No edge-type filter here, and that is deliberate rather than an
- # oversight. ``file_to_box`` is keyed on file paths only, so a containment
- # edge cannot survive the two lookups below: ``defines`` is file → symbol
- # and loses its target, ``has_method`` is symbol → symbol and loses its
- # source. Measured across the 41 indexed corpus repos, excluding
- # containment changes no relation, no count and no coupling band on any of
- # them. Temporal edges do survive, and a "co-changes" arrow is a labeled
- # relation this view means to draw (see ``_EDGE_VERB``), not leakage.
+ # No containment filter needed: ``file_to_box`` is keyed on file paths, so
+ # ``defines`` (file → symbol) and ``has_method`` (symbol → symbol) always
+ # lose an endpoint below. Co-change is filtered: it is history, not a
+ # dependency, and drawn only when the caller asks for the overlay.
counts: dict[tuple[str, str], int] = defaultdict(int)
types: dict[tuple[str, str], set[str]] = defaultdict(set)
for src, tgt, etype in edges:
+ if etype == "co_changes" and not include_co_changes:
+ continue
src_box = file_to_box.get(src)
if src_box is None:
continue
@@ -95,6 +102,8 @@ async def aggregate_relations(
continue
if src_box == tgt_box:
continue
+ if is_config_path(src) or is_config_path(tgt):
+ continue
key = (src_box, tgt_box)
counts[key] += 1
types[key].add(etype)
diff --git a/tests/unit/server/services/test_c4_builder.py b/tests/unit/server/services/test_c4_builder.py
index 1b55db8da..9a71b9d7e 100644
--- a/tests/unit/server/services/test_c4_builder.py
+++ b/tests/unit/server/services/test_c4_builder.py
@@ -251,3 +251,133 @@ async def test_relation_labels_are_readable_with_coupling(async_session):
assert cross.label == "imports"
assert cross.coupling in {"loose", "moderate", "tight"}
assert "+1" not in cross.label
+
+
+async def _seed_nodes_and_edges(session, local_path, nodes, edges=()):
+ repo = await upsert_repository(session, name="shop", local_path=local_path)
+ await batch_upsert_graph_nodes(
+ session,
+ repo.id,
+ [{"node_id": n, "node_type": "file", "language": lang, "symbol_count": 1} for n, lang in nodes],
+ )
+ if edges:
+ await batch_upsert_graph_edges(
+ session,
+ repo.id,
+ [{"source_node_id": s, "target_node_id": t, "edge_type": e} for s, t, e in edges],
+ )
+ await session.commit()
+ return repo
+
+
+async def test_dominant_language_ignores_markdown(async_session):
+ nodes = [
+ ("packages/docs-kit/package.json", "json"),
+ ("packages/docs-kit/a.md", "markdown"),
+ ("packages/docs-kit/b.md", "markdown"),
+ ("packages/docs-kit/c.md", "markdown"),
+ ("packages/docs-kit/index.ts", "typescript"),
+ ]
+ repo = await _seed_nodes_and_edges(async_session, "/nonexistent/shop", nodes)
+ view = await c4_builder.build_l2(async_session, repo.id)
+ (container,) = view.containers
+ assert container.language == "typescript"
+ assert container.file_count == 5
+
+
+async def test_fixture_manifest_not_container(async_session):
+ nodes = [
+ ("package.json", "json"),
+ ("src/index.ts", "typescript"),
+ ("tests/fixtures/app/package.json", "json"),
+ ("tests/fixtures/app/index.ts", "typescript"),
+ ("examples/demo/package.json", "json"),
+ ("examples/demo/main.ts", "typescript"),
+ ("packages/core/package.json", "json"),
+ ("packages/core/src/lib.ts", "typescript"),
+ ("packages/core/test/fixture/package.json", "json"),
+ ]
+ repo = await _seed_nodes_and_edges(async_session, "/nonexistent/shop", nodes)
+ view = await c4_builder.build_l2(async_session, repo.id)
+ by_path = {c.path: c for c in view.containers}
+ assert set(by_path) == {"", "packages/core"}
+ # Fixture and sample files fold into the nearest remaining container.
+ assert by_path[""].file_count == 6
+ assert by_path["packages/core"].file_count == 3
+ # With no readable manifest a container is named by its relative path.
+ assert by_path["packages/core"].name == "packages/core"
+ assert by_path[""].name == "shop"
+
+
+async def test_fallback_top_level_dirs_skip_tests_and_examples(async_session):
+ nodes = [
+ ("app/main.go", "go"),
+ ("tests/app_test.go", "go"),
+ ("examples/hello/main.go", "go"),
+ ]
+ repo = await _seed_nodes_and_edges(async_session, "/nonexistent/shop", nodes)
+ view = await c4_builder.build_l2(async_session, repo.id)
+ assert [c.path for c in view.containers] == ["app"]
+
+
+async def test_container_named_from_manifest(async_session, tmp_path):
+ (tmp_path / "packages" / "core").mkdir(parents=True)
+ (tmp_path / "packages" / "core" / "package.json").write_text('{"name": "@shop/core"}')
+ (tmp_path / "pyproject.toml").write_text('[project]\nname = "shop-root"\n')
+ nodes = [
+ ("pyproject.toml", "toml"),
+ ("shop/app.py", "python"),
+ ("packages/core/package.json", "json"),
+ ("packages/core/index.ts", "typescript"),
+ ]
+ repo = await _seed_nodes_and_edges(async_session, str(tmp_path), nodes)
+ names = {c.path: c.name for c in (await c4_builder.build_l2(async_session, repo.id)).containers}
+ assert names == {"": "shop-root", "packages/core": "@shop/core"}
+
+
+async def _seed_two_containers_with_edges(session, edges):
+ nodes = [
+ ("packages/a/package.json", "json"),
+ ("packages/a/index.ts", "typescript"),
+ ("packages/a/tsconfig.json", "json"),
+ ("packages/b/package.json", "json"),
+ ("packages/b/index.ts", "typescript"),
+ ]
+ return await _seed_nodes_and_edges(session, "/nonexistent/shop", nodes, edges)
+
+
+async def test_l2_excludes_cochange(async_session):
+ repo = await _seed_two_containers_with_edges(
+ async_session, [("packages/a/index.ts", "packages/b/index.ts", "co_changes")]
+ )
+ assert (await c4_builder.build_l2(async_session, repo.id)).relations == []
+ # The shared roll-up drops history everywhere, the export included.
+ model = await c4_builder.build_model(async_session, repo.id)
+ assert model.container_relations == [] and model.component_relations == []
+
+ overlay = await c4_builder.build_l2(async_session, repo.id, include_co_changes=True)
+ assert [(r.source_id, r.target_id, r.label) for r in overlay.relations] == [
+ ("pkg:packages/a", "pkg:packages/b", "co-changes")
+ ]
+
+
+async def test_l2_skips_edges_to_config_files(async_session):
+ repo = await _seed_two_containers_with_edges(
+ async_session,
+ [
+ ("packages/b/index.ts", "packages/a/tsconfig.json", "imports"),
+ ("packages/b/index.ts", "packages/a/index.ts", "imports"),
+ ],
+ )
+ (relation,) = (await c4_builder.build_l2(async_session, repo.id)).relations
+ assert (relation.source_id, relation.target_id, relation.edge_count) == (
+ "pkg:packages/b",
+ "pkg:packages/a",
+ 1,
+ )