Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.
Python aicode-complexitycode-healthcode-intelligencecode-qualitydead-codedeveloper-toolsdocumentationgit-analyticsmcpopen-sourcepythonrefactoringstatic-analysistechnical-debt
fix(c4): system context shows runtime services, not the repo's own packages (#2699)
Swati Ahuja448cf926837e
diff --git a/packages/core/src/repowise/core/generation/context/readme_digest.py b/packages/core/src/repowise/core/generation/context/readme_digest.pyindex f53a2c946..883a9d0ce 100644--- a/packages/core/src/repowise/core/generation/context/readme_digest.py+++ b/packages/core/src/repowise/core/generation/context/readme_digest.py@@ -60,14 +60,17 @@ _IMAGE = re.compile(r"!\[[^\]]*\]\([^)]*\)")_LINK = re.compile(r"\[([^\]]+)\]\([^)]*\)")#: Named because a backslash cannot appear inside an f-string before 3.12._LINK_TEXT = r"\1"+#: A list item that is only a link: a table-of-contents entry, not prose.+_TOC_ENTRY = re.compile(r"^\s*(?:[-*+]|\d+[.)])\s+\[[^\]]+\]\([^)]*\)\s*$")-def _opening_paragraph(lines: list[str], start: int) -> str:+def _opening_paragraph(lines: list[str], start: int, *, past_headings: bool = False) -> str:"""The first prose paragraph at or after *start*, before the next heading.Skips fenced code, decoration and blank lines. Returns "" when the sectionopens straight onto another heading, which is the common shape of a table- of contents.+ of contents, unless *past_headings* lets it read on past headings and+ link-only list items to the first prose."""i = startfenced = False@@ -86,8 +89,11 @@ def _opening_paragraph(lines: list[str], start: int) -> str:i += 1continueif _HEADING.match(line):- break- if not stripped or _DECORATION.match(line):+ if body or not past_headings:+ break+ i += 1+ continue+ if not stripped or _DECORATION.match(line) or (past_headings and _TOC_ENTRY.match(line)):# Blank lines before the paragraph are skipped; one after it ends# the paragraph, so that a section contributes one paragraph.if body:@@ -144,3 +150,16 @@ def readme_digest(repo_root: Path, *, max_chars: int = _MAX_CHARS) -> str:out.append(entry)budget -= costreturn "\n\n".join(out)+++def readme_opening(repo_root: Path) -> str:+ """The README's first prose paragraph, headed or not; "" when there is none."""+ for rel in _DIGEST_DOCS:+ try:+ text = (repo_root / rel).read_text(encoding="utf-8", errors="replace")+ except OSError:+ continue+ paragraph = _opening_paragraph(text.splitlines(), 0, past_headings=True)+ if paragraph:+ return paragraph+ return ""diff --git a/packages/core/src/repowise/core/ingestion/external_systems/__init__.py b/packages/core/src/repowise/core/ingestion/external_systems/__init__.pyindex e5cc0e728..19eab5155 100644--- a/packages/core/src/repowise/core/ingestion/external_systems/__init__.py+++ b/packages/core/src/repowise/core/ingestion/external_systems/__init__.py@@ -14,11 +14,13 @@ never raise on malformed input.from __future__ import annotations+import refrom collections.abc import Iterablefrom contextlib import suppressfrom pathlib import Pathfrom types import ModuleType+from ..workspace_members import first_party_package_namesfrom . import bazel, cargo, cmake, go, maven, npm, nuget, pypifrom .base import ExternalSystemRecord, ManifestParser@@ -93,7 +95,22 @@ def extract_external_systems(if maven_paths:with suppress(Exception): # parser bugs must not break ingestionrecords.extend(maven.parse_many(maven_paths, repo_root))- return _deduplicate(records)+ return _deduplicate(_drop_first_party(records, repo_root))+++def _dist_key(name: str) -> str:+ """PEP 503 spelling, so ``repowise_core`` and ``repowise-core`` compare equal."""+ return re.sub(r"[-_.]+", "-", name).lower()+++def _drop_first_party(+ records: list[ExternalSystemRecord], repo_root: Path+) -> list[ExternalSystemRecord]:+ """Drop deps on the repo's own packages: they are containers, not external systems."""+ if not records:+ return records+ own = {_dist_key(n) for n in first_party_package_names(Path(repo_root))}+ return [r for r in records if _dist_key(r.name) not in own]def _discover(repo_root: Path) -> list[Path]:diff --git a/packages/core/src/repowise/core/ingestion/external_systems/npm.py b/packages/core/src/repowise/core/ingestion/external_systems/npm.pyindex a594444c4..0a12d5af5 100644--- a/packages/core/src/repowise/core/ingestion/external_systems/npm.py+++ b/packages/core/src/repowise/core/ingestion/external_systems/npm.py@@ -1,8 +1,9 @@"""Parse npm/yarn/pnpm ``package.json`` manifests.Captures ``dependencies``, ``devDependencies``, ``peerDependencies``, and-``optionalDependencies``. Workspace metadata is ignored — workspace packages-are containers, not external systems.+``optionalDependencies``. A dep pinned by a local protocol (``workspace:``,+``link:``, ``file:``) is first-party; named workspace members are dropped+repo-wide by the extractor."""from __future__ import annotations@@ -34,7 +35,6 @@ def parse(manifest_path: Path, repo_root: Path) -> list[ExternalSystemRecord]:return []declared_in = manifest_path.relative_to(repo_root).as_posix()- workspace_names = _collect_workspace_names(data, manifest_path, repo_root)records: list[ExternalSystemRecord] = []seen: set[str] = set()@@ -44,7 +44,7 @@ def parse(manifest_path: Path, repo_root: Path) -> list[ExternalSystemRecord]:continuefor raw_name, raw_version in block.items():name = str(raw_name).strip()- if not name or name in seen or name in workspace_names:+ if not name or name in seen or _is_local(raw_version):continueseen.add(name)version = _normalize_version(raw_version)@@ -63,42 +63,15 @@ def parse(manifest_path: Path, repo_root: Path) -> list[ExternalSystemRecord]:return records+_LOCAL_PROTOCOLS = ("workspace:", "link:", "file:")+++def _is_local(version: object) -> bool:+ return isinstance(version, str) and version.strip().startswith(_LOCAL_PROTOCOLS)++def _normalize_version(value: object) -> str | None:if not isinstance(value, str):return Nonev = value.strip()return v or None---def _collect_workspace_names(- data: dict[str, object], manifest_path: Path, repo_root: Path-) -> set[str]:- """Read ``workspaces`` globs and return the names of each sibling package.-- We rely on this set to skip workspace deps that appear as version ``*``- or ``workspace:*`` in a parent package.json — those are first-party- containers, not external systems.- """- workspaces = data.get("workspaces")- patterns: list[str] = []- if isinstance(workspaces, list):- patterns = [str(p) for p in workspaces if isinstance(p, str)]- elif isinstance(workspaces, dict):- packages = workspaces.get("packages")- if isinstance(packages, list):- patterns = [str(p) for p in packages if isinstance(p, str)]- if not patterns:- return set()-- root = manifest_path.parent- names: set[str] = set()- for pat in patterns:- for match in root.glob(pat + "/package.json"):- try:- inner = json.loads(match.read_text(encoding="utf-8"))- except (OSError, json.JSONDecodeError):- continue- inner_name = inner.get("name") if isinstance(inner, dict) else None- if isinstance(inner_name, str) and inner_name:- names.add(inner_name)- return namesdiff --git a/packages/core/src/repowise/core/ingestion/workspace_members.py b/packages/core/src/repowise/core/ingestion/workspace_members.pyindex de26d62a5..484ed21e1 100644--- a/packages/core/src/repowise/core/ingestion/workspace_members.py+++ b/packages/core/src/repowise/core/ingestion/workspace_members.py@@ -116,22 +116,48 @@ def declared_workspace_members(repo_root: Path) -> dict[str, set[str]]:return {name: dirs for name, dirs in found.items() if dirs is not None}-def manifest_package_name(manifest: Path) -> str | None:- """The package name *manifest* declares, or None when it names none."""- name: object = None+def _manifest_field(manifest: Path, key: str) -> str | None:+ """A top-level package field (``name``, ``description``) *manifest* declares."""+ value: object = Noneif manifest.name == "package.json":try:data = json.loads(manifest.read_text(encoding="utf-8", errors="ignore"))except (OSError, ValueError):return None- name = data.get("name") if isinstance(data, dict) else None+ value = data.get(key) if isinstance(data, dict) else Noneelif manifest.name in ("pyproject.toml", "Cargo.toml"):data = _toml(manifest)section = data.get("project") or data.get("package")- if not isinstance(section, dict) or not section.get("name"):+ if not isinstance(section, dict) or not section.get(key):tool = data.get("tool")section = tool.get("poetry") if isinstance(tool, dict) else None- name = section.get("name") if isinstance(section, dict) else None- elif manifest.name == "go.mod":+ value = section.get(key) if isinstance(section, dict) else None+ return value.strip() if isinstance(value, str) and value.strip() else None+++def manifest_package_name(manifest: Path) -> str | None:+ """The package name *manifest* declares, or None when it names none."""+ if manifest.name == "go.mod":name = _read_module_directive(manifest)- return name.strip() if isinstance(name, str) and name.strip() else None+ return name.strip() if name and name.strip() else None+ return _manifest_field(manifest, "name")+++_ROOT_MANIFESTS = ("package.json", "pyproject.toml", "Cargo.toml", "go.mod")+++def root_description(repo_root: Path) -> str | None:+ """The ``description`` the repo's root manifest gives itself, if any."""+ for filename in _ROOT_MANIFESTS:+ text = _manifest_field(repo_root / filename, "description")+ if text:+ return text+ return None+++def first_party_package_names(repo_root: Path) -> set[str]:+ """Names of the root package and every declared workspace member."""+ manifests = [repo_root / f for f in _ROOT_MANIFESTS]+ for filename, dirs in declared_workspace_members(repo_root).items():+ manifests.extend(repo_root / rel / filename for rel in dirs)+ return {name for m in manifests if m.is_file() and (name := manifest_package_name(m))}diff --git a/packages/server/src/repowise/server/services/c4_builder/__init__.py b/packages/server/src/repowise/server/services/c4_builder/__init__.pyindex 4cb6f2d4c..edcc245fd 100644--- a/packages/server/src/repowise/server/services/c4_builder/__init__.py+++ b/packages/server/src/repowise/server/services/c4_builder/__init__.py@@ -10,7 +10,8 @@ Public APIAll three reuse the same persisted graph (``graph_nodes``, ``graph_edges``)and the ``external_systems`` table populated during ingestion. No on-disk-re-scan is performed at request time.+re-scan is performed at request time; only the L1 system description reads+the root manifest and README."""from __future__ import annotations@@ -18,11 +19,13 @@ from __future__ import annotationsimport jsonfrom collections import defaultdictfrom dataclasses import replace+from pathlib import Pathfrom sqlalchemy import selectfrom sqlalchemy.ext.asyncio import AsyncSessionfrom repowise.core.ids import ExternalSystemId, SystemId, file_path_of, parse, render+from repowise.core.ingestion.workspace_members import root_descriptionfrom repowise.core.persistence import ExternalSystem, Repositoryfrom repowise.core.persistence.crud import (code_file_rows,@@ -31,6 +34,8 @@ from repowise.core.persistence.crud import (get_kg_tour_steps,)from repowise.core.persistence.models import DeadCodeFinding, GitMetadata+from repowise.core.support_paths import is_support_path+from repowise.core.test_paths import is_test_related_pathfrom .actors import derive_actorsfrom .components import detect_components, detect_components_for_all@@ -77,20 +82,36 @@ async def load_repo(session: AsyncSession, repo_id: str) -> Repository | None:return result.scalar_one_or_none()+#: What a System Context shows: runtime services and frameworks. Libraries+#: and tools are implementation detail and stay on L2/L3.+_CONTEXT_CATEGORIES = ("service", "framework")++async def _external_views(- session: AsyncSession, repo_id: str+ session: AsyncSession, repo_id: str, *, context_only: bool = False) -> tuple[list[ExternalSystemView], dict[str, str]]:"""Return the deduplicated list of ExternalSystemView (one per name) anda map from each name → ``ext:<name>`` id.Multi-manifest deps collapse: we pick the highest-category entry(framework > service > tool > library) so the UI shows the most- interesting label.+ interesting label. ``context_only`` keeps the non-dev service and+ framework rows an L1 shows, declared by the system itself rather than+ its tests, examples or docs site."""- result = await session.execute(- select(ExternalSystem).where(ExternalSystem.repository_id == repo_id)- )+ query = select(ExternalSystem).where(ExternalSystem.repository_id == repo_id)+ if context_only:+ query = query.where(+ ExternalSystem.is_dev_dep.is_(False),+ ExternalSystem.category.in_(_CONTEXT_CATEGORIES),+ )+ result = await session.execute(query)rows = list(result.scalars())+ if context_only:+ rows = [+ r for r in rows+ if not (is_support_path(r.declared_in) or is_test_related_path(r.declared_in))+ ]priority = {"framework": 3, "service": 2, "tool": 1, "library": 0}by_name: dict[str, ExternalSystem] = {}for row in rows:@@ -126,7 +147,25 @@ def _is_external_box(box_id: str) -> bool:def _system_for(repo: Repository | None, repo_id: str) -> System:if repo is None:return System(id=render(SystemId(repo_id)), name=repo_id)- return System(id=render(SystemId(repo.id)), name=repo.name, description="")+ return System(+ id=render(SystemId(repo.id)),+ name=repo.name,+ description=_system_description(repo.local_path),+ )+++def _system_description(local_path: str | None) -> str:+ """The root manifest's description, else the README's first prose paragraph."""+ if not local_path:+ return ""+ root = Path(local_path)+ text = root_description(root)+ if text:+ return text+ # Lazy: the generation package is heavy to import for one file read.+ from repowise.core.generation.context.readme_digest import readme_opening++ return readme_opening(root)# ---------------------------------------------------------------------------@@ -158,7 +197,7 @@ async def _actors_for(async def build_l1(session: AsyncSession, repo_id: str) -> C4L1:repo = await load_repo(session, repo_id)system = _system_for(repo, repo_id)- externals, _ = await _external_views(session, repo_id)+ externals, _ = await _external_views(session, repo_id, context_only=True)people, relations = await _actors_for(session, repo_id, system)for ext in externals:diff --git a/packages/server/src/repowise/server/services/c4_builder/mermaid.py b/packages/server/src/repowise/server/services/c4_builder/mermaid.pyindex 5c720b4c9..eae350e6c 100644--- a/packages/server/src/repowise/server/services/c4_builder/mermaid.py+++ b/packages/server/src/repowise/server/services/c4_builder/mermaid.py@@ -78,6 +78,8 @@ def to_mermaid_l1(view: C4L1) -> str:)lines.extend(_emit_externals(view.external_systems))+ if not view.external_systems:+ lines.append(" %% No external service dependencies detected")if view.relations:lines.append("")diff --git a/packages/ui/src/c4/C4Diagram.tsx b/packages/ui/src/c4/C4Diagram.tsxindex dda2a5ee9..2cc31f69f 100644--- a/packages/ui/src/c4/C4Diagram.tsx+++ b/packages/ui/src/c4/C4Diagram.tsx@@ -210,6 +210,24 @@ function C4DiagramInner({{!error && !isLoading && nodes.length === 0 && (<CenteredMessage tone="empty" text="Nothing to show at this level." />)}+ {!error && level === 1 && l1View && l1View.external_systems.length === 0 && (+ <div+ role="status"+ style={{+ position: "absolute",+ left: 0,+ right: 0,+ bottom: 16,+ textAlign: "center",+ color: "var(--color-text-tertiary)",+ fontSize: 13,+ pointerEvents: "none",+ zIndex: 3,+ }}+ >+ No external service dependencies detected+ </div>+ )}<ReactFlownodes={nodes}diff --git a/tests/unit/ingestion/external_systems/test_npm.py b/tests/unit/ingestion/external_systems/test_npm.pyindex 85d36a4e6..69b3e8de0 100644--- a/tests/unit/ingestion/external_systems/test_npm.py+++ b/tests/unit/ingestion/external_systems/test_npm.py@@ -5,7 +5,7 @@ from __future__ import annotationsimport jsonfrom pathlib import Path-from repowise.core.ingestion.external_systems import npm+from repowise.core.ingestion.external_systems import extract_external_systems, npmdef _write(tmp_path: Path, rel: str, data: dict) -> Path:@@ -95,3 +95,37 @@ def test_declared_in_is_repo_relative_posix(tmp_path):)records = npm.parse(nested, tmp_path)assert records[0].declared_in == "packages/web/package.json"+++def test_local_protocol_versions_are_not_external(tmp_path):+ manifest = _write(+ tmp_path,+ "package.json",+ {+ "dependencies": {+ "@org/a": "workspace:^",+ "@org/b": "link:../b",+ "@org/c": "file:../c",+ "react": "^18",+ },+ },+ )+ assert {r.name for r in npm.parse(manifest, tmp_path)} == {"react"}+++def test_pnpm_workspace_deps_not_external(tmp_path):+ # A member three levels down depends on a sibling by plain version: only+ # the root pnpm declaration can tell it is first-party.+ (tmp_path / "pnpm-workspace.yaml").write_text(+ "packages:\n - 'packages/*'\n - 'packages/providers/*'\n", encoding="utf-8"+ )+ _write(tmp_path, "package.json", {"name": "monorepo", "devDependencies": {"vitest": "^1"}})+ _write(tmp_path, "packages/core/package.json", {"name": "@org/core"})+ _write(+ tmp_path,+ "packages/providers/openai/package.json",+ {"name": "@org/openai", "dependencies": {"@org/core": "1.0.0", "zod": "^3"}},+ )+ names = {r.name for r in extract_external_systems(tmp_path)}+ assert names == {"vitest", "zod"}+diff --git a/tests/unit/ingestion/external_systems/test_pypi.py b/tests/unit/ingestion/external_systems/test_pypi.pyindex c0d48c84b..4361959af 100644--- a/tests/unit/ingestion/external_systems/test_pypi.py+++ b/tests/unit/ingestion/external_systems/test_pypi.py@@ -4,7 +4,7 @@ from __future__ import annotationsfrom pathlib import Path-from repowise.core.ingestion.external_systems import pypi+from repowise.core.ingestion.external_systems import extract_external_systems, pypidef _write(tmp_path: Path, rel: str, content: str) -> Path:@@ -82,3 +82,17 @@ def test_requirements_txt(tmp_path):def test_malformed_pyproject_returns_empty(tmp_path):p = _write(tmp_path, "pyproject.toml", "not [ valid toml")assert pypi.parse(p, tmp_path) == []+++def test_uv_workspace_members_not_external(tmp_path):+ (tmp_path / "pyproject.toml").write_text(+ '[project]\nname = "app"\ndependencies = ["app-core", "httpx"]\n'+ '[tool.uv.workspace]\nmembers = ["packages/*"]\n',+ encoding="utf-8",+ )+ core = tmp_path / "packages" / "core"+ core.mkdir(parents=True)+ (core / "pyproject.toml").write_text(+ '[project]\nname = "app_core"\ndependencies = ["app"]\n', encoding="utf-8"+ )+ assert {r.name for r in extract_external_systems(tmp_path)} == {"httpx"}diff --git a/tests/unit/server/services/test_c4_l1_context.py b/tests/unit/server/services/test_c4_l1_context.pynew file mode 100644index 000000000..9f7fc0442--- /dev/null+++ b/tests/unit/server/services/test_c4_l1_context.py@@ -0,0 +1,131 @@+"""L1 System Context: which externals it shows and what it says the system is."""++from __future__ import annotations++import json++import pytest++from repowise.core.persistence import bulk_upsert_external_systems, upsert_repository+from repowise.server.services import c4_builder+from repowise.server.services.c4_builder.mermaid import to_mermaid_l1++_EMPTY_NOTE = "No external service dependencies detected"+++def _ext(name: str, category: str, *, dev: bool = False, declared_in: str = "package.json"):+ return {+ "name": name,+ "display_name": name,+ "ecosystem": "npm",+ "category": category,+ "version": "1",+ "declared_in": declared_in,+ "is_dev_dep": dev,+ }+++async def _repo(session, tmp_path, externals):+ repo = await upsert_repository(session, name="demo", local_path=str(tmp_path))+ if externals:+ await bulk_upsert_external_systems(session, repo.id, externals)+ await session.commit()+ return repo+++async def test_l1_shows_only_runtime_services_and_frameworks(async_session, tmp_path):+ repo = await _repo(+ async_session,+ tmp_path,+ [+ _ext("stripe", "service"),+ _ext("react", "framework"),+ _ext("zod", "library"),+ _ext("vitest", "tool", dev=True),+ _ext("@sentry/node", "service", dev=True),+ # Dev in one manifest, runtime in another: still a runtime dep.+ _ext("express", "framework", dev=True, declared_in="a/package.json"),+ _ext("express", "framework", declared_in="b/package.json"),+ ],+ )+ view = await c4_builder.build_l1(async_session, repo.id)+ assert {e.name for e in view.external_systems} == {"stripe", "react", "express"}+ system_edges = [r for r in view.relations if r.source_id == view.system.id]+ assert len(system_edges) == 3++ # Libraries still reach the lower levels' registry.+ all_views, _ = await c4_builder._external_views(async_session, repo.id)+ assert "zod" in {e.name for e in all_views}+++async def test_l1_ignores_deps_declared_by_examples_tests_and_docs(async_session, tmp_path):+ repo = await _repo(+ async_session,+ tmp_path,+ [+ _ext("redis", "service", declared_in="examples/celery/pyproject.toml"),+ _ext("next", "framework", declared_in="docs/package.json"),+ _ext("stripe", "service", declared_in="tests/fixtures/app/package.json"),+ _ext("openai", "service", declared_in="packages/core/package.json"),+ ],+ )+ view = await c4_builder.build_l1(async_session, repo.id)+ assert {e.name for e in view.external_systems} == {"openai"}+++async def test_empty_l1_says_so(async_session, tmp_path):+ repo = await _repo(async_session, tmp_path, [_ext("zod", "library")])+ view = await c4_builder.build_l1(async_session, repo.id)+ assert view.external_systems == []+ assert _EMPTY_NOTE in to_mermaid_l1(view)+++async def test_system_description_from_root_manifest(async_session, tmp_path):+ (tmp_path / "package.json").write_text(+ json.dumps({"name": "demo", "description": "Streams model output."}), encoding="utf-8"+ )+ (tmp_path / "README.md").write_text("# Demo\n\nSomething else.\n", encoding="utf-8")+ repo = await _repo(async_session, tmp_path, [])+ view = await c4_builder.build_l1(async_session, repo.id)+ assert view.system.description == "Streams model output."+++async def test_system_description_falls_back_to_readme(async_session, tmp_path):+ (tmp_path / "pyproject.toml").write_text('[project]\nname = "demo"\n', encoding="utf-8")+ (tmp_path / "README.md").write_text(+ "# Demo\n\n\n\nA tiny web framework.\n\n## Install\n\npip install demo\n",+ encoding="utf-8",+ )+ repo = await _repo(async_session, tmp_path, [])+ view = await c4_builder.build_l1(async_session, repo.id)+ assert view.system.description == "A tiny web framework."+++ "readme",+ [+ "A tiny web framework.\n\nMore detail.\n",+ "\n\nA tiny web framework.\n\n## Install\n\nUse pip.\n",+ "# Web\n\n## Contents\n\n- [Install](#install)\n 1. [Usage](#usage)\n\n"+ "A tiny web framework.\n",+ ],+ ids=["no-headings", "intro-before-first-heading", "toc-before-intro"],+)+async def test_system_description_reads_unheaded_readme_intro(async_session, tmp_path, readme):+ (tmp_path / "README.md").write_text(readme, encoding="utf-8")+ repo = await _repo(async_session, tmp_path, [])+ view = await c4_builder.build_l1(async_session, repo.id)+ assert view.system.description == "A tiny web framework."+++async def test_system_description_empty_without_sources(async_session, tmp_path):+ repo = await _repo(async_session, tmp_path / "missing", [])+ view = await c4_builder.build_l1(async_session, repo.id)+ assert view.system.description == ""