Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.
Python aicode-complexitycode-healthcode-intelligencecode-qualitydead-codedeveloper-toolsdocumentationgit-analyticsmcpopen-sourcepythonrefactoringstatic-analysistechnical-debt
fix(ingestion): workspace packages at any depth, checked against the declared members (#2685)
Swati Ahuja64d8a38e7f74
diff --git a/packages/core/src/repowise/core/analysis/knowledge_graph.py b/packages/core/src/repowise/core/analysis/knowledge_graph.pyindex e14766cfa..36789270d 100644--- a/packages/core/src/repowise/core/analysis/knowledge_graph.py+++ b/packages/core/src/repowise/core/analysis/knowledge_graph.py@@ -281,6 +281,10 @@ def build_knowledge_graph_skeleton("is_monorepo": repo_structure.is_monorepo if repo_structure else False,"total_files": repo_structure.total_files if repo_structure else len(parsed_files),"entry_points": orientation_entry_points(repo_structure),+ "packages": [+ {"name": p.name, "path": p.path, "declared": p.declared}+ for p in getattr(repo_structure, "packages", None) or []+ ],"tech_stack": tech_stack[:20],}diff --git a/packages/core/src/repowise/core/ingestion/models.py b/packages/core/src/repowise/core/ingestion/models.pyindex 107fe78e5..e5ec83785 100644--- a/packages/core/src/repowise/core/ingestion/models.py+++ b/packages/core/src/repowise/core/ingestion/models.py@@ -168,6 +168,8 @@ class PackageInfo:language: LanguageTagentry_points: list[str]manifest_file: str # pyproject.toml | package.json | Cargo.toml | go.mod+ # A member of a root workspace declaration (pnpm/npm/yarn, Cargo, uv, go.work).+ declared: bool = False@dataclassdiff --git a/packages/core/src/repowise/core/ingestion/package_roots.py b/packages/core/src/repowise/core/ingestion/package_roots.pyindex 3f4b0dcce..31dd1154a 100644--- a/packages/core/src/repowise/core/ingestion/package_roots.py+++ b/packages/core/src/repowise/core/ingestion/package_roots.py@@ -64,9 +64,25 @@ def scan_package_roots(repo_root: str | os.PathLike[str],*,is_pruned: Callable[[Path], bool] | None = None,+ prune_nested_git: bool = True,) -> set[str]:"""Repo-relative POSIX directories holding a package manifest.+ The directory set of :func:`scan_package_manifests`; see it for the walk.+ """+ return set(+ scan_package_manifests(repo_root, is_pruned=is_pruned, prune_nested_git=prune_nested_git)+ )+++def scan_package_manifests(+ repo_root: str | os.PathLike[str],+ *,+ is_pruned: Callable[[Path], bool] | None = None,+ prune_nested_git: bool = True,+) -> dict[str, frozenset[str]]:+ """Each package root mapped to the manifest filenames found in it.+A manifest at the repo root is omitted: it would put every file in onebucket, which is the degenerate case package attribution exists to avoid.@@ -77,21 +93,24 @@ def scan_package_roots(beneath it. It defaults to a stock :class:`FileTraverser`'s boundary test;pass a configured traverser's :meth:`~FileTraverser.dir_chain_skipped` (oruse :meth:`FileTraverser.package_root_dirs`) when the repo has submodule- or exclude settings.+ or exclude settings. ``prune_nested_git=False`` descends into nested+ checkouts, for a traverser opted into indexing them."""root = Path(repo_root)names = package_manifest_names()prune = is_pruned if is_pruned is not None else _default_prune(root)- roots: set[str] = set()+ roots: dict[str, frozenset[str]] = {}- for dirpath, dirnames, filenames in walk_repo(root):+ for dirpath, dirnames, filenames in walk_repo(root, prune_nested_git=prune_nested_git):rel_dir = Path(dirpath).relative_to(root)# Prune in place so the walk never descends — the point of the scan is# to be cheap enough to run on every update.dirnames[:] = [d for d in dirnames if not prune(rel_dir / d)]- if rel_dir == Path() or not names.intersection(filenames):+ if rel_dir == Path():continue- roots.add(rel_dir.as_posix())+ matched = names.intersection(filenames)+ if matched:+ roots[rel_dir.as_posix()] = matchedreturn rootsdiff --git a/packages/core/src/repowise/core/ingestion/traverser.py b/packages/core/src/repowise/core/ingestion/traverser.pyindex 9c31d8860..01a151324 100644--- a/packages/core/src/repowise/core/ingestion/traverser.py+++ b/packages/core/src/repowise/core/ingestion/traverser.py@@ -805,61 +805,79 @@ class FileTraverser:# ------------------------------------------------------------------def _detect_monorepo(self) -> tuple[list[PackageInfo], bool]:- """Detect package sub-directories by looking for manifest files.-- Candidate dirs the main traversal would never enter (nested git- repos, submodules, gitignored/blocked dirs) are rejected up front, so- a package the walk skips is neither reported nor scanned.+ """Packages: manifest directories at any depth, checked against workspaces.++ Candidates come from :meth:`package_root_dirs`, so a package the walk+ skips is neither reported nor scanned. Roots under test or example trees+ are dropped: a fixture or sample carries a manifest without being one+ of the repo's packages.++ A root workspace declaration (pnpm/npm/yarn, Cargo, uv, go.work) is the+ authority for its manifest kind: its members are ``declared``, and an+ undeclared root is dropped when one of its manifests is a declared kind+ (the package manager does not build it) or when it sits inside a+ declared member (a template or fixture shipped with that package).+ Undeclared roots of kinds no declaration covers are kept."""+ from ..support_paths import is_example_path+ from .workspace_members import declared_workspace_members, manifest_package_name+packages: list[PackageInfo] = []- seen_paths: set[str] = set()+ members = declared_workspace_members(self.repo_root)+ all_members = set().union(*members.values())# Mirrors GraphBuilder._prune_nested_git.prune_nested = not (self._include_submodules or self._include_nested_repos)- for depth in (1, 2):- pattern = "/".join(["*"] * depth) + "/*"- for candidate in self.repo_root.glob(pattern):- if candidate.name not in _MANIFEST_FILES:- continue- pkg_dir = candidate.parent- rel_pkg_path = pkg_dir.relative_to(self.repo_root)- rel_pkg = rel_pkg_path.as_posix()- if rel_pkg in seen_paths:- continue- if self.dir_chain_skipped(rel_pkg_path):- continue- seen_paths.add(rel_pkg)- lang, entry_pts = _scan_package_dir(- pkg_dir,- self.repo_root,- prune_nested_git=prune_nested,- is_pruned=self.dir_chain_skipped,- )- packages.append(- PackageInfo(- name=pkg_dir.name,- path=rel_pkg,- language=lang,- entry_points=entry_pts,- manifest_file=candidate.name,- )+ for rel_pkg, found in sorted(self.package_manifests().items()):+ pkg_dir = self.repo_root / rel_pkg+ manifests = sorted(found)+ probe = f"{rel_pkg}/{manifests[0]}"+ if is_test_related_path(probe) or is_example_path(probe):+ continue+ declaring = [m for m in manifests if rel_pkg in members.get(m, ())]+ if not declaring and (+ any(m in members for m in manifests)+ or any(p.as_posix() in all_members for p in Path(rel_pkg).parents)+ ):+ continue+ manifest = (declaring or manifests)[0]+ lang, entry_pts = _scan_package_dir(+ pkg_dir,+ self.repo_root,+ prune_nested_git=prune_nested,+ is_pruned=self.dir_chain_skipped,+ )+ packages.append(+ PackageInfo(+ name=manifest_package_name(pkg_dir / manifest) or pkg_dir.name,+ path=rel_pkg,+ language=lang,+ entry_points=entry_pts,+ manifest_file=manifest,+ declared=bool(declaring),)+ )- packages.sort(key=lambda p: p.path)return packages, len(packages) > 1def package_root_dirs(self) -> set[str]:"""Every directory holding a package manifest, at any depth.Shares :func:`.package_roots.scan_package_roots` with health's module- attribution, and this traverser's own skip semantics, so the two agree- on what a package is. Distinct from :meth:`get_repo_structure`'s- ``packages``, which stops at depth 2 and pays for language and- entry-point detection per package.+ attribution, and this traverser's own skip semantics (nested-repo+ opt-ins included), so the two agree on what a package is."""- from .package_roots import scan_package_roots+ return set(self.package_manifests())- return scan_package_roots(self.repo_root, is_pruned=self.dir_chain_skipped)+ def package_manifests(self) -> dict[str, frozenset[str]]:+ """:meth:`package_root_dirs` with the manifest filenames each one holds."""+ from .package_roots import scan_package_manifests++ return scan_package_manifests(+ self.repo_root,+ is_pruned=self.dir_chain_skipped,+ prune_nested_git=not (self._include_submodules or self._include_nested_repos),+ )def dir_chain_skipped(self, rel_dir: Path) -> bool:"""True if *rel_dir* (or any ancestor) would be pruned by ``_walk``.diff --git a/packages/core/src/repowise/core/ingestion/workspace_members.py b/packages/core/src/repowise/core/ingestion/workspace_members.pynew file mode 100644index 000000000..de26d62a5--- /dev/null+++ b/packages/core/src/repowise/core/ingestion/workspace_members.py@@ -0,0 +1,137 @@+"""Which directories a repo's root workspace manifests declare as members.++A package manager's own member list is the authority on what the repo's+packages are. A manifest found on disk is only a candidate: fixture+``package.json`` files, scaffolding templates and docs sites all carry one and+none of them is a package the workspace builds.++Keyed by the manifest filename a member carries, so the caller can tell which+ecosystem declared a workspace:++- ``package.json``: ``pnpm-workspace.yaml``, else ``package.json``'s+ ``workspaces`` (npm, yarn); read by the TypeScript resolver's reader.+- ``Cargo.toml``: ``[workspace] members`` minus ``exclude``.+- ``pyproject.toml``: ``[tool.uv.workspace] members`` minus ``exclude``.+- ``go.mod``: ``go.work``'s ``use`` directives.++Only the repo root's manifests are read. Ceiling: a workspace declared in a+subdirectory (a JS workspace nested in a Python repo) is not seen, so its+members stay undeclared; reading nested declarations is the upgrade path.+"""++from __future__ import annotations++import json+import posixpath+import re+import tomllib+from pathlib import Path++from .resolvers.go import _read_module_directive+from .resolvers.ts_workspace import (+ _expand_member_dirs,+ _read_workspace_declaration,+ _WorkspaceDeclaration,+)+++def _globbed(repo_root: Path, includes: list, excludes: list) -> set[str]:+ """Member globs expanded with the JS workspace globber, as repo-relative dirs."""+ declared = _WorkspaceDeclaration(+ tuple(p for p in includes if isinstance(p, str)),+ tuple(p for p in excludes if isinstance(p, str)),+ include_root=False,+ )+ out: set[str] = set()+ for d in _expand_member_dirs(repo_root, declared):+ try:+ rel = d.relative_to(repo_root).as_posix()+ except ValueError:+ continue+ if rel != ".":+ out.add(rel)+ return out+++def _toml(path: Path) -> dict:+ try:+ with open(path, "rb") as f:+ return tomllib.load(f)+ except (OSError, tomllib.TOMLDecodeError):+ return {}+++def _toml_workspace(repo_root: Path, manifest: str, *table: str) -> set[str] | None:+ """``members``/``exclude`` of a TOML workspace table (Cargo and uv share the shape).++ None without a ``members`` list: Cargo then makes the root's path+ dependencies members implicitly, which this does not read.+ """+ data: object = _toml(repo_root / manifest)+ for key in table:+ data = data.get(key) if isinstance(data, dict) else None+ if not isinstance(data, dict):+ return None+ members, exclude = data.get("members"), data.get("exclude")+ if not isinstance(members, list):+ return None+ return _globbed(repo_root, members, exclude if isinstance(exclude, list) else [])+++_GO_WORK_USE = re.compile(r"^\s*use\s*(?:\(([^)]*)\)|(\S+))", re.MULTILINE)+++def _go_work_members(repo_root: Path) -> set[str] | None:+ """Directories ``go.work`` pulls in with ``use`` (single or block form)."""+ try:+ text = (repo_root / "go.work").read_text(encoding="utf-8", errors="ignore")+ except OSError:+ return None+ text = re.sub(r"//[^\n]*", "", text)+ out: set[str] = set()+ for block, single in _GO_WORK_USE.findall(text):+ for entry in block.split() if block else [single]:+ rel = posixpath.normpath(entry.strip('"`'))+ if rel != "." and not rel.startswith("..") and (repo_root / rel).is_dir():+ out.add(rel)+ return out+++def declared_workspace_members(repo_root: Path) -> dict[str, set[str]]:+ """``{manifest filename: member dirs}`` for every root declaration.++ A declaration whose members expand to nothing (a root-only pnpm workspace,+ ``members = []``) is kept with an empty set: it still says no other+ manifest of its kind is a member.+ """+ js = _read_workspace_declaration(repo_root)+ found = {+ "package.json": (+ None if js.is_empty else _globbed(repo_root, list(js.includes), list(js.excludes))+ ),+ "Cargo.toml": _toml_workspace(repo_root, "Cargo.toml", "workspace"),+ "pyproject.toml": _toml_workspace(repo_root, "pyproject.toml", "tool", "uv", "workspace"),+ "go.mod": _go_work_members(repo_root),+ }+ return {name: dirs for name, dirs in found.items() if dirs is not None}+++def manifest_package_name(manifest: Path) -> str | None:+ """The package name *manifest* declares, or None when it names none."""+ name: object = None+ if manifest.name == "package.json":+ try:+ data = json.loads(manifest.read_text(encoding="utf-8", errors="ignore"))+ except (OSError, ValueError):+ return None+ name = data.get("name") if isinstance(data, dict) else None+ elif manifest.name in ("pyproject.toml", "Cargo.toml"):+ data = _toml(manifest)+ section = data.get("project") or data.get("package")+ if not isinstance(section, dict) or not section.get("name"):+ tool = data.get("tool")+ section = tool.get("poetry") if isinstance(tool, dict) else None+ name = section.get("name") if isinstance(section, dict) else None+ elif manifest.name == "go.mod":+ name = _read_module_directive(manifest)+ return name.strip() if isinstance(name, str) and name.strip() else Nonediff --git a/tests/unit/ingestion/test_workspace_packages.py b/tests/unit/ingestion/test_workspace_packages.pynew file mode 100644index 000000000..ffddc8c64--- /dev/null+++ b/tests/unit/ingestion/test_workspace_packages.py@@ -0,0 +1,136 @@+"""Package detection: any depth, workspace-declared members, no fixtures."""++from __future__ import annotations++import json+from pathlib import Path++from repowise.core.ingestion.traverser import FileTraverser+++def _write(root: Path, rel: str, text: str = "") -> None:+ path = root / rel+ path.parent.mkdir(parents=True, exist_ok=True)+ path.write_text(text)+++def _packages(root: Path) -> dict[str, tuple[str, bool]]:+ packages, _ = FileTraverser(root)._detect_monorepo()+ return {p.path: (p.name, p.declared) for p in packages}+++def _npm(root: Path, rel: str, name: str) -> None:+ _write(root, f"{rel}/package.json", json.dumps({"name": name}))+ _write(root, f"{rel}/src/index.ts", "export {}")+++def test_detect_monorepo_depth3_pnpm(tmp_path: Path) -> None:+ _write(tmp_path, "package.json", json.dumps({"name": "root", "private": True}))+ _write(+ tmp_path,+ "pnpm-workspace.yaml",+ "packages:\n - 'js/libs/*'\n - 'js/libs/plugins/*'\n - '!js/libs/skip'\n",+ )+ _npm(tmp_path, "js/libs/core", "@acme/core")+ _npm(tmp_path, "js/libs/plugins/alpha", "@acme/alpha")+ _npm(tmp_path, "js/libs/skip", "@acme/skip")+ # A package.json the pnpm workspace does not list is not one of its packages.+ _npm(tmp_path, "ts/tools/stray", "stray")++ structure = FileTraverser(tmp_path).get_repo_structure()+ found = {p.path: (p.name, p.declared) for p in structure.packages}+ assert found == {+ "js/libs/core": ("@acme/core", True),+ "js/libs/plugins/alpha": ("@acme/alpha", True),+ }+ assert structure.is_monorepo is True+++def test_fixture_manifests_not_packages(tmp_path: Path) -> None:+ _write(tmp_path, "package.json", json.dumps({"workspaces": ["packages/*"]}))+ _npm(tmp_path, "packages/app", "app")+ # Fixtures and samples carry manifests without being packages.+ _npm(tmp_path, "packages/app/test/fixtures/proj", "fixture-proj")+ _npm(tmp_path, "packages/app/__fixtures__/other", "fixture-other")+ _npm(tmp_path, "examples/demo-app", "demo-app")+ # A template shipped inside a declared member is not a member either.+ _npm(tmp_path, "packages/app/templates/starter", "starter")+ # Other ecosystems with no workspace declaration keep their packages.+ _write(tmp_path, "tools/lint/pyproject.toml", "[project]\nname = 'acme-lint'\n")+ _write(tmp_path, "tools/lint/lint.py", "pass")++ assert _packages(tmp_path) == {+ "packages/app": ("app", True),+ "tools/lint": ("acme-lint", False),+ }+++def test_uv_workspace_members(tmp_path: Path) -> None:+ _write(+ tmp_path,+ "pyproject.toml",+ "[project]\nname = 'root'\n\n[tool.uv.workspace]\n"+ "members = ['libs/*', 'apps/api']\nexclude = ['libs/legacy']\n",+ )+ for rel, name in (("libs/core", "acme-core"), ("libs/legacy", "legacy"), ("apps/api", "api")):+ _write(tmp_path, f"{rel}/pyproject.toml", f"[project]\nname = '{name}'\n")+ _write(tmp_path, f"{rel}/main.py", "pass")+ _write(tmp_path, "scratch/pyproject.toml", "[tool.poetry]\nname = 'scratch'\n")++ assert _packages(tmp_path) == {+ "apps/api": ("api", True),+ "libs/core": ("acme-core", True),+ }+++def test_cargo_workspace_members(tmp_path: Path) -> None:+ _write(+ tmp_path,+ "Cargo.toml",+ "[workspace]\nmembers = ['crates/*']\nexclude = ['crates/old']\n",+ )+ for rel, name in (("crates/cli", "acme-cli"), ("crates/old", "old")):+ _write(tmp_path, f"{rel}/Cargo.toml", f"[package]\nname = '{name}'\n")+ _write(tmp_path, f"{rel}/src/lib.rs", "")+ # A Python eval fixture inside a crate is not a package of the repo.+ _write(tmp_path, "crates/cli/truth/case-one/pyproject.toml", "[project]\nname = 'case'\n")++ assert _packages(tmp_path) == {"crates/cli": ("acme-cli", True)}+++def test_go_work_members(tmp_path: Path) -> None:+ _write(+ tmp_path, "go.work", "go 1.22\n\nuse (\n\t./svc/api // the API\n\t./lib\n)\nuse ./tool\n"+ )+ for rel in ("svc/api", "lib", "tool", "unused"):+ _write(tmp_path, f"{rel}/go.mod", f"module example.com/{rel}\n\ngo 1.22\n")+ _write(tmp_path, f"{rel}/main.go", "package main\n")++ assert _packages(tmp_path) == {+ "lib": ("example.com/lib", True),+ "svc/api": ("example.com/svc/api", True),+ "tool": ("example.com/tool", True),+ }+++def test_no_declaration_keeps_every_root_at_any_depth(tmp_path: Path) -> None:+ _write(tmp_path, "a/b/c/pyproject.toml", "[project]\nname = 'deep'\n")+ _write(tmp_path, "svc/package.json", "{}")++ # Unnamed manifests fall back to the directory name.+ assert _packages(tmp_path) == {"a/b/c": ("deep", False), "svc": ("svc", False)}+++def test_empty_declaration_still_governs_its_kind(tmp_path: Path) -> None:+ # A root-only pnpm workspace and a uv workspace whose globs match nothing+ # are still declarations: nested manifests of their kinds are not members.+ _write(tmp_path, "package.json", json.dumps({"name": "root"}))+ _write(tmp_path, "pnpm-workspace.yaml", "onlyBuiltDependencies:\n - esbuild\n")+ _npm(tmp_path, "scripts/tool", "tool")+ _write(tmp_path, "pyproject.toml", "[tool.uv.workspace]\nmembers = ['libs/*']\n")+ _write(tmp_path, "scratch/pyproject.toml", "[project]\nname = 'scratch'\n")+ # A Cargo workspace table with no members list leaves crates undeclared.+ _write(tmp_path, "Cargo.toml", "[workspace]\n")+ _write(tmp_path, "crates/cli/Cargo.toml", "[package]\nname = 'acme-cli'\n")++ assert _packages(tmp_path) == {"crates/cli": ("acme-cli", False)}