Skip to content

Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.

fix(ingestion): workspace packages at any depth, checked against the declared members (#2685)

Swati Ahuja64d8a38e7f74
diff --git a/packages/core/src/repowise/core/analysis/knowledge_graph.py b/packages/core/src/repowise/core/analysis/knowledge_graph.py
index e14766cfa..36789270d 100644
--- a/packages/core/src/repowise/core/analysis/knowledge_graph.py
+++ b/packages/core/src/repowise/core/analysis/knowledge_graph.py
@@ -281,6 +281,10 @@ def build_knowledge_graph_skeleton(
"is_monorepo": repo_structure.is_monorepo if repo_structure else False,
"total_files": repo_structure.total_files if repo_structure else len(parsed_files),
"entry_points": orientation_entry_points(repo_structure),
+ "packages": [
+ {"name": p.name, "path": p.path, "declared": p.declared}
+ for p in getattr(repo_structure, "packages", None) or []
+ ],
"tech_stack": tech_stack[:20],
}
diff --git a/packages/core/src/repowise/core/ingestion/models.py b/packages/core/src/repowise/core/ingestion/models.py
index 107fe78e5..e5ec83785 100644
--- a/packages/core/src/repowise/core/ingestion/models.py
+++ b/packages/core/src/repowise/core/ingestion/models.py
@@ -168,6 +168,8 @@ class PackageInfo:
language: LanguageTag
entry_points: list[str]
manifest_file: str # pyproject.toml | package.json | Cargo.toml | go.mod
+ # A member of a root workspace declaration (pnpm/npm/yarn, Cargo, uv, go.work).
+ declared: bool = False
@dataclass
diff --git a/packages/core/src/repowise/core/ingestion/package_roots.py b/packages/core/src/repowise/core/ingestion/package_roots.py
index 3f4b0dcce..31dd1154a 100644
--- a/packages/core/src/repowise/core/ingestion/package_roots.py
+++ b/packages/core/src/repowise/core/ingestion/package_roots.py
@@ -64,9 +64,25 @@ def scan_package_roots(
repo_root: str | os.PathLike[str],
*,
is_pruned: Callable[[Path], bool] | None = None,
+ prune_nested_git: bool = True,
) -> set[str]:
"""Repo-relative POSIX directories holding a package manifest.
+ The directory set of :func:`scan_package_manifests`; see it for the walk.
+ """
+ return set(
+ scan_package_manifests(repo_root, is_pruned=is_pruned, prune_nested_git=prune_nested_git)
+ )
+
+
+def scan_package_manifests(
+ repo_root: str | os.PathLike[str],
+ *,
+ is_pruned: Callable[[Path], bool] | None = None,
+ prune_nested_git: bool = True,
+) -> dict[str, frozenset[str]]:
+ """Each package root mapped to the manifest filenames found in it.
+
A manifest at the repo root is omitted: it would put every file in one
bucket, which is the degenerate case package attribution exists to avoid.
@@ -77,21 +93,24 @@ def scan_package_roots(
beneath it. It defaults to a stock :class:`FileTraverser`'s boundary test;
pass a configured traverser's :meth:`~FileTraverser.dir_chain_skipped` (or
use :meth:`FileTraverser.package_root_dirs`) when the repo has submodule
- or exclude settings.
+ or exclude settings. ``prune_nested_git=False`` descends into nested
+ checkouts, for a traverser opted into indexing them.
"""
root = Path(repo_root)
names = package_manifest_names()
prune = is_pruned if is_pruned is not None else _default_prune(root)
- roots: set[str] = set()
+ roots: dict[str, frozenset[str]] = {}
- for dirpath, dirnames, filenames in walk_repo(root):
+ for dirpath, dirnames, filenames in walk_repo(root, prune_nested_git=prune_nested_git):
rel_dir = Path(dirpath).relative_to(root)
# Prune in place so the walk never descends — the point of the scan is
# to be cheap enough to run on every update.
dirnames[:] = [d for d in dirnames if not prune(rel_dir / d)]
- if rel_dir == Path() or not names.intersection(filenames):
+ if rel_dir == Path():
continue
- roots.add(rel_dir.as_posix())
+ matched = names.intersection(filenames)
+ if matched:
+ roots[rel_dir.as_posix()] = matched
return roots
diff --git a/packages/core/src/repowise/core/ingestion/traverser.py b/packages/core/src/repowise/core/ingestion/traverser.py
index 9c31d8860..01a151324 100644
--- a/packages/core/src/repowise/core/ingestion/traverser.py
+++ b/packages/core/src/repowise/core/ingestion/traverser.py
@@ -805,61 +805,79 @@ class FileTraverser:
# ------------------------------------------------------------------
def _detect_monorepo(self) -> tuple[list[PackageInfo], bool]:
- """Detect package sub-directories by looking for manifest files.
-
- Candidate dirs the main traversal would never enter (nested git
- repos, submodules, gitignored/blocked dirs) are rejected up front, so
- a package the walk skips is neither reported nor scanned.
+ """Packages: manifest directories at any depth, checked against workspaces.
+
+ Candidates come from :meth:`package_root_dirs`, so a package the walk
+ skips is neither reported nor scanned. Roots under test or example trees
+ are dropped: a fixture or sample carries a manifest without being one
+ of the repo's packages.
+
+ A root workspace declaration (pnpm/npm/yarn, Cargo, uv, go.work) is the
+ authority for its manifest kind: its members are ``declared``, and an
+ undeclared root is dropped when one of its manifests is a declared kind
+ (the package manager does not build it) or when it sits inside a
+ declared member (a template or fixture shipped with that package).
+ Undeclared roots of kinds no declaration covers are kept.
"""
+ from ..support_paths import is_example_path
+ from .workspace_members import declared_workspace_members, manifest_package_name
+
packages: list[PackageInfo] = []
- seen_paths: set[str] = set()
+ members = declared_workspace_members(self.repo_root)
+ all_members = set().union(*members.values())
# Mirrors GraphBuilder._prune_nested_git.
prune_nested = not (self._include_submodules or self._include_nested_repos)
- for depth in (1, 2):
- pattern = "/".join(["*"] * depth) + "/*"
- for candidate in self.repo_root.glob(pattern):
- if candidate.name not in _MANIFEST_FILES:
- continue
- pkg_dir = candidate.parent
- rel_pkg_path = pkg_dir.relative_to(self.repo_root)
- rel_pkg = rel_pkg_path.as_posix()
- if rel_pkg in seen_paths:
- continue
- if self.dir_chain_skipped(rel_pkg_path):
- continue
- seen_paths.add(rel_pkg)
- lang, entry_pts = _scan_package_dir(
- pkg_dir,
- self.repo_root,
- prune_nested_git=prune_nested,
- is_pruned=self.dir_chain_skipped,
- )
- packages.append(
- PackageInfo(
- name=pkg_dir.name,
- path=rel_pkg,
- language=lang,
- entry_points=entry_pts,
- manifest_file=candidate.name,
- )
+ for rel_pkg, found in sorted(self.package_manifests().items()):
+ pkg_dir = self.repo_root / rel_pkg
+ manifests = sorted(found)
+ probe = f"{rel_pkg}/{manifests[0]}"
+ if is_test_related_path(probe) or is_example_path(probe):
+ continue
+ declaring = [m for m in manifests if rel_pkg in members.get(m, ())]
+ if not declaring and (
+ any(m in members for m in manifests)
+ or any(p.as_posix() in all_members for p in Path(rel_pkg).parents)
+ ):
+ continue
+ manifest = (declaring or manifests)[0]
+ lang, entry_pts = _scan_package_dir(
+ pkg_dir,
+ self.repo_root,
+ prune_nested_git=prune_nested,
+ is_pruned=self.dir_chain_skipped,
+ )
+ packages.append(
+ PackageInfo(
+ name=manifest_package_name(pkg_dir / manifest) or pkg_dir.name,
+ path=rel_pkg,
+ language=lang,
+ entry_points=entry_pts,
+ manifest_file=manifest,
+ declared=bool(declaring),
)
+ )
- packages.sort(key=lambda p: p.path)
return packages, len(packages) > 1
def package_root_dirs(self) -> set[str]:
"""Every directory holding a package manifest, at any depth.
Shares :func:`.package_roots.scan_package_roots` with health's module
- attribution, and this traverser's own skip semantics, so the two agree
- on what a package is. Distinct from :meth:`get_repo_structure`'s
- ``packages``, which stops at depth 2 and pays for language and
- entry-point detection per package.
+ attribution, and this traverser's own skip semantics (nested-repo
+ opt-ins included), so the two agree on what a package is.
"""
- from .package_roots import scan_package_roots
+ return set(self.package_manifests())
- return scan_package_roots(self.repo_root, is_pruned=self.dir_chain_skipped)
+ def package_manifests(self) -> dict[str, frozenset[str]]:
+ """:meth:`package_root_dirs` with the manifest filenames each one holds."""
+ from .package_roots import scan_package_manifests
+
+ return scan_package_manifests(
+ self.repo_root,
+ is_pruned=self.dir_chain_skipped,
+ prune_nested_git=not (self._include_submodules or self._include_nested_repos),
+ )
def dir_chain_skipped(self, rel_dir: Path) -> bool:
"""True if *rel_dir* (or any ancestor) would be pruned by ``_walk``.
diff --git a/packages/core/src/repowise/core/ingestion/workspace_members.py b/packages/core/src/repowise/core/ingestion/workspace_members.py
new file mode 100644
index 000000000..de26d62a5
--- /dev/null
+++ b/packages/core/src/repowise/core/ingestion/workspace_members.py
@@ -0,0 +1,137 @@
+"""Which directories a repo's root workspace manifests declare as members.
+
+A package manager's own member list is the authority on what the repo's
+packages are. A manifest found on disk is only a candidate: fixture
+``package.json`` files, scaffolding templates and docs sites all carry one and
+none of them is a package the workspace builds.
+
+Keyed by the manifest filename a member carries, so the caller can tell which
+ecosystem declared a workspace:
+
+- ``package.json``: ``pnpm-workspace.yaml``, else ``package.json``'s
+ ``workspaces`` (npm, yarn); read by the TypeScript resolver's reader.
+- ``Cargo.toml``: ``[workspace] members`` minus ``exclude``.
+- ``pyproject.toml``: ``[tool.uv.workspace] members`` minus ``exclude``.
+- ``go.mod``: ``go.work``'s ``use`` directives.
+
+Only the repo root's manifests are read. Ceiling: a workspace declared in a
+subdirectory (a JS workspace nested in a Python repo) is not seen, so its
+members stay undeclared; reading nested declarations is the upgrade path.
+"""
+
+from __future__ import annotations
+
+import json
+import posixpath
+import re
+import tomllib
+from pathlib import Path
+
+from .resolvers.go import _read_module_directive
+from .resolvers.ts_workspace import (
+ _expand_member_dirs,
+ _read_workspace_declaration,
+ _WorkspaceDeclaration,
+)
+
+
+def _globbed(repo_root: Path, includes: list, excludes: list) -> set[str]:
+ """Member globs expanded with the JS workspace globber, as repo-relative dirs."""
+ declared = _WorkspaceDeclaration(
+ tuple(p for p in includes if isinstance(p, str)),
+ tuple(p for p in excludes if isinstance(p, str)),
+ include_root=False,
+ )
+ out: set[str] = set()
+ for d in _expand_member_dirs(repo_root, declared):
+ try:
+ rel = d.relative_to(repo_root).as_posix()
+ except ValueError:
+ continue
+ if rel != ".":
+ out.add(rel)
+ return out
+
+
+def _toml(path: Path) -> dict:
+ try:
+ with open(path, "rb") as f:
+ return tomllib.load(f)
+ except (OSError, tomllib.TOMLDecodeError):
+ return {}
+
+
+def _toml_workspace(repo_root: Path, manifest: str, *table: str) -> set[str] | None:
+ """``members``/``exclude`` of a TOML workspace table (Cargo and uv share the shape).
+
+ None without a ``members`` list: Cargo then makes the root's path
+ dependencies members implicitly, which this does not read.
+ """
+ data: object = _toml(repo_root / manifest)
+ for key in table:
+ data = data.get(key) if isinstance(data, dict) else None
+ if not isinstance(data, dict):
+ return None
+ members, exclude = data.get("members"), data.get("exclude")
+ if not isinstance(members, list):
+ return None
+ return _globbed(repo_root, members, exclude if isinstance(exclude, list) else [])
+
+
+_GO_WORK_USE = re.compile(r"^\s*use\s*(?:\(([^)]*)\)|(\S+))", re.MULTILINE)
+
+
+def _go_work_members(repo_root: Path) -> set[str] | None:
+ """Directories ``go.work`` pulls in with ``use`` (single or block form)."""
+ try:
+ text = (repo_root / "go.work").read_text(encoding="utf-8", errors="ignore")
+ except OSError:
+ return None
+ text = re.sub(r"//[^\n]*", "", text)
+ out: set[str] = set()
+ for block, single in _GO_WORK_USE.findall(text):
+ for entry in block.split() if block else [single]:
+ rel = posixpath.normpath(entry.strip('"`'))
+ if rel != "." and not rel.startswith("..") and (repo_root / rel).is_dir():
+ out.add(rel)
+ return out
+
+
+def declared_workspace_members(repo_root: Path) -> dict[str, set[str]]:
+ """``{manifest filename: member dirs}`` for every root declaration.
+
+ A declaration whose members expand to nothing (a root-only pnpm workspace,
+ ``members = []``) is kept with an empty set: it still says no other
+ manifest of its kind is a member.
+ """
+ js = _read_workspace_declaration(repo_root)
+ found = {
+ "package.json": (
+ None if js.is_empty else _globbed(repo_root, list(js.includes), list(js.excludes))
+ ),
+ "Cargo.toml": _toml_workspace(repo_root, "Cargo.toml", "workspace"),
+ "pyproject.toml": _toml_workspace(repo_root, "pyproject.toml", "tool", "uv", "workspace"),
+ "go.mod": _go_work_members(repo_root),
+ }
+ return {name: dirs for name, dirs in found.items() if dirs is not None}
+
+
+def manifest_package_name(manifest: Path) -> str | None:
+ """The package name *manifest* declares, or None when it names none."""
+ name: object = None
+ if manifest.name == "package.json":
+ try:
+ data = json.loads(manifest.read_text(encoding="utf-8", errors="ignore"))
+ except (OSError, ValueError):
+ return None
+ name = data.get("name") if isinstance(data, dict) else None
+ elif manifest.name in ("pyproject.toml", "Cargo.toml"):
+ data = _toml(manifest)
+ section = data.get("project") or data.get("package")
+ if not isinstance(section, dict) or not section.get("name"):
+ tool = data.get("tool")
+ section = tool.get("poetry") if isinstance(tool, dict) else None
+ name = section.get("name") if isinstance(section, dict) else None
+ elif manifest.name == "go.mod":
+ name = _read_module_directive(manifest)
+ return name.strip() if isinstance(name, str) and name.strip() else None
diff --git a/tests/unit/ingestion/test_workspace_packages.py b/tests/unit/ingestion/test_workspace_packages.py
new file mode 100644
index 000000000..ffddc8c64
--- /dev/null
+++ b/tests/unit/ingestion/test_workspace_packages.py
@@ -0,0 +1,136 @@
+"""Package detection: any depth, workspace-declared members, no fixtures."""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+from repowise.core.ingestion.traverser import FileTraverser
+
+
+def _write(root: Path, rel: str, text: str = "") -> None:
+ path = root / rel
+ path.parent.mkdir(parents=True, exist_ok=True)
+ path.write_text(text)
+
+
+def _packages(root: Path) -> dict[str, tuple[str, bool]]:
+ packages, _ = FileTraverser(root)._detect_monorepo()
+ return {p.path: (p.name, p.declared) for p in packages}
+
+
+def _npm(root: Path, rel: str, name: str) -> None:
+ _write(root, f"{rel}/package.json", json.dumps({"name": name}))
+ _write(root, f"{rel}/src/index.ts", "export {}")
+
+
+def test_detect_monorepo_depth3_pnpm(tmp_path: Path) -> None:
+ _write(tmp_path, "package.json", json.dumps({"name": "root", "private": True}))
+ _write(
+ tmp_path,
+ "pnpm-workspace.yaml",
+ "packages:\n - 'js/libs/*'\n - 'js/libs/plugins/*'\n - '!js/libs/skip'\n",
+ )
+ _npm(tmp_path, "js/libs/core", "@acme/core")
+ _npm(tmp_path, "js/libs/plugins/alpha", "@acme/alpha")
+ _npm(tmp_path, "js/libs/skip", "@acme/skip")
+ # A package.json the pnpm workspace does not list is not one of its packages.
+ _npm(tmp_path, "ts/tools/stray", "stray")
+
+ structure = FileTraverser(tmp_path).get_repo_structure()
+ found = {p.path: (p.name, p.declared) for p in structure.packages}
+ assert found == {
+ "js/libs/core": ("@acme/core", True),
+ "js/libs/plugins/alpha": ("@acme/alpha", True),
+ }
+ assert structure.is_monorepo is True
+
+
+def test_fixture_manifests_not_packages(tmp_path: Path) -> None:
+ _write(tmp_path, "package.json", json.dumps({"workspaces": ["packages/*"]}))
+ _npm(tmp_path, "packages/app", "app")
+ # Fixtures and samples carry manifests without being packages.
+ _npm(tmp_path, "packages/app/test/fixtures/proj", "fixture-proj")
+ _npm(tmp_path, "packages/app/__fixtures__/other", "fixture-other")
+ _npm(tmp_path, "examples/demo-app", "demo-app")
+ # A template shipped inside a declared member is not a member either.
+ _npm(tmp_path, "packages/app/templates/starter", "starter")
+ # Other ecosystems with no workspace declaration keep their packages.
+ _write(tmp_path, "tools/lint/pyproject.toml", "[project]\nname = 'acme-lint'\n")
+ _write(tmp_path, "tools/lint/lint.py", "pass")
+
+ assert _packages(tmp_path) == {
+ "packages/app": ("app", True),
+ "tools/lint": ("acme-lint", False),
+ }
+
+
+def test_uv_workspace_members(tmp_path: Path) -> None:
+ _write(
+ tmp_path,
+ "pyproject.toml",
+ "[project]\nname = 'root'\n\n[tool.uv.workspace]\n"
+ "members = ['libs/*', 'apps/api']\nexclude = ['libs/legacy']\n",
+ )
+ for rel, name in (("libs/core", "acme-core"), ("libs/legacy", "legacy"), ("apps/api", "api")):
+ _write(tmp_path, f"{rel}/pyproject.toml", f"[project]\nname = '{name}'\n")
+ _write(tmp_path, f"{rel}/main.py", "pass")
+ _write(tmp_path, "scratch/pyproject.toml", "[tool.poetry]\nname = 'scratch'\n")
+
+ assert _packages(tmp_path) == {
+ "apps/api": ("api", True),
+ "libs/core": ("acme-core", True),
+ }
+
+
+def test_cargo_workspace_members(tmp_path: Path) -> None:
+ _write(
+ tmp_path,
+ "Cargo.toml",
+ "[workspace]\nmembers = ['crates/*']\nexclude = ['crates/old']\n",
+ )
+ for rel, name in (("crates/cli", "acme-cli"), ("crates/old", "old")):
+ _write(tmp_path, f"{rel}/Cargo.toml", f"[package]\nname = '{name}'\n")
+ _write(tmp_path, f"{rel}/src/lib.rs", "")
+ # A Python eval fixture inside a crate is not a package of the repo.
+ _write(tmp_path, "crates/cli/truth/case-one/pyproject.toml", "[project]\nname = 'case'\n")
+
+ assert _packages(tmp_path) == {"crates/cli": ("acme-cli", True)}
+
+
+def test_go_work_members(tmp_path: Path) -> None:
+ _write(
+ tmp_path, "go.work", "go 1.22\n\nuse (\n\t./svc/api // the API\n\t./lib\n)\nuse ./tool\n"
+ )
+ for rel in ("svc/api", "lib", "tool", "unused"):
+ _write(tmp_path, f"{rel}/go.mod", f"module example.com/{rel}\n\ngo 1.22\n")
+ _write(tmp_path, f"{rel}/main.go", "package main\n")
+
+ assert _packages(tmp_path) == {
+ "lib": ("example.com/lib", True),
+ "svc/api": ("example.com/svc/api", True),
+ "tool": ("example.com/tool", True),
+ }
+
+
+def test_no_declaration_keeps_every_root_at_any_depth(tmp_path: Path) -> None:
+ _write(tmp_path, "a/b/c/pyproject.toml", "[project]\nname = 'deep'\n")
+ _write(tmp_path, "svc/package.json", "{}")
+
+ # Unnamed manifests fall back to the directory name.
+ assert _packages(tmp_path) == {"a/b/c": ("deep", False), "svc": ("svc", False)}
+
+
+def test_empty_declaration_still_governs_its_kind(tmp_path: Path) -> None:
+ # A root-only pnpm workspace and a uv workspace whose globs match nothing
+ # are still declarations: nested manifests of their kinds are not members.
+ _write(tmp_path, "package.json", json.dumps({"name": "root"}))
+ _write(tmp_path, "pnpm-workspace.yaml", "onlyBuiltDependencies:\n - esbuild\n")
+ _npm(tmp_path, "scripts/tool", "tool")
+ _write(tmp_path, "pyproject.toml", "[tool.uv.workspace]\nmembers = ['libs/*']\n")
+ _write(tmp_path, "scratch/pyproject.toml", "[project]\nname = 'scratch'\n")
+ # A Cargo workspace table with no members list leaves crates undeclared.
+ _write(tmp_path, "Cargo.toml", "[workspace]\n")
+ _write(tmp_path, "crates/cli/Cargo.toml", "[package]\nname = 'acme-cli'\n")
+
+ assert _packages(tmp_path) == {"crates/cli": ("acme-cli", False)}