Skip to content

Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.

feat(workspace): link a Python dependency to the sibling repo that publishes it (#4585)

Raghav Chamadiya8e9bd5afe6d0
diff --git a/docs/reference/WORKSPACE_CONTRACTS.md b/docs/reference/WORKSPACE_CONTRACTS.md
index d030344c9..2b0ef7b0d 100644
--- a/docs/reference/WORKSPACE_CONTRACTS.md
+++ b/docs/reference/WORKSPACE_CONTRACTS.md
@@ -194,6 +194,14 @@ directories, tests, fixtures and examples, which covers split packages such as
`src/Illuminate/Support/composer.json`. A name two repos publish links to
neither.
+Python dependencies match the same way. A repo publishes the distribution its root
+`pyproject.toml` names and those of its `[tool.uv.workspace]` members. A dependency
+is read from each of those projects' `pyproject.toml` (`[project]` dependencies and
+optional dependencies, Poetry dependencies and groups) and the `requirements*.txt`
+beside it, and names compare in their PEP 503 spelling (`Acme.Core` is `acme-core`).
+A Poetry path dependency or a `[tool.uv.sources]` path source resolves by its path
+alone. Lock files and `setup.cfg` are not read.
+
Maven matching is filesystem-only and coordinate-based. Repowise resolves local
reactor modules, local parents, properties, and dependency-management versions,
then links an active direct compile/runtime dependency only when exactly one
diff --git a/docs/scale/WORKSPACES.md b/docs/scale/WORKSPACES.md
index 2fe70fd71..76b257100 100644
--- a/docs/scale/WORKSPACES.md
+++ b/docs/scale/WORKSPACES.md
@@ -253,9 +253,9 @@ Reads package manifests (`package.json`, `composer.json`, `pyproject.toml`,
`Cargo.toml`, `go.mod`, `.csproj`, and Maven `pom.xml`) to detect when one repo
depends on another as a package or project.
-npm and composer dependencies match through a local path that resolves into a
-sibling repo, or a package name that exactly one sibling repo publishes; a name
-two repos publish links to neither. Maven matching is filesystem-only: it links
+npm, composer and Python dependencies match through a local path that resolves
+into a sibling repo, or a package name that exactly one sibling repo publishes; a
+name two repos publish links to neither. Maven matching is filesystem-only: it links
an active direct compile/runtime dependency when exactly one workspace project
publishes that `groupId:artifactId`, and never runs Maven or downloads
artifacts. Details: [contract matching reference](../reference/WORKSPACE_CONTRACTS.md#package-matching).
diff --git a/packages/core/src/repowise/core/workspace/manifests.py b/packages/core/src/repowise/core/workspace/manifests.py
index 32dad5ebc..e75a668b5 100644
--- a/packages/core/src/repowise/core/workspace/manifests.py
+++ b/packages/core/src/repowise/core/workspace/manifests.py
@@ -12,9 +12,10 @@ without being a package, while build files such as ``*.csproj`` mark packages
inside one service.
:func:`_index_manifests` reads every repo once up front: one pruned walk finds
-the .NET, Maven and composer manifests, and the declared npm workspace gives
-the npm packages the repo publishes, so a dependency can match a sibling repo
-by package *name*, not only by a relative path.
+the .NET, Maven and composer manifests, the declared npm workspace gives the
+npm packages the repo publishes, and the root ``pyproject.toml`` with its uv
+workspace members gives the Python distributions, so a dependency can match a
+sibling repo by package *name*, not only by a relative path.
"""
from __future__ import annotations
@@ -32,10 +33,15 @@ from repowise.core.ingestion.composer import (
is_package_dir,
read_composer,
)
+from repowise.core.ingestion.external_systems import _dist_key, pypi
from repowise.core.ingestion.resolvers.ts_workspace import (
build_workspace_map,
read_workspaces_field,
)
+from repowise.core.ingestion.workspace_members import (
+ declared_workspace_members,
+ manifest_package_name,
+)
_log = logging.getLogger("repowise.workspace.manifests")
@@ -55,7 +61,7 @@ class CrossRepoPackageDep:
target_repo: str
source_manifest: str
# npm_local_path, npm_workspace, npm_package, composer_path, composer_package,
- # pip_path, cargo_path, go_replace, dotnet_project_ref, dotnet_nuget_internal,
+ # pip_path, pip_package, cargo_path, go_replace, dotnet_project_ref, dotnet_nuget_internal,
# maven_coordinate
kind: str
target_package: str = ""
@@ -101,6 +107,8 @@ class _ManifestIndex:
#: alias -> [(repo-relative manifest path, decoded package.json)]
npm_by_repo: dict[str, list[tuple[str, dict]]] = field(default_factory=dict)
composer_by_repo: dict[str, list[ComposerManifest]] = field(default_factory=dict)
+ #: alias -> repo-relative dirs of the Python projects it ships ("" is the root)
+ python_by_repo: dict[str, list[str]] = field(default_factory=dict)
#: (ecosystem, published name) -> (alias, manifest), None once two repos claim it
published: dict[tuple[str, str], tuple[str, str] | None] = field(default_factory=dict)
@@ -144,6 +152,21 @@ def _npm_packages(repo_path: Path) -> list[tuple[str, dict]]:
return out
+def _python_projects(repo_path: Path) -> list[str]:
+ """The repo root and every declared uv workspace member.
+
+ The same rule as npm: the package manager's member list says which
+ ``pyproject.toml`` files are projects the repo ships, so a fixture or an
+ example project never claims a distribution name.
+ """
+ members = declared_workspace_members(repo_path).get("pyproject.toml", set())
+ return ["", *sorted(members)]
+
+
+def _in_dir(rel_dir: str, filename: str) -> str:
+ return f"{rel_dir}/{filename}" if rel_dir else filename
+
+
def _walk_project_manifests(
repo_root: Path,
index: _ManifestIndex,
@@ -220,6 +243,11 @@ def _index_manifests(repo_paths: dict[str, Path]) -> _ManifestIndex:
for manifest in index.composer_by_repo[alias]:
if manifest.name:
index.claim("composer", manifest.name, alias, _composer_path(manifest))
+ index.python_by_repo[alias] = _python_projects(path)
+ for rel_dir in index.python_by_repo[alias]:
+ rel = _in_dir(rel_dir, "pyproject.toml")
+ if name := manifest_package_name(path / rel):
+ index.claim("pypi", _dist_key(name), alias, rel)
return index
@@ -287,7 +315,7 @@ def _name_dep(
alias: str,
ecosystem: str,
name: str,
- version: str,
+ version: str | None,
manifest: str,
kind: str,
) -> CrossRepoPackageDep | None:
@@ -351,22 +379,75 @@ def _load_toml(path: Path) -> dict:
return {}
-def _scan_pyproject(index: _ManifestIndex, alias: str, repo_path: Path) -> _Deps:
- """Poetry and PEP 621 path dependencies."""
- toml_path = repo_path / "pyproject.toml"
- if not toml_path.is_file():
+def _python_path_sources(data: dict) -> Iterator[tuple[str, str]]:
+ """``(name, path)`` for each Poetry path dependency (groups too) and uv path source."""
+ tool = data.get("tool")
+ tool = tool if isinstance(tool, dict) else {}
+ poetry = tool.get("poetry")
+ poetry = poetry if isinstance(poetry, dict) else {}
+ uv = tool.get("uv")
+ groups = poetry.get("group")
+ groups = groups.values() if isinstance(groups, dict) else ()
+ tables = [
+ poetry.get("dependencies"),
+ uv.get("sources") if isinstance(uv, dict) else None,
+ *(g.get("dependencies") for g in groups if isinstance(g, dict)),
+ ]
+ for table in tables:
+ if not isinstance(table, dict):
+ continue
+ for name, spec in table.items():
+ # uv allows a list of sources split by markers.
+ for one in spec if isinstance(spec, list) else [spec]:
+ if isinstance(one, dict) and isinstance(one.get("path"), str):
+ yield str(name), one["path"]
+
+
+def _is_python_manifest(filename: str) -> bool:
+ lower = filename.lower()
+ return filename == "pyproject.toml" or (
+ lower.startswith("requirements") and lower.endswith(".txt")
+ )
+
+
+def _python_requirements(base: Path, repo_path: Path) -> Iterator[tuple[str, str | None, str]]:
+ """``(PEP 503 name, version, manifest)`` for every requirement in *base*'s manifests."""
+ try:
+ filenames = sorted(f.name for f in base.iterdir() if f.is_file())
+ except OSError:
return
- data = _load_toml(toml_path)
- poetry_deps = data.get("tool", {}).get("poetry", {}).get("dependencies", {})
- specs = list(poetry_deps.values()) if isinstance(poetry_deps, dict) else []
- # PEP 621 dependencies with path (uncommon but possible via tool configs)
- for group_key in ("dependencies", "optional-dependencies"):
- group = data.get("project", {}).get(group_key, {})
- if isinstance(group, dict):
- specs.extend(group.values())
- for spec in specs:
- if isinstance(spec, dict) and "path" in spec:
- yield _path_dep(index, alias, repo_path, spec["path"], "pyproject.toml", "pip_path")
+ for filename in filter(_is_python_manifest, filenames):
+ try:
+ records = pypi.parse(base / filename, repo_path)
+ except Exception: # a malformed manifest must not fail the workspace pass
+ _log.debug("Could not read %s", base / filename, exc_info=True)
+ continue
+ for record in records:
+ # A URL or VCS line (`git+https://...`, `https://...`) reads as
+ # the name `git` / `https`; it names no distribution.
+ if not (record.version or "").startswith(("+", ":")):
+ yield _dist_key(record.name), record.version, record.declared_in
+
+
+def _scan_python(index: _ManifestIndex, alias: str, repo_path: Path) -> _Deps:
+ """Path dependencies into a sibling, and requirements of a sibling's distribution.
+
+ Each project the repo ships is read: its ``pyproject.toml`` (PEP 621,
+ optional dependencies, Poetry and its groups) and the ``requirements*.txt``
+ beside it. Names compare in their PEP 503 spelling. A Poetry path
+ dependency or a uv path source resolves by that path alone, as an npm
+ ``file:`` spec does.
+ """
+ for rel_dir in index.python_by_repo.get(alias, ()):
+ base = repo_path / rel_dir
+ pathed: set[str] = set()
+ rel = _in_dir(rel_dir, "pyproject.toml")
+ for name, ref in _python_path_sources(_load_toml(base / "pyproject.toml")):
+ pathed.add(_dist_key(name))
+ yield _path_dep(index, alias, base, ref, rel, "pip_path")
+ for key, version, manifest in _python_requirements(base, repo_path):
+ if key not in pathed:
+ yield _name_dep(index, alias, "pypi", key, version, manifest, "pip_package")
def _scan_cargo(index: _ManifestIndex, alias: str, repo_path: Path) -> _Deps:
@@ -514,7 +595,7 @@ class Ecosystem:
ECOSYSTEMS: tuple[Ecosystem, ...] = (
Ecosystem("npm", (PACKAGE_JSON,), _scan_npm),
Ecosystem("composer", (COMPOSER_JSON,), _scan_composer),
- Ecosystem("python", ("pyproject.toml", "requirements.txt"), _scan_pyproject),
+ Ecosystem("python", ("pyproject.toml", "requirements.txt"), _scan_python),
Ecosystem("cargo", ("Cargo.toml",), _scan_cargo),
Ecosystem("go", ("go.mod",), _scan_go_mod),
Ecosystem("dotnet", (), _scan_dotnet),
diff --git a/tests/unit/workspace/test_manifests.py b/tests/unit/workspace/test_manifests.py
index 75ed401bd..7b045de9b 100644
--- a/tests/unit/workspace/test_manifests.py
+++ b/tests/unit/workspace/test_manifests.py
@@ -1,4 +1,4 @@
-"""The ecosystem table: package-name edges for npm and composer, and service markers."""
+"""The ecosystem table: package-name edges for npm, composer and Python, and service markers."""
from __future__ import annotations
@@ -141,6 +141,103 @@ class TestComposer:
]
+def _text(path: Path, text: str) -> None:
+ path.parent.mkdir(parents=True, exist_ok=True)
+ path.write_text(text, encoding="utf-8")
+
+
+class TestPythonNames:
+ def test_pinned_dependency_on_a_sibling_distribution(self, tmp_path: Path) -> None:
+ lib, api = tmp_path / "lib", tmp_path / "api"
+ _text(lib / "pyproject.toml", '[project]\nname = "Acme.Core"\n')
+ _text(
+ api / "pyproject.toml",
+ '[project]\nname = "api"\ndependencies = ["acme-core==1.2.0", "httpx>=0.27"]\n',
+ )
+ deps = detect_package_dependencies({"lib": lib, "api": api})
+ assert [(d.source_repo, d.target_repo, d.kind, d.target_package) for d in deps] == [
+ ("api", "lib", "pip_package", "acme-core")
+ ]
+ assert deps[0].target_manifest == "pyproject.toml"
+ assert deps[0].requested_version == "==1.2.0"
+
+ def test_uv_members_publish_and_every_dependency_table_is_read(self, tmp_path: Path) -> None:
+ lib, api = tmp_path / "lib", tmp_path / "api"
+ _text(
+ lib / "pyproject.toml",
+ '[project]\nname = "lib-root"\n[tool.uv.workspace]\nmembers = ["packages/*"]\n',
+ )
+ _text(lib / "packages" / "money" / "pyproject.toml", '[project]\nname = "acme_money"\n')
+ _text(lib / "packages" / "tax" / "pyproject.toml", '[tool.poetry]\nname = "acme-tax"\n')
+ _text(
+ api / "pyproject.toml",
+ '[project]\nname = "api"\n'
+ '[project.optional-dependencies]\nbilling = ["Acme-Money[fast]>=2"]\n'
+ '[tool.poetry.group.dev.dependencies]\nlib-root = "*"\n',
+ )
+ _text(api / "requirements-prod.txt", "acme.tax==3 ; python_version >= '3.11'\n")
+ assert _deps({"lib": lib, "api": api}) == [
+ ("api", "lib", "pyproject.toml", "pip_package", "acme-money"),
+ ("api", "lib", "pyproject.toml", "pip_package", "lib-root"),
+ ("api", "lib", "requirements-prod.txt", "pip_package", "acme-tax"),
+ ]
+
+ def test_non_member_projects_publish_nothing(self, tmp_path: Path) -> None:
+ lib, api = tmp_path / "lib", tmp_path / "api"
+ _text(lib / "pyproject.toml", '[project]\nname = "lib"\n')
+ _text(lib / "tests" / "fixtures" / "pyproject.toml", '[project]\nname = "fixture-app"\n')
+ _text(api / "requirements.txt", "fixture-app\n")
+ assert _deps({"lib": lib, "api": api}) == []
+
+ def test_a_name_two_repos_publish_links_neither(self, tmp_path: Path) -> None:
+ a, b, api = tmp_path / "a", tmp_path / "b", tmp_path / "api"
+ _text(a / "pyproject.toml", '[project]\nname = "shared"\n')
+ _text(b / "pyproject.toml", '[project]\nname = "Shared"\n')
+ _text(api / "requirements.txt", "shared\n")
+ assert _deps({"a": a, "b": b, "api": api}) == []
+
+ def test_a_requirements_only_repo_publishes_nothing(self, tmp_path: Path) -> None:
+ svc, api = tmp_path / "svc", tmp_path / "api"
+ _text(svc / "requirements.txt", "flask\n")
+ _text(api / "requirements.txt", "svc\nflask\n")
+ assert _deps({"svc": svc, "api": api}) == []
+
+ def test_a_url_line_names_no_distribution(self, tmp_path: Path) -> None:
+ git, api = tmp_path / "git", tmp_path / "api"
+ _text(git / "pyproject.toml", '[project]\nname = "git"\n')
+ _text(api / "requirements.txt", "git+https://example.com/lib.git#egg=lib\n")
+ assert _deps({"git": git, "api": api}) == []
+
+ def test_a_poetry_group_path_dependency_resolves_by_path(self, tmp_path: Path) -> None:
+ lib, api = tmp_path / "lib", tmp_path / "api"
+ _text(lib / "pyproject.toml", '[project]\nname = "lib"\n')
+ _text(
+ api / "pyproject.toml",
+ '[tool.poetry]\nname = "api"\n'
+ '[tool.poetry.group.dev.dependencies]\nlib = { path = "../lib", develop = true }\n',
+ )
+ assert _deps({"lib": lib, "api": api}) == [("api", "lib", "pyproject.toml", "pip_path", "")]
+
+ def test_a_path_source_resolves_by_path_alone(self, tmp_path: Path) -> None:
+ lib, api = tmp_path / "lib", tmp_path / "api"
+ _text(lib / "pyproject.toml", '[project]\nname = "lib"\n')
+ _text(
+ api / "pyproject.toml",
+ '[project]\nname = "api"\ndependencies = ["lib"]\n'
+ '[tool.uv.sources]\nlib = { path = "../lib", editable = true }\n',
+ )
+ assert _deps({"lib": lib, "api": api}) == [("api", "lib", "pyproject.toml", "pip_path", "")]
+
+ def test_a_malformed_manifest_yields_nothing_from_it(self, tmp_path: Path) -> None:
+ lib, api = tmp_path / "lib", tmp_path / "api"
+ _text(lib / "pyproject.toml", '[project]\nname = "lib"\n')
+ _text(api / "pyproject.toml", "[project\nname = ")
+ _text(api / "requirements.txt", "lib\n")
+ assert _deps({"lib": lib, "api": api}) == [
+ ("api", "lib", "requirements.txt", "pip_package", "lib")
+ ]
+
+
class TestServiceMarkers:
def test_markers_come_from_the_table(self) -> None:
assert {m for eco in ECOSYSTEMS for m in eco.markers} == SERVICE_MARKERS