Skip to content

The agent that grows with you

fix(pm): apply lock overrides to requirements builds; read pip check uncoloured

Maciej Lisiewski2b31d12aded2
diff --git a/pm/build_env.py b/pm/build_env.py
index 10e29fd30da..067b6a69372 100644
--- a/pm/build_env.py
+++ b/pm/build_env.py
@@ -67,6 +67,8 @@ def main(argv: Sequence[str] | None = None) -> int:
parser.add_argument("--wheelhouse", type=Path)
parser.add_argument("--requirements", type=Path)
parser.add_argument("--requirement", action="append", default=[])
+ parser.add_argument("--lock", type=Path,
+ help="uv.lock the requirements were exported from; its overrides apply")
args = parser.parse_args(argv)
requirements = list(args.requirement)
if args.requirements is not None:
@@ -121,6 +123,7 @@ def main(argv: Sequence[str] | None = None) -> int:
executable = pm.build_requirements_environment(
requirements, out=args.out, python=args.python, cache=args.cache,
wheelhouse=args.wheelhouse, offline=args.offline, sealed=args.sealed, explicit=True,
+ lock=args.lock,
)
else:
executable = pm.build_environment(
diff --git a/pm/build_operations.py b/pm/build_operations.py
index ccb23fcdb1c..63d54a2483b 100644
--- a/pm/build_operations.py
+++ b/pm/build_operations.py
@@ -68,12 +68,14 @@ def build_requirements_environment(
requirements: Sequence[str], *, out: Path, python: Path | None = None,
cache: Path | None = None, env: Mapping[str, str] | None = None,
wheelhouse: Path | None = None, offline: bool = False, sealed: bool = False,
- explicit: bool = False,
+ explicit: bool = False, lock: Path | None = None,
) -> Path:
"""Create and check a fresh environment. Never mutate an existing destination.
Wheelhouse builds disable indexes and source builds. Failure removes only
- this invocation's exclusively claimed output, not prior builds.
+ this invocation's exclusively claimed output, not prior builds. *lock* is the
+ uv.lock the requirements were exported from: its overrides apply to the
+ install and the closing check.
"""
from pm.environment import _fresh_build, managed_environment
from pm.operations import _require_install_allowed, _requirements
@@ -81,6 +83,7 @@ def build_requirements_environment(
_require_install_allowed(explicit)
requirements = _requirements(requirements) if requirements or isinstance(requirements, str) else []
out = Path(out).absolute()
+ lock = Path(lock).absolute() if lock is not None else None
wheelhouse = Path(wheelhouse).absolute() if wheelhouse is not None else None
if wheelhouse is not None and not wheelhouse.is_dir():
raise InstallError("venv", f"wheelhouse directory is missing: {wheelhouse}")
@@ -91,8 +94,8 @@ def build_requirements_environment(
cache=Path(cache) if cache is not None else None, env=env,
offline=offline, explicit=explicit, output=sys.stderr,
)
- with _fresh_build(environment, sealed=sealed):
- environment.install_requirements(requirements, wheelhouse=wheelhouse)
+ with _fresh_build(environment, sealed=sealed, lock=lock):
+ environment.install_requirements(requirements, wheelhouse=wheelhouse, lock=lock)
return environment.executable
diff --git a/pm/client.py b/pm/client.py
index 8d06f12b230..b52f8041f88 100644
--- a/pm/client.py
+++ b/pm/client.py
@@ -402,13 +402,13 @@ def build_requirements_environment(requirements: Sequence[str], *, out: Path,
python: Path | None = None, cache: Path | None = None,
env: Mapping[str, str] | None = None, wheelhouse: Path | None = None,
offline: bool = False, sealed: bool = False,
- explicit: bool = False) -> Path:
+ explicit: bool = False, lock: Path | None = None) -> Path:
if isinstance(requirements, str):
raise TypeError("requirements must be a sequence, not a string")
return Path(_python_operation("build_requirements_environment", {
"requirements": list(requirements), "out": Path(out), "python": python, "cache": cache,
"env": dict(env) if env is not None else None, "wheelhouse": wheelhouse,
- "offline": offline, "sealed": sealed, "explicit": explicit,
+ "offline": offline, "sealed": sealed, "explicit": explicit, "lock": lock,
}))
diff --git a/pm/environment.py b/pm/environment.py
index 3eaa1ab8efe..35e2b65547b 100644
--- a/pm/environment.py
+++ b/pm/environment.py
@@ -451,19 +451,33 @@ class PythonEnvironment:
if result.returncode:
raise classify_uv_failure("export", result.returncode, result.stderr or result.stdout)
- def install_requirements(self, requirements: Sequence[str], *, wheelhouse: Path | None = None) -> None:
+ def install_requirements(self, requirements: Sequence[str], *, wheelhouse: Path | None = None,
+ lock: Path | None = None) -> None:
if not requirements:
return
# A file avoids command-line length limits and shell/marker quoting.
with tempfile.TemporaryDirectory(prefix="pm-requirements-") as temporary:
requirements_file = Path(temporary) / "requirements.txt"
requirements_file.write_text("\n".join(requirements) + "\n", encoding="utf-8")
- self._install_requirements_file(requirements_file, wheelhouse=wheelhouse)
+ self._install_requirements_file(requirements_file, wheelhouse=wheelhouse, lock=lock)
def _install_requirements_file(self, requirements: Path, *, wheelhouse: Path | None = None,
- timeout: int = 1800) -> None:
+ lock: Path | None = None, timeout: int = 1800) -> None:
+ """Install *requirements*; *lock* is the uv.lock they were exported from.
+
+ `uv pip install` re-resolves the export, so the lock's manifest overrides
+ must come back as `--overrides` or the graph the lock resolved is rejected.
+ """
command = ["pip", "install", "--no-config", "--python", str(self.executable),
"--requirements", str(requirements)]
+ if lock is not None:
+ from pm.pip_check import override_requirements
+
+ overrides = override_requirements(lock)
+ if overrides:
+ overrides_file = requirements.with_name("overrides.txt")
+ overrides_file.write_text("\n".join(overrides) + "\n", encoding="utf-8")
+ command += ["--overrides", str(overrides_file)]
if wheelhouse is not None:
command += ["--no-index", "--only-binary", ":all:",
"--find-links", str(wheelhouse.absolute())]
@@ -475,7 +489,8 @@ class PythonEnvironment:
"""Install rebuilt wheels whose hashes the bundle manifest owns, not uv.lock."""
requirements = source / "requirements.txt"
self.export_requirements(source, requirements, timeout=timeout)
- self._install_requirements_file(requirements, wheelhouse=wheelhouse, timeout=timeout)
+ self._install_requirements_file(requirements, wheelhouse=wheelhouse, lock=source / "uv.lock",
+ timeout=timeout)
self.check(source / "uv.lock")
def prune_cache(self, *, ci: bool = False) -> None:
@@ -494,7 +509,8 @@ class PythonEnvironment:
a streamed run keeps only the output's tail, which can drop the report's head.
"""
result = replace(self, output=None)._run(
- ["pip", "check", "--no-config", "--python", str(self.executable)],
+ # Forced colour (FORCE_COLOR/CLICOLOR_FORCE) would wrap the report in ANSI pm.pip_check cannot read.
+ ["pip", "check", "--no-config", "--color", "never", "--python", str(self.executable)],
cwd=self.destination.parent, timeout=60,
)
if not result.returncode:
diff --git a/pm/pip_check.py b/pm/pip_check.py
index 35a309ce466..cebb57b5621 100644
--- a/pm/pip_check.py
+++ b/pm/pip_check.py
@@ -7,6 +7,9 @@ the overrides it applied (`[manifest] overrides`) and the version it resolved fo
each, so a report line is expected exactly when it names an overridden
dependency installed at the locked version. Everything else still fails.
+Installs from the lock's requirements export (`uv pip install`) re-resolve, so they
+need the same overrides back (override_requirements) to reach the locked graph.
+
Stdlib only: the bootstrap runner loads pm.environment on a system python that
may predate tomllib.
"""
@@ -28,26 +31,42 @@ def _normalize(name: str) -> str:
return re.sub(r"[-_.]+", "-", name).lower()
-def _parse_lock(text: str) -> tuple[list[str], list[tuple[str, str]]]:
- """Override names and (name, version) packages from uv.lock text."""
+def _parse_lock(text: str) -> tuple[list[dict[str, str]], list[tuple[str, str]]]:
+ """Manifest overrides ({name, specifier?, marker?}) and (name, version) packages from uv.lock text."""
try:
import tomllib
except ModuleNotFoundError:
manifest = re.search(r"(?ms)^\[manifest\]\s*$(.*?)(?=^\[)", text + "\n[")
overrides = re.search(r"(?ms)^overrides\s*=\s*\[(.*?)^\]", manifest.group(1)) if manifest else None
- names = re.findall(r'\{\s*name\s*=\s*"([^"]+)"', overrides.group(1)) if overrides else []
+ tables = re.findall(r"\{([^}]*)\}", overrides.group(1)) if overrides else []
+ entries = [dict(re.findall(r'(\w+)\s*=\s*"([^"]*)"', table)) for table in tables]
packages = re.findall(r'(?m)^\[\[package\]\]\s*\nname = "([^"]+)"\s*\nversion = "([^"]+)"', text)
- return names, packages
+ return entries, packages
data = tomllib.loads(text)
- names = [override["name"] for override in data.get("manifest", {}).get("overrides", [])]
+ entries = data.get("manifest", {}).get("overrides", [])
packages = [(package["name"], package["version"]) for package in data.get("package", []) if "version" in package]
- return names, packages
+ return entries, packages
+
+
+def override_requirements(lock: Path) -> list[str]:
+ """The lock's manifest overrides as requirement lines for `uv pip install --overrides`."""
+ entries, _ = _parse_lock(lock.read_text(encoding="utf-8"))
+ lines = []
+ for entry in entries:
+ if set(entry) - {"name", "specifier", "marker"}:
+ # A source override (URL, path, git) has no requirement-line form here.
+ raise ValueError(f"unsupported uv.lock override: {entry}")
+ line = entry["name"] + entry.get("specifier", "")
+ if entry.get("marker"):
+ line += " ; " + entry["marker"]
+ lines.append(line)
+ return lines
def overridden_versions(lock: Path) -> dict[str, set[str]]:
"""Locked versions of every dependency the lock's manifest overrides."""
- names, packages = _parse_lock(lock.read_text(encoding="utf-8"))
- overridden = {_normalize(name) for name in names}
+ entries, packages = _parse_lock(lock.read_text(encoding="utf-8"))
+ overridden = {_normalize(entry["name"]) for entry in entries}
versions: dict[str, set[str]] = {}
for name, version in packages:
if _normalize(name) in overridden:
diff --git a/pyproject.toml b/pyproject.toml
index 907d0ab787b..3885a6c8055 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -631,7 +631,8 @@ override-dependencies = [
# equal to the direct dependency (uv overrides replace that requirement
# too). PM's install-time `uv pip check` accepts the resulting mismatch
# because uv.lock records this override (pm/pip_check.py).
- # Remove this line when browser-harness allows Pillow 13.
+ # Remove this line when browser-harness allows Pillow 13. Plugin workspaces
+ # inherit [tool.uv], so a plugin requiring pillow<13 also gets 13.0.0.
"Pillow==13.0.0",
# misaki[en] (pulled by [kittentts]) lists spacy-curated-transformers.
# The newest release that fits spacy 3.8 (2.x needs thinc 9), 0.3.1, caps
diff --git a/scripts/termux/build_environment.py b/scripts/termux/build_environment.py
index dd27c278651..4c18fe0729a 100644
--- a/scripts/termux/build_environment.py
+++ b/scripts/termux/build_environment.py
@@ -30,6 +30,7 @@ def application(root: Path, requirements: Path, python: Path) -> None:
build_requirements_environment(
requirements.read_text(encoding="utf-8-sig").splitlines(), out=root / "venv",
python=python, wheelhouse=root / "wheelhouse", offline=True, sealed=True, explicit=True,
+ lock=root / "app" / "uv.lock",
)
diff --git a/scripts/termux/build_wheels.py b/scripts/termux/build_wheels.py
index db9fb107e7f..e2be7300345 100644
--- a/scripts/termux/build_wheels.py
+++ b/scripts/termux/build_wheels.py
@@ -4,7 +4,7 @@ against THIS container's bionic python, retag, and run the offline
completeness + import gates. Invoked by termux_build.sh inside the
digest-pinned termux/termux-docker container.
-Usage: build_wheels.py --resolved RESOLVED.txt --build-set BUILD_SET.txt \
+Usage: build_wheels.py --resolved RESOLVED.txt --lock UV.LOCK --build-set BUILD_SET.txt \
--wheelhouse DIR --retag RETAG_SCRIPT --platform-tag TAG
Everything here runs under the container's termux python (bionic): pip
@@ -32,6 +32,7 @@ from urllib.parse import urlsplit
def parse_args() -> argparse.Namespace:
p = argparse.ArgumentParser(description=__doc__)
p.add_argument("--resolved", required=True, help="tab-separated name/spec/marker file")
+ p.add_argument("--lock", required=True, help="uv.lock the resolved graph was exported from")
p.add_argument("--build-set", required=True, help="names needing sdist builds (one per line)")
p.add_argument("--wheelhouse", required=True)
p.add_argument("--retag", required=True, help="path to retag_wheel.py")
@@ -346,7 +347,7 @@ def import_native_modules(build_set: list[str]) -> None:
print(" imported", module, flush=True)
-def wheelhouse_gates(resolved: Path, wheelhouse: Path, build_set: list[str]) -> None:
+def wheelhouse_gates(resolved: Path, wheelhouse: Path, build_set: list[str], lock: Path) -> None:
"""Offline install of every marker-admitted dep into ONE clean venv,
shared by both gates (the full offline install runs once, not twice).
@@ -367,6 +368,7 @@ def wheelhouse_gates(resolved: Path, wheelhouse: Path, build_set: list[str]) ->
vp = build_requirements_environment(
reqs.read_text(encoding="utf-8-sig").splitlines(), out=tmp / "venv",
python=Path(sys.executable), wheelhouse=wheelhouse, offline=True, explicit=True,
+ lock=lock,
)
print(" completeness gate: offline install of the marker-admitted graph OK")
subprocess.run(
@@ -403,7 +405,7 @@ def main() -> int:
repaired = repair_wheel(wheel, library)
if repaired:
print(f" linked {repaired} native extensions to {library.name}: {wheel.name}")
- wheelhouse_gates(resolved, wheelhouse, build_set)
+ wheelhouse_gates(resolved, wheelhouse, build_set, Path(args.lock))
print(f"wheelhouse complete: {len(list(wheelhouse.glob('*.whl')))} wheels in {wheelhouse}")
return 0
diff --git a/scripts/termux/termux_build.sh b/scripts/termux/termux_build.sh
index b351618ecb8..83544eea3c6 100755
--- a/scripts/termux/termux_build.sh
+++ b/scripts/termux/termux_build.sh
@@ -137,6 +137,7 @@ if [ "${1:-}" = "--in-container" ]; then
log "Building the android wheel set from sdist (bionic, payload ABI)"
"$BUILD_VENV/bin/python" -u "$HERE/build_wheels.py" \
--resolved "$RESOLVED" --build-set "$BUILD_SET" \
+ --lock "$PAYLOAD_ROOT/.work/tree/uv.lock" \
--wheelhouse "$WHEELHOUSE" --retag "$HERE/retag_wheel.py" \
--platform-tag "$PLATFORM_TAG" \
|| fail "wheel building failed"
diff --git a/tests/pm/test_pip_check.py b/tests/pm/test_pip_check.py
index 1ac39d57584..9d034cce5e7 100644
--- a/tests/pm/test_pip_check.py
+++ b/tests/pm/test_pip_check.py
@@ -17,7 +17,7 @@ import sys
import pytest
from pm.package import InstallError
-from pm.pip_check import overridden_versions, unexpected_incompatibilities
+from pm.pip_check import override_requirements, overridden_versions, unexpected_incompatibilities
from tests.pm._fixtures import _run, _wheel
REPO_LOCK = Path(__file__).resolve().parents[2] / "uv.lock"
@@ -98,11 +98,17 @@ def test_unrecognised_report_is_never_read_as_clean(lock, report):
assert unexpected_incompatibilities(report, lock) is None
+def test_overrides_round_trip_as_requirement_lines(lock):
+ assert override_requirements(lock) == ["Shared_Dep==2.0", "unused ; sys_platform == 'never'"]
+
+
def test_stdlib_fallback_reads_the_real_lock_like_tomllib(monkeypatch):
"""The bootstrap runner may lack tomllib; its parser must agree on Hermes's own lock."""
with_tomllib = overridden_versions(REPO_LOCK)
+ requirements = override_requirements(REPO_LOCK)
monkeypatch.setitem(sys.modules, "tomllib", None)
assert overridden_versions(REPO_LOCK) == with_tomllib
+ assert override_requirements(REPO_LOCK) == requirements
assert with_tomllib, "Hermes's lock declares overrides; an empty read means the parser missed them"
@@ -143,11 +149,14 @@ def _overridden_project(tmp_path, overridden, extra_wheels=(), extra_dependencie
return source, env
-def test_build_accepts_the_override_and_check_still_rejects_what_it_does_not_cover(overridden_project, tmp_path):
[email protected]("color", [{}, {"FORCE_COLOR": "1", "CLICOLOR_FORCE": "1"}], ids=["plain", "forced-color"])
+def test_build_accepts_the_override_and_check_still_rejects_what_it_does_not_cover(overridden_project, tmp_path, color):
from pm import build_environment
from pm.environment import managed_environment
source, env = overridden_project
+ # Forced colour must not turn uv's report into ANSI the parser cannot read.
+ env = {**env, **color}
out = tmp_path / "environment"
executable = build_environment(explicit=True, source=source, python=Path(sys.executable), out=out,
cache=tmp_path / "cache", env=env, offline=True, frozen=True)
@@ -169,6 +178,36 @@ def test_build_accepts_the_override_and_check_still_rejects_what_it_does_not_cov
environment.check(stale)
+def test_requirements_builds_from_an_overridden_lock_apply_its_overrides(overridden_project, tmp_path):
+ """Termux and the PM runtime install the lock's export with `uv pip install`, which
+ re-resolves: without the lock's overrides it rejects the graph the lock resolved."""
+ from pm import build_requirements_environment, export_requirements
+ from pm.environment import ResolutionConflict, managed_environment
+
+ source, env = overridden_project
+ wheels = tmp_path / "wheels"
+ exported = tmp_path / "requirements.txt"
+ export_requirements(source, exported, cache=tmp_path / "cache", env=env, explicit=True)
+ requirements = exported.read_text(encoding="utf-8").splitlines()
+ assert "shared-dep==2.0" in requirements
+
+ def build(out, **kwargs):
+ return build_requirements_environment(requirements, out=tmp_path / out, python=Path(sys.executable),
+ wheelhouse=wheels, cache=tmp_path / "cache", env=env,
+ offline=True, explicit=True, **kwargs)
+
+ with pytest.raises(ResolutionConflict, match="shared-dep"):
+ build("without-lock")
+ executable = build("with-lock", lock=source / "uv.lock")
+ assert _run([str(executable), "-I", "-c", "import shared_dep; print(shared_dep.__version__)"],
+ cwd=tmp_path, env=env) == "2.0"
+
+ environment = managed_environment(tmp_path / "wheelhouse-install", python=Path(sys.executable),
+ cache=tmp_path / "cache", env=env, offline=True, explicit=True)
+ environment.create()
+ environment.install_wheelhouse(source, wheels)
+
+
def test_streamed_check_still_sees_failures_ahead_of_a_long_expected_report(tmp_path):
"""Interactive builds stream uv output and keep only its tail. An unrelated failure
reported before 2,000+ characters of override-covered lines must still fail the check."""
diff --git a/tests/scripts/test_pm_build_consumers.py b/tests/scripts/test_pm_build_consumers.py
index 745c55fd094..171b0e1cb80 100644
--- a/tests/scripts/test_pm_build_consumers.py
+++ b/tests/scripts/test_pm_build_consumers.py
@@ -115,7 +115,11 @@ def test_termux_gate_checks_real_offline_wheels_and_application_uses_same_graph(
# This adapter runs on the actual host; no platform faking is needed.
if os.name == "nt":
resolved.write_text('native-probe\t==1.0\t\t\n', encoding="utf-8")
- build_wheels.wheelhouse_gates(resolved, wheels, ["native-probe"])
+ # The application build reads the lock from the staged app tree, as in the deb.
+ lock = tmp_path / "app" / "uv.lock"
+ lock.parent.mkdir()
+ lock.write_text('version = 1\n\n[[package]]\nname = "native-probe"\nversion = "1.0"\n', encoding="utf-8")
+ build_wheels.wheelhouse_gates(resolved, wheels, ["native-probe"], lock)
requirements = tmp_path / "requirements.txt"
build_wheels.write_reqs_file(resolved, requirements)
requirements.write_bytes(b"\xef\xbb\xbf" + requirements.read_bytes())
@@ -125,4 +129,4 @@ def test_termux_gate_checks_real_offline_wheels_and_application_uses_same_graph(
for wheel in wheels.iterdir():
wheel.unlink()
with pytest.raises(InstallError):
- build_wheels.wheelhouse_gates(resolved, wheels, ["native-probe"])
+ build_wheels.wheelhouse_gates(resolved, wheels, ["native-probe"], lock)