The agent that grows with you
Python aiai-agentai-agentsanthropicchatgptclaudeclaude-codecodexhermeshermes-agentllmnous-researchopenai
fix(pm): apply lock overrides to requirements builds; read pip check uncoloured
Maciej Lisiewski2b31d12aded2
diff --git a/pm/build_env.py b/pm/build_env.pyindex 10e29fd30da..067b6a69372 100644--- a/pm/build_env.py+++ b/pm/build_env.py@@ -67,6 +67,8 @@ def main(argv: Sequence[str] | None = None) -> int:parser.add_argument("--wheelhouse", type=Path)parser.add_argument("--requirements", type=Path)parser.add_argument("--requirement", action="append", default=[])+ parser.add_argument("--lock", type=Path,+ help="uv.lock the requirements were exported from; its overrides apply")args = parser.parse_args(argv)requirements = list(args.requirement)if args.requirements is not None:@@ -121,6 +123,7 @@ def main(argv: Sequence[str] | None = None) -> int:executable = pm.build_requirements_environment(requirements, out=args.out, python=args.python, cache=args.cache,wheelhouse=args.wheelhouse, offline=args.offline, sealed=args.sealed, explicit=True,+ lock=args.lock,)else:executable = pm.build_environment(diff --git a/pm/build_operations.py b/pm/build_operations.pyindex ccb23fcdb1c..63d54a2483b 100644--- a/pm/build_operations.py+++ b/pm/build_operations.py@@ -68,12 +68,14 @@ def build_requirements_environment(requirements: Sequence[str], *, out: Path, python: Path | None = None,cache: Path | None = None, env: Mapping[str, str] | None = None,wheelhouse: Path | None = None, offline: bool = False, sealed: bool = False,- explicit: bool = False,+ explicit: bool = False, lock: Path | None = None,) -> Path:"""Create and check a fresh environment. Never mutate an existing destination.Wheelhouse builds disable indexes and source builds. Failure removes only- this invocation's exclusively claimed output, not prior builds.+ this invocation's exclusively claimed output, not prior builds. *lock* is the+ uv.lock the requirements were exported from: its overrides apply to the+ install and the closing check."""from pm.environment import _fresh_build, managed_environmentfrom pm.operations import _require_install_allowed, _requirements@@ -81,6 +83,7 @@ def build_requirements_environment(_require_install_allowed(explicit)requirements = _requirements(requirements) if requirements or isinstance(requirements, str) else []out = Path(out).absolute()+ lock = Path(lock).absolute() if lock is not None else Nonewheelhouse = Path(wheelhouse).absolute() if wheelhouse is not None else Noneif wheelhouse is not None and not wheelhouse.is_dir():raise InstallError("venv", f"wheelhouse directory is missing: {wheelhouse}")@@ -91,8 +94,8 @@ def build_requirements_environment(cache=Path(cache) if cache is not None else None, env=env,offline=offline, explicit=explicit, output=sys.stderr,)- with _fresh_build(environment, sealed=sealed):- environment.install_requirements(requirements, wheelhouse=wheelhouse)+ with _fresh_build(environment, sealed=sealed, lock=lock):+ environment.install_requirements(requirements, wheelhouse=wheelhouse, lock=lock)return environment.executablediff --git a/pm/client.py b/pm/client.pyindex 8d06f12b230..b52f8041f88 100644--- a/pm/client.py+++ b/pm/client.py@@ -402,13 +402,13 @@ def build_requirements_environment(requirements: Sequence[str], *, out: Path,python: Path | None = None, cache: Path | None = None,env: Mapping[str, str] | None = None, wheelhouse: Path | None = None,offline: bool = False, sealed: bool = False,- explicit: bool = False) -> Path:+ explicit: bool = False, lock: Path | None = None) -> Path:if isinstance(requirements, str):raise TypeError("requirements must be a sequence, not a string")return Path(_python_operation("build_requirements_environment", {"requirements": list(requirements), "out": Path(out), "python": python, "cache": cache,"env": dict(env) if env is not None else None, "wheelhouse": wheelhouse,- "offline": offline, "sealed": sealed, "explicit": explicit,+ "offline": offline, "sealed": sealed, "explicit": explicit, "lock": lock,}))diff --git a/pm/environment.py b/pm/environment.pyindex 3eaa1ab8efe..35e2b65547b 100644--- a/pm/environment.py+++ b/pm/environment.py@@ -451,19 +451,33 @@ class PythonEnvironment:if result.returncode:raise classify_uv_failure("export", result.returncode, result.stderr or result.stdout)- def install_requirements(self, requirements: Sequence[str], *, wheelhouse: Path | None = None) -> None:+ def install_requirements(self, requirements: Sequence[str], *, wheelhouse: Path | None = None,+ lock: Path | None = None) -> None:if not requirements:return# A file avoids command-line length limits and shell/marker quoting.with tempfile.TemporaryDirectory(prefix="pm-requirements-") as temporary:requirements_file = Path(temporary) / "requirements.txt"requirements_file.write_text("\n".join(requirements) + "\n", encoding="utf-8")- self._install_requirements_file(requirements_file, wheelhouse=wheelhouse)+ self._install_requirements_file(requirements_file, wheelhouse=wheelhouse, lock=lock)def _install_requirements_file(self, requirements: Path, *, wheelhouse: Path | None = None,- timeout: int = 1800) -> None:+ lock: Path | None = None, timeout: int = 1800) -> None:+ """Install *requirements*; *lock* is the uv.lock they were exported from.++ `uv pip install` re-resolves the export, so the lock's manifest overrides+ must come back as `--overrides` or the graph the lock resolved is rejected.+ """command = ["pip", "install", "--no-config", "--python", str(self.executable),"--requirements", str(requirements)]+ if lock is not None:+ from pm.pip_check import override_requirements++ overrides = override_requirements(lock)+ if overrides:+ overrides_file = requirements.with_name("overrides.txt")+ overrides_file.write_text("\n".join(overrides) + "\n", encoding="utf-8")+ command += ["--overrides", str(overrides_file)]if wheelhouse is not None:command += ["--no-index", "--only-binary", ":all:","--find-links", str(wheelhouse.absolute())]@@ -475,7 +489,8 @@ class PythonEnvironment:"""Install rebuilt wheels whose hashes the bundle manifest owns, not uv.lock."""requirements = source / "requirements.txt"self.export_requirements(source, requirements, timeout=timeout)- self._install_requirements_file(requirements, wheelhouse=wheelhouse, timeout=timeout)+ self._install_requirements_file(requirements, wheelhouse=wheelhouse, lock=source / "uv.lock",+ timeout=timeout)self.check(source / "uv.lock")def prune_cache(self, *, ci: bool = False) -> None:@@ -494,7 +509,8 @@ class PythonEnvironment:a streamed run keeps only the output's tail, which can drop the report's head."""result = replace(self, output=None)._run(- ["pip", "check", "--no-config", "--python", str(self.executable)],+ # Forced colour (FORCE_COLOR/CLICOLOR_FORCE) would wrap the report in ANSI pm.pip_check cannot read.+ ["pip", "check", "--no-config", "--color", "never", "--python", str(self.executable)],cwd=self.destination.parent, timeout=60,)if not result.returncode:diff --git a/pm/pip_check.py b/pm/pip_check.pyindex 35a309ce466..cebb57b5621 100644--- a/pm/pip_check.py+++ b/pm/pip_check.py@@ -7,6 +7,9 @@ the overrides it applied (`[manifest] overrides`) and the version it resolved foeach, so a report line is expected exactly when it names an overriddendependency installed at the locked version. Everything else still fails.+Installs from the lock's requirements export (`uv pip install`) re-resolve, so they+need the same overrides back (override_requirements) to reach the locked graph.+Stdlib only: the bootstrap runner loads pm.environment on a system python thatmay predate tomllib."""@@ -28,26 +31,42 @@ def _normalize(name: str) -> str:return re.sub(r"[-_.]+", "-", name).lower()-def _parse_lock(text: str) -> tuple[list[str], list[tuple[str, str]]]:- """Override names and (name, version) packages from uv.lock text."""+def _parse_lock(text: str) -> tuple[list[dict[str, str]], list[tuple[str, str]]]:+ """Manifest overrides ({name, specifier?, marker?}) and (name, version) packages from uv.lock text."""try:import tomllibexcept ModuleNotFoundError:manifest = re.search(r"(?ms)^\[manifest\]\s*$(.*?)(?=^\[)", text + "\n[")overrides = re.search(r"(?ms)^overrides\s*=\s*\[(.*?)^\]", manifest.group(1)) if manifest else None- names = re.findall(r'\{\s*name\s*=\s*"([^"]+)"', overrides.group(1)) if overrides else []+ tables = re.findall(r"\{([^}]*)\}", overrides.group(1)) if overrides else []+ entries = [dict(re.findall(r'(\w+)\s*=\s*"([^"]*)"', table)) for table in tables]packages = re.findall(r'(?m)^\[\[package\]\]\s*\nname = "([^"]+)"\s*\nversion = "([^"]+)"', text)- return names, packages+ return entries, packagesdata = tomllib.loads(text)- names = [override["name"] for override in data.get("manifest", {}).get("overrides", [])]+ entries = data.get("manifest", {}).get("overrides", [])packages = [(package["name"], package["version"]) for package in data.get("package", []) if "version" in package]- return names, packages+ return entries, packages+++def override_requirements(lock: Path) -> list[str]:+ """The lock's manifest overrides as requirement lines for `uv pip install --overrides`."""+ entries, _ = _parse_lock(lock.read_text(encoding="utf-8"))+ lines = []+ for entry in entries:+ if set(entry) - {"name", "specifier", "marker"}:+ # A source override (URL, path, git) has no requirement-line form here.+ raise ValueError(f"unsupported uv.lock override: {entry}")+ line = entry["name"] + entry.get("specifier", "")+ if entry.get("marker"):+ line += " ; " + entry["marker"]+ lines.append(line)+ return linesdef overridden_versions(lock: Path) -> dict[str, set[str]]:"""Locked versions of every dependency the lock's manifest overrides."""- names, packages = _parse_lock(lock.read_text(encoding="utf-8"))- overridden = {_normalize(name) for name in names}+ entries, packages = _parse_lock(lock.read_text(encoding="utf-8"))+ overridden = {_normalize(entry["name"]) for entry in entries}versions: dict[str, set[str]] = {}for name, version in packages:if _normalize(name) in overridden:diff --git a/pyproject.toml b/pyproject.tomlindex 907d0ab787b..3885a6c8055 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -631,7 +631,8 @@ override-dependencies = [# equal to the direct dependency (uv overrides replace that requirement# too). PM's install-time `uv pip check` accepts the resulting mismatch# because uv.lock records this override (pm/pip_check.py).- # Remove this line when browser-harness allows Pillow 13.+ # Remove this line when browser-harness allows Pillow 13. Plugin workspaces+ # inherit [tool.uv], so a plugin requiring pillow<13 also gets 13.0.0."Pillow==13.0.0",# misaki[en] (pulled by [kittentts]) lists spacy-curated-transformers.# The newest release that fits spacy 3.8 (2.x needs thinc 9), 0.3.1, capsdiff --git a/scripts/termux/build_environment.py b/scripts/termux/build_environment.pyindex dd27c278651..4c18fe0729a 100644--- a/scripts/termux/build_environment.py+++ b/scripts/termux/build_environment.py@@ -30,6 +30,7 @@ def application(root: Path, requirements: Path, python: Path) -> None:build_requirements_environment(requirements.read_text(encoding="utf-8-sig").splitlines(), out=root / "venv",python=python, wheelhouse=root / "wheelhouse", offline=True, sealed=True, explicit=True,+ lock=root / "app" / "uv.lock",)diff --git a/scripts/termux/build_wheels.py b/scripts/termux/build_wheels.pyindex db9fb107e7f..e2be7300345 100644--- a/scripts/termux/build_wheels.py+++ b/scripts/termux/build_wheels.py@@ -4,7 +4,7 @@ against THIS container's bionic python, retag, and run the offlinecompleteness + import gates. Invoked by termux_build.sh inside thedigest-pinned termux/termux-docker container.-Usage: build_wheels.py --resolved RESOLVED.txt --build-set BUILD_SET.txt \+Usage: build_wheels.py --resolved RESOLVED.txt --lock UV.LOCK --build-set BUILD_SET.txt \--wheelhouse DIR --retag RETAG_SCRIPT --platform-tag TAGEverything here runs under the container's termux python (bionic): pip@@ -32,6 +32,7 @@ from urllib.parse import urlsplitdef parse_args() -> argparse.Namespace:p = argparse.ArgumentParser(description=__doc__)p.add_argument("--resolved", required=True, help="tab-separated name/spec/marker file")+ p.add_argument("--lock", required=True, help="uv.lock the resolved graph was exported from")p.add_argument("--build-set", required=True, help="names needing sdist builds (one per line)")p.add_argument("--wheelhouse", required=True)p.add_argument("--retag", required=True, help="path to retag_wheel.py")@@ -346,7 +347,7 @@ def import_native_modules(build_set: list[str]) -> None:print(" imported", module, flush=True)-def wheelhouse_gates(resolved: Path, wheelhouse: Path, build_set: list[str]) -> None:+def wheelhouse_gates(resolved: Path, wheelhouse: Path, build_set: list[str], lock: Path) -> None:"""Offline install of every marker-admitted dep into ONE clean venv,shared by both gates (the full offline install runs once, not twice).@@ -367,6 +368,7 @@ def wheelhouse_gates(resolved: Path, wheelhouse: Path, build_set: list[str]) ->vp = build_requirements_environment(reqs.read_text(encoding="utf-8-sig").splitlines(), out=tmp / "venv",python=Path(sys.executable), wheelhouse=wheelhouse, offline=True, explicit=True,+ lock=lock,)print(" completeness gate: offline install of the marker-admitted graph OK")subprocess.run(@@ -403,7 +405,7 @@ def main() -> int:repaired = repair_wheel(wheel, library)if repaired:print(f" linked {repaired} native extensions to {library.name}: {wheel.name}")- wheelhouse_gates(resolved, wheelhouse, build_set)+ wheelhouse_gates(resolved, wheelhouse, build_set, Path(args.lock))print(f"wheelhouse complete: {len(list(wheelhouse.glob('*.whl')))} wheels in {wheelhouse}")return 0diff --git a/scripts/termux/termux_build.sh b/scripts/termux/termux_build.shindex b351618ecb8..83544eea3c6 100755--- a/scripts/termux/termux_build.sh+++ b/scripts/termux/termux_build.sh@@ -137,6 +137,7 @@ if [ "${1:-}" = "--in-container" ]; thenlog "Building the android wheel set from sdist (bionic, payload ABI)""$BUILD_VENV/bin/python" -u "$HERE/build_wheels.py" \--resolved "$RESOLVED" --build-set "$BUILD_SET" \+ --lock "$PAYLOAD_ROOT/.work/tree/uv.lock" \--wheelhouse "$WHEELHOUSE" --retag "$HERE/retag_wheel.py" \--platform-tag "$PLATFORM_TAG" \|| fail "wheel building failed"diff --git a/tests/pm/test_pip_check.py b/tests/pm/test_pip_check.pyindex 1ac39d57584..9d034cce5e7 100644--- a/tests/pm/test_pip_check.py+++ b/tests/pm/test_pip_check.py@@ -17,7 +17,7 @@ import sysimport pytestfrom pm.package import InstallError-from pm.pip_check import overridden_versions, unexpected_incompatibilities+from pm.pip_check import override_requirements, overridden_versions, unexpected_incompatibilitiesfrom tests.pm._fixtures import _run, _wheelREPO_LOCK = Path(__file__).resolve().parents[2] / "uv.lock"@@ -98,11 +98,17 @@ def test_unrecognised_report_is_never_read_as_clean(lock, report):assert unexpected_incompatibilities(report, lock) is None+def test_overrides_round_trip_as_requirement_lines(lock):+ assert override_requirements(lock) == ["Shared_Dep==2.0", "unused ; sys_platform == 'never'"]++def test_stdlib_fallback_reads_the_real_lock_like_tomllib(monkeypatch):"""The bootstrap runner may lack tomllib; its parser must agree on Hermes's own lock."""with_tomllib = overridden_versions(REPO_LOCK)+ requirements = override_requirements(REPO_LOCK)monkeypatch.setitem(sys.modules, "tomllib", None)assert overridden_versions(REPO_LOCK) == with_tomllib+ assert override_requirements(REPO_LOCK) == requirementsassert with_tomllib, "Hermes's lock declares overrides; an empty read means the parser missed them"@@ -143,11 +149,14 @@ def _overridden_project(tmp_path, overridden, extra_wheels=(), extra_dependenciereturn source, env-def test_build_accepts_the_override_and_check_still_rejects_what_it_does_not_cover(overridden_project, tmp_path):[email protected]("color", [{}, {"FORCE_COLOR": "1", "CLICOLOR_FORCE": "1"}], ids=["plain", "forced-color"])+def test_build_accepts_the_override_and_check_still_rejects_what_it_does_not_cover(overridden_project, tmp_path, color):from pm import build_environmentfrom pm.environment import managed_environmentsource, env = overridden_project+ # Forced colour must not turn uv's report into ANSI the parser cannot read.+ env = {**env, **color}out = tmp_path / "environment"executable = build_environment(explicit=True, source=source, python=Path(sys.executable), out=out,cache=tmp_path / "cache", env=env, offline=True, frozen=True)@@ -169,6 +178,36 @@ def test_build_accepts_the_override_and_check_still_rejects_what_it_does_not_covenvironment.check(stale)+def test_requirements_builds_from_an_overridden_lock_apply_its_overrides(overridden_project, tmp_path):+ """Termux and the PM runtime install the lock's export with `uv pip install`, which+ re-resolves: without the lock's overrides it rejects the graph the lock resolved."""+ from pm import build_requirements_environment, export_requirements+ from pm.environment import ResolutionConflict, managed_environment++ source, env = overridden_project+ wheels = tmp_path / "wheels"+ exported = tmp_path / "requirements.txt"+ export_requirements(source, exported, cache=tmp_path / "cache", env=env, explicit=True)+ requirements = exported.read_text(encoding="utf-8").splitlines()+ assert "shared-dep==2.0" in requirements++ def build(out, **kwargs):+ return build_requirements_environment(requirements, out=tmp_path / out, python=Path(sys.executable),+ wheelhouse=wheels, cache=tmp_path / "cache", env=env,+ offline=True, explicit=True, **kwargs)++ with pytest.raises(ResolutionConflict, match="shared-dep"):+ build("without-lock")+ executable = build("with-lock", lock=source / "uv.lock")+ assert _run([str(executable), "-I", "-c", "import shared_dep; print(shared_dep.__version__)"],+ cwd=tmp_path, env=env) == "2.0"++ environment = managed_environment(tmp_path / "wheelhouse-install", python=Path(sys.executable),+ cache=tmp_path / "cache", env=env, offline=True, explicit=True)+ environment.create()+ environment.install_wheelhouse(source, wheels)++def test_streamed_check_still_sees_failures_ahead_of_a_long_expected_report(tmp_path):"""Interactive builds stream uv output and keep only its tail. An unrelated failurereported before 2,000+ characters of override-covered lines must still fail the check."""diff --git a/tests/scripts/test_pm_build_consumers.py b/tests/scripts/test_pm_build_consumers.pyindex 745c55fd094..171b0e1cb80 100644--- a/tests/scripts/test_pm_build_consumers.py+++ b/tests/scripts/test_pm_build_consumers.py@@ -115,7 +115,11 @@ def test_termux_gate_checks_real_offline_wheels_and_application_uses_same_graph(# This adapter runs on the actual host; no platform faking is needed.if os.name == "nt":resolved.write_text('native-probe\t==1.0\t\t\n', encoding="utf-8")- build_wheels.wheelhouse_gates(resolved, wheels, ["native-probe"])+ # The application build reads the lock from the staged app tree, as in the deb.+ lock = tmp_path / "app" / "uv.lock"+ lock.parent.mkdir()+ lock.write_text('version = 1\n\n[[package]]\nname = "native-probe"\nversion = "1.0"\n', encoding="utf-8")+ build_wheels.wheelhouse_gates(resolved, wheels, ["native-probe"], lock)requirements = tmp_path / "requirements.txt"build_wheels.write_reqs_file(resolved, requirements)requirements.write_bytes(b"\xef\xbb\xbf" + requirements.read_bytes())@@ -125,4 +129,4 @@ def test_termux_gate_checks_real_offline_wheels_and_application_uses_same_graph(for wheel in wheels.iterdir():wheel.unlink()with pytest.raises(InstallError):- build_wheels.wheelhouse_gates(resolved, wheels, ["native-probe"])+ build_wheels.wheelhouse_gates(resolved, wheels, ["native-probe"], lock)