Skip to content

The agent that grows with you

fix(tests): share the account tool store only when pm install would not rewrite it

Teknium4bc973abd36a
diff --git a/pm/install.py b/pm/install.py
index c418aa491e5..8ecf63041b8 100644
--- a/pm/install.py
+++ b/pm/install.py
@@ -68,12 +68,15 @@ def _store() -> Store:
return Store(paths.store_root())
-def _heal_exec_bit(binary: Path) -> bool:
+def _heal_exec_bit(binary: Path, *, heal: bool = True) -> bool:
"""Store entries staged from archives that drop the exec bit sit at 0644 and fail every
launch with PermissionError. Modes are not part of the pinned digest, so restore it in place;
- an entry we cannot chmod (sealed store) is treated as not installed."""
+ an entry we cannot chmod (sealed store), or may not (``heal=False``), is treated as not
+ installed."""
if os.name == "nt" or os.access(binary, os.X_OK):
return True
+ if not heal:
+ return False
try:
binary.chmod(binary.stat().st_mode | 0o111)
except OSError:
@@ -83,8 +86,9 @@ def _heal_exec_bit(binary: Path) -> bool:
def _installed_location(package: Package, lockfile: Lockfile, target: str, *,
verify: bool = False, allow_outdated: bool = False,
- roots: tuple[Path, ...] | None = None):
- """Prefer the current pin. An explicit read may retain a prior PM install."""
+ roots: tuple[Path, ...] | None = None, heal: bool = True):
+ """Prefer the current pin. An explicit read may retain a prior PM install. ``heal=False``
+ keeps the lookup read-only (scripts/tool_store_pins.py asks it about a store it must not touch)."""
search_roots = dict.fromkeys(roots if roots is not None else (paths.store_root(), paths.writable_store_root()))
fallback = None
for root in search_roots:
@@ -96,7 +100,7 @@ def _installed_location(package: Package, lockfile: Lockfile, target: str, *,
binary = package.binary(store.entry(fact["entry"]), target)
if binary is not None and not binary.is_file():
continue
- if binary is not None and target == current_target() and not _heal_exec_bit(binary):
+ if binary is not None and target == current_target() and not _heal_exec_bit(binary, heal=heal):
continue
if verify and not _entry_verified(package, fact, store, target):
continue
diff --git a/scripts/run_tests.sh b/scripts/run_tests.sh
index 543f2b31868..4609511eadd 100755
--- a/scripts/run_tests.sh
+++ b/scripts/run_tests.sh
@@ -57,42 +57,50 @@ _has_pytest() { [ -n "$1" ] && [ -x "$1" ] && "$1" -c 'import pytest' 2>/dev/nul
# A test run under a faked HOME (worktree lanes, probe homes) provisions its
# test environment from the OS account's tool store instead of downloading
# ~1 GB of tools into every temporary home, but only while that store already
-# records this checkout's pins (_store_has_pins). Only this runner's own
-# provisioning shares: the pytest
-# children below start from `env -i` without HERMES_RUNTIME_DIR, so tests that
-# exercise setup, install or update keep their own stores. An explicit
+# records this checkout's pins (_store_has_pins). The check runs unlocked
+# before `pm install` takes the store's install lock, so it keeps runs that
+# START on different pins apart; it does not cover the window in between, where
+# the account's own `hermes update` (or a run on other pins) can re-point
+# facts.json and this run's `pm install` then re-stages its pins over them.
+# Only this runner's own provisioning gets HERMES_RUNTIME_DIR: the pytest
+# children below start from `env -i` without it, so tests that exercise setup,
+# install or update resolve their own stores. Their PATH still holds the
+# account store's tool dirs, so a test writing through a PATH tool
+# (`pip install`, `npm -g`) would write into the account store. An explicit
# HERMES_RUNTIME_DIR, a custom HERMES_HOME, a home that already holds a store,
# or an account without a usable store keep today's per-home layout.
+# Prints the account home, then the pinned Python minor.
_account_tool_store() {
- local account real
+ local account py_minor
[ -z "${HERMES_RUNTIME_DIR:-}" ] || return 1
case "${HERMES_HOME:-}" in "" | "$HOME/.hermes" | "$HOME/.hermes/"*) ;; *) return 1 ;; esac
[ ! -f "$HOME/.hermes/tools/facts.json" ] || return 1
account="$(getent passwd "$(id -u)" 2>/dev/null | cut -d: -f6)"
- [ -n "$account" ] || account="$(dscl . -read "/Users/$(id -un)" NFSHomeDirectory 2>/dev/null | awk '{print $2}')"
- [ -n "$account" ] && [ -d "$account" ] || return 1
- real="$(cd "$account" 2>/dev/null && pwd -P)" || return 1
- [ "$real" != "$(cd "$HOME" 2>/dev/null && pwd -P)" ] || return 1
+ [ -n "$account" ] || account="$(dscl . -read "/Users/$(id -un)" NFSHomeDirectory 2>/dev/null | sed -n 's/^NFSHomeDirectory: *//p')"
+ [ -n "$account" ] && [ -d "$account" ] && [ ! "$account" -ef "$HOME" ] || return 1
[ -f "$account/.hermes/tools/facts.json" ] && [ -w "$account/.hermes/tools" ] || return 1
- _store_has_pins "$account/.hermes/tools" || return 1
- printf '%s\n' "$account"
+ py_minor="$(_store_has_pins "$account/.hermes/tools")" || return 1
+ printf '%s\n%s\n' "$account" "$py_minor"
}
# Read-only check that the store already records this checkout's pins
-# (scripts/tool_store_pins.py says why and what counts).
+# (scripts/tool_store_pins.py says why and what counts); prints the Python minor.
_store_has_pins() {
local python
python="$(HERMES_RUNTIME_DIR="$1" hermes_bootstrap_python "$REPO_ROOT")" || return 1
- PYTHONHOME= "$python" -I -B "$REPO_ROOT/scripts/tool_store_pins.py" "$REPO_ROOT" "$1" >/dev/null 2>&1
+ PYTHONHOME= "$python" -I -B "$REPO_ROOT/scripts/tool_store_pins.py" "$REPO_ROOT" "$1" 2>/dev/null
}
-if _account="$(_account_tool_store)"; then
+if _shared="$(_account_tool_store)"; then
+ _account="${_shared%%$'\n'*}"
export HERMES_RUNTIME_DIR="$_account/.hermes/tools"
- # setup-hermes.sh's uv-managed bootstrap interpreter is the same kind of cache.
- if [ -z "${UV_PYTHON_INSTALL_DIR:-}" ] && [ -w "$_account/.local/share/uv/python" ]; then
+ # setup-hermes.sh's uv-managed bootstrap interpreter is the same kind of
+ # cache, but only reused: a missing pinned minor would download into it.
+ if [ -z "${UV_PYTHON_INSTALL_DIR:-}" ] && [ -w "$_account/.local/share/uv/python" ] \
+ && compgen -G "$_account/.local/share/uv/python/cpython-${_shared#*$'\n'}[.-]*" >/dev/null; then
export UV_PYTHON_INSTALL_DIR="$_account/.local/share/uv/python"
fi
echo "▶ faked HOME: using the account's tool store $HERMES_RUNTIME_DIR"
fi
-unset _account
+unset _account _shared
if [ -z "${__HERMES_ACTIVATED:-}" ] && _has_pytest "${HERMES_PYTHON:-}"; then
PYTHON="$HERMES_PYTHON"
echo "▶ not activated — using HERMES_PYTHON: $PYTHON"
@@ -213,6 +221,8 @@ done
# ── Run in hermetic env ──────────────────────────────────────────────────────
# env -i: start with empty environment, opt-in only what we need.
# No credential var can leak — you'd have to explicitly add it here.
+# Never forward HERMES_RUNTIME_DIR or UV_PYTHON_INSTALL_DIR: under a faked HOME
+# they point at the account's shared stores (see _account_tool_store).
echo "▶ running per-file parallel test suite via run_tests_parallel.py"
echo " (TZ=UTC LANG=C.UTF-8 PYTHONHASHSEED=0; clean env)"
diff --git a/scripts/tool_store_pins.py b/scripts/tool_store_pins.py
index 4486b3888a4..b7463faa0ca 100644
--- a/scripts/tool_store_pins.py
+++ b/scripts/tool_store_pins.py
@@ -1,28 +1,26 @@
"""Exit 0 only when a tool store already holds this checkout's pins.
Usage: python -I -B scripts/tool_store_pins.py REPO STORE
+(on success prints the pinned Python major.minor)
scripts/run_tests.sh asks this before sharing the account's tool store with a
faked HOME. facts.json selects ONE entry per package, not one per version:
provisioning a checkout with other pins re-points the account's own runtime at
them and pulls the selection from under a concurrent run on the store's pins.
So share only when every tool `pm install` would provision here (cmd_install's
-roots, defaults and the venv's deps) is already recorded at this checkout's
-pins AND runnable as is: a recorded entry whose binary is missing or not
-executable sends setup-hermes.sh (uv) or `pm install` back into a re-stage
-whose unlocked remove + publish races other runs on the shared store. Unlike
-PM's own lookup this never heals an exec bit. Read-only: no lock, no install,
-no bytecode; any unreadable record or error means "do not share".
+roots, defaults and the venv's deps) passes PM's own installed-at-the-pin lookup
+without an exec-bit heal, and records the env this checkout's package code
+would record: a stale env makes `pm install` re-record the shared facts.json,
+and a missing or non-executable binary (setup-hermes.sh's uv copy included)
+sends setup or `pm install` into a re-stage whose unlocked remove + publish
+races other runs on the shared store. Read-only: no lock, no install, no
+bytecode; any unreadable record or error means "do not share".
"""
import os
import sys
from pathlib import Path
-def _runnable(binary: Path) -> bool:
- return binary.is_file() and (os.name == "nt" or os.access(binary, os.X_OK))
-
-
def _setup_uv(store: Path, target: str, version: str) -> Path:
"""setup-hermes.sh's exact uv path: $store/uv-$uv_version-$os-$arch/uv[.exe]."""
os_arch = "-".join(target.split("-")[:2])
@@ -32,24 +30,32 @@ def _setup_uv(store: Path, target: str, version: str) -> Path:
def main(repo: str, store: Path) -> int:
sys.path.insert(0, repo)
from pm.defaults import default_packages
+ from pm.install import _installed_location
from pm.lock import Facts, Lockfile
from pm.package import StatePackage
from pm.registry import tool_roots, walk
from pm.store import current_target
lock = Lockfile(Path(repo, "pm", "lock.json"))
- facts = Facts(store / "facts.json", strict=True)
+ # Strict first: PM's lenient read would quarantine an unparsable file as .corrupt.
+ Facts(store / "facts.json", strict=True)
names, target = lock.names(), current_target()
for package in walk(tool_roots(names) + default_packages(names) + ["venv"]):
if isinstance(package, StatePackage):
continue
- shas = tuple(a["sha256"] for a in lock.artifacts(package.name, target))
- if not facts.installed(package.name, lock.version(package.name), store, (target, shas) if shas else None):
+ location = _installed_location(package, lock, target, roots=(store,), heal=False)
+ if location is None:
return 1
- binary = package.binary(store / facts.get(package.name)["entry"], target)
- if binary is not None and not _runnable(binary):
+ facts, found = location
+ entry = found.entry(facts.get(package.name)["entry"])
+ if facts.env_for(package.name, found.root) != package.env(entry, target):
return 1
- return 0 if _runnable(_setup_uv(store, target, lock.version("uv"))) else 1
+ uv = _setup_uv(store, target, lock.version("uv"))
+ if not uv.is_file() or not (os.name == "nt" or os.access(uv, os.X_OK)):
+ return 1
+ # The runner reuses uv's interpreter cache only when it already holds this minor.
+ print(".".join(lock.version("python").split("+")[0].split(".")[:2]))
+ return 0
if __name__ == "__main__":
diff --git a/tests/scripts/test_tool_store_pins.py b/tests/scripts/test_tool_store_pins.py
index d5c353b1e08..844af73c448 100644
--- a/tests/scripts/test_tool_store_pins.py
+++ b/tests/scripts/test_tool_store_pins.py
@@ -1,8 +1,10 @@
"""run_tests.sh shares the account's tool store only when it can run from it as is.
-A store whose facts match the checkout's pins but whose uv binary is missing or
-not executable must not be shared: setup-hermes.sh would then re-stage uv into
-the shared store with an unlocked ``rm -rf`` + publish that races other runs.
+A store whose facts match the checkout's pins must still not be shared when
+running from it would write to it: a missing or non-executable binary sends
+setup-hermes.sh (uv) or `pm install` into an unlocked re-stage, and an env that
+differs from what this checkout's package code records makes `pm install`
+re-record the shared facts.json.
"""
import json
import subprocess
@@ -12,39 +14,41 @@ from pathlib import Path
import pytest
from pm.defaults import default_packages
-from pm.lock import Lockfile
+from pm.lock import Facts, Lockfile
from pm.package import StatePackage
-from pm.registry import tool_roots, walk
+from pm.registry import get_package, tool_roots, walk
from pm.store import current_target
ROOT = Path(__file__).resolve().parents[2]
+def _executable(path: Path) -> Path:
+ path.parent.mkdir(parents=True, exist_ok=True)
+ path.write_text("#!/bin/sh\n")
+ path.chmod(0o755)
+ return path
+
+
def _matching_store(store: Path) -> Path:
- """A store whose facts record every walked tool at the lock's pins, each
- entry holding an executable binary. Returns setup-hermes.sh's uv path."""
+ """A store recording every walked tool at the lock's pins with the env PM
+ would record, each entry holding an executable binary, plus setup-hermes.sh's
+ own uv copy. PM's entries are named so none is the setup path, which keeps
+ the two uv checks apart. Returns setup-hermes.sh's uv path."""
lock = Lockfile(ROOT / "pm" / "lock.json")
names, target = lock.names(), current_target()
- setup_target = "-".join(target.split("-")[:2]) # setup-hermes.sh: "$os-$arch"
- packages = {}
+ facts = Facts(store / "facts.json")
for package in walk(tool_roots(names) + default_packages(names) + ["venv"]):
if isinstance(package, StatePackage):
continue
version = lock.version(package.name)
- entry_name = f"{package.name}-{version}-{setup_target if package.name == 'uv' else target}"
- entry = store / entry_name
+ entry = store / f"pm-{package.name}-{version}-{target}"
entry.mkdir(parents=True)
binary = package.binary(entry, target)
- if binary is not None:
- binary.parent.mkdir(parents=True, exist_ok=True)
- binary.write_text("#!/bin/sh\n")
- binary.chmod(0o755)
- packages[package.name] = {
- "entry": entry_name, "version": version, "target": target, "env": {},
- "artifacts": [a["sha256"] for a in lock.artifacts(package.name, target)],
- }
- (store / "facts.json").write_text(json.dumps({"schema": 1, "packages": packages}))
- return store / packages["uv"]["entry"] / "uv"
+ _executable(binary if binary is not None else entry / "chrome") # chromium's is discovered
+ facts.record(package.name, version, entry.name, package.env(entry, target), store,
+ target=target, artifacts=[a["sha256"] for a in lock.artifacts(package.name, target)])
+ # setup-hermes.sh: entry="$store/uv-$uv_version-$os-$arch"
+ return _executable(store / f"uv-{lock.version('uv')}-{'-'.join(target.split('-')[:2])}" / "uv")
def _shares(store: Path) -> bool:
@@ -55,18 +59,37 @@ def _shares(store: Path) -> bool:
return result.returncode == 0
+def _entry_binary(store: Path, name: str) -> Path:
+ fact = Facts(store / "facts.json", strict=True).get(name) or {}
+ binary = get_package(name).binary(store / fact["entry"], current_target())
+ assert binary is not None
+ return binary
+
+
+def _stale_env(store: Path, name: str) -> None:
+ data = json.loads((store / "facts.json").read_text())
+ data["packages"][name]["env"] = {}
+ (store / "facts.json").write_text(json.dumps(data))
+
+
+BREAKAGES = {
+ "setup_uv_missing": lambda store, uv: uv.unlink(),
+ "setup_uv_not_executable": lambda store, uv: uv.chmod(0o644),
+ "ripgrep_not_executable": lambda store, uv: _entry_binary(store, "ripgrep").chmod(0o644),
+ "node_env_stale": lambda store, uv: _stale_env(store, "node"),
+}
+
+
@pytest.mark.platforms("posix")
[email protected]("breakage", ["missing", "not_executable"])
-def test_store_without_a_runnable_uv_is_not_shared(tmp_path, breakage):
[email protected]("breakage", sorted(BREAKAGES))
+def test_store_running_from_which_would_write_to_it_is_not_shared(tmp_path, breakage):
store = tmp_path / "tools"
uv = _matching_store(store)
assert _shares(store), "control: a complete store at this checkout's pins must be shared"
- if breakage == "missing":
- uv.unlink()
- else:
- uv.chmod(0o644)
- facts_before = (store / "facts.json").read_bytes()
+ BREAKAGES[breakage](store, uv)
+ before = sorted((p.relative_to(store), p.is_file() and p.read_bytes(), p.stat().st_mode)
+ for p in store.rglob("*"))
assert not _shares(store)
- assert uv.exists() == (breakage == "not_executable")
- assert breakage == "missing" or not uv.stat().st_mode & 0o111, "the check must stay read-only"
- assert (store / "facts.json").read_bytes() == facts_before
+ after = sorted((p.relative_to(store), p.is_file() and p.read_bytes(), p.stat().st_mode)
+ for p in store.rglob("*"))
+ assert after == before, "the check must stay read-only (no heal, no re-record, no .corrupt)"