Skip to content

HarnessRouter Community Edition: the self-hosted, Apache-2.0 edition of the unified interface for agent harnesses. Run Codex, Claude Code, Hermes, PI, DSH, and more through one API, with sessions, streaming, files, cancellation, and failure handling. Implements the Unified Harness Protocol (UHP), an open standard. Your keys, your infrastructure.

Environments: a build source snapshot is held against every change to the source (refs #425) (#447)

hahahahahayesyeseys23a84e5db2f9
diff --git a/runner/environments.py b/runner/environments.py
index a41c6c2b..4fdfbfd3 100644
--- a/runner/environments.py
+++ b/runner/environments.py
@@ -29,6 +29,7 @@ with EACCES; that is the enforcement, not an instruction.
"""
from __future__ import annotations
+import asyncio
import grp
import hashlib
import json
@@ -85,6 +86,17 @@ _gid_lock = threading.Lock()
router = APIRouter()
_builds_lock = threading.Lock()
_builds: dict[str, threading.Thread] = {} # "<id>:<n>" -> the thread building it
+_source_locks: dict[str, threading.Lock] = {}
+_source_locks_guard = threading.Lock()
+
+
+def _source_lock(env_id: str) -> threading.Lock:
+ """Orders every change to an environment's source against the snapshot a build takes of it, so
+ a version is one whole state of the source and never a mix of two. The runner serves
+ environments from this one process, which is why a threading.Lock per environment id is enough;
+ a runner that served them from several processes would need the same ordering across them."""
+ with _source_locks_guard:
+ return _source_locks.setdefault(str(env_id), threading.Lock())
# ── paths ───────────────────────────────────────────────────────────────────────────────────────
@@ -315,28 +327,31 @@ def write_file(env_id: str, rel: str, data: bytes) -> dict:
p = _safe_rel(src, rel)
if p.is_dir():
raise HTTPException(409, "a directory is at that path")
- p.parent.mkdir(parents=True, exist_ok=True)
- tmp = p.with_name(p.name + ".hr-tmp")
- tmp.write_bytes(data)
- os.replace(tmp, p)
+ with _source_lock(env_id):
+ p.parent.mkdir(parents=True, exist_ok=True)
+ tmp = p.with_name(p.name + ".hr-tmp")
+ tmp.write_bytes(data)
+ os.replace(tmp, p)
return {"path": rel.strip("/"), "bytes": len(data)}
def make_dir(env_id: str, rel: str) -> dict:
src = _source(env_id)
p = _safe_rel(src, rel)
- p.mkdir(parents=True, exist_ok=True)
+ with _source_lock(env_id):
+ p.mkdir(parents=True, exist_ok=True)
return {"path": rel.strip("/"), "dir": True}
def delete_path(env_id: str, rel: str) -> dict:
p = _safe_rel(source_dir(env_id), rel)
- if p.is_dir() and not p.is_symlink():
- shutil.rmtree(p)
- elif p.exists() or p.is_symlink():
- p.unlink()
- else:
- raise HTTPException(404, "nothing at that path")
+ with _source_lock(env_id):
+ if p.is_dir() and not p.is_symlink():
+ shutil.rmtree(p)
+ elif p.exists() or p.is_symlink():
+ p.unlink()
+ else:
+ raise HTTPException(404, "nothing at that path")
return {"path": rel.strip("/"), "deleted": True}
@@ -369,7 +384,15 @@ def _member_ok(name: str) -> str | None:
def import_archive(env_id: str, data_path: str, *, replace: bool = False) -> dict:
"""Extract a zip or tar archive into the source. Only regular files and directories land:
symlinks, hard links, devices and any member that names a path outside the source are
- dropped and counted, never written. A single wrapping directory is stripped."""
+ dropped and counted, never written. A single wrapping directory is stripped.
+
+ The lock is held from the replace through the last member, so a build snapshotting while this
+ runs waits instead of copying a tree half-refilled. The wait is bounded by the archive."""
+ with _source_lock(env_id):
+ return _import_archive(env_id, data_path, replace=replace)
+
+
+def _import_archive(env_id: str, data_path: str, *, replace: bool = False) -> dict:
src = source_dir(env_id)
if replace and src.exists():
shutil.rmtree(src)
@@ -513,10 +536,13 @@ def import_git(env_id: str, url: str, ref: str = "", *, replace: bool = False, p
except subprocess.TimeoutExpired:
raise HTTPException(504, "git clone did not finish in 10 minutes")
shutil.rmtree(tmp + "/repo/.git", ignore_errors=True)
- src = source_dir(env_id)
- if replace and src.exists():
- shutil.rmtree(src)
- written, skipped = _copy_without_links(tmp + "/repo", _source(env_id))
+ # The clone ran outside the lock, into its own temporary directory, so a slow network never
+ # holds up a build. Only swapping the source in needs ordering against a snapshot.
+ with _source_lock(env_id):
+ src = source_dir(env_id)
+ if replace and src.exists():
+ shutil.rmtree(src)
+ written, skipped = _copy_without_links(tmp + "/repo", _source(env_id))
return {"imported": "git", "url": url, "ref": ref, "written": written, "skipped": skipped, **source_stat(env_id)}
@@ -889,9 +915,12 @@ def _build(env_id: str, n: int, slug: str, activate: bool, spec: dict | None = N
deadline = time.time() + BUILD_TIMEOUT
env = _tool_env()
stage("copying files")
- log.append(f"copying the project ({source_stat(env_id)['count']} files)")
- shutil.copytree(src, dst, symlinks=False, dirs_exist_ok=True,
- ignore=lambda d, names: [x for x in names if x in NOT_COPIED])
+ # The count and the copy are one reading of the tree: a source write waits here, so a version
+ # holds one whole state of the source and the log counts what this build actually copied.
+ with _source_lock(env_id):
+ log.append(f"copying the project ({source_stat(env_id)['count']} files)")
+ shutil.copytree(src, dst, symlinks=False, dirs_exist_ok=True,
+ ignore=lambda d, names: [x for x in names if x in NOT_COPIED])
as_build = _build_as(env_id)
with tempfile.TemporaryDirectory(prefix="hr-env-build-") as scratch:
env["HOME"] = env["TMPDIR"] = scratch
@@ -1152,7 +1181,10 @@ def r_read(env_id: str, path: str) -> Response:
@router.put("/environments/{env_id}/source")
async def r_write(env_id: str, path: str, request: Request) -> dict:
- return write_file(env_id, path, await request.body())
+ data = await request.body()
+ # Off the event loop, as the import is: a write waits for the source lock while a build
+ # snapshots the source, and the runner answers every turn from this one loop.
+ return await asyncio.to_thread(write_file, env_id, path, data)
@router.post("/environments/{env_id}/mkdir")
@@ -1167,17 +1199,21 @@ def r_delete(env_id: str, path: str) -> dict:
@router.post("/environments/{env_id}/import")
async def r_import(env_id: str, request: Request, replace: int = 0, git_url: str = "", git_ref: str = "") -> dict:
+ # The clone, the extraction and the wait for the source lock all run in a worker thread. Called
+ # here directly they ran on the event loop the runner answers every turn from, so a git import
+ # froze the whole runner for as long as the clone took (its /healthz answered in 4.3 s during
+ # a 4.6 s import of django; a clone may take up to ten minutes).
if git_url:
# git_pin: host:port:address the gateway classified, on a shared deployment (_git_pin_args)
- return import_git(env_id, git_url, git_ref, replace=bool(replace),
- pin=str(request.query_params.get("git_pin") or ""))
+ return await asyncio.to_thread(import_git, env_id, git_url, git_ref, replace=bool(replace),
+ pin=str(request.query_params.get("git_pin") or ""))
fd, spool = tempfile.mkstemp(prefix="hr-env-import-")
try:
with os.fdopen(fd, "wb") as out:
async for chunk in request.stream():
if chunk:
out.write(chunk)
- return import_archive(env_id, spool, replace=bool(replace))
+ return await asyncio.to_thread(import_archive, env_id, spool, replace=bool(replace))
finally:
try:
os.unlink(spool)
diff --git a/runner/tests/test_environments.py b/runner/tests/test_environments.py
index 1e844e4d..114d80d9 100644
--- a/runner/tests/test_environments.py
+++ b/runner/tests/test_environments.py
@@ -9,6 +9,7 @@ import pathlib
import stat
import sys
import tarfile
+import threading
import time
import zipfile
@@ -185,6 +186,125 @@ def test_when_the_record_says_ready_the_version_is_already_the_active_one(store,
assert seen == [(1, 1)]
+def test_a_source_write_during_a_build_snapshot_waits_for_the_snapshot(store, monkeypatch):
+ """#425: `_build` copied the source in its own thread while `write_file` could still land, so a
+ version could hold `a/` from before a write and `z/` from after it — a state the source was
+ never in. The snapshot takes the environment's source lock, and this asks the question at the
+ moment the copy is standing between two directories."""
+ E.write_file("henv_k", "a/one.py", b"old a\n")
+ E.write_file("henv_k", "z/two.py", b"old z\n")
+ inside_copy, release = threading.Event(), threading.Event()
+ real_copytree = E.shutil.copytree
+
+ def slow_copytree(src, dst, **kw):
+ real_copytree(pathlib.Path(src) / "a", pathlib.Path(dst) / "a", dirs_exist_ok=True)
+ inside_copy.set() # the build is mid-copy, past a/, before z/
+ release.wait(30)
+ real_copytree(pathlib.Path(src) / "z", pathlib.Path(dst) / "z", dirs_exist_ok=True)
+ monkeypatch.setattr(E.shutil, "copytree", slow_copytree)
+
+ # the count in the build log is read with the same hold the copy takes, so it describes the
+ # tree this build copied rather than whatever the source looked like a moment later
+ reads = []
+ real_count = E.source_stat
+
+ def spy_count(env):
+ out = real_count(env)
+ lock = getattr(E, "_source_lock", None) # absent before the fix: the read is not held
+ reads.append((out["count"], bool(lock is not None and lock(env).locked())))
+ return out
+ monkeypatch.setattr(E, "source_stat", spy_count)
+
+ landed = threading.Event()
+
+ def writer():
+ E.write_file("henv_k", "z/three.py", b"new z\n")
+ landed.set()
+ thread = threading.Thread(target=writer)
+ E.start_build("henv_k", 1, "mixed")
+ assert inside_copy.wait(30), "the build never reached the copy"
+ thread.start()
+ assert not landed.wait(0.5), "the write landed while the build was inside the snapshot"
+ release.set()
+ thread.join(30)
+ assert landed.is_set()
+
+ rec = _wait("henv_k", 1)
+ assert rec["status"] == "ready", rec["log"]
+ v1 = E.version_dir("henv_k", 1)
+ assert (v1 / "a" / "one.py").read_bytes() == b"old a\n"
+ assert (v1 / "z" / "two.py").read_bytes() == b"old z\n"
+ assert not (v1 / "z" / "three.py").exists() # one whole state, not a mix of two
+ assert (2, True) in reads, reads # a/one.py + z/two.py, counted under the hold
+ # the write is not lost, only ordered: the next build holds it
+ E.start_build("henv_k", 2, "mixed")
+ assert _wait("henv_k", 2)["status"] == "ready"
+ assert (E.version_dir("henv_k", 2) / "z" / "three.py").read_bytes() == b"new z\n"
+
+
[email protected]("change,env_id", [("make_dir", "henv_md"), ("delete_path", "henv_dp"),
+ ("import_archive", "henv_ia"), ("import_git", "henv_ig")])
+def test_the_snapshot_holds_every_change_to_the_source_not_only_a_write(store, monkeypatch, change, env_id):
+ """#425 asks for the interleaving to be tested rather than for a lock to exist. `make_dir`,
+ `delete_path` and the two imports change the source just as `write_file` does, so each is
+ another way a version can hold a state the source was never in."""
+ E.write_file(env_id, "a/one.py", b"old a\n")
+ E.write_file(env_id, "z/two.py", b"old z\n")
+ archive = store / "one.zip"
+ buf = io.BytesIO()
+ with zipfile.ZipFile(buf, "w") as zf:
+ zf.writestr("from_archive.txt", "in archive\n")
+ archive.write_bytes(buf.getvalue())
+ if change == "import_git":
+ # the clone is the part that must stay OUTSIDE the hold, so it is not what is being asked
+ # here: git is answered without a network and the swap-in is what the test watches
+ def no_clone(cmd, **kw):
+ class R:
+ returncode, stdout, stderr = 0, "", ""
+ return R()
+ monkeypatch.setattr(E.subprocess, "run", no_clone)
+
+ inside_copy, release, landed = threading.Event(), threading.Event(), threading.Event()
+ real_copytree = E.shutil.copytree
+
+ def slow_copytree(src, dst, **kw):
+ real_copytree(pathlib.Path(src) / "a", pathlib.Path(dst) / "a", dirs_exist_ok=True)
+ inside_copy.set()
+ release.wait(30)
+ real_copytree(pathlib.Path(src) / "z", pathlib.Path(dst) / "z", dirs_exist_ok=True)
+ monkeypatch.setattr(E.shutil, "copytree", slow_copytree)
+
+ def worker():
+ if change == "make_dir":
+ E.make_dir(env_id, "z/deep")
+ elif change == "delete_path":
+ E.delete_path(env_id, "z/two.py")
+ elif change == "import_git":
+ E.import_git(env_id, "https://example.invalid/repository.git")
+ else:
+ E.import_archive(env_id, str(archive), replace=False)
+ landed.set()
+ thread = threading.Thread(target=worker)
+ E.start_build(env_id, 1, "held")
+ assert inside_copy.wait(30), "the build never reached the copy"
+ thread.start()
+ assert not landed.wait(0.5), f"{change} landed while the build was inside the snapshot"
+ release.set()
+ thread.join(30)
+ assert landed.is_set(), f"{change} never finished after the snapshot released"
+
+ paths = {e["path"] for e in E.tree(env_id)}
+ if change == "make_dir":
+ assert "z/deep" in paths
+ elif change == "delete_path":
+ assert "z/two.py" not in paths
+ elif change == "import_git":
+ assert {"a/one.py", "z/two.py"} <= paths # swapped in, and the source is still there
+ else:
+ assert "from_archive.txt" in paths
+ assert _wait(env_id, 1)["status"] == "ready"
+
+
def test_a_build_whose_last_write_fails_leaves_the_active_version_as_it_was(store, monkeypatch):
E.write_file("henv_l", "setup.sh", b"#!/bin/sh\necho preparing\n")
E.start_build("henv_l", 1, "last")
@@ -534,3 +654,55 @@ def test_a_pinned_git_clone_connects_to_the_address_the_gateway_classified(store
with pytest.raises(HTTPException):
E.import_git("henv_" + "c" * 32, "[email protected]:team/repo.git" if " " in bad else
"https://git.example/team/repo.git", pin=bad)
+
+
+def test_a_write_waiting_on_a_snapshot_or_a_slow_import_leaves_the_runner_answering(store, monkeypatch):
+ """The runner answers every turn from one event loop. A write waiting for the source lock, or a
+ git clone, run on that loop froze every other request until it ended (on 0.32.8 a 4.6 s import
+ kept the runner's /healthz waiting 4.3 s); both now run in a worker thread."""
+ from fastapi import FastAPI
+ from fastapi.testclient import TestClient
+ app = FastAPI()
+ app.include_router(E.router)
+
+ @app.get("/ping")
+ async def ping():
+ return {}
+
+ eid = "henv_" + "d" * 32
+ E.write_file(eid, "a.txt", b"1")
+ done: dict = {}
+
+ def ping_seconds(c):
+ """How long the loop took to answer, or None when it had not answered in 2 s."""
+ out: dict = {}
+ th = threading.Thread(target=lambda: (t0 := time.time(), c.get("/ping"), out.update(s=time.time() - t0)), daemon=True)
+ th.start()
+ th.join(2.0)
+ return out.get("s")
+
+ with TestClient(app) as c: # one client, one event loop, as in the runner
+ lock = E._source_lock(eid)
+ lock.acquire() # a build is inside its snapshot
+ try:
+ w = threading.Thread(target=lambda: done.update(w=c.put(f"/environments/{eid}/source",
+ params={"path": "a.txt"}, content=b"2")), daemon=True)
+ w.start()
+ time.sleep(0.3)
+ waited = ping_seconds(c)
+ still_waiting = "w" not in done # the write is held until the snapshot ends
+ finally:
+ lock.release()
+ w.join(10)
+ assert waited is not None and waited < 1.0, f"the runner did not answer while a write waited ({waited})"
+ assert still_waiting and done["w"].status_code == 200 and E.read_file(eid, "a.txt")[0] == b"2"
+
+ monkeypatch.setattr(E, "import_git", lambda *a, **k: (time.sleep(1.5), {"imported": "git"})[1])
+ g = threading.Thread(target=lambda: done.update(g=c.post(f"/environments/{eid}/import",
+ params={"git_url": "https://git.example/x.git"})), daemon=True)
+ g.start()
+ time.sleep(0.3)
+ waited = ping_seconds(c)
+ g.join(10)
+ assert waited is not None and waited < 1.0, f"the runner did not answer while a git import ran ({waited})"
+ assert done["g"].status_code == 200