Skip to content

HarnessRouter Community Edition: the self-hosted, Apache-2.0 edition of the unified interface for agent harnesses. Run Codex, Claude Code, Hermes, PI, DSH, and more through one API, with sessions, streaming, files, cancellation, and failure handling. Implements the Unified Harness Protocol (UHP), an open standard. Your keys, your infrastructure.

fix(workspace): a write the app makes must outlive the live workspace

richard-epsilla5a8d5dadec8a
diff --git a/gateway/app.py b/gateway/app.py
index 76bd87ae..22f72eca 100644
--- a/gateway/app.py
+++ b/gateway/app.py
@@ -1826,6 +1826,12 @@ async def _hydrate(sid: str, rec: dict) -> None:
r = await _hydrate_relay(sid, params)
rec["hydrated"] = r.status_code < 400
rec["hydrate"] = r.json() if r.headers.get("content-type", "").startswith("application/json") else None
+ if rec["hydrated"]:
+ # The workspace is now EXACTLY the checkpoint, which is only ever taken at the end of a
+ # turn. Anything an app wrote since then has just been wiped out of it, so put it back
+ # before the agent opens the file. The warm-resume branch above returns early and never
+ # reaches this: that workspace was never wiped, so it still holds those writes.
+ rec["app_writes_reapplied"] = await _reapply_app_writes(sid)
if not rec["hydrated"] and want_sha:
# A checkpoint EXISTED (ws_sha on the vertex) but restoring it failed. Do NOT let this
# turn checkpoint over the good blob from a workspace that isn't that checkpoint.
@@ -4999,7 +5005,8 @@ async def _collect_produced(sid: str, exclude: set[str] | None = None) -> list[d
fname = path.lstrip("./")
if await _blob_put(f"containers/{sid}/{cfile}", fr.content, kb=RESP_BLOB_KB):
await _blob_put(f"containers/{sid}/{cfile}.meta",
- json.dumps({"filename": fname, "media_type": media}).encode(), kb=RESP_BLOB_KB)
+ json.dumps({"filename": fname, "media_type": media,
+ "written_at": time.time()}).encode(), kb=RESP_BLOB_KB)
out.append({"container_id": sid, "file_id": cfile, "filename": fname, "bytes": len(fr.content)})
except Exception: # noqa: BLE001
continue
@@ -6881,13 +6888,19 @@ async def _ws_files(sid: str) -> dict[str, bytes] | None:
return out
-async def _cfile_by_name(sid: str) -> dict[str, str]:
- """filename -> captured `cfile_…` id for this session, from the blob metas.
+async def _newest_captures(sid: str) -> dict[str, tuple[str, bool]]:
+ """filename -> (captured `cfile_…` id, written_by_an_app) for this session, from the blob metas.
- ONE resolver for both file routes. The listing built this inline while the by-path read knew
+ ONE resolver for every caller. The listing built this inline while the by-path read knew
nothing about it, which is how a session could LIST dashboard.json and 404 it in the same
- breath once the live workspace aged out."""
- out: dict[str, str] = {}
+ breath once the live workspace aged out.
+
+ NEWEST WINS. Two captures can name the same file — the agent's, taken when a turn ended, and
+ the app's, taken when somebody saved between turns — and serving the older one hands back work
+ the person already replaced. Ties keep the previous last-one-wins order, so a pair of legacy
+ metas with no stamp resolves exactly as it did before."""
+ out: dict[str, tuple[str, bool]] = {}
+ newest: dict[str, float] = {}
listing = await _blob_list(f"containers/{sid}/", limit=200, kb=RESP_BLOB_KB)
metas = [i for i in (listing.get("items") or []) if str(i.get("file_id", "")).endswith(".meta")]
for it in metas[:200]:
@@ -6895,14 +6908,57 @@ async def _cfile_by_name(sid: str) -> dict[str, str]:
if not meta_b:
continue
try:
- fname = json.loads(meta_b).get("filename")
+ meta = json.loads(meta_b)
except Exception: # noqa: BLE001
continue
- if fname:
- out[str(fname)] = it["file_id"].rsplit("/", 1)[-1][: -len(".meta")]
+ fname = meta.get("filename")
+ if not fname:
+ continue
+ ts = float(meta.get("written_at") or 0.0)
+ if str(fname) in newest and ts < newest[str(fname)]:
+ continue
+ newest[str(fname)] = ts
+ out[str(fname)] = (it["file_id"].rsplit("/", 1)[-1][: -len(".meta")],
+ meta.get("source") == "app")
return out
+async def _cfile_by_name(sid: str) -> dict[str, str]:
+ """filename -> newest captured `cfile_…` id. The file routes want only the id."""
+ return {name: cid for name, (cid, _app) in (await _newest_captures(sid)).items()}
+
+
+async def _reapply_app_writes(sid: str) -> int:
+ """Put back the files an APP wrote since the last checkpoint, into a freshly hydrated workspace.
+
+ Hydrate restores the checkpoint, and the checkpoint is only ever taken when a TURN ends. So a
+ sheet somebody edited between turns is not in it: without this the agent would open the older
+ copy, write it back, and the person's rows would be gone — the same loss the durable capture
+ fixes for reads, arriving one turn later instead.
+
+ The test is deliberately conservative: re-apply ONLY where the app's capture is the newest one
+ for that filename. If the agent touched the same file in its last turn, its capture is newer,
+ the checkpoint already holds it, and this leaves it alone. That way the repair can never be the
+ thing that overwrites the agent's work."""
+ try:
+ caps = await _newest_captures(sid)
+ except Exception: # noqa: BLE001 — blob store unavailable: the turn still runs
+ return 0
+ n = 0
+ for name, (cid, from_app) in caps.items():
+ if not from_app:
+ continue
+ try:
+ got = await _container_file_bytes(sid, cid)
+ if got and await BACKING.workspace.write(sid, name, got[0]):
+ n += 1
+ except Exception: # noqa: BLE001
+ continue
+ if n:
+ print(f"[hydrate] re-applied {n} app write(s) sid={sid}", flush=True)
+ return n
+
+
async def _container_file_bytes(container_id: str, file_id: str) -> tuple[bytes, str, str] | None:
"""→ (data, media_type, filename) for either id form: a captured `cfile_…` blob, or a synthetic
`wf_…` workspace-path id extracted from the session checkpoint."""
@@ -6977,20 +7033,10 @@ async def session_workspace_files(sid: str, request: Request, changed: bool = Fa
if tf is None:
raise HTTPException(404, "no workspace for this session yet — run a task first")
# Captured-output ids by filename (so listed entries reuse the id the response already cited).
- cfile_by_name: dict[str, str] = {}
- listing = await _blob_list(f"containers/{sid}/", limit=200, kb=RESP_BLOB_KB)
- metas = [i for i in (listing.get("items") or []) if str(i.get("file_id", "")).endswith(".meta")]
- for it in metas[:200]:
- meta_b = await _blob_get(it["file_id"], kb=RESP_BLOB_KB)
- if not meta_b:
- continue
- try:
- fname = json.loads(meta_b).get("filename")
- except Exception: # noqa: BLE001
- continue
- cfile = it["file_id"].rsplit("/", 1)[-1][: -len(".meta")]
- if fname:
- cfile_by_name.setdefault(str(fname), cfile)
+ # THE SHARED RESOLVER, not a second copy of it. This route used to walk the metas itself and
+ # keep the FIRST id per filename while the by-path read kept the NEWEST, so the listing could
+ # hand out the id of a superseded capture and the two routes disagreed about the same file.
+ cfile_by_name = await _cfile_by_name(sid)
files = []
with tf:
for m in tf.getmembers():
diff --git a/gateway/backing.py b/gateway/backing.py
index dd2e27ae..04c2373b 100644
--- a/gateway/backing.py
+++ b/gateway/backing.py
@@ -30,6 +30,7 @@ import hashlib
import secrets as secrets_mod
import io
import json
+import mimetypes
import os
import posixpath
import re
@@ -381,9 +382,10 @@ class RunnerWorkspaceFiles:
Live, unlike the checkpoint the listing route reads: an app sees a file the moment a tool call
creates it, rather than when the turn ends."""
- def __init__(self, endpoint: str, key: str):
+ def __init__(self, endpoint: str, key: str, blob=None, kb: str = ""):
self.endpoint = (endpoint or "").rstrip("/")
self.key = key
+ self.blob, self.kb = blob, kb
self._client: httpx.AsyncClient | None = None
def _c(self) -> httpx.AsyncClient:
@@ -415,7 +417,42 @@ class RunnerWorkspaceFiles:
content=data)
except Exception: # noqa: BLE001
return False
- return r.status_code == 200
+ if r.status_code != 200:
+ return False
+ await self._capture(sid, member, data)
+ return True
+
+ async def _capture(self, sid: str, member: str, data: bytes) -> None:
+ """Persist the app's copy of this file so it outlives the live directory.
+
+ THIS is why the class takes a blob store. The live workspace is reaped after
+ HR_WORKSPACE_TTL_HOURS and is only ever refreshed by a TURN's checkpoint, so a file an app
+ wrote BETWEEN turns lived in exactly one place and was deleted with it: a sheet somebody
+ typed into and never asked the agent about again came back empty, and so did every agent
+ column they had bound by hand. The hosted sibling writes straight into the durable tarball
+ and never had the hole, which is the tell — one Protocol, two meanings for `write`.
+
+ The id is derived from the path, so a sheet saved a thousand times keeps ONE blob instead
+ of leaking one per save. `written_at` is what makes the read side deterministic: the
+ agent's own capture of the same filename carries one too, and the newer of the two wins
+ rather than whichever the blob listing happened to return last."""
+ if not self.blob or not self.kb:
+ return
+ cfile = "cfile_" + hashlib.sha256(f"app:{member}".encode()).hexdigest()[:32]
+ meta = json.dumps({"filename": member,
+ "media_type": mimetypes.guess_type(member)[0] or "application/octet-stream",
+ "written_at": time.time(),
+ "source": "app"}).encode()
+ try:
+ if await self.blob.put(self.kb, f"containers/{sid}/{cfile}", data):
+ await self.blob.put(self.kb, f"containers/{sid}/{cfile}.meta", meta)
+ return
+ except Exception as e: # noqa: BLE001
+ print(f"[workspace] durable capture failed sid={sid} path={member}: {e}", flush=True)
+ return
+ # A live write that was not captured is the old silent-loss shape. Say so rather than
+ # letting it look like a clean save.
+ print(f"[workspace] durable capture rejected sid={sid} path={member}", flush=True)
class CheckpointWorkspaceFiles:
@@ -505,10 +542,13 @@ def make_backing(*, client_getter, vg_url: str, vg_key: str, vg_tenant: str,
mode="vg",
workspace=CheckpointWorkspaceFiles(blob, os.environ.get("HARNESS_BLOB_KB", "harness-sessions"), ws_key))
data = os.environ.get("HR_DATA_DIR", ".hr-data")
+ local_blob = FileBlobStore(os.path.join(data, "blobs"))
return Backing(
graph=SqliteGraphStore(os.path.join(data, "graph.db")),
- blob=FileBlobStore(os.path.join(data, "blobs")),
+ blob=local_blob,
secrets=FileSecretStore(os.path.join(data, "secrets")),
mode="local",
workspace=RunnerWorkspaceFiles(os.environ.get("POOL_MGMT_ENDPOINT", "http://127.0.0.1:8081"),
- os.environ.get("HARNESS_INTERNAL_KEY", "")))
+ os.environ.get("HARNESS_INTERNAL_KEY", ""),
+ local_blob,
+ os.environ.get("HARNESS_RESP_BLOB_KB", "harness-responses")))
diff --git a/gateway/tests/test_session_file_fallback.py b/gateway/tests/test_session_file_fallback.py
index 197aa218..5d24a859 100644
--- a/gateway/tests/test_session_file_fallback.py
+++ b/gateway/tests/test_session_file_fallback.py
@@ -69,3 +69,125 @@ async def test_a_file_that_never_existed_still_404s(monkeypatch):
with pytest.raises(Exception):
await A.read_session_file("hsess_x", "nope.json", request=None)
+
+
+async def test_the_newest_capture_of_a_filename_wins(monkeypatch):
+ """Two captures can name the same file: the agent's, taken when a turn ended, and the app's,
+ taken when somebody saved between turns. Serving the older one hands back work the person
+ already replaced, and which one that was used to depend on blob-listing order."""
+ import json as _json
+
+ async def _list(_prefix, **_k):
+ return {"items": [{"file_id": "containers/s/cfile_old.meta"},
+ {"file_id": "containers/s/cfile_new.meta"}]}
+
+ async def _get(fid, **_k):
+ stamps = {"containers/s/cfile_old.meta": 100.0, "containers/s/cfile_new.meta": 200.0}
+ return _json.dumps({"filename": "sheet.json", "written_at": stamps[fid]}).encode()
+
+ monkeypatch.setattr(A, "_blob_list", _list)
+ monkeypatch.setattr(A, "_blob_get", _get)
+ assert (await A._cfile_by_name("s"))["sheet.json"] == "cfile_new"
+
+ # ...and the same regardless of the order the listing happened to return them in.
+ async def _list_rev(_prefix, **_k):
+ return {"items": [{"file_id": "containers/s/cfile_new.meta"},
+ {"file_id": "containers/s/cfile_old.meta"}]}
+
+ monkeypatch.setattr(A, "_blob_list", _list_rev)
+ assert (await A._cfile_by_name("s"))["sheet.json"] == "cfile_new"
+
+
+async def test_legacy_metas_with_no_stamp_resolve_as_they_always_did(monkeypatch):
+ """Captures written before the stamp existed must not change meaning: last one still wins."""
+ import json as _json
+
+ async def _list(_prefix, **_k):
+ return {"items": [{"file_id": "containers/s/cfile_a.meta"},
+ {"file_id": "containers/s/cfile_b.meta"}]}
+
+ async def _get(_fid, **_k):
+ return _json.dumps({"filename": "sheet.json"}).encode()
+
+ monkeypatch.setattr(A, "_blob_list", _list)
+ monkeypatch.setattr(A, "_blob_get", _get)
+ assert (await A._cfile_by_name("s"))["sheet.json"] == "cfile_b"
+
+
+def _captures(monkeypatch, entries):
+ """entries: [(cfile_id, filename, written_at, source)]"""
+ import json as _json
+
+ async def _list(_prefix, **_k):
+ return {"items": [{"file_id": f"containers/s/{cid}.meta"} for cid, *_ in entries]}
+
+ async def _get(fid, **_k):
+ cid = fid.rsplit("/", 1)[-1][: -len(".meta")]
+ for c, name, ts, src in entries:
+ if c == cid:
+ m = {"filename": name, "written_at": ts}
+ if src:
+ m["source"] = src
+ return _json.dumps(m).encode()
+ return None
+
+ monkeypatch.setattr(A, "_blob_list", _list)
+ monkeypatch.setattr(A, "_blob_get", _get)
+
+
+async def test_hydrate_puts_back_what_the_app_wrote_since_the_checkpoint(monkeypatch):
+ """The checkpoint is only taken when a turn ENDS, so a sheet edited between turns is not in it.
+ Without this the agent opens the older copy and writes the person's rows away."""
+ _captures(monkeypatch, [("cfile_app", "sheet.json", 200.0, "app")])
+
+ async def _bytes(_sid, cid):
+ return (b'{"rows":["typed by the person"]}', "application/json", "sheet.json") if cid == "cfile_app" else None
+
+ written = {}
+
+ class _Ws:
+ async def write(self, _sid, name, data):
+ written[name] = data
+ return True
+
+ monkeypatch.setattr(A, "_container_file_bytes", _bytes)
+ monkeypatch.setattr(A.BACKING, "workspace", _Ws(), raising=False)
+
+ assert await A._reapply_app_writes("s") == 1
+ assert written == {"sheet.json": b'{"rows":["typed by the person"]}'}
+
+
+async def test_it_never_overwrites_a_newer_agent_capture(monkeypatch):
+ """The repair must not become the thing that destroys work. If the agent touched the file in
+ its last turn, the checkpoint already holds that, and the older app copy is left alone."""
+ _captures(monkeypatch, [("cfile_app", "sheet.json", 100.0, "app"),
+ ("cfile_agent", "sheet.json", 300.0, None)])
+
+ written = {}
+
+ class _Ws:
+ async def write(self, _sid, name, data):
+ written[name] = data
+ return True
+
+ async def _bytes(_sid, _cid):
+ return (b"x", "application/json", "sheet.json")
+
+ monkeypatch.setattr(A, "_container_file_bytes", _bytes)
+ monkeypatch.setattr(A.BACKING, "workspace", _Ws(), raising=False)
+
+ assert await A._reapply_app_writes("s") == 0
+ assert written == {}
+
+
+async def test_a_broken_blob_store_does_not_stop_the_turn(monkeypatch):
+ async def _boom(*_a, **_k):
+ raise RuntimeError("blob store down")
+
+ monkeypatch.setattr(A, "_blob_list", _boom)
+ assert await A._reapply_app_writes("s") == 0
diff --git a/gateway/tests/test_workspace_write_durability.py b/gateway/tests/test_workspace_write_durability.py
new file mode 100644
index 00000000..44587a12
--- /dev/null
+++ b/gateway/tests/test_workspace_write_durability.py
@@ -0,0 +1,109 @@
+"""A workspace write made through the API must outlive the live workspace.
+
+Self-hosted, a session's workspace is a live directory the runner owns, and it is reaped after
+HR_WORKSPACE_TTL_HOURS. Only a TURN's checkpoint ever refreshed the durable copy, so a file an APP
+wrote between turns existed in exactly one place and was deleted with it. The sheets kit keeps the
+whole spreadsheet in sheet.json, so a sheet somebody typed rows into and never asked the agent
+about again came back with the agent's original empty structure, and every agent column they had
+bound by hand came back unbound. The hosted sibling (CheckpointWorkspaceFiles) writes into the
+durable tarball and never had the hole, which is the tell: one Protocol, two meanings for `write`.
+
+Unlike the app-level tests next door, these import only backing.py, so they run anywhere.
+"""
+import pathlib
+import sys
+
+import pytest
+
+sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[1]))
+import backing as B # noqa: E402
+
+
+class _Blob:
+ """The durable side, reduced to what the capture actually needs."""
+
+ def __init__(self):
+ self.objects: dict[tuple[str, str], bytes] = {}
+
+ async def put(self, kb, key, data):
+ self.objects[(kb, key)] = data
+ return True
+
+
+class _OK:
+ status_code = 200
+
+
+def _runner(blob):
+ ws = B.RunnerWorkspaceFiles("http://runner", "k", blob, "harness-responses")
+
+ class _Client:
+ async def put(self, *_a, **_k):
+ return _OK()
+
+ ws._c = lambda: _Client() # the live write always succeeds here
+ return ws
+
+
+async def test_a_write_is_captured_durably():
+ blob = _Blob()
+ assert await _runner(blob).write("hsess1", "sheet.json", b'{"rows":[1]}') is True
+
+ metas = [k for k in blob.objects if k[1].endswith(".meta")]
+ assert len(metas) == 1, "the write must leave exactly one durable copy plus its meta"
+ meta = __import__("json").loads(blob.objects[metas[0]])
+ assert meta["filename"] == "sheet.json"
+ assert meta["written_at"] > 0, "without a stamp the read side cannot tell which copy is newer"
+
+ body = [k for k in blob.objects if not k[1].endswith(".meta")][0]
+ assert blob.objects[body] == b'{"rows":[1]}'
+ assert body[1].startswith("containers/hsess1/cfile_")
+
+
+async def test_saving_the_same_file_again_does_not_leak_a_blob_per_save():
+ """A spreadsheet autosaves. One blob per keystroke would be a storage leak, so the id is
+ derived from the path rather than minted fresh."""
+ blob = _Blob()
+ ws = _runner(blob)
+ for i in range(5):
+ assert await ws.write("hsess1", "sheet.json", f'{{"n":{i}}}'.encode()) is True
+
+ assert len(blob.objects) == 2, "five saves, one blob and one meta"
+ body = [k for k in blob.objects if not k[1].endswith(".meta")][0]
+ assert blob.objects[body] == b'{"n":4}', "the newest save must be the one kept"
+
+
+async def test_two_files_keep_two_blobs():
+ blob = _Blob()
+ ws = _runner(blob)
+ await ws.write("hsess1", "sheet.json", b"a")
+ await ws.write("hsess1", "notes.md", b"b")
+ assert len({k for k in blob.objects if not k[1].endswith(".meta")}) == 2
+
+
+async def test_a_refused_live_write_captures_nothing():
+ """Durability must not invent a file the workspace itself rejected."""
+ blob = _Blob()
+ ws = _runner(blob)
+
+ class _Refused:
+ status_code = 409
+
+ class _Client:
+ async def put(self, *_a, **_k):
+ return _Refused()
+
+ ws._c = lambda: _Client()
+ assert await ws.write("hsess1", "sheet.json", b"x") is False
+ assert blob.objects == {}
+
+
+async def test_a_path_that_escapes_the_workspace_captures_nothing():
+ blob = _Blob()
+ assert await _runner(blob).write("hsess1", "../../etc/passwd", b"x") is False
+ assert blob.objects == {}