Skip to content

HarnessRouter Community Edition: the self-hosted, Apache-2.0 edition of the unified interface for agent harnesses. Run Codex, Claude Code, Hermes, PI, DSH, and more through one API, with sessions, streaming, files, cancellation, and failure handling. Implements the Unified Harness Protocol (UHP), an open standard. Your keys, your infrastructure.

52.6 KB
Raw Blame
"""Pluggable backing stores for the harness gateway.

The gateway needs three capabilities from the outside world, and nothing else:

  GraphStore   — session/harness/response/api-key records + ownership edges
  BlobStore    — traces, response records, workspace checkpoints, produced files
  SecretStore  — model-provider connections/policies ("bring your own keys")

Two implementations of each:

  local (this file)      — SQLite + filesystem + env/file, zero cloud dependencies. The whole
                           gateway runs on a laptop: HR_BACKING=local (or just leave
                           VG_GATEWAY_URL unset). Point POOL_MGMT_ENDPOINT at a single runner
                           process (the pool is only an identifier-routing proxy in front of
                           runner replicas — the routes are identical).
  vg (backing_vg.py)     — the production AgentStudio infra: vg-gateway (Cosmos graph + Azure
                           blob) and the vault service. Verbatim the code that always ran here.

Selection is one env var: HR_BACKING=vg|local; unset = vg when VG_GATEWAY_URL is set, local
otherwise. The interface is deliberately SEMANTIC (get/upsert/find/add_edge), not query-shaped:
call sites never build backend query strings, so an implementation is free to be a graph, a
relational table, or anything else that can look up records by label + property equality.
"""
from __future__ import annotations

import asyncio
import base64
import contextlib
import hashlib
import secrets as secrets_mod
import io
import json
import mimetypes
import os
import posixpath
import re
import shutil
import sqlite3
import tarfile
import time
import uuid
import weakref
from pathlib import Path
from typing import Protocol

import httpx


# ── interfaces ────────────────────────────────────────────────────────────────────
class GraphStore(Protocol):
    async def get(self, vid: str, label: str | None = None) -> dict | None: ...
    async def upsert(self, label: str, vid: str, props: dict, *, raise_on_fail: bool = False) -> None: ...
    async def add_edge(self, label: str, src: str, dst: str) -> None: ...
    async def find(self, label: str, eq: dict | None = None, neq: dict | None = None) -> list[dict]: ...
    async def find_strict(self, label: str, eq: dict | None = None, neq: dict | None = None) -> list[dict]:
        """`find`, raising when the store cannot answer: an empty list means nothing matches."""
        ...
    async def delete(self, vid: str, label: str | None = None) -> bool:
        """Remove one vertex and its edges; False when there was none (or the label differs)."""
        ...
    async def probe(self) -> None:
        """Raise if the store is unreachable/misconfigured. Used by /readyz, which must FAIL on a
        broken plane — the normal read paths swallow errors and return empty."""
        ...


class BlobUnavailable(RuntimeError):
    """The blob store could not answer a read (as opposed to: the object is not there)."""


class BlobStore(Protocol):
    async def get(self, kb: str, file_id: str) -> bytes | None: ...
    async def get_or_raise(self, kb: str, file_id: str) -> bytes | None:
        """`get`, except that only a missing object answers None: a read that failed raises
        BlobUnavailable, for a reader that must not show a failure as nothing there."""
        ...
    async def put(self, kb: str, file_id: str, data: bytes) -> bool: ...
    async def delete(self, kb: str, file_id: str) -> bool: ...
    async def list(self, kb: str, prefix: str, limit: int = 20, cursor: str | None = None) -> dict: ...
    async def probe(self) -> None: ...


class SecretStore(Protocol):
    async def get(self, tenant: str, name: str) -> str | None: ...
    async def delete(self, tenant: str, name: str) -> None:
        """Forget a stored secret; nothing happens when there is none."""
        ...

    async def put(self, tenant: str, name: str, value: str, *,
                  require_encryption: bool = False) -> None:
        """`require_encryption=True` is the caller saying "this is a customer credential, not a
        provider key injected by the operator" — a database connection string, for instance. A
        store that cannot encrypt must raise SecretsNotConfigured rather than write it in the
        clear. Part of the interface, not of one implementation: a datasource must be refused
        the same way whichever backing is configured."""
        ...


class WorkspaceFiles(Protocol):
    """Reading and writing one file inside a session's workspace.

    Two environments, genuinely different storage, one interface:

    * **Self-hosted** — the runner shares this container and the workspace is a live directory on
      the data volume. A write is a write.
    * **Hosted** — sandboxes are ephemeral, so between turns a workspace exists ONLY as the
      checkpoint tarball in blob storage (`sessions/<sid>/workspace.tgz`). A write has to rewrite
      that archive, because there is no filesystem to write to until the next turn hydrates one.

    This exists so an app built on a Harness can own state the agent also edits — the slides kit
    keeps deck.json here — without either side needing to know which deployment it is running in.
    """
    async def read(self, sid: str, path: str) -> bytes | None: ...
    async def write(self, sid: str, path: str, data: bytes) -> bool: ...
    # The three below answer one of the WS_* outcomes rather than a bool: a route has to tell "nothing
    # there" (404) from "already there" (409) from a path it must refuse (400) from storage failing.
    # mkdir answers WS_UNCHANGED for a directory that was already there.
    async def delete(self, sid: str, path: str) -> str: ...
    async def mkdir(self, sid: str, path: str) -> str: ...
    async def move(self, sid: str, path: str, to: str) -> str: ...


WS_OK, WS_NOT_FOUND, WS_EXISTS, WS_BAD_PATH, WS_FAILED = "ok", "not_found", "exists", "bad_path", "failed"
WS_UNCHANGED = "unchanged"

# The name a nesting base gives its agent's directory in `.harness/agent-subdir` (nanobot's
# `workspace`): ONE plain directory name, or no subdir at all. The gateway's listing reads the same
# marker with this same rule (app.py, _ws_rules), so the files a change can reach are the files the
# listing shows.
WS_SUBDIR_RE = re.compile(r"[A-Za-z0-9][A-Za-z0-9._-]{0,63}")


def ws_protected(rel: str, pats: tuple[str, ...]) -> bool:
    """Is `rel` (relative to the session directory) a base's own machinery, as the runner lists it
    in `.git/info/exclude` — or a directory with some inside it? nanobot's skills and memory,
    opensquilla's SOUL.md: the listing never shows them, and a delete or a move must not reach them
    by naming them, or by naming a folder that holds them."""
    for q in pats:
        d = q.rstrip("/")
        if rel == d or (q.endswith("/") and rel.startswith(q)) or d.startswith(rel + "/"):
            return True
    return False


def keyed_lock(registry: "weakref.WeakValueDictionary[str, asyncio.Lock]", key: str) -> asyncio.Lock:
    """The asyncio.Lock for `key` in `registry`, made on first use and dropped once nobody holds it.
    IN THIS PROCESS only: replicas do not share it."""
    lock = registry.get(key)
    if lock is None:
        lock = registry[key] = asyncio.Lock()
    return lock


# One rewrite of a session's checkpoint at a time. Every change reads the whole archive and writes a
# new one, so two at once would each start from the same archive and the second write would drop the
# first change while both answered 200.
_CKPT_LOCKS: "weakref.WeakValueDictionary[str, asyncio.Lock]" = weakref.WeakValueDictionary()


def _ckpt_lock(sid: str) -> asyncio.Lock:
    return keyed_lock(_CKPT_LOCKS, sid)


class Backing:
    def __init__(self, graph: GraphStore, blob: BlobStore, secrets: SecretStore, mode: str,
                 workspace: "WorkspaceFiles"):
        self.graph, self.blob, self.secrets, self.mode = graph, blob, secrets, mode
        self.workspace = workspace


# ── local: SQLite graph ───────────────────────────────────────────────────────────
class SqliteGraphStore:
    """Vertices/edges in one SQLite file. VG upsert semantics preserved: property() sets the
    listed props and keeps the rest (merge, not replace); edge ids are the same uuid5(src|label|dst)
    so re-adds are idempotent. find() filters by property equality in Python — dev-scale data,
    no json_extract dialect games."""

    def __init__(self, path: str):
        self._path = path
        Path(path).parent.mkdir(parents=True, exist_ok=True)
        with self._conn() as c:
            c.execute("CREATE TABLE IF NOT EXISTS vertices (vid TEXT PRIMARY KEY, label TEXT NOT NULL, props TEXT NOT NULL)")
            c.execute("CREATE INDEX IF NOT EXISTS idx_vertices_label ON vertices(label)")
            c.execute("CREATE TABLE IF NOT EXISTS edges (eid TEXT PRIMARY KEY, label TEXT NOT NULL, src TEXT NOT NULL, dst TEXT NOT NULL)")

    def _conn(self) -> sqlite3.Connection:
        c = sqlite3.connect(self._path, timeout=30)
        c.execute("PRAGMA journal_mode=WAL")
        return c

    async def get(self, vid: str, label: str | None = None) -> dict | None:
        def _do():
            with self._conn() as c:
                row = c.execute("SELECT vid, label, props FROM vertices WHERE vid=?", (vid,)).fetchone()
            if not row:
                return None
            if label is not None and row[1] != label:
                return None      # label filter is load-bearing where the id is caller-controlled
            return {"id": row[0], **json.loads(row[2])}
        return await asyncio.to_thread(_do)

    async def upsert(self, label: str, vid: str, props: dict, *, raise_on_fail: bool = False) -> None:
        clean = {k: str(v) for k, v in props.items()}   # VG stores string props; keep parity

        def _do():
            with self._conn() as c:
                row = c.execute("SELECT props FROM vertices WHERE vid=?", (vid,)).fetchone()
                merged = {**(json.loads(row[0]) if row else {}), **clean}
                c.execute("INSERT INTO vertices(vid,label,props) VALUES(?,?,?) "
                          "ON CONFLICT(vid) DO UPDATE SET props=excluded.props",
                          (vid, label, json.dumps(merged)))
        try:
            await asyncio.to_thread(_do)
        except Exception:
            if raise_on_fail:
                raise RuntimeError("graph write failed")

    async def add_edge(self, label: str, src: str, dst: str) -> None:
        if not (src and dst):
            return
        eid = str(uuid.uuid5(uuid.NAMESPACE_URL, f"{src}|{label}|{dst}"))

        def _do():
            with self._conn() as c:
                c.execute("INSERT OR IGNORE INTO edges(eid,label,src,dst) VALUES(?,?,?,?)",
                          (eid, label, src, dst))
        try:
            await asyncio.to_thread(_do)
        except Exception:  # noqa: BLE001 — best-effort, matches VG edge semantics
            pass

    async def delete(self, vid: str, label: str | None = None) -> bool:
        def _do():
            with self._conn() as c:
                row = c.execute("SELECT label FROM vertices WHERE vid=?", (vid,)).fetchone()
                if not row or (label is not None and row[0] != label):
                    return False
                c.execute("DELETE FROM vertices WHERE vid=?", (vid,))
                c.execute("DELETE FROM edges WHERE src=? OR dst=?", (vid, vid))
                return True
        try:
            return await asyncio.to_thread(_do)
        except Exception:  # noqa: BLE001
            return False

    async def find(self, label: str, eq: dict | None = None, neq: dict | None = None) -> list[dict]:
        try:
            return await self.find_strict(label, eq, neq)
        except Exception:  # noqa: BLE001
            return []

    async def find_strict(self, label: str, eq: dict | None = None, neq: dict | None = None) -> list[dict]:
        def _do():
            with self._conn() as c:
                rows = c.execute("SELECT vid, props FROM vertices WHERE label=?", (label,)).fetchall()
            out = []
            for vid, props_json in rows:
                p = json.loads(props_json)
                if eq and any(str(p.get(k, "")) != str(v) for k, v in eq.items()):
                    continue
                if neq and any(str(p.get(k, "")) == str(v) for k, v in neq.items()):
                    continue
                out.append({"id": vid, **p})
            return out
        return await asyncio.to_thread(_do)

    async def probe(self) -> None:
        def _do():
            with self._conn() as c:
                c.execute("SELECT 1 FROM vertices LIMIT 1").fetchone()
        await asyncio.to_thread(_do)


# ── local: filesystem blobs ───────────────────────────────────────────────────────
class FileBlobStore:
    """Objects as plain files under {root}/{kb}/{file_id}. Listing is ascending-lexical over the
    relative keys — the SAME ordering contract the trace machinery relies on from Azure blob
    listing (inverted-timestamp keys ⇒ newest-first)."""

    def __init__(self, root: str):
        self._root = Path(root)

    def _p(self, kb: str, file_id: str) -> Path:
        p = (self._root / kb / file_id).resolve()
        base = (self._root / kb).resolve()
        if not str(p).startswith(str(base)):        # refuse path escape
            raise ValueError(f"blob id escapes store: {file_id!r}")
        return p

    async def get(self, kb: str, file_id: str) -> bytes | None:
        def _do():
            try:
                return self._p(kb, file_id).read_bytes()
            except (OSError, ValueError):
                return None
        return await asyncio.to_thread(_do)

    async def get_or_raise(self, kb: str, file_id: str) -> bytes | None:
        def _do():
            try:
                return self._p(kb, file_id).read_bytes()
            except (FileNotFoundError, NotADirectoryError, ValueError):
                return None                         # not there (or an id that is no key at all)
            except OSError as e:
                raise BlobUnavailable(type(e).__name__) from e
        return await asyncio.to_thread(_do)

    async def put(self, kb: str, file_id: str, data: bytes) -> bool:
        if not data:
            return False

        def _do():
            try:
                p = self._p(kb, file_id)
                p.parent.mkdir(parents=True, exist_ok=True)
                tmp = p.with_suffix(p.suffix + ".tmp")
                tmp.write_bytes(data)
                tmp.replace(p)                      # atomic: a torn write never replaces a good blob
                return True
            except (OSError, ValueError):
                return False
        return await asyncio.to_thread(_do)

    async def delete(self, kb: str, file_id: str) -> bool:
        def _do():
            try:
                path = self._p(kb, file_id)
                path.unlink(missing_ok=True)
            except (OSError, ValueError):
                return False
            # The directories a key's slashes created go with the last object in them, up to the
            # store root. A blob store has no directories; this one only borrows them from the file
            # system, and left alone they outlive every object: hr-test held 3,696 empty
            # sessions/<sid>/ shells on 2026-09-16, one per session ever deleted. rmdir refuses a
            # directory that still holds anything, which is the whole check.
            base = (self._root / kb).resolve()
            for parent in path.parents:
                if parent == base or base not in parent.parents:
                    break
                try:
                    parent.rmdir()
                except OSError:
                    break
            return True
        return await asyncio.to_thread(_do)

    async def purge(self, kb: str, prefix: str) -> None:
        """Remove everything under a folder-shaped prefix ("sessions/<sid>/"), including the
        temporary files a torn write leaves and `list` does not show. A deleted session's folder
        held a lone `changed.json.tmp` after every listed object was gone (hr-test copy,
        2026-10-08)."""
        def _do():
            base = (self._root / kb).resolve()
            if not prefix.endswith("/") or prefix.strip("/") == "":
                return
            top = (base / prefix).resolve()
            if top == base or base not in top.parents or not top.is_dir():
                return
            shutil.rmtree(top, ignore_errors=True)
            for parent in top.parents:
                if parent == base or base not in parent.parents:
                    break
                try:
                    parent.rmdir()
                except OSError:
                    break
        await asyncio.to_thread(_do)

    async def list(self, kb: str, prefix: str, limit: int = 20, cursor: str | None = None) -> dict:
        # Walk only the subtree the prefix names: everything before its last "/" is a directory
        # under the store, the rest a name prefix inside it. Walking the whole store cost every
        # blob on every call, and a paged listing repeated the walk per page: with 15k blobs on a
        # self-hosted volume one turns read took 55 s and the gateway sat at 150% CPU (2026-09-06).
        def _do():
            base = (self._root / kb).resolve()
            dir_part, _, _ = prefix.rpartition("/")
            top = (base / dir_part).resolve() if dir_part else base
            if not str(top).startswith(str(base)) or not top.is_dir():   # escape or nothing there
                return {"items": [], "cursor": None}
            root_len = len(str(base)) + 1
            keys: list[str] = []
            for root, _dirs, files in os.walk(top):
                rel = root[root_len:].replace(os.sep, "/") if len(root) > root_len else ""
                for name in files:
                    if name.endswith(".tmp"):
                        continue
                    key = f"{rel}/{name}" if rel else name
                    if key.startswith(prefix) and (not cursor or key > cursor):
                        keys.append(key)
            keys.sort()
            page = keys[:limit]
            nxt = page[-1] if len(keys) > limit else None
            return {"items": [{"file_id": k} for k in page], "cursor": nxt}
        return await asyncio.to_thread(_do)

    async def probe(self) -> None:
        def _do():
            self._root.mkdir(parents=True, exist_ok=True)
            if not os.access(self._root, os.W_OK):
                raise RuntimeError(f"blob root not writable: {self._root}")
        await asyncio.to_thread(_do)


# ── local: env/file secrets ("bring your own provider keys") ──────────────────────
class SecretsNotConfigured(RuntimeError):
    """Raised when a secret must be written but there is nowhere safe to write it.

    Deliberately an error rather than a fallback. A provider key injected read-only through the
    environment is one risk; a customer's production database connection string written to a file
    in plaintext is another entirely, and the difference is not something to decide silently on
    the operator's behalf."""


class FileSecretStore:
    """Reads HR_SECRET_{TENANT}_{NAME} from the environment first (read-only injection — the
    open-repo way to supply provider credentials without any vault), then falls back to files
    under {root}/{tenant}/{name}.

    Files are ENCRYPTED when HR_SECRET_KEY is set: AES-256-GCM under a key derived from it, with
    a random nonce per write, stored as `hrenc1:<b64 nonce||ciphertext>`. Reads accept both forms,
    so a store written before this existed keeps working and re-encrypts on its next write.

    Without HR_SECRET_KEY, `put(..., require_encryption=True)` REFUSES. That is the whole point:
    the caller that stores a database credential says so, and on an instance with no key it gets
    an error telling the operator what to set, instead of a plaintext file nobody knew about."""

    _MAGIC = "hrenc1:"

    def __init__(self, root: str):
        self._root = Path(root)
        self._key = self._derive(os.environ.get("HR_SECRET_KEY", ""))

    @property
    def encrypts(self) -> bool:
        return self._key is not None

    @staticmethod
    def _derive(passphrase: str) -> bytes | None:
        """A 32-byte key from the operator's passphrase. Scrypt with a fixed salt: the salt's job
        is to stop cross-deployment rainbow tables, and there is nowhere to keep a random one that
        would not sit beside the ciphertext anyway."""
        if not passphrase:
            return None
        return hashlib.scrypt(passphrase.encode(), salt=b"harnessrouter.secret.v1",
                              n=2 ** 14, r=8, p=1, dklen=32)

    def _seal(self, value: str) -> str:
        from cryptography.hazmat.primitives.ciphers.aead import AESGCM
        nonce = secrets_mod.token_bytes(12)
        blob = nonce + AESGCM(self._key).encrypt(nonce, value.encode(), None)
        return self._MAGIC + base64.b64encode(blob).decode()

    def _open(self, raw: str) -> str:
        if not raw.startswith(self._MAGIC):
            return raw                      # written before encryption existed
        if self._key is None:
            raise SecretsNotConfigured(
                "This secret is encrypted but HR_SECRET_KEY is not set. Set it to the passphrase "
                "used when the secret was stored.")
        from cryptography.hazmat.primitives.ciphers.aead import AESGCM
        blob = base64.b64decode(raw[len(self._MAGIC):])
        return AESGCM(self._key).decrypt(blob[:12], blob[12:], None).decode()

    @staticmethod
    def _env_key(tenant: str, name: str) -> str:
        return "HR_SECRET_" + re.sub(r"[^A-Za-z0-9]", "_", f"{tenant}_{name}").upper()

    def _p(self, tenant: str, name: str) -> Path:
        safe = re.sub(r"[^A-Za-z0-9._-]", "_", name)
        return self._root / re.sub(r"[^A-Za-z0-9._-]", "_", tenant) / safe

    async def get(self, tenant: str, name: str) -> str | None:
        v = os.environ.get(self._env_key(tenant, name))
        if v:
            return v

        def _do():
            try:
                return self._p(tenant, name).read_text()
            except OSError:
                return None
        raw = await asyncio.to_thread(_do)
        return self._open(raw) if raw is not None else None

    async def put(self, tenant: str, name: str, value: str, *, require_encryption: bool = False) -> None:
        if require_encryption and self._key is None:
            raise SecretsNotConfigured(
                "Refusing to store this credential: HR_SECRET_KEY is not set, so it could only be "
                "written to disk in plaintext. Set HR_SECRET_KEY to a passphrase and restart.")
        body = self._seal(value) if self._key is not None else value

        def _do():
            p = self._p(tenant, name)
            p.parent.mkdir(parents=True, exist_ok=True)
            p.write_text(body)
            with contextlib.suppress(OSError):
                p.chmod(0o600)              # belt to the encryption's braces
        await asyncio.to_thread(_do)

    async def delete(self, tenant: str, name: str) -> None:
        def _do():
            with contextlib.suppress(OSError):
                self._p(tenant, name).unlink()
        await asyncio.to_thread(_do)

    async def blank(self) -> list[tuple[str, str]]:
        """Every stored secret whose value is empty, as (tenant, name). Before 0.32.0 a removed
        credential was overwritten with nothing rather than deleted; the retention sweep removes
        those files. A value that cannot be opened (encrypted, and no key) is not reported."""
        def _do():
            out: list[tuple[str, str]] = []
            if not self._root.is_dir():
                return out
            for tdir in sorted(self._root.iterdir()):
                if not tdir.is_dir():
                    continue
                for f in sorted(tdir.iterdir()):
                    if not f.is_file():
                        continue
                    try:
                        if self._open(f.read_text()) == "":
                            out.append((tdir.name, f.name))
                    except Exception:  # noqa: BLE001 — unreadable is not empty
                        continue
            return out
        return await asyncio.to_thread(_do)


# ── selection ─────────────────────────────────────────────────────────────────────

# ── workspace files ───────────────────────────────────────────────────────────────
def _safe_member(path: str) -> str | None:
    """A workspace-relative path, or None if it tries to leave the workspace.

    Rejected rather than sanitised: a path that needed cleaning was not the path the caller meant,
    and silently writing somewhere else is worse than refusing."""
    p = (path or "").strip().lstrip("/")
    # normpath leaves "." and ".." as they are, so they are named here: written into a checkpoint,
    # a member called ".." is one that tar refuses to unpack, and every later restore would fail
    if not p or p in (".", "..") or p != posixpath.normpath(p) or p.startswith("../") or "\\" in p:
        return None
    return p


def app_capture_id(member: str) -> str:
    """The durable id of an APP's copy of one workspace path. Derived from the path, so saving the
    same file again replaces its one blob, and a delete (the gateway's _put_tombstone) can find and
    drop the app's copy of a path by its name alone."""
    return "cfile_" + hashlib.sha256(f"app:{member}".encode()).hexdigest()[:32]


def _tar_name(m: tarfile.TarInfo) -> str:
    return m.name[2:] if m.name.startswith("./") else m.name.rstrip("/")


def _under(name: str, member: str) -> bool:
    return name == member or name.startswith(member + "/")


def _tar_subdir(tf: tarfile.TarFile) -> str:
    """The nesting base's agent directory, from the archive's `.harness/agent-subdir`, or ""."""
    for name in (".harness/agent-subdir", "./.harness/agent-subdir"):
        try:
            m = tf.getmember(name)
        except KeyError:
            continue
        f = tf.extractfile(m) if m.isreg() and m.size <= 64 * 1024 else None
        sub = f.read().decode(errors="replace").strip() if f else ""
        return sub if WS_SUBDIR_RE.fullmatch(sub) else ""
    return ""


def tar_change(raw: bytes | None, op: str, member: str, to: str = "") -> tuple[str, bytes | None]:
    """Apply delete, mkdir or move to a checkpoint tarball → (WS_* outcome, the new archive or None
    when nothing changed).

    A path names a file or a directory, and a directory is everything under it: a tarball made by
    `tar -C ws .` lists the directory's own entry, but one rewritten by `write` may hold only the
    files, so both shapes count as the directory existing.

    WHICH paths a change may reach is the listing's rule, read from the same two files the listing
    reads (`.harness/agent-subdir`, `.git/info/exclude`). The base that nests its workspace (nanobot)
    keeps the user's files one level down, so a path is looked for there first. The session root is
    second, for a file an app wrote between turns (it lands there until the next turn adopts it),
    but never the agent's directory itself: `workspace` at the root IS nanobot's whole working
    directory, its skills, memory and instruction file with the user's files. A move stays on the
    side its source was found. Nothing the base excludes as its own machinery is touched.

    Streamed, as `write` is: the member list is read first and each member's bytes are copied
    straight across on the second pass, so memory is the two compressed archives and one member,
    never the whole unpacked workspace."""
    src = tarfile.open(fileobj=io.BytesIO(raw), mode="r:gz") if raw else None
    try:
        members = src.getmembers() if src else []
        names = [_tar_name(m) for m in members]

        def _small(name: str) -> str:
            for m, n in zip(members, names):
                if n == name and m.isreg() and m.size <= 64 * 1024:
                    f = src.extractfile(m)
                    return f.read().decode(errors="replace") if f else ""
            return ""

        sub = _small(".harness/agent-subdir").strip()
        if not WS_SUBDIR_RE.fullmatch(sub):
            sub = ""
        pats = tuple(ln.strip() for ln in _small(".git/info/exclude").splitlines()
                     if ln.strip() and not ln.startswith("#"))
        files = {n for m, n in zip(members, names) if not m.isdir()}

        def _protected(p: str) -> bool:
            # a path at the root of a nesting base is ADOPTED into the agent's directory next turn,
            # so it is also checked as it will be there
            return ws_protected(p, pats) or bool(sub and not _under(p, sub) and ws_protected(f"{sub}/{p}", pats))

        def _exists(p: str) -> bool:
            return any(_under(n, p) for n in names)

        def _file_above(p: str) -> bool:
            """A FILE where one of p's directories would be: an archive holding both `notes.txt` and
            `notes.txt/a.md` cannot be unpacked, and every later restore of the session would fail."""
            parts = p.split("/")
            return any("/".join(parts[:k]) in files for k in range(1, len(parts)))

        cands = ([f"{sub}/{member}"] if sub else []) + \
                ([member] if not sub or member.split("/", 1)[0] != sub else [])
        present = [c for c in cands if _exists(c)]
        found = present[0] if present else ""
        drop, rename, add = (), ("", ""), None
        if op == "delete":
            if not found:
                return WS_NOT_FOUND, None
            if any(_protected(c) for c in present):
                return WS_BAD_PATH, None
            # every copy: one left at the root of a nesting base would be adopted back next turn
            drop = tuple(present)
        elif op == "mkdir":
            # a new folder goes where the runner makes it: in the agent's directory on a nesting base
            target = found or (f"{sub}/{member}" if sub and member.split("/", 1)[0] != sub else member)
            if _protected(target):
                return WS_BAD_PATH, None
            if found:
                return (WS_EXISTS, None) if found in files else (WS_UNCHANGED, None)
            if _file_above(target):
                return WS_EXISTS, None
            add = tarfile.TarInfo(target)
            add.type, add.mode, add.mtime = tarfile.DIRTYPE, 0o755, int(time.time())
        elif op == "move":
            if not found:
                return WS_NOT_FOUND, None
            dst = f"{sub}/{to}" if sub and found != member else to
            if _protected(found) or _protected(dst) or _under(dst, found):
                return WS_BAD_PATH, None
            # On a nesting base the name can be taken on the OTHER side too: a file an app put at the
            # root (it waits there for adoption) or the agent's own. Moved in beside it, one of the two
            # shadows the other for good (adoption never moves over a name the agent's directory has),
            # which is an overwrite by another name.
            taken = ([f"{sub}/{to}"] if sub else []) + ([to] if not sub or to.split("/", 1)[0] != sub else [])
            if _exists(dst) or _file_above(dst) or any(_exists(c) for c in taken):
                return WS_EXISTS, None
            if any(_protected(c) for c in present[1:]):
                return WS_BAD_PATH, None
            rename = (found, dst)
            # a second copy (at the root of a nesting base) would be adopted back under the old name
            drop = tuple(present[1:])
        else:
            raise ValueError(op)
        regular = {n: m for m, n in zip(members, names) if m.isreg()}
        carried: dict[str, str] = {}      # a removed file -> the link that now holds its bytes

        def _renamed(n: str) -> str:
            return rename[1] + n[len(rename[0]):] if rename[0] and _under(n, rename[0]) else n

        out = io.BytesIO()
        with tarfile.open(fileobj=out, mode="w:gz") as dst_tar:
            for m, n in zip(members, names):
                if any(_under(n, d) for d in drop):
                    continue
                dot = "./" if m.name.startswith("./") else ""
                new = _renamed(n)
                m.name = dot + new
                data = None
                if m.islnk():
                    # A hard link is stored as a pointer to an EARLIER member's name. One left pointing
                    # at a removed or renamed member makes an archive no restore can unpack.
                    ldot = "./" if m.linkname.startswith("./") else ""
                    target = m.linkname[2:] if ldot else m.linkname
                    if any(_under(target, d) for d in drop):
                        if target in carried:
                            m.linkname = ldot + carried[target]
                        elif target in regular:
                            data = src.extractfile(m)        # resolves to the target's bytes
                            m.type, m.linkname, m.size = tarfile.REGTYPE, "", regular[target].size
                            carried[target] = new
                        else:
                            continue
                    else:
                        m.linkname = ldot + _renamed(target)
                elif m.isfile():
                    data = src.extractfile(m)
                dst_tar.addfile(m, data)
            if add is not None:
                dst_tar.addfile(add)
        return WS_OK, out.getvalue()
    finally:
        if src is not None:
            src.close()


class RunnerWorkspaceFiles:
    """Self-hosted: a session's workspace is a live directory on the data volume, and the RUNNER is
    the only principal that touches it. Behind the write-wall each session directory belongs to that
    session's own uid and the product deliberately cannot read or write there (see _isolate_session
    in the runner), so a live read or write goes through the runner, which acts as the session,
    rather than through this process's own filesystem access. One door, whether the wall is up or
    not: the runner is beside the gateway in every self-hosted deployment.

    Live, unlike the checkpoint the listing route reads: an app sees a file the moment a tool call
    creates it, rather than when the turn ends."""

    def __init__(self, endpoint: str, key: str, blob=None, kb: str = "", ckpt_kb: str = "", key_for=None):
        self.endpoint = (endpoint or "").rstrip("/")
        self.key = key
        self.blob, self.kb = blob, kb
        self.ckpt_kb, self.key_for = ckpt_kb, key_for
        self._client: httpx.AsyncClient | None = None

    def _c(self) -> httpx.AsyncClient:
        if self._client is None:
            self._client = httpx.AsyncClient(timeout=httpx.Timeout(60.0, connect=5.0))
        return self._client

    def _headers(self) -> dict:
        return {"X-Harness-Internal": self.key} if self.key else {}

    async def read(self, sid: str, path: str) -> bytes | None:
        member = _safe_member(path)
        if not member or not self.endpoint:
            return None
        try:
            r = await self._c().get(f"{self.endpoint}/file", params={"identifier": sid, "path": member},
                                    headers=self._headers())
        except Exception:      # noqa: BLE001 — a runner that is down reads as "no file", never a 500
            return None
        return r.content if r.status_code == 200 else None

    async def write(self, sid: str, path: str, data: bytes) -> bool:
        member = _safe_member(path)
        if not member or not self.endpoint:
            return False
        try:
            r = await self._c().put(f"{self.endpoint}/file", params={"identifier": sid, "path": member},
                                    headers={**self._headers(), "content-type": "application/octet-stream"},
                                    content=data)
        except Exception:      # noqa: BLE001
            return False
        if r.status_code != 200:
            return False
        await self._capture(sid, member, data)
        return True

    async def delete(self, sid: str, path: str) -> str:
        return await self._change(sid, "delete", path)

    async def mkdir(self, sid: str, path: str) -> str:
        return await self._change(sid, "mkdir", path)

    async def move(self, sid: str, path: str, to: str) -> str:
        return await self._change(sid, "move", path, to)

    async def _change(self, sid: str, op: str, path: str, to: str = "") -> str:
        """The live directory first, through the runner (which acts as the session), then the
        checkpoint tarball.

        The checkpoint is the second half because the live directory is not the only copy: it is
        reaped after HR_WORKSPACE_TTL_HOURS, and a sandbox that does not hold the last checkpoint
        is wiped and restored from that tarball. A delete made only in the live directory would
        come back on that restore, and a move would come back as two copies. `write` closes the
        same hole with a per-file capture; a delete, a directory or a rename has no file to
        capture, so the archive itself is changed, as the hosted backing does."""
        member = _safe_member(path)
        dest = _safe_member(to) if op == "move" else ""
        if not member or (op == "move" and not dest) or not self.endpoint:
            return WS_BAD_PATH
        params = {"identifier": sid, "path": member, **({"to": dest} if op == "move" else {})}
        url = f"{self.endpoint}/file" + ("" if op == "delete" else f"/{op}")
        try:
            r = await self._c().request("DELETE" if op == "delete" else "POST", url, params=params,
                                        headers=self._headers())
        except Exception:      # noqa: BLE001
            return WS_FAILED
        outcome = {200: WS_OK, 404: WS_NOT_FOUND, 409: WS_EXISTS, 400: WS_BAD_PATH}.get(r.status_code, WS_FAILED)
        if outcome == WS_NOT_FOUND:
            # Not in the live directory. That directory is reaped after HR_WORKSPACE_TTL_HOURS while
            # the checkpoint keeps the file, and the next turn restores from the checkpoint, so the
            # change is the checkpoint's to make. A path in neither is a 404.
            return await self._change_checkpoint(sid, op, member, dest, live=False)
        if outcome != WS_OK:
            return outcome
        try:
            got = r.json()
        except ValueError:
            got = {}
        # The checkpoint is changed by the caller's path, not the one the runner answers: tar_change
        # resolves it the way the runner did (the agent's directory first on a nesting base), and
        # the runner's session-relative spelling would be looked for one level down a second time.
        ckpt = await self._change_checkpoint(sid, op, member, dest)
        if ckpt in (WS_EXISTS, WS_BAD_PATH):
            return ckpt
        return WS_UNCHANGED if op == "mkdir" and got.get("created") is False else WS_OK

    async def _change_checkpoint(self, sid: str, op: str, member: str, to: str, live: bool = True) -> str:
        """Apply the change to the checkpoint → its outcome. After a live change (`live`) the change
        already stands, so an archive that cannot be stored is logged rather than failed; without
        one, the archive is the only copy and its outcome is the answer.

        Two answers hold even after a live change:
        - A folder made while the session has no checkpoint (its turn ended without one) has no
          archive to go into, and the next turn's restore empties the live directory (the captures
          put files back, and a folder has none). So the archive is started with it, as the hosted
          backing starts one.
        - An archive that REFUSES what the live directory allowed (a file where the folder or the
          destination would go) means the live directory is not the session's workspace: it is a
          reaped one the runner made anew, which the next turn wipes and restores from this archive.
          The refusal is the answer then, not a 200 that the next turn undoes."""
        if not self.blob or not self.ckpt_kb or not self.key_for:
            return WS_OK if live else WS_NOT_FOUND
        key = self.key_for(sid)
        try:
            async with _ckpt_lock(sid):
                raw = await self.blob.get(self.ckpt_kb, key)
                if not raw and op != "mkdir":
                    # no checkpoint yet: the live directory is the only copy
                    return WS_OK if live else WS_NOT_FOUND
                outcome, tar = await asyncio.to_thread(tar_change, raw, op, member, to)
                if outcome in (WS_NOT_FOUND, WS_UNCHANGED):
                    # not in the checkpoint (made since it was taken), or already so
                    return WS_OK if live else outcome
                if outcome in (WS_EXISTS, WS_BAD_PATH):
                    if live:
                        print(f"[workspace] checkpoint refused a {op} the live directory allowed "
                              f"sid={sid} path={member}: {outcome}", flush=True)
                    return outcome
                if await self.blob.put(self.ckpt_kb, key, tar):
                    return WS_OK
                if not live:
                    return WS_FAILED
            why = "store refused"
        except Exception as e:   # noqa: BLE001
            if not live:
                return WS_FAILED
            why = f"{type(e).__name__}: {e}"
        # The live change stands, but a restore from this checkpoint would undo it. Say so.
        print(f"[workspace] checkpoint {op} not applied sid={sid} path={member}: {why}", flush=True)
        return WS_OK

    async def _capture(self, sid: str, member: str, data: bytes) -> None:
        """Persist the app's copy of this file so it outlives the live directory.

        THIS is why the class takes a blob store. The live workspace is reaped after
        HR_WORKSPACE_TTL_HOURS and is only ever refreshed by a TURN's checkpoint, so a file an app
        wrote BETWEEN turns lived in exactly one place and was deleted with it: a sheet somebody
        typed into and never asked the agent about again came back empty, and so did every agent
        column they had bound by hand. The hosted sibling writes straight into the durable tarball
        and never had the hole, which is the tell — one Protocol, two meanings for `write`.

        The id is derived from the path, so a sheet saved a thousand times keeps ONE blob instead
        of leaking one per save. `written_at` is what makes the read side deterministic: the
        agent's own capture of the same filename carries one too, and the newer of the two wins
        rather than whichever the blob listing happened to return last."""
        if not self.blob or not self.kb:
            return
        await put_app_capture(self.blob, self.kb, sid, member, data)


async def put_app_capture(blob, kb: str, sid: str, member: str, data: bytes) -> bool:
    """Store an app's durable copy of one workspace file (RunnerWorkspaceFiles._capture says why).
    Also how a move carries an app's copy to the file's new name."""
    cfile = app_capture_id(member)
    meta = json.dumps({"filename": member,
                       "media_type": mimetypes.guess_type(member)[0] or "application/octet-stream",
                       "written_at": time.time(),
                       "source": "app"}).encode()
    try:
        if await blob.put(kb, f"containers/{sid}/{cfile}", data):
            await blob.put(kb, f"containers/{sid}/{cfile}.meta", meta)
            return True
    except Exception as e:   # noqa: BLE001
        print(f"[workspace] durable capture failed sid={sid} path={member}: {e}", flush=True)
        return False
    # A live write that was not captured is the old silent-loss shape. Say so rather than
    # letting it look like a clean save.
    print(f"[workspace] durable capture rejected sid={sid} path={member}", flush=True)
    return False


class CheckpointWorkspaceFiles:
    """Hosted: between turns the workspace is only the checkpoint tarball, so a write rewrites
    that archive — copy every member across, substituting the one being written.

    Rewriting rather than appending is deliberate: tar allows duplicate names and readers take the
    LAST one, so appending would work until something read the archive differently, and the bug
    would be a stale file appearing at random."""

    def __init__(self, blob, kb: str, key_for, on_rewrite=None):
        self.blob, self.kb, self.key_for = blob, kb, key_for
        # async (sid, sha256 of the new archive) -> None, called after every rewrite. The gateway
        # sets it to record the archive's sha as the session's checkpoint sha (see _store).
        self.on_rewrite = on_rewrite

    async def _store(self, sid: str, key: str, tar: bytes) -> bool:
        """Put a rewritten archive. A sandbox kept warm from the last turn still holds the OLD
        workspace, and hydrate skips the restore when the sandbox's sha matches the session's. Left as
        it was, the next turn would run on the old files and its checkpoint would write them back
        over this change: a deleted file restored, a moved one at both names.

        So the new archive's sha is recorded FIRST, and the archive is stored only once it is. A sha
        that cannot be recorded stops the change before anything moved (the caller sees a failure,
        and a retry starts clean); an archive that cannot be stored after it leaves a sha that
        matches no sandbox, which costs the next turn a full restore of the old archive and nothing
        else."""
        if self.on_rewrite is not None:
            try:
                await self.on_rewrite(sid, hashlib.sha256(tar).hexdigest())
            except Exception as e:   # noqa: BLE001
                print(f"[workspace] checkpoint sha not recorded sid={sid}: {e}; not rewritten", flush=True)
                return False
        return bool(await self.blob.put(self.kb, key, tar))

    async def read(self, sid: str, path: str) -> bytes | None:
        member = _safe_member(path)
        if not member:
            return None
        raw = await self.blob.get(self.kb, self.key_for(sid))
        if not raw:
            return None

        def _extract() -> bytes | None:
            with tarfile.open(fileobj=io.BytesIO(raw), mode="r:gz") as tf:
                # the agent's directory first on a nesting base, the session root second: the
                # order the runner's GET /file reads in
                sub = _tar_subdir(tf)
                for name in ([f"{sub}/{member}", f"./{sub}/{member}"] if sub else []) + [member, f"./{member}"]:
                    try:
                        f = tf.extractfile(name)
                    except KeyError:
                        continue
                    if f:
                        return f.read()
            return None
        try:
            return await asyncio.to_thread(_extract)
        except Exception:      # noqa: BLE001 — a corrupt archive reads as "not there"
            return None

    async def write(self, sid: str, path: str, data: bytes) -> bool:
        member = _safe_member(path)
        if not member:
            return False
        key = self.key_for(sid)
        async with _ckpt_lock(sid):
            return await self._write_locked(sid, key, member, data)

    async def _write_locked(self, sid: str, key: str, member: str, data: bytes) -> bool:
        raw = await self.blob.get(self.kb, key)

        def _rewrite() -> bytes:
            out = io.BytesIO()
            wrote = False
            with tarfile.open(fileobj=out, mode="w:gz") as dst:
                target = member
                if raw:
                    with tarfile.open(fileobj=io.BytesIO(raw), mode="r:gz") as src:
                        members = src.getmembers()
                        names = [m.name[2:] if m.name.startswith("./") else m.name for m in members]
                        # a file the agent already has, on a nesting base, is replaced where it is
                        # (in its directory), not shadowed by a second copy at the root
                        sub = _tar_subdir(src)
                        top = member.split("/", 1)[0]
                        if sub and (f"{sub}/{member}" in names or (
                                "/" in member and member not in names
                                and any(_under(n, f"{sub}/{top}") for n in names))):
                            # where it is, or — a new file — in a folder the agent already has, which
                            # adoption would not move a root copy into
                            target = f"{sub}/{member}"
                        for m, name in zip(members, names):
                            if name == target:
                                continue          # replaced below
                            f = src.extractfile(m) if m.isfile() else None
                            dst.addfile(m, f)
                info = tarfile.TarInfo(target)
                info.size = len(data)
                info.mtime = int(time.time())
                info.mode = 0o644
                dst.addfile(info, io.BytesIO(data))
                wrote = True
            return out.getvalue() if wrote else b""

        try:
            tar = await asyncio.to_thread(_rewrite)
        except Exception:      # noqa: BLE001
            return False
        return bool(tar) and await self._store(sid, key, tar)

    async def delete(self, sid: str, path: str) -> str:
        return await self._change(sid, "delete", path)

    async def mkdir(self, sid: str, path: str) -> str:
        return await self._change(sid, "mkdir", path)

    async def move(self, sid: str, path: str, to: str) -> str:
        return await self._change(sid, "move", path, to)

    async def _change(self, sid: str, op: str, path: str, to: str = "") -> str:
        """Rewrite the archive with the change applied, as `write` does: between turns the archive
        IS the workspace."""
        member = _safe_member(path)
        dest = _safe_member(to) if op == "move" else ""
        if not member or (op == "move" and not dest):
            return WS_BAD_PATH
        key = self.key_for(sid)
        async with _ckpt_lock(sid):
            try:
                raw = await self.blob.get(self.kb, key)
                outcome, tar = await asyncio.to_thread(tar_change, raw, op, member, dest)
            except Exception:      # noqa: BLE001
                return WS_FAILED
            if outcome != WS_OK or tar is None:
                return outcome
            return WS_OK if await self._store(sid, key, tar) else WS_FAILED


def make_backing(*, client_getter, vg_url: str, vg_key: str, vg_tenant: str,
                 vault_url: str, vault_key: str) -> Backing:
    """Build the configured backing. HR_BACKING=vg|local; unset ⇒ vg when VG_GATEWAY_URL is
    configured (production), local otherwise (a laptop clone works with zero env)."""
    mode = os.environ.get("HR_BACKING", "").strip().lower() or ("vg" if vg_url else "local")
    # Where a session's workspace actually lives, which is the one thing that differs most between
    # the two deployments: a live directory the runner owns and serves, or a checkpoint tarball
    # that is the only copy between turns.
    ws_key = lambda sid: f"sessions/{sid}/workspace.tgz"   # noqa: E731 — matches gateway._ws_blob
    if mode == "vg":
        from backing_vg import VgBlobStore, VgGraphStore, VaultSecretStore
        blob = VgBlobStore(client_getter, vg_url, vg_key)
        return Backing(
            graph=VgGraphStore(client_getter, vg_url, vg_key, vg_tenant),
            blob=blob,
            secrets=VaultSecretStore(client_getter, vault_url, vault_key),
            mode="vg",
            workspace=CheckpointWorkspaceFiles(blob, os.environ.get("HARNESS_BLOB_KB", "harness-sessions"), ws_key))
    data = os.environ.get("HR_DATA_DIR", ".hr-data")
    local_blob = FileBlobStore(os.path.join(data, "blobs"))
    return Backing(
        graph=SqliteGraphStore(os.path.join(data, "graph.db")),
        blob=local_blob,
        secrets=FileSecretStore(os.path.join(data, "secrets")),
        mode="local",
        workspace=RunnerWorkspaceFiles(os.environ.get("POOL_MGMT_ENDPOINT", "http://127.0.0.1:8081"),
                                       os.environ.get("HARNESS_INTERNAL_KEY", ""),
                                       local_blob,
                                       os.environ.get("HARNESS_RESP_BLOB_KB", "harness-responses"),
                                       os.environ.get("HARNESS_BLOB_KB", "harness-sessions"), ws_key))