Skip to content

"Vibe-Trading: Your Personal Trading Agent"

feat(agent): persist background task outcomes and reconcile them after restart

Yufeng Heb4de393bcd70
diff --git a/agent/src/session/service.py b/agent/src/session/service.py
index 32731aa27..e36fc592a 100644
--- a/agent/src/session/service.py
+++ b/agent/src/session/service.py
@@ -119,6 +119,11 @@ class SessionService:
self._search_index = get_shared_index()
self._background_handles: Dict[str, Any] = {}
self._shutting_down = False
+ # Durable records live under the runtime root that owns runs_dir, so a
+ # test pointing runs_dir at a temp root gets its own isolated store.
+ from src.tools.background_tools import init_background_store
+
+ self._background_store = init_background_store(Path(runs_dir).parent)
self._recover_interrupted_attempts()
def _recover_interrupted_attempts(self) -> None:
diff --git a/agent/src/tools/background_store.py b/agent/src/tools/background_store.py
new file mode 100644
index 000000000..fb63257ae
--- /dev/null
+++ b/agent/src/tools/background_store.py
@@ -0,0 +1,223 @@
+"""Durable records for background tasks, and conservative restart recovery.
+
+A background task currently lives only in memory, so a process restart loses
+the fact that a command ever ran and the user is left guessing. This stores one
+small record per task under the canonical runtime root and, on a new process,
+reconciles what it finds.
+
+Recovery is deliberately one-way and never executes anything. A record left in
+``running`` or ``cancelling`` belongs to a process that is gone; it becomes
+``interrupted`` or ``outcome_unknown`` with a recovery reason. It is never
+re-run, and a stored PID is descriptive only: it is not used to signal a
+process, because that pid may have been reused by an unrelated program.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import os
+import re
+import tempfile
+import time
+from dataclasses import asdict, dataclass, field, replace
+from pathlib import Path
+from typing import Any, Iterable, Optional
+
+from src.tools.redaction import redact_text
+
+SCHEMA_VERSION = 1
+#: Preview length kept per field. Enough to identify a command or outcome
+#: without becoming a second copy of the run's full output.
+MAX_PREVIEW_CHARS = 2_000
+_TASK_ID_RE = re.compile(r"^[0-9a-f]{8,64}$")
+
+#: Statuses a task can hold while its owning process is still alive.
+LIVE_STATUSES = frozenset({"running", "cancelling"})
+#: Statuses whose result is worth reading back after a restart.
+TERMINAL_STATUSES = frozenset({"completed", "error", "timeout", "cancelled"})
+
+
+class BackgroundStoreError(RuntimeError):
+ """A record could not be stored or read back safely."""
+
+
+@dataclass
+class TaskRecord:
+ """What survives a restart for one background task."""
+
+ schema_version: int
+ task_id: str
+ owner_id: str
+ status: str
+ command_preview: str
+ attempt_id: Optional[str] = None
+ exit_code: Optional[int] = None
+ result_preview: str = ""
+ started_at: Optional[float] = None
+ ended_at: Optional[float] = None
+ duration_seconds: Optional[float] = None
+ recovery_reason: Optional[str] = None
+ #: Descriptive only. Never used to signal a process after a restart.
+ pid: Optional[int] = None
+ updated_at: float = field(default_factory=time.time)
+
+ @property
+ def is_live(self) -> bool:
+ return self.status in LIVE_STATUSES
+
+ @property
+ def is_terminal(self) -> bool:
+ return self.status in TERMINAL_STATUSES
+
+
+def _preview(value: object) -> str:
+ """Bound and redact one untrusted field before it reaches disk."""
+ return redact_text(str(value or ""))[:MAX_PREVIEW_CHARS]
+
+
+def _owner_token(owner_id: str) -> str:
+ """Stable layout-only token for an owner id.
+
+ A hash rather than the raw id: the owner carries a session identifier that
+ is not a secret but is not a filename either, and the raw form would put
+ user-supplied separators into the path.
+ """
+ return hashlib.sha256(owner_id.encode("utf-8")).hexdigest()[:16]
+
+
+class BackgroundTaskStore:
+ """One JSON record per task, written atomically under a fixed root."""
+
+ def __init__(self, root: Path) -> None:
+ self.root = Path(root)
+ self._directory = self.root / "background-tasks"
+
+ def _path_for(self, record: TaskRecord) -> Path:
+ if not _TASK_ID_RE.match(record.task_id):
+ raise BackgroundStoreError(f"unsafe background task id: {record.task_id!r}")
+ return self._directory / f"{_owner_token(record.owner_id)}-{record.task_id}.json"
+
+ def save(self, record: TaskRecord) -> Path:
+ """Persist one record, replacing any previous copy of it."""
+ path = self._path_for(record)
+ self._write_json_atomically(path, asdict(record))
+ return path
+
+ def load_all(self) -> tuple[list[TaskRecord], list[str]]:
+ """Return readable records and the paths that could not be read.
+
+ A corrupt or unknown-version file is reported, never overwritten and
+ never guessed at: silently dropping it would hide the task, and
+ silently reinterpreting it would invent an outcome.
+ """
+ records: list[TaskRecord] = []
+ unreadable: list[str] = []
+ if not self._directory.exists():
+ return records, unreadable
+ for path in sorted(self._directory.glob("*.json")):
+ record = self._load_one(path)
+ if record is None:
+ unreadable.append(str(path))
+ else:
+ records.append(record)
+ return records, unreadable
+
+ def _load_one(self, path: Path) -> Optional[TaskRecord]:
+ try:
+ payload = json.loads(path.read_text(encoding="utf-8"))
+ except (OSError, ValueError):
+ return None
+ if not isinstance(payload, dict):
+ return None
+ if payload.get("schema_version") != SCHEMA_VERSION:
+ return None
+ try:
+ return TaskRecord(**payload)
+ except TypeError:
+ return None
+
+ @staticmethod
+ def _write_json_atomically(path: Path, payload: dict[str, Any]) -> None:
+ path.parent.mkdir(parents=True, exist_ok=True, mode=0o700)
+ content = (json.dumps(payload, ensure_ascii=False, indent=2) + "\n").encode("utf-8")
+ fd, temporary = tempfile.mkstemp(dir=path.parent, prefix=f".{path.name}.", suffix=".tmp")
+ try:
+ try:
+ with os.fdopen(fd, "wb") as handle:
+ fd = -1
+ handle.write(content)
+ handle.flush()
+ if hasattr(os, "fchmod"):
+ os.fchmod(handle.fileno(), 0o600)
+ os.fsync(handle.fileno())
+ finally:
+ if fd >= 0:
+ os.close(fd)
+ os.replace(temporary, path)
+ except BaseException:
+ try:
+ os.unlink(temporary)
+ except OSError:
+ pass
+ raise
+ try:
+ os.chmod(path, 0o600)
+ except OSError:
+ pass
+
+
+def record_from_task(task_id: str, task: dict, *, started_at: Optional[float] = None) -> TaskRecord:
+ """Project an in-memory task entry onto its durable shape."""
+ started = task.get("started_at")
+ return TaskRecord(
+ schema_version=SCHEMA_VERSION,
+ task_id=task_id,
+ owner_id=str(task.get("owner_id") or ""),
+ attempt_id=task.get("attempt_id"),
+ status=str(task.get("status") or "error"),
+ command_preview=_preview(task.get("command")),
+ result_preview=_preview(task.get("result")),
+ exit_code=task.get("exit_code") if isinstance(task.get("exit_code"), int) else None,
+ started_at=float(started) if isinstance(started, (int, float)) else started_at,
+ ended_at=task.get("ended_at"),
+ duration_seconds=task.get("duration_seconds"),
+ pid=task.get("pid") if isinstance(task.get("pid"), int) else None,
+ )
+
+
+def reconcile_after_restart(
+ records: Iterable[TaskRecord],
+) -> tuple[list[TaskRecord], list[TaskRecord]]:
+ """Turn a previous process's live records into explicit outcomes.
+
+ Nothing is re-run and nothing is signalled. A record left ``running`` cannot
+ be distinguished from one whose output was never written, so it becomes
+ ``interrupted``; a record left ``cancelling`` cannot be distinguished from
+ one whose reaping never finished, so it becomes ``outcome_unknown``. Both
+ carry a recovery reason rather than being dropped.
+ """
+ recovered: list[TaskRecord] = []
+ unchanged: list[TaskRecord] = []
+ for record in records:
+ if record.status == "running":
+ recovered.append(
+ replace(
+ record,
+ status="interrupted",
+ recovery_reason="service_restart",
+ ended_at=time.time(),
+ )
+ )
+ elif record.status == "cancelling":
+ recovered.append(
+ replace(
+ record,
+ status="outcome_unknown",
+ recovery_reason="service_restart_before_reap_completed",
+ ended_at=time.time(),
+ )
+ )
+ else:
+ unchanged.append(record)
+ return recovered, unchanged
diff --git a/agent/src/tools/background_tools.py b/agent/src/tools/background_tools.py
index 37c50732e..8d49aeefc 100644
--- a/agent/src/tools/background_tools.py
+++ b/agent/src/tools/background_tools.py
@@ -18,6 +18,10 @@ from src.agent.tools import BaseTool
from src.tools._shell_safety import broad_python_kill_error
WORKDIR = Path(__file__).resolve().parents[2]
+#: Task ids with a live process anywhere in this interpreter. Restart recovery
+#: must not mistake a sibling manager's running task for a previous process's.
+_LIVE_TASK_IDS: set[str] = set()
+_LIVE_TASK_IDS_LOCK = threading.Lock()
_COMMAND_TIMEOUT_SECONDS = 300.0
_TERMINATION_GRACE_SECONDS = 2.0
_MAX_OUTPUT_CHARS = 50_000
@@ -141,8 +145,10 @@ class BackgroundManager:
scope: Optional[BackgroundScope] = None,
*,
on_completion: Optional[Callable[[str, dict], None]] = None,
+ store: Optional[Any] = None,
) -> None:
self.tasks: Dict[str, dict] = {}
+ self._store = store
# (owner_id, payload). Kept as pairs so the payload shape consumers and
# existing tests see stays exactly what it was; routing is internal.
self._notifications: List[tuple[str, dict]] = []
@@ -170,6 +176,7 @@ class BackgroundManager:
other._lock = self._lock
other._condition = self._condition
other._accepting = self._accepting
+ other._store = self._store
other._on_completion = (
self._on_completion if on_completion is None else on_completion
)
@@ -178,6 +185,43 @@ class BackgroundManager:
def _owns(self, task: dict) -> bool:
return task.get("owner_id") == self._scope.owner_id
+ def _persist(self, task_id: str) -> None:
+ """Best effort: a full or unwritable runtime root must not break the run."""
+ if self._store is None:
+ return
+ task = self.tasks.get(task_id)
+ if task is None:
+ return
+ try:
+ from src.tools.background_store import record_from_task
+
+ self._store.save(record_from_task(task_id, task))
+ except Exception:
+ logging.getLogger(__name__).exception("Could not persist background task %s", task_id)
+
+ def recover_on_start(self) -> dict:
+ """Reconcile records left by a previous process, once and without reruns.
+
+ A task id this interpreter is already running is left alone: a second
+ service constructed in the same process must not declare the first
+ one's live work interrupted.
+ """
+ if self._store is None:
+ return {"recovered": 0, "unchanged": 0, "unreadable": []}
+ from src.tools.background_store import reconcile_after_restart
+
+ records, unreadable = self._store.load_all()
+ with _LIVE_TASK_IDS_LOCK:
+ live = set(_LIVE_TASK_IDS)
+ stale = [record for record in records if record.task_id not in live]
+ recovered, unchanged = reconcile_after_restart(stale)
+ for record in recovered:
+ try:
+ self._store.save(record)
+ except Exception:
+ logging.getLogger(__name__).exception("Could not persist recovered task %s", record.task_id)
+ return {"recovered": len(recovered), "unchanged": len(unchanged), "unreadable": unreadable}
+
def run(self, command: str) -> str:
"""Start a background task and return its task_id.
@@ -210,6 +254,9 @@ class BackgroundManager:
"owner_id": self._scope.owner_id,
"attempt_id": self._scope.attempt_id,
}
+ with _LIVE_TASK_IDS_LOCK:
+ _LIVE_TASK_IDS.add(task_id)
+ self._persist(task_id)
threading.Thread(target=self._execute, args=(task_id, command), daemon=True).start()
return json.dumps({"status": "ok", "task_id": task_id, "message": f"Started: {command[:80]}"})
@@ -283,7 +330,12 @@ class BackgroundManager:
"exit_code": exit_code,
}
self._notifications.append((task["owner_id"], payload))
+ task["ended_at"] = time.time()
+ task["pid"] = getattr(process, "pid", None)
self._condition.notify_all()
+ with _LIVE_TASK_IDS_LOCK:
+ _LIVE_TASK_IDS.discard(task_id)
+ self._persist(task_id)
if self._on_completion is not None:
try:
self._on_completion(task["owner_id"], {**payload, "attempt_id": task.get("attempt_id")})
@@ -436,6 +488,25 @@ class BackgroundManager:
_BG = BackgroundManager()
+_BG_STORE: Optional[Any] = None
+
+
+def init_background_store(root: Path) -> Any:
+ """Attach durable task records to the process-wide table, once per process.
+
+ Recovery runs here and only here, so a second service constructed in the
+ same interpreter cannot declare the first one's live work interrupted.
+ """
+ global _BG_STORE
+ with _LIVE_TASK_IDS_LOCK:
+ if _BG_STORE is not None:
+ return _BG_STORE
+ from src.tools.background_store import BackgroundTaskStore
+
+ _BG_STORE = BackgroundTaskStore(Path(root))
+ _BG._store = _BG_STORE
+ _BG.recover_on_start()
+ return _BG_STORE
def get_background_manager() -> BackgroundManager:
diff --git a/agent/tests/test_background_store_recovery.py b/agent/tests/test_background_store_recovery.py
new file mode 100644
index 000000000..b10183873
--- /dev/null
+++ b/agent/tests/test_background_store_recovery.py
@@ -0,0 +1,245 @@
+"""Durable task records and the restart reconciliation that must never re-run."""
+
+from __future__ import annotations
+
+import json
+import os
+import stat
+import time
+from pathlib import Path
+
+import pytest
+
+from src.tools import background_store as store_mod
+from src.tools.background_store import (
+ SCHEMA_VERSION,
+ BackgroundStoreError,
+ BackgroundTaskStore,
+ TaskRecord,
+ record_from_task,
+ reconcile_after_restart,
+)
+
+
+def make(**overrides) -> TaskRecord:
+ values = dict(
+ schema_version=SCHEMA_VERSION,
+ task_id="deadbeef",
+ owner_id="session:s1",
+ status="completed",
+ command_preview="echo hi",
+ exit_code=0,
+ result_preview="hi",
+ )
+ values.update(overrides)
+ return TaskRecord(**values)
+
+
+class TestRecoveryIsConservative:
+ @pytest.fixture(autouse=True)
+ def forbid_side_effects(self, monkeypatch):
+ """Recovery must not spawn, signal, fetch or call a model."""
+
+ def explode(*args, **kwargs):
+ raise AssertionError(f"recovery must not call {kwargs or args}")
+
+ for module, names in [
+ (store_mod, []),
+ ]:
+ for name in names:
+ monkeypatch.setattr(module, name, explode)
+ monkeypatch.setattr(store_mod.os, "kill", explode)
+ monkeypatch.setattr(store_mod.os, "killpg", explode)
+ monkeypatch.setattr(store_mod.time, "sleep", explode)
+
+ def test_running_becomes_interrupted_without_rerun(self):
+ recovered, unchanged = reconcile_after_restart([make(status="running")])
+ assert unchanged == []
+ assert recovered[0].status == "interrupted"
+ assert recovered[0].recovery_reason == "service_restart"
+ assert recovered[0].command_preview == "echo hi"
+
+ def test_cancelling_stays_outcome_unknown(self):
+ recovered, _ = reconcile_after_restart([make(status="cancelling")])
+ assert recovered[0].status == "outcome_unknown"
+ assert recovered[0].recovery_reason == "service_restart_before_reap_completed"
+
+ @pytest.mark.parametrize("status", ["completed", "error", "timeout", "cancelled"])
+ def test_terminal_outcomes_are_kept_queryable(self, status):
+ recovered, unchanged = reconcile_after_restart([make(status=status, exit_code=3)])
+ assert recovered == []
+ assert unchanged[0].status == status
+ assert unchanged[0].exit_code == 3
+
+ def test_recovery_is_idempotent(self):
+ once, _ = reconcile_after_restart([make(status="running")])
+ twice, unchanged = reconcile_after_restart(once)
+ assert twice == []
+ assert unchanged[0].status == "interrupted"
+
+ def test_stored_pid_is_descriptive_only(self):
+ record = make(status="running", pid=os.getpid())
+ recovered, _ = reconcile_after_restart([record])
+ assert recovered[0].pid == os.getpid()
+ # No signal path reads it; nothing above touches os.kill.
+
+
+class TestStore:
+ def test_round_trip_and_layout_is_stable_per_owner(self, tmp_path):
+ store = BackgroundTaskStore(tmp_path)
+ record = make()
+ path = store.save(record)
+ assert path.parent == tmp_path / "background-tasks"
+ assert path.name == path.name # deterministic
+ assert store.save(record).name == path.name
+ records, unreadable = store.load_all()
+ assert unreadable == []
+ assert records == [record]
+
+ def test_owner_id_is_not_a_filename(self, tmp_path):
+ store = BackgroundTaskStore(tmp_path)
+ path = store.save(make(owner_id="session:../escape"))
+ assert ".." not in path.name
+ assert path.parent == tmp_path / "background-tasks"
+
+ def test_unsafe_task_id_is_rejected(self, tmp_path):
+ store = BackgroundTaskStore(tmp_path)
+ with pytest.raises(BackgroundStoreError):
+ store.save(make(task_id="../../etc/passwd"))
+
+ def test_private_permissions(self, tmp_path):
+ store = BackgroundTaskStore(tmp_path)
+ path = store.save(make())
+ assert stat.S_IMODE(path.stat().st_mode) == 0o600
+ assert stat.S_IMODE(path.parent.stat().st_mode) == 0o700
+
+ def test_unknown_version_is_reported_not_overwritten(self, tmp_path):
+ store = BackgroundTaskStore(tmp_path)
+ store.save(make())
+ for path in (tmp_path / "background-tasks").glob("*.json"):
+ payload = json.loads(path.read_text())
+ payload["schema_version"] = SCHEMA_VERSION + 1
+ path.write_text(json.dumps(payload))
+ records, unreadable = store.load_all()
+ assert records == []
+ assert len(unreadable) == 1
+ assert json.loads(Path(unreadable[0]).read_text())["schema_version"] == SCHEMA_VERSION + 1
+
+ def test_half_written_file_is_reported_not_invented(self, tmp_path):
+ store = BackgroundTaskStore(tmp_path)
+ directory = tmp_path / "background-tasks"
+ directory.mkdir()
+ (directory / "deadbeef.json").write_text('{"schema_version": 1, "task_id"')
+ records, unreadable = store.load_all()
+ assert records == []
+ assert len(unreadable) == 1
+
+ def test_symlink_is_refused(self, tmp_path):
+ store = BackgroundTaskStore(tmp_path)
+ directory = tmp_path / "background-tasks"
+ directory.mkdir()
+ outside = tmp_path / "outside.json"
+ outside.write_text(json.dumps({"schema_version": SCHEMA_VERSION}))
+ link = directory / f"{'0' * 16}-deadbeef.json"
+ link.symlink_to(outside)
+ _records, unreadable = store.load_all()
+ assert len(unreadable) == 1
+
+ def test_previews_are_bounded_and_redacted(self, tmp_path):
+ record = make(
+ command_preview=store_mod._preview(" ".join(["x" * 100] * 50)),
+ result_preview=store_mod._preview(f"token sk-{'a' * 40} end"),
+ )
+ assert len(record.command_preview) <= store_mod.MAX_PREVIEW_CHARS
+ assert len(record.result_preview) <= store_mod.MAX_PREVIEW_CHARS
+ assert f"sk-{'a' * 40}" not in record.result_preview
+
+ def test_two_local_processes_can_share_the_directory(self, tmp_path):
+ """One process's records stay readable after another writes beside them."""
+ first = BackgroundTaskStore(tmp_path)
+ second = BackgroundTaskStore(tmp_path)
+ first.save(make(task_id="aaaaaaaa", owner_id="session:a"))
+ second.save(make(task_id="bbbbbbbb", owner_id="session:b"))
+ records, unreadable = first.load_all()
+ assert unreadable == []
+ assert {r.task_id for r in records} == {"aaaaaaaa", "bbbbbbbb"}
+
+ def test_recovery_survives_a_process_that_never_wrote(self, tmp_path):
+ """A crash before the first save leaves no record, and none is invented."""
+ store = BackgroundTaskStore(tmp_path)
+ assert store.load_all() == ([], [])
+ recovered, unchanged = reconcile_after_restart([])
+ assert recovered == [] and unchanged == []
+
+ def test_record_from_task_projects_the_live_shape(self):
+ task = {
+ "status": "running",
+ "owner_id": "session:s1",
+ "attempt_id": "a1",
+ "command": "echo hi",
+ "result": None,
+ "exit_code": None,
+ "started_at": 10.0,
+ "duration_seconds": 2.5,
+ }
+ record = record_from_task("deadbeef", task)
+ assert record.owner_id == "session:s1"
+ assert record.attempt_id == "a1"
+ assert record.status == "running"
+ assert record.started_at == 10.0
+ assert record.duration_seconds == 2.5
+ assert record.is_live and not record.is_terminal
+
+
+class TestManagerWiring:
+ def test_a_completed_run_leaves_a_queryable_record(self, tmp_path, monkeypatch):
+ import json as _json
+ import shlex
+ import sys as _sys
+
+ from src.tools import background_tools as bg
+
+ monkeypatch.setattr(bg, "WORKDIR", tmp_path)
+ manager = bg.BackgroundManager(bg.BackgroundScope("session:s1"), store=BackgroundTaskStore(tmp_path))
+ command = shlex.join([_sys.executable, "-c", "print('stored')"])
+ task_id = _json.loads(manager.run(command))["task_id"]
+ deadline = time.monotonic() + 15
+ while time.monotonic() < deadline and _json.loads(manager.check(task_id))["status"] == "running":
+ time.sleep(0.01)
+ records, unreadable = manager._store.load_all()
+ assert unreadable == []
+ assert records[0].task_id == task_id
+ assert records[0].status == "completed"
+ assert records[0].attempt_id is None
+ assert records[0].command_preview
+
+ def test_restart_recovery_leaves_this_processes_live_tasks_alone(self, tmp_path):
+ from src.tools import background_tools as bg
+
+ store = BackgroundTaskStore(tmp_path)
+ live = make(task_id="aaaaaaaa", status="running", owner_id="session:live")
+ stale = make(task_id="bbbbbbbb", status="running", owner_id="session:stale")
+ store.save(live)
+ store.save(stale)
+ bg._LIVE_TASK_IDS.add("aaaaaaaa")
+ try:
+ first = BackgroundTaskStore(tmp_path)
+ manager = bg.BackgroundManager(bg.BackgroundScope("session:x"), store=first)
+ summary = manager.recover_on_start()
+ finally:
+ bg._LIVE_TASK_IDS.discard("aaaaaaaa")
+ assert summary["recovered"] == 1
+ records, _ = first.load_all()
+ by_id = {record.task_id: record for record in records}
+ assert by_id["aaaaaaaa"].status == "running"
+ assert by_id["bbbbbbbb"].status == "interrupted"
+
+ def test_recovery_does_not_rerun_a_previous_commands(self, tmp_path, monkeypatch):
+ from src.tools import background_tools as bg
+
+ monkeypatch.setattr(bg, "WORKDIR", tmp_path)
+ store = BackgroundTaskStore(tmp_path)
+ store.save(make(task_id="cccccccc", status="running", command_preview="touch should-not-exist"))
+ manager = bg.BackgroundManager(bg.BackgroundScope("session:x"), store=store)
+ assert manager.recover_on_start()["recovered"] == 1
+ assert not (tmp_path / "should-not-exist").exists()