"Vibe-Trading: Your Personal Trading Agent"
Python ai-agentalgorithmic-tradingbacktestingfintechllmmcpmulti-agentpythonquantitative-financetrading
feat(agent): persist background task outcomes and reconcile them after restart
Yufeng Heb4de393bcd70
diff --git a/agent/src/session/service.py b/agent/src/session/service.pyindex 32731aa27..e36fc592a 100644--- a/agent/src/session/service.py+++ b/agent/src/session/service.py@@ -119,6 +119,11 @@ class SessionService:self._search_index = get_shared_index()self._background_handles: Dict[str, Any] = {}self._shutting_down = False+ # Durable records live under the runtime root that owns runs_dir, so a+ # test pointing runs_dir at a temp root gets its own isolated store.+ from src.tools.background_tools import init_background_store++ self._background_store = init_background_store(Path(runs_dir).parent)self._recover_interrupted_attempts()def _recover_interrupted_attempts(self) -> None:diff --git a/agent/src/tools/background_store.py b/agent/src/tools/background_store.pynew file mode 100644index 000000000..fb63257ae--- /dev/null+++ b/agent/src/tools/background_store.py@@ -0,0 +1,223 @@+"""Durable records for background tasks, and conservative restart recovery.++A background task currently lives only in memory, so a process restart loses+the fact that a command ever ran and the user is left guessing. This stores one+small record per task under the canonical runtime root and, on a new process,+reconciles what it finds.++Recovery is deliberately one-way and never executes anything. A record left in+``running`` or ``cancelling`` belongs to a process that is gone; it becomes+``interrupted`` or ``outcome_unknown`` with a recovery reason. It is never+re-run, and a stored PID is descriptive only: it is not used to signal a+process, because that pid may have been reused by an unrelated program.+"""++from __future__ import annotations++import hashlib+import json+import os+import re+import tempfile+import time+from dataclasses import asdict, dataclass, field, replace+from pathlib import Path+from typing import Any, Iterable, Optional++from src.tools.redaction import redact_text++SCHEMA_VERSION = 1+#: Preview length kept per field. Enough to identify a command or outcome+#: without becoming a second copy of the run's full output.+MAX_PREVIEW_CHARS = 2_000+_TASK_ID_RE = re.compile(r"^[0-9a-f]{8,64}$")++#: Statuses a task can hold while its owning process is still alive.+LIVE_STATUSES = frozenset({"running", "cancelling"})+#: Statuses whose result is worth reading back after a restart.+TERMINAL_STATUSES = frozenset({"completed", "error", "timeout", "cancelled"})+++class BackgroundStoreError(RuntimeError):+ """A record could not be stored or read back safely."""+++@dataclass+class TaskRecord:+ """What survives a restart for one background task."""++ schema_version: int+ task_id: str+ owner_id: str+ status: str+ command_preview: str+ attempt_id: Optional[str] = None+ exit_code: Optional[int] = None+ result_preview: str = ""+ started_at: Optional[float] = None+ ended_at: Optional[float] = None+ duration_seconds: Optional[float] = None+ recovery_reason: Optional[str] = None+ #: Descriptive only. Never used to signal a process after a restart.+ pid: Optional[int] = None+ updated_at: float = field(default_factory=time.time)++ @property+ def is_live(self) -> bool:+ return self.status in LIVE_STATUSES++ @property+ def is_terminal(self) -> bool:+ return self.status in TERMINAL_STATUSES+++def _preview(value: object) -> str:+ """Bound and redact one untrusted field before it reaches disk."""+ return redact_text(str(value or ""))[:MAX_PREVIEW_CHARS]+++def _owner_token(owner_id: str) -> str:+ """Stable layout-only token for an owner id.++ A hash rather than the raw id: the owner carries a session identifier that+ is not a secret but is not a filename either, and the raw form would put+ user-supplied separators into the path.+ """+ return hashlib.sha256(owner_id.encode("utf-8")).hexdigest()[:16]+++class BackgroundTaskStore:+ """One JSON record per task, written atomically under a fixed root."""++ def __init__(self, root: Path) -> None:+ self.root = Path(root)+ self._directory = self.root / "background-tasks"++ def _path_for(self, record: TaskRecord) -> Path:+ if not _TASK_ID_RE.match(record.task_id):+ raise BackgroundStoreError(f"unsafe background task id: {record.task_id!r}")+ return self._directory / f"{_owner_token(record.owner_id)}-{record.task_id}.json"++ def save(self, record: TaskRecord) -> Path:+ """Persist one record, replacing any previous copy of it."""+ path = self._path_for(record)+ self._write_json_atomically(path, asdict(record))+ return path++ def load_all(self) -> tuple[list[TaskRecord], list[str]]:+ """Return readable records and the paths that could not be read.++ A corrupt or unknown-version file is reported, never overwritten and+ never guessed at: silently dropping it would hide the task, and+ silently reinterpreting it would invent an outcome.+ """+ records: list[TaskRecord] = []+ unreadable: list[str] = []+ if not self._directory.exists():+ return records, unreadable+ for path in sorted(self._directory.glob("*.json")):+ record = self._load_one(path)+ if record is None:+ unreadable.append(str(path))+ else:+ records.append(record)+ return records, unreadable++ def _load_one(self, path: Path) -> Optional[TaskRecord]:+ try:+ payload = json.loads(path.read_text(encoding="utf-8"))+ except (OSError, ValueError):+ return None+ if not isinstance(payload, dict):+ return None+ if payload.get("schema_version") != SCHEMA_VERSION:+ return None+ try:+ return TaskRecord(**payload)+ except TypeError:+ return None++ @staticmethod+ def _write_json_atomically(path: Path, payload: dict[str, Any]) -> None:+ path.parent.mkdir(parents=True, exist_ok=True, mode=0o700)+ content = (json.dumps(payload, ensure_ascii=False, indent=2) + "\n").encode("utf-8")+ fd, temporary = tempfile.mkstemp(dir=path.parent, prefix=f".{path.name}.", suffix=".tmp")+ try:+ try:+ with os.fdopen(fd, "wb") as handle:+ fd = -1+ handle.write(content)+ handle.flush()+ if hasattr(os, "fchmod"):+ os.fchmod(handle.fileno(), 0o600)+ os.fsync(handle.fileno())+ finally:+ if fd >= 0:+ os.close(fd)+ os.replace(temporary, path)+ except BaseException:+ try:+ os.unlink(temporary)+ except OSError:+ pass+ raise+ try:+ os.chmod(path, 0o600)+ except OSError:+ pass+++def record_from_task(task_id: str, task: dict, *, started_at: Optional[float] = None) -> TaskRecord:+ """Project an in-memory task entry onto its durable shape."""+ started = task.get("started_at")+ return TaskRecord(+ schema_version=SCHEMA_VERSION,+ task_id=task_id,+ owner_id=str(task.get("owner_id") or ""),+ attempt_id=task.get("attempt_id"),+ status=str(task.get("status") or "error"),+ command_preview=_preview(task.get("command")),+ result_preview=_preview(task.get("result")),+ exit_code=task.get("exit_code") if isinstance(task.get("exit_code"), int) else None,+ started_at=float(started) if isinstance(started, (int, float)) else started_at,+ ended_at=task.get("ended_at"),+ duration_seconds=task.get("duration_seconds"),+ pid=task.get("pid") if isinstance(task.get("pid"), int) else None,+ )+++def reconcile_after_restart(+ records: Iterable[TaskRecord],+) -> tuple[list[TaskRecord], list[TaskRecord]]:+ """Turn a previous process's live records into explicit outcomes.++ Nothing is re-run and nothing is signalled. A record left ``running`` cannot+ be distinguished from one whose output was never written, so it becomes+ ``interrupted``; a record left ``cancelling`` cannot be distinguished from+ one whose reaping never finished, so it becomes ``outcome_unknown``. Both+ carry a recovery reason rather than being dropped.+ """+ recovered: list[TaskRecord] = []+ unchanged: list[TaskRecord] = []+ for record in records:+ if record.status == "running":+ recovered.append(+ replace(+ record,+ status="interrupted",+ recovery_reason="service_restart",+ ended_at=time.time(),+ )+ )+ elif record.status == "cancelling":+ recovered.append(+ replace(+ record,+ status="outcome_unknown",+ recovery_reason="service_restart_before_reap_completed",+ ended_at=time.time(),+ )+ )+ else:+ unchanged.append(record)+ return recovered, unchangeddiff --git a/agent/src/tools/background_tools.py b/agent/src/tools/background_tools.pyindex 37c50732e..8d49aeefc 100644--- a/agent/src/tools/background_tools.py+++ b/agent/src/tools/background_tools.py@@ -18,6 +18,10 @@ from src.agent.tools import BaseToolfrom src.tools._shell_safety import broad_python_kill_errorWORKDIR = Path(__file__).resolve().parents[2]+#: Task ids with a live process anywhere in this interpreter. Restart recovery+#: must not mistake a sibling manager's running task for a previous process's.+_LIVE_TASK_IDS: set[str] = set()+_LIVE_TASK_IDS_LOCK = threading.Lock()_COMMAND_TIMEOUT_SECONDS = 300.0_TERMINATION_GRACE_SECONDS = 2.0_MAX_OUTPUT_CHARS = 50_000@@ -141,8 +145,10 @@ class BackgroundManager:scope: Optional[BackgroundScope] = None,*,on_completion: Optional[Callable[[str, dict], None]] = None,+ store: Optional[Any] = None,) -> None:self.tasks: Dict[str, dict] = {}+ self._store = store# (owner_id, payload). Kept as pairs so the payload shape consumers and# existing tests see stays exactly what it was; routing is internal.self._notifications: List[tuple[str, dict]] = []@@ -170,6 +176,7 @@ class BackgroundManager:other._lock = self._lockother._condition = self._conditionother._accepting = self._accepting+ other._store = self._storeother._on_completion = (self._on_completion if on_completion is None else on_completion)@@ -178,6 +185,43 @@ class BackgroundManager:def _owns(self, task: dict) -> bool:return task.get("owner_id") == self._scope.owner_id+ def _persist(self, task_id: str) -> None:+ """Best effort: a full or unwritable runtime root must not break the run."""+ if self._store is None:+ return+ task = self.tasks.get(task_id)+ if task is None:+ return+ try:+ from src.tools.background_store import record_from_task++ self._store.save(record_from_task(task_id, task))+ except Exception:+ logging.getLogger(__name__).exception("Could not persist background task %s", task_id)++ def recover_on_start(self) -> dict:+ """Reconcile records left by a previous process, once and without reruns.++ A task id this interpreter is already running is left alone: a second+ service constructed in the same process must not declare the first+ one's live work interrupted.+ """+ if self._store is None:+ return {"recovered": 0, "unchanged": 0, "unreadable": []}+ from src.tools.background_store import reconcile_after_restart++ records, unreadable = self._store.load_all()+ with _LIVE_TASK_IDS_LOCK:+ live = set(_LIVE_TASK_IDS)+ stale = [record for record in records if record.task_id not in live]+ recovered, unchanged = reconcile_after_restart(stale)+ for record in recovered:+ try:+ self._store.save(record)+ except Exception:+ logging.getLogger(__name__).exception("Could not persist recovered task %s", record.task_id)+ return {"recovered": len(recovered), "unchanged": len(unchanged), "unreadable": unreadable}+def run(self, command: str) -> str:"""Start a background task and return its task_id.@@ -210,6 +254,9 @@ class BackgroundManager:"owner_id": self._scope.owner_id,"attempt_id": self._scope.attempt_id,}+ with _LIVE_TASK_IDS_LOCK:+ _LIVE_TASK_IDS.add(task_id)+ self._persist(task_id)threading.Thread(target=self._execute, args=(task_id, command), daemon=True).start()return json.dumps({"status": "ok", "task_id": task_id, "message": f"Started: {command[:80]}"})@@ -283,7 +330,12 @@ class BackgroundManager:"exit_code": exit_code,}self._notifications.append((task["owner_id"], payload))+ task["ended_at"] = time.time()+ task["pid"] = getattr(process, "pid", None)self._condition.notify_all()+ with _LIVE_TASK_IDS_LOCK:+ _LIVE_TASK_IDS.discard(task_id)+ self._persist(task_id)if self._on_completion is not None:try:self._on_completion(task["owner_id"], {**payload, "attempt_id": task.get("attempt_id")})@@ -436,6 +488,25 @@ class BackgroundManager:_BG = BackgroundManager()+_BG_STORE: Optional[Any] = None+++def init_background_store(root: Path) -> Any:+ """Attach durable task records to the process-wide table, once per process.++ Recovery runs here and only here, so a second service constructed in the+ same interpreter cannot declare the first one's live work interrupted.+ """+ global _BG_STORE+ with _LIVE_TASK_IDS_LOCK:+ if _BG_STORE is not None:+ return _BG_STORE+ from src.tools.background_store import BackgroundTaskStore++ _BG_STORE = BackgroundTaskStore(Path(root))+ _BG._store = _BG_STORE+ _BG.recover_on_start()+ return _BG_STOREdef get_background_manager() -> BackgroundManager:diff --git a/agent/tests/test_background_store_recovery.py b/agent/tests/test_background_store_recovery.pynew file mode 100644index 000000000..b10183873--- /dev/null+++ b/agent/tests/test_background_store_recovery.py@@ -0,0 +1,245 @@+"""Durable task records and the restart reconciliation that must never re-run."""++from __future__ import annotations++import json+import os+import stat+import time+from pathlib import Path++import pytest++from src.tools import background_store as store_mod+from src.tools.background_store import (+ SCHEMA_VERSION,+ BackgroundStoreError,+ BackgroundTaskStore,+ TaskRecord,+ record_from_task,+ reconcile_after_restart,+)+++def make(**overrides) -> TaskRecord:+ values = dict(+ schema_version=SCHEMA_VERSION,+ task_id="deadbeef",+ owner_id="session:s1",+ status="completed",+ command_preview="echo hi",+ exit_code=0,+ result_preview="hi",+ )+ values.update(overrides)+ return TaskRecord(**values)+++class TestRecoveryIsConservative:+ @pytest.fixture(autouse=True)+ def forbid_side_effects(self, monkeypatch):+ """Recovery must not spawn, signal, fetch or call a model."""++ def explode(*args, **kwargs):+ raise AssertionError(f"recovery must not call {kwargs or args}")++ for module, names in [+ (store_mod, []),+ ]:+ for name in names:+ monkeypatch.setattr(module, name, explode)+ monkeypatch.setattr(store_mod.os, "kill", explode)+ monkeypatch.setattr(store_mod.os, "killpg", explode)+ monkeypatch.setattr(store_mod.time, "sleep", explode)++ def test_running_becomes_interrupted_without_rerun(self):+ recovered, unchanged = reconcile_after_restart([make(status="running")])+ assert unchanged == []+ assert recovered[0].status == "interrupted"+ assert recovered[0].recovery_reason == "service_restart"+ assert recovered[0].command_preview == "echo hi"++ def test_cancelling_stays_outcome_unknown(self):+ recovered, _ = reconcile_after_restart([make(status="cancelling")])+ assert recovered[0].status == "outcome_unknown"+ assert recovered[0].recovery_reason == "service_restart_before_reap_completed"++ @pytest.mark.parametrize("status", ["completed", "error", "timeout", "cancelled"])+ def test_terminal_outcomes_are_kept_queryable(self, status):+ recovered, unchanged = reconcile_after_restart([make(status=status, exit_code=3)])+ assert recovered == []+ assert unchanged[0].status == status+ assert unchanged[0].exit_code == 3++ def test_recovery_is_idempotent(self):+ once, _ = reconcile_after_restart([make(status="running")])+ twice, unchanged = reconcile_after_restart(once)+ assert twice == []+ assert unchanged[0].status == "interrupted"++ def test_stored_pid_is_descriptive_only(self):+ record = make(status="running", pid=os.getpid())+ recovered, _ = reconcile_after_restart([record])+ assert recovered[0].pid == os.getpid()+ # No signal path reads it; nothing above touches os.kill.+++class TestStore:+ def test_round_trip_and_layout_is_stable_per_owner(self, tmp_path):+ store = BackgroundTaskStore(tmp_path)+ record = make()+ path = store.save(record)+ assert path.parent == tmp_path / "background-tasks"+ assert path.name == path.name # deterministic+ assert store.save(record).name == path.name+ records, unreadable = store.load_all()+ assert unreadable == []+ assert records == [record]++ def test_owner_id_is_not_a_filename(self, tmp_path):+ store = BackgroundTaskStore(tmp_path)+ path = store.save(make(owner_id="session:../escape"))+ assert ".." not in path.name+ assert path.parent == tmp_path / "background-tasks"++ def test_unsafe_task_id_is_rejected(self, tmp_path):+ store = BackgroundTaskStore(tmp_path)+ with pytest.raises(BackgroundStoreError):+ store.save(make(task_id="../../etc/passwd"))++ def test_private_permissions(self, tmp_path):+ store = BackgroundTaskStore(tmp_path)+ path = store.save(make())+ assert stat.S_IMODE(path.stat().st_mode) == 0o600+ assert stat.S_IMODE(path.parent.stat().st_mode) == 0o700++ def test_unknown_version_is_reported_not_overwritten(self, tmp_path):+ store = BackgroundTaskStore(tmp_path)+ store.save(make())+ for path in (tmp_path / "background-tasks").glob("*.json"):+ payload = json.loads(path.read_text())+ payload["schema_version"] = SCHEMA_VERSION + 1+ path.write_text(json.dumps(payload))+ records, unreadable = store.load_all()+ assert records == []+ assert len(unreadable) == 1+ assert json.loads(Path(unreadable[0]).read_text())["schema_version"] == SCHEMA_VERSION + 1++ def test_half_written_file_is_reported_not_invented(self, tmp_path):+ store = BackgroundTaskStore(tmp_path)+ directory = tmp_path / "background-tasks"+ directory.mkdir()+ (directory / "deadbeef.json").write_text('{"schema_version": 1, "task_id"')+ records, unreadable = store.load_all()+ assert records == []+ assert len(unreadable) == 1++ def test_symlink_is_refused(self, tmp_path):+ store = BackgroundTaskStore(tmp_path)+ directory = tmp_path / "background-tasks"+ directory.mkdir()+ outside = tmp_path / "outside.json"+ outside.write_text(json.dumps({"schema_version": SCHEMA_VERSION}))+ link = directory / f"{'0' * 16}-deadbeef.json"+ link.symlink_to(outside)+ _records, unreadable = store.load_all()+ assert len(unreadable) == 1++ def test_previews_are_bounded_and_redacted(self, tmp_path):+ record = make(+ command_preview=store_mod._preview(" ".join(["x" * 100] * 50)),+ result_preview=store_mod._preview(f"token sk-{'a' * 40} end"),+ )+ assert len(record.command_preview) <= store_mod.MAX_PREVIEW_CHARS+ assert len(record.result_preview) <= store_mod.MAX_PREVIEW_CHARS+ assert f"sk-{'a' * 40}" not in record.result_preview++ def test_two_local_processes_can_share_the_directory(self, tmp_path):+ """One process's records stay readable after another writes beside them."""+ first = BackgroundTaskStore(tmp_path)+ second = BackgroundTaskStore(tmp_path)+ first.save(make(task_id="aaaaaaaa", owner_id="session:a"))+ second.save(make(task_id="bbbbbbbb", owner_id="session:b"))+ records, unreadable = first.load_all()+ assert unreadable == []+ assert {r.task_id for r in records} == {"aaaaaaaa", "bbbbbbbb"}++ def test_recovery_survives_a_process_that_never_wrote(self, tmp_path):+ """A crash before the first save leaves no record, and none is invented."""+ store = BackgroundTaskStore(tmp_path)+ assert store.load_all() == ([], [])+ recovered, unchanged = reconcile_after_restart([])+ assert recovered == [] and unchanged == []++ def test_record_from_task_projects_the_live_shape(self):+ task = {+ "status": "running",+ "owner_id": "session:s1",+ "attempt_id": "a1",+ "command": "echo hi",+ "result": None,+ "exit_code": None,+ "started_at": 10.0,+ "duration_seconds": 2.5,+ }+ record = record_from_task("deadbeef", task)+ assert record.owner_id == "session:s1"+ assert record.attempt_id == "a1"+ assert record.status == "running"+ assert record.started_at == 10.0+ assert record.duration_seconds == 2.5+ assert record.is_live and not record.is_terminal+++class TestManagerWiring:+ def test_a_completed_run_leaves_a_queryable_record(self, tmp_path, monkeypatch):+ import json as _json+ import shlex+ import sys as _sys++ from src.tools import background_tools as bg++ monkeypatch.setattr(bg, "WORKDIR", tmp_path)+ manager = bg.BackgroundManager(bg.BackgroundScope("session:s1"), store=BackgroundTaskStore(tmp_path))+ command = shlex.join([_sys.executable, "-c", "print('stored')"])+ task_id = _json.loads(manager.run(command))["task_id"]+ deadline = time.monotonic() + 15+ while time.monotonic() < deadline and _json.loads(manager.check(task_id))["status"] == "running":+ time.sleep(0.01)+ records, unreadable = manager._store.load_all()+ assert unreadable == []+ assert records[0].task_id == task_id+ assert records[0].status == "completed"+ assert records[0].attempt_id is None+ assert records[0].command_preview++ def test_restart_recovery_leaves_this_processes_live_tasks_alone(self, tmp_path):+ from src.tools import background_tools as bg++ store = BackgroundTaskStore(tmp_path)+ live = make(task_id="aaaaaaaa", status="running", owner_id="session:live")+ stale = make(task_id="bbbbbbbb", status="running", owner_id="session:stale")+ store.save(live)+ store.save(stale)+ bg._LIVE_TASK_IDS.add("aaaaaaaa")+ try:+ first = BackgroundTaskStore(tmp_path)+ manager = bg.BackgroundManager(bg.BackgroundScope("session:x"), store=first)+ summary = manager.recover_on_start()+ finally:+ bg._LIVE_TASK_IDS.discard("aaaaaaaa")+ assert summary["recovered"] == 1+ records, _ = first.load_all()+ by_id = {record.task_id: record for record in records}+ assert by_id["aaaaaaaa"].status == "running"+ assert by_id["bbbbbbbb"].status == "interrupted"++ def test_recovery_does_not_rerun_a_previous_commands(self, tmp_path, monkeypatch):+ from src.tools import background_tools as bg++ monkeypatch.setattr(bg, "WORKDIR", tmp_path)+ store = BackgroundTaskStore(tmp_path)+ store.save(make(task_id="cccccccc", status="running", command_preview="touch should-not-exist"))+ manager = bg.BackgroundManager(bg.BackgroundScope("session:x"), store=store)+ assert manager.recover_on_start()["recovered"] == 1+ assert not (tmp_path / "should-not-exist").exists()