"Vibe-Trading: Your Personal Trading Agent"
Python ai-agentalgorithmic-tradingbacktestingfintechllmmcpmulti-agentpythonquantitative-financetrading
Merge PR #2100: persist scoped background tasks and expose recovered outcomes
Haozhe Wu0104ff97d455
diff --cc .gitignoreindex fe3223ab7,fe3223ab7..4aeb80a8b--- a/.gitignore+++ b/.gitignore@@@ -188,3 -188,3 +188,6 @@@ agent/tests/artifacts# Local simulated account ledgers/paper-trading/++++# Runtime background command previews and restart outcomes++/background-tasks/diff --cc agent/src/session/service.pyindex 0af58474e,e36fc592a..2d0bbcbac--- a/agent/src/session/service.py+++ b/agent/src/session/service.py@@@ -650,7 -646,7 +655,7 @@@ class SessionServicebackground_scope = BackgroundScope(owner_id=f"session:{session_id}", attempt_id=attempt_id)-- background = get_background_manager().for_scope(++ background = get_background_manager(Path(self.runs_dir).parent).for_scope(background_scope,on_completion=self._publish_background_completion,)diff --cc agent/src/tools/background_store.pyindex 000000000,fb63257ae..5a7b00510mode 000000,100644..100644--- a/agent/src/tools/background_store.py+++ b/agent/src/tools/background_store.py@@@ -1,0 -1,223 +1,238 @@@+ """Durable records for background tasks, and conservative restart recovery.++ A background task currently lives only in memory, so a process restart loses+ the fact that a command ever ran and the user is left guessing. This stores one+ small record per task under the canonical runtime root and, on a new process,+ reconciles what it finds.++ Recovery is deliberately one-way and never executes anything. A record left in+ ``running`` or ``cancelling`` belongs to a process that is gone; it becomes+ ``interrupted`` or ``outcome_unknown`` with a recovery reason. It is never+ re-run, and a stored PID is descriptive only: it is not used to signal a+ process, because that pid may have been reused by an unrelated program.+ """++ from __future__ import annotations++ import hashlib+ import json++import math+ import os+ import re+ import tempfile+ import time+ from dataclasses import asdict, dataclass, field, replace+ from pathlib import Path+ from typing import Any, Iterable, Optional++ from src.tools.redaction import redact_text++ SCHEMA_VERSION = 1+ #: Preview length kept per field. Enough to identify a command or outcome+ #: without becoming a second copy of the run's full output.+ MAX_PREVIEW_CHARS = 2_000+ _TASK_ID_RE = re.compile(r"^[0-9a-f]{8,64}$")++ #: Statuses a task can hold while its owning process is still alive.+ LIVE_STATUSES = frozenset({"running", "cancelling"})+ #: Statuses whose result is worth reading back after a restart.+ TERMINAL_STATUSES = frozenset({"completed", "error", "timeout", "cancelled"})+++ class BackgroundStoreError(RuntimeError):+ """A record could not be stored or read back safely."""+++ @dataclass+ class TaskRecord:+ """What survives a restart for one background task."""++ schema_version: int+ task_id: str+ owner_id: str+ status: str+ command_preview: str+ attempt_id: Optional[str] = None+ exit_code: Optional[int] = None+ result_preview: str = ""+ started_at: Optional[float] = None+ ended_at: Optional[float] = None+ duration_seconds: Optional[float] = None+ recovery_reason: Optional[str] = None+ #: Descriptive only. Never used to signal a process after a restart.+ pid: Optional[int] = None+ updated_at: float = field(default_factory=time.time)++ @property+ def is_live(self) -> bool:+ return self.status in LIVE_STATUSES++ @property+ def is_terminal(self) -> bool:+ return self.status in TERMINAL_STATUSES+++ def _preview(value: object) -> str:+ """Bound and redact one untrusted field before it reaches disk."""+ return redact_text(str(value or ""))[:MAX_PREVIEW_CHARS]+++ def _owner_token(owner_id: str) -> str:+ """Stable layout-only token for an owner id.++ A hash rather than the raw id: the owner carries a session identifier that+ is not a secret but is not a filename either, and the raw form would put+ user-supplied separators into the path.+ """+ return hashlib.sha256(owner_id.encode("utf-8")).hexdigest()[:16]+++ class BackgroundTaskStore:+ """One JSON record per task, written atomically under a fixed root."""++ def __init__(self, root: Path) -> None:+ self.root = Path(root)+ self._directory = self.root / "background-tasks"++ def _path_for(self, record: TaskRecord) -> Path:+ if not _TASK_ID_RE.match(record.task_id):+ raise BackgroundStoreError(f"unsafe background task id: {record.task_id!r}")+ return self._directory / f"{_owner_token(record.owner_id)}-{record.task_id}.json"++ def save(self, record: TaskRecord) -> Path:+ """Persist one record, replacing any previous copy of it."""+ path = self._path_for(record)+ self._write_json_atomically(path, asdict(record))+ return path++ def load_all(self) -> tuple[list[TaskRecord], list[str]]:+ """Return readable records and the paths that could not be read.++ A corrupt or unknown-version file is reported, never overwritten and+ never guessed at: silently dropping it would hide the task, and+ silently reinterpreting it would invent an outcome.+ """+ records: list[TaskRecord] = []+ unreadable: list[str] = []+ if not self._directory.exists():+ return records, unreadable+ for path in sorted(self._directory.glob("*.json")):+ record = self._load_one(path)+ if record is None:+ unreadable.append(str(path))+ else:+ records.append(record)+ return records, unreadable++ def _load_one(self, path: Path) -> Optional[TaskRecord]:+ try:++ if path.is_symlink() or path.stat().st_size > 64_000:++ return None+ payload = json.loads(path.read_text(encoding="utf-8"))+ except (OSError, ValueError):+ return None+ if not isinstance(payload, dict):+ return None+ if payload.get("schema_version") != SCHEMA_VERSION:+ return None+ try:- return TaskRecord(**payload)- except TypeError:++ record = TaskRecord(**payload)++ if (not isinstance(record.task_id, str) or not _TASK_ID_RE.fullmatch(record.task_id)++ or not isinstance(record.owner_id, str) or not record.owner_id.strip()++ or record.status not in LIVE_STATUSES | TERMINAL_STATUSES | {"interrupted", "outcome_unknown"}++ or path.name != self._path_for(record).name):++ return None++ for value in (record.started_at, record.ended_at, record.duration_seconds, record.updated_at):++ if value is not None and (isinstance(value, bool) or not isinstance(value, (int, float))++ or not math.isfinite(value)):++ return None++ record.command_preview = _preview(record.command_preview)++ record.result_preview = _preview(record.result_preview)++ return record++ except (TypeError, ValueError, BackgroundStoreError):+ return None++ @staticmethod+ def _write_json_atomically(path: Path, payload: dict[str, Any]) -> None:+ path.parent.mkdir(parents=True, exist_ok=True, mode=0o700)+ content = (json.dumps(payload, ensure_ascii=False, indent=2) + "\n").encode("utf-8")+ fd, temporary = tempfile.mkstemp(dir=path.parent, prefix=f".{path.name}.", suffix=".tmp")+ try:+ try:+ with os.fdopen(fd, "wb") as handle:+ fd = -1+ handle.write(content)+ handle.flush()+ if hasattr(os, "fchmod"):+ os.fchmod(handle.fileno(), 0o600)+ os.fsync(handle.fileno())+ finally:+ if fd >= 0:+ os.close(fd)+ os.replace(temporary, path)+ except BaseException:+ try:+ os.unlink(temporary)+ except OSError:+ pass+ raise+ try:+ os.chmod(path, 0o600)+ except OSError:+ pass+++ def record_from_task(task_id: str, task: dict, *, started_at: Optional[float] = None) -> TaskRecord:+ """Project an in-memory task entry onto its durable shape."""- started = task.get("started_at")++ started = task.get("started_wall_time", task.get("started_at"))+ return TaskRecord(+ schema_version=SCHEMA_VERSION,+ task_id=task_id,+ owner_id=str(task.get("owner_id") or ""),+ attempt_id=task.get("attempt_id"),+ status=str(task.get("status") or "error"),+ command_preview=_preview(task.get("command")),+ result_preview=_preview(task.get("result")),+ exit_code=task.get("exit_code") if isinstance(task.get("exit_code"), int) else None,+ started_at=float(started) if isinstance(started, (int, float)) else started_at,+ ended_at=task.get("ended_at"),+ duration_seconds=task.get("duration_seconds"),+ pid=task.get("pid") if isinstance(task.get("pid"), int) else None,+ )+++ def reconcile_after_restart(+ records: Iterable[TaskRecord],+ ) -> tuple[list[TaskRecord], list[TaskRecord]]:+ """Turn a previous process's live records into explicit outcomes.++ Nothing is re-run and nothing is signalled. A record left ``running`` cannot+ be distinguished from one whose output was never written, so it becomes+ ``interrupted``; a record left ``cancelling`` cannot be distinguished from+ one whose reaping never finished, so it becomes ``outcome_unknown``. Both+ carry a recovery reason rather than being dropped.+ """+ recovered: list[TaskRecord] = []+ unchanged: list[TaskRecord] = []+ for record in records:+ if record.status == "running":+ recovered.append(+ replace(+ record,+ status="interrupted",+ recovery_reason="service_restart",+ ended_at=time.time(),+ )+ )+ elif record.status == "cancelling":+ recovered.append(+ replace(+ record,+ status="outcome_unknown",+ recovery_reason="service_restart_before_reap_completed",+ ended_at=time.time(),+ )+ )+ else:+ unchanged.append(record)+ return recovered, unchangeddiff --cc agent/src/tools/background_tools.pyindex 37c50732e,8d49aeefc..1d913f8d5--- a/agent/src/tools/background_tools.py+++ b/agent/src/tools/background_tools.py@@@ -141,8 -145,10 +145,11 @@@ class BackgroundManagerscope: Optional[BackgroundScope] = None,*,on_completion: Optional[Callable[[str, dict], None]] = None,+ store: Optional[Any] = None,) -> None:self.tasks: Dict[str, dict] = {}+ self._store = store++ self._persist_lock = threading.Lock()# (owner_id, payload). Kept as pairs so the payload shape consumers and# existing tests see stays exactly what it was; routing is internal.self._notifications: List[tuple[str, dict]] = []@@@ -170,6 -176,7 +177,8 @@@other._lock = self._lockother._condition = self._conditionother._accepting = self._accepting+ other._store = self._store++ other._persist_lock = self._persist_lockother._on_completion = (self._on_completion if on_completion is None else on_completion)@@@ -178,6 -185,43 +187,57 @@@def _owns(self, task: dict) -> bool:return task.get("owner_id") == self._scope.owner_id+ def _persist(self, task_id: str) -> None:+ """Best effort: a full or unwritable runtime root must not break the run."""+ if self._store is None:+ return- task = self.tasks.get(task_id)- if task is None:- return- try:- from src.tools.background_store import record_from_task-- self._store.save(record_from_task(task_id, task))- except Exception:- logging.getLogger(__name__).exception("Could not persist background task %s", task_id)++ with self._persist_lock:++ with self._lock:++ task = dict(self.tasks.get(task_id) or {})++ if not task:++ return++ try:++ from src.tools.background_store import record_from_task++ self._store.save(record_from_task(task_id, task))++ except Exception:++ logging.getLogger(__name__).exception("Could not persist background task %s", task_id)++ def recover_on_start(self) -> dict:+ """Reconcile records left by a previous process, once and without reruns.++ A task id this interpreter is already running is left alone: a second+ service constructed in the same process must not declare the first+ one's live work interrupted.+ """+ if self._store is None:+ return {"recovered": 0, "unchanged": 0, "unreadable": []}+ from src.tools.background_store import reconcile_after_restart++ records, unreadable = self._store.load_all()+ with _LIVE_TASK_IDS_LOCK:+ live = set(_LIVE_TASK_IDS)+ stale = [record for record in records if record.task_id not in live]+ recovered, unchanged = reconcile_after_restart(stale)+ for record in recovered:+ try:+ self._store.save(record)+ except Exception:+ logging.getLogger(__name__).exception("Could not persist recovered task %s", record.task_id)++ with self._lock:++ for record in recovered + unchanged:++ if record.task_id in self.tasks:++ continue++ self.tasks[record.task_id] = {++ "status": record.status, "command": record.command_preview,++ "result": record.result_preview, "exit_code": record.exit_code,++ "owner_id": record.owner_id, "attempt_id": record.attempt_id,++ "process": None, "started_at": None,++ "started_wall_time": record.started_at, "ended_at": record.ended_at,++ "duration_seconds": record.duration_seconds,++ "recovery_reason": record.recovery_reason,++ }+ return {"recovered": len(recovered), "unchanged": len(unchanged), "unreadable": unreadable}+def run(self, command: str) -> str:"""Start a background task and return its task_id.@@@ -207,6 -251,6 +267,7 @@@"process": None,"cancel_requested": False,"started_at": started_at,++ "started_wall_time": time.time(),"owner_id": self._scope.owner_id,"attempt_id": self._scope.attempt_id,}@@@ -227,7 -274,7 +291,9 @@@with self._lock:task = self.tasks[task_id]task["process"] = process++ task["pid"] = process.pidcancel_requested = bool(task.get("cancel_requested"))++ self._persist(task_id)if cancel_requested:_force_stop_process_tree(process)@@@ -283,7 -330,12 +349,12 @@@"exit_code": exit_code,}self._notifications.append((task["owner_id"], payload))+ task["ended_at"] = time.time()+ task["pid"] = getattr(process, "pid", None)self._condition.notify_all()++ self._persist(task_id)+ with _LIVE_TASK_IDS_LOCK:+ _LIVE_TASK_IDS.discard(task_id)- self._persist(task_id)if self._on_completion is not None:try:self._on_completion(task["owner_id"], {**payload, "attempt_id": task.get("attempt_id")})@@@ -361,6 -413,6 +432,7 @@@task["cancel_requested"] = Truetask["status"] = "cancelling"++ self._persist(task_id)if process is not None:_force_stop_process_tree(process)return json.dumps({@@@ -397,6 -449,6 +469,10 @@@"exit_code": t.get("exit_code"),"attempt_id": t.get("attempt_id"),}++ if t.get("recovery_reason"):++ payload["recovery_reason"] = t["recovery_reason"]++ if t.get("started_wall_time") is not None:++ payload["started_at"] = t["started_wall_time"]started_at = t.get("started_at")if isinstance(started_at, (int, float)):duration = t.get("duration_seconds")@@@ -435,19 -487,38 +511,35 @@@return mine--_BG = BackgroundManager()-_BG_STORE: Optional[Any] = None++_STORE_MANAGERS: dict[Path, BackgroundManager] = {}++_STORE_MANAGERS_LOCK = threading.Lock()++- def get_background_manager() -> BackgroundManager:- """Return an unowned handle over the process-wide task table.++def _manager_for_root(root: Path) -> BackgroundManager:++ from src.tools.background_store import BackgroundTaskStore+- Every call binds its own isolated scope, so two callers each get a private- view and cannot see, read or cancel each other's tasks. A host that wants a- session or worker to receive its own completion notifications must build a- scoped handle with :meth:`BackgroundManager.for_scope` and hand the same- object to both the tool registry and ``AgentLoop``.- """- return _BG.for_scope(BackgroundScope(owner_id=f"isolated:{uuid.uuid4().hex}"))++ key = Path(root).resolve()++ with _STORE_MANAGERS_LOCK:++ if key not in _STORE_MANAGERS:++ manager = BackgroundManager(store=BackgroundTaskStore(key))++ summary = manager.recover_on_start()++ if summary["unreadable"]:++ logging.getLogger(__name__).warning("Unreadable background records: %s", summary["unreadable"])++ _STORE_MANAGERS[key] = manager++ return _STORE_MANAGERS[key]+++ def init_background_store(root: Path) -> Any:- """Attach durable task records to the process-wide table, once per process.++ """Initialize and reconcile one runtime root exactly once in this process."""++ return _manager_for_root(root)._store+- Recovery runs here and only here, so a second service constructed in the- same interpreter cannot declare the first one's live work interrupted.- """- global _BG_STORE- with _LIVE_TASK_IDS_LOCK:- if _BG_STORE is not None:- return _BG_STORE- from src.tools.background_store import BackgroundTaskStore-- _BG_STORE = BackgroundTaskStore(Path(root))- _BG._store = _BG_STORE- _BG.recover_on_start()- return _BG_STORE---def get_background_manager() -> BackgroundManager:- """Return an unowned handle over the process-wide task table.-- Every call binds its own isolated scope, so two callers each get a private- view and cannot see, read or cancel each other's tasks. A host that wants a- session or worker to receive its own completion notifications must build a- scoped handle with :meth:`BackgroundManager.for_scope` and hand the same- object to both the tool registry and ``AgentLoop``.- """- return _BG.for_scope(BackgroundScope(owner_id=f"isolated:{uuid.uuid4().hex}"))++++def get_background_manager(root: Path | None = None) -> BackgroundManager:++ """Return a private handle over this runtime root's durable task table."""++ if root is None:++ from src.config.paths import get_runtime_root++ root = get_runtime_root()++ return _manager_for_root(root).for_scope(BackgroundScope(owner_id=f"isolated:{uuid.uuid4().hex}"))class BackgroundRunTool(BaseTool):diff --cc agent/tests/test_background_store_recovery.pyindex 000000000,87d3aa9cc..6d55efde4mode 000000,100644..100644--- a/agent/tests/test_background_store_recovery.py+++ b/agent/tests/test_background_store_recovery.py@@@ -1,0 -1,250 +1,288 @@@+ """Durable task records and the restart reconciliation that must never re-run."""++ from __future__ import annotations++ import json+ import os+ import stat+ import time+ from pathlib import Path++ import pytest++ from src.tools import background_store as store_mod+ from src.tools.background_store import (+ SCHEMA_VERSION,+ BackgroundStoreError,+ BackgroundTaskStore,+ TaskRecord,+ record_from_task,+ reconcile_after_restart,+ )+++ def make(**overrides) -> TaskRecord:+ values = dict(+ schema_version=SCHEMA_VERSION,+ task_id="deadbeef",+ owner_id="session:s1",+ status="completed",+ command_preview="echo hi",+ exit_code=0,+ result_preview="hi",+ )+ values.update(overrides)+ return TaskRecord(**values)+++ class TestRecoveryIsConservative:+ @pytest.fixture(autouse=True)+ def forbid_side_effects(self, monkeypatch):+ """Recovery must not spawn, signal, fetch or call a model."""++ def explode(*args, **kwargs):+ raise AssertionError(f"recovery must not call {kwargs or args}")++ from tests.module_os_helpers import patch_module_os++ patch_module_os(monkeypatch, store_mod, kill=explode, killpg=explode)+ monkeypatch.setattr(store_mod.time, "sleep", explode)++ def test_running_becomes_interrupted_without_rerun(self):+ recovered, unchanged = reconcile_after_restart([make(status="running")])+ assert unchanged == []+ assert recovered[0].status == "interrupted"+ assert recovered[0].recovery_reason == "service_restart"+ assert recovered[0].command_preview == "echo hi"++ def test_cancelling_stays_outcome_unknown(self):+ recovered, _ = reconcile_after_restart([make(status="cancelling")])+ assert recovered[0].status == "outcome_unknown"+ assert recovered[0].recovery_reason == "service_restart_before_reap_completed"++ @pytest.mark.parametrize("status", ["completed", "error", "timeout", "cancelled"])+ def test_terminal_outcomes_are_kept_queryable(self, status):+ recovered, unchanged = reconcile_after_restart([make(status=status, exit_code=3)])+ assert recovered == []+ assert unchanged[0].status == status+ assert unchanged[0].exit_code == 3++ def test_recovery_is_idempotent(self):+ once, _ = reconcile_after_restart([make(status="running")])+ twice, unchanged = reconcile_after_restart(once)+ assert twice == []+ assert unchanged[0].status == "interrupted"++ def test_stored_pid_is_descriptive_only(self):+ record = make(status="running", pid=os.getpid())+ recovered, _ = reconcile_after_restart([record])+ assert recovered[0].pid == os.getpid()+ # No signal path reads it; nothing above touches os.kill.+++ class TestStore:+ def test_round_trip_and_layout_is_stable_per_owner(self, tmp_path):+ store = BackgroundTaskStore(tmp_path)+ record = make()+ path = store.save(record)+ assert path.parent == tmp_path / "background-tasks"+ assert path.name == path.name # deterministic+ assert store.save(record).name == path.name+ records, unreadable = store.load_all()+ assert unreadable == []+ assert records == [record]++ def test_owner_id_is_not_a_filename(self, tmp_path):+ store = BackgroundTaskStore(tmp_path)+ path = store.save(make(owner_id="session:../escape"))+ assert ".." not in path.name+ assert path.parent == tmp_path / "background-tasks"++ def test_unsafe_task_id_is_rejected(self, tmp_path):+ store = BackgroundTaskStore(tmp_path)+ with pytest.raises(BackgroundStoreError):+ store.save(make(task_id="../../etc/passwd"))++ def test_private_permissions(self, tmp_path):+ store = BackgroundTaskStore(tmp_path)+ path = store.save(make())+ assert stat.S_IMODE(path.stat().st_mode) == 0o600+ assert stat.S_IMODE(path.parent.stat().st_mode) == 0o700++ def test_unknown_version_is_reported_not_overwritten(self, tmp_path):+ store = BackgroundTaskStore(tmp_path)+ store.save(make())+ for path in (tmp_path / "background-tasks").glob("*.json"):+ payload = json.loads(path.read_text())+ payload["schema_version"] = SCHEMA_VERSION + 1+ path.write_text(json.dumps(payload))+ records, unreadable = store.load_all()+ assert records == []+ assert len(unreadable) == 1+ assert json.loads(Path(unreadable[0]).read_text())["schema_version"] == SCHEMA_VERSION + 1++ def test_half_written_file_is_reported_not_invented(self, tmp_path):+ store = BackgroundTaskStore(tmp_path)+ directory = tmp_path / "background-tasks"+ directory.mkdir()+ (directory / "deadbeef.json").write_text('{"schema_version": 1, "task_id"')+ records, unreadable = store.load_all()+ assert records == []+ assert len(unreadable) == 1++ def test_symlink_is_refused(self, tmp_path):+ store = BackgroundTaskStore(tmp_path)+ directory = tmp_path / "background-tasks"+ directory.mkdir()+ outside = tmp_path / "outside.json"- outside.write_text(json.dumps({"schema_version": SCHEMA_VERSION}))++ outside.write_text(json.dumps(__import__("dataclasses").asdict(make(task_id="deadbeef"))))+ link = directory / f"{'0' * 16}-deadbeef.json"+ link.symlink_to(outside)+ _records, unreadable = store.load_all()+ assert len(unreadable) == 1++ def test_previews_are_bounded_and_redacted(self, tmp_path):+ record = make(+ command_preview=store_mod._preview(" ".join(["x" * 100] * 50)),+ result_preview=store_mod._preview(f"token sk-{'a' * 40} end"),+ )+ assert len(record.command_preview) <= store_mod.MAX_PREVIEW_CHARS+ assert len(record.result_preview) <= store_mod.MAX_PREVIEW_CHARS+ assert f"sk-{'a' * 40}" not in record.result_preview++ def test_two_local_processes_can_share_the_directory(self, tmp_path):+ """One process's records stay readable after another writes beside them."""+ first = BackgroundTaskStore(tmp_path)+ second = BackgroundTaskStore(tmp_path)+ first.save(make(task_id="aaaaaaaa", owner_id="session:a"))+ second.save(make(task_id="bbbbbbbb", owner_id="session:b"))+ records, unreadable = first.load_all()+ assert unreadable == []+ assert {r.task_id for r in records} == {"aaaaaaaa", "bbbbbbbb"}++ def test_recovery_survives_a_process_that_never_wrote(self, tmp_path):+ """A crash before the first save leaves no record, and none is invented."""+ store = BackgroundTaskStore(tmp_path)+ assert store.load_all() == ([], [])+ recovered, unchanged = reconcile_after_restart([])+ assert recovered == [] and unchanged == []++ def test_record_from_task_projects_the_live_shape(self):+ task = {+ "status": "running",+ "owner_id": "session:s1",+ "attempt_id": "a1",+ "command": "echo hi",+ "result": None,+ "exit_code": None,+ "started_at": 10.0,+ "duration_seconds": 2.5,+ }+ record = record_from_task("deadbeef", task)+ assert record.owner_id == "session:s1"+ assert record.attempt_id == "a1"+ assert record.status == "running"+ assert record.started_at == 10.0+ assert record.duration_seconds == 2.5+ assert record.is_live and not record.is_terminal+++ class TestManagerWiring:+ def test_a_completed_run_leaves_a_queryable_record(self, tmp_path, monkeypatch):+ import json as _json+ import shlex+ import sys as _sys++ from src.tools import background_tools as bg++ monkeypatch.setattr(bg, "WORKDIR", tmp_path)+ manager = bg.BackgroundManager(bg.BackgroundScope("session:s1"), store=BackgroundTaskStore(tmp_path))+ command = shlex.join([_sys.executable, "-c", "print('stored')"])+ task_id = _json.loads(manager.run(command))["task_id"]+ deadline = time.monotonic() + 15+ while time.monotonic() < deadline and _json.loads(manager.check(task_id))["status"] == "running":+ time.sleep(0.01)+ # The in-memory status flips inside the lock while the best-effort write+ # happens after it, so reading the store the moment check() moves on+ # races the persist. Wait for the record to land.+ records: list = []+ unreadable: list = []+ while time.monotonic() < deadline:+ records, unreadable = manager._store.load_all()+ if records and records[0].status != "running":+ break+ time.sleep(0.01)+ assert unreadable == []+ assert records[0].task_id == task_id+ assert records[0].status == "completed"+ assert records[0].attempt_id is None+ assert records[0].command_preview++ def test_restart_recovery_leaves_this_processes_live_tasks_alone(self, tmp_path):+ from src.tools import background_tools as bg++ store = BackgroundTaskStore(tmp_path)+ live = make(task_id="aaaaaaaa", status="running", owner_id="session:live")+ stale = make(task_id="bbbbbbbb", status="running", owner_id="session:stale")+ store.save(live)+ store.save(stale)+ bg._LIVE_TASK_IDS.add("aaaaaaaa")+ try:+ first = BackgroundTaskStore(tmp_path)+ manager = bg.BackgroundManager(bg.BackgroundScope("session:x"), store=first)+ summary = manager.recover_on_start()+ finally:+ bg._LIVE_TASK_IDS.discard("aaaaaaaa")+ assert summary["recovered"] == 1+ records, _ = first.load_all()+ by_id = {record.task_id: record for record in records}+ assert by_id["aaaaaaaa"].status == "running"+ assert by_id["bbbbbbbb"].status == "interrupted"++ def test_recovery_does_not_rerun_a_previous_commands(self, tmp_path, monkeypatch):+ from src.tools import background_tools as bg++ monkeypatch.setattr(bg, "WORKDIR", tmp_path)+ store = BackgroundTaskStore(tmp_path)+ store.save(make(task_id="cccccccc", status="running", command_preview="touch should-not-exist"))+ manager = bg.BackgroundManager(bg.BackgroundScope("session:x"), store=store)+ assert manager.recover_on_start()["recovered"] == 1+ assert not (tmp_path / "should-not-exist").exists()++++++++def test_recovered_task_is_queryable_by_owner_not_other_sessions(tmp_path):++ from src.tools import background_tools as bg++ store = BackgroundTaskStore(tmp_path)++ store.save(make(task_id="abcdef12", owner_id="session:a", status="running"))++ manager = bg.BackgroundManager(bg.BackgroundScope("session:a"), store=store)++ manager.recover_on_start()++ result = json.loads(manager.check("abcdef12"))++ assert result["status"] == "interrupted"++ assert result["recovery_reason"] == "service_restart"++ assert "abcdef12" in manager.check()++ sibling = manager.for_scope(bg.BackgroundScope("session:b"))++ assert json.loads(sibling.check("abcdef12"))["status"] == "error"++++++def test_runtime_roots_keep_their_records_and_tables_separate(tmp_path):++ from src.tools import background_tools as bg++ first = bg.get_background_manager(tmp_path / "a").for_scope(bg.BackgroundScope("session:a"))++ second = bg.get_background_manager(tmp_path / "b").for_scope(bg.BackgroundScope("session:a"))++ first.tasks["deadbeef"] = {"owner_id": "session:a", "status": "completed", "command": "echo done", "result": "done"}++ first._persist("deadbeef")++ assert json.loads(second.check("deadbeef"))["status"] == "error"++ assert second._store.load_all() == ([], [])++++++def test_cancel_intent_is_persisted_before_reaping(tmp_path, monkeypatch):++ from src.tools import background_tools as bg++ store = BackgroundTaskStore(tmp_path)++ manager = bg.BackgroundManager(bg.BackgroundScope("session:a"), store=store)++ manager.tasks["deadbeef"] = {"owner_id": "session:a", "status": "running", "command": "echo done", "process": None}++ manager.cancel("deadbeef")++ records, unreadable = store.load_all()++ assert not unreadable++ assert records[0].status == "cancelling"++ recovered, _ = reconcile_after_restart(records)++ assert recovered[0].status == "outcome_unknown"