Skip to content

"Vibe-Trading: Your Personal Trading Agent"

Merge PR #2100: persist scoped background tasks and expose recovered outcomes

Haozhe Wu0104ff97d455
diff --cc .gitignore
index fe3223ab7,fe3223ab7..4aeb80a8b
--- a/.gitignore
+++ b/.gitignore
@@@ -188,3 -188,3 +188,6 @@@ agent/tests/artifacts
# Local simulated account ledgers
/paper-trading/
++
++# Runtime background command previews and restart outcomes
++/background-tasks/
diff --cc agent/src/session/service.py
index 0af58474e,e36fc592a..2d0bbcbac
--- a/agent/src/session/service.py
+++ b/agent/src/session/service.py
@@@ -650,7 -646,7 +655,7 @@@ class SessionService
background_scope = BackgroundScope(
owner_id=f"session:{session_id}", attempt_id=attempt_id
)
-- background = get_background_manager().for_scope(
++ background = get_background_manager(Path(self.runs_dir).parent).for_scope(
background_scope,
on_completion=self._publish_background_completion,
)
diff --cc agent/src/tools/background_store.py
index 000000000,fb63257ae..5a7b00510
mode 000000,100644..100644
--- a/agent/src/tools/background_store.py
+++ b/agent/src/tools/background_store.py
@@@ -1,0 -1,223 +1,238 @@@
+ """Durable records for background tasks, and conservative restart recovery.
+
+ A background task currently lives only in memory, so a process restart loses
+ the fact that a command ever ran and the user is left guessing. This stores one
+ small record per task under the canonical runtime root and, on a new process,
+ reconciles what it finds.
+
+ Recovery is deliberately one-way and never executes anything. A record left in
+ ``running`` or ``cancelling`` belongs to a process that is gone; it becomes
+ ``interrupted`` or ``outcome_unknown`` with a recovery reason. It is never
+ re-run, and a stored PID is descriptive only: it is not used to signal a
+ process, because that pid may have been reused by an unrelated program.
+ """
+
+ from __future__ import annotations
+
+ import hashlib
+ import json
++import math
+ import os
+ import re
+ import tempfile
+ import time
+ from dataclasses import asdict, dataclass, field, replace
+ from pathlib import Path
+ from typing import Any, Iterable, Optional
+
+ from src.tools.redaction import redact_text
+
+ SCHEMA_VERSION = 1
+ #: Preview length kept per field. Enough to identify a command or outcome
+ #: without becoming a second copy of the run's full output.
+ MAX_PREVIEW_CHARS = 2_000
+ _TASK_ID_RE = re.compile(r"^[0-9a-f]{8,64}$")
+
+ #: Statuses a task can hold while its owning process is still alive.
+ LIVE_STATUSES = frozenset({"running", "cancelling"})
+ #: Statuses whose result is worth reading back after a restart.
+ TERMINAL_STATUSES = frozenset({"completed", "error", "timeout", "cancelled"})
+
+
+ class BackgroundStoreError(RuntimeError):
+ """A record could not be stored or read back safely."""
+
+
+ @dataclass
+ class TaskRecord:
+ """What survives a restart for one background task."""
+
+ schema_version: int
+ task_id: str
+ owner_id: str
+ status: str
+ command_preview: str
+ attempt_id: Optional[str] = None
+ exit_code: Optional[int] = None
+ result_preview: str = ""
+ started_at: Optional[float] = None
+ ended_at: Optional[float] = None
+ duration_seconds: Optional[float] = None
+ recovery_reason: Optional[str] = None
+ #: Descriptive only. Never used to signal a process after a restart.
+ pid: Optional[int] = None
+ updated_at: float = field(default_factory=time.time)
+
+ @property
+ def is_live(self) -> bool:
+ return self.status in LIVE_STATUSES
+
+ @property
+ def is_terminal(self) -> bool:
+ return self.status in TERMINAL_STATUSES
+
+
+ def _preview(value: object) -> str:
+ """Bound and redact one untrusted field before it reaches disk."""
+ return redact_text(str(value or ""))[:MAX_PREVIEW_CHARS]
+
+
+ def _owner_token(owner_id: str) -> str:
+ """Stable layout-only token for an owner id.
+
+ A hash rather than the raw id: the owner carries a session identifier that
+ is not a secret but is not a filename either, and the raw form would put
+ user-supplied separators into the path.
+ """
+ return hashlib.sha256(owner_id.encode("utf-8")).hexdigest()[:16]
+
+
+ class BackgroundTaskStore:
+ """One JSON record per task, written atomically under a fixed root."""
+
+ def __init__(self, root: Path) -> None:
+ self.root = Path(root)
+ self._directory = self.root / "background-tasks"
+
+ def _path_for(self, record: TaskRecord) -> Path:
+ if not _TASK_ID_RE.match(record.task_id):
+ raise BackgroundStoreError(f"unsafe background task id: {record.task_id!r}")
+ return self._directory / f"{_owner_token(record.owner_id)}-{record.task_id}.json"
+
+ def save(self, record: TaskRecord) -> Path:
+ """Persist one record, replacing any previous copy of it."""
+ path = self._path_for(record)
+ self._write_json_atomically(path, asdict(record))
+ return path
+
+ def load_all(self) -> tuple[list[TaskRecord], list[str]]:
+ """Return readable records and the paths that could not be read.
+
+ A corrupt or unknown-version file is reported, never overwritten and
+ never guessed at: silently dropping it would hide the task, and
+ silently reinterpreting it would invent an outcome.
+ """
+ records: list[TaskRecord] = []
+ unreadable: list[str] = []
+ if not self._directory.exists():
+ return records, unreadable
+ for path in sorted(self._directory.glob("*.json")):
+ record = self._load_one(path)
+ if record is None:
+ unreadable.append(str(path))
+ else:
+ records.append(record)
+ return records, unreadable
+
+ def _load_one(self, path: Path) -> Optional[TaskRecord]:
+ try:
++ if path.is_symlink() or path.stat().st_size > 64_000:
++ return None
+ payload = json.loads(path.read_text(encoding="utf-8"))
+ except (OSError, ValueError):
+ return None
+ if not isinstance(payload, dict):
+ return None
+ if payload.get("schema_version") != SCHEMA_VERSION:
+ return None
+ try:
- return TaskRecord(**payload)
- except TypeError:
++ record = TaskRecord(**payload)
++ if (not isinstance(record.task_id, str) or not _TASK_ID_RE.fullmatch(record.task_id)
++ or not isinstance(record.owner_id, str) or not record.owner_id.strip()
++ or record.status not in LIVE_STATUSES | TERMINAL_STATUSES | {"interrupted", "outcome_unknown"}
++ or path.name != self._path_for(record).name):
++ return None
++ for value in (record.started_at, record.ended_at, record.duration_seconds, record.updated_at):
++ if value is not None and (isinstance(value, bool) or not isinstance(value, (int, float))
++ or not math.isfinite(value)):
++ return None
++ record.command_preview = _preview(record.command_preview)
++ record.result_preview = _preview(record.result_preview)
++ return record
++ except (TypeError, ValueError, BackgroundStoreError):
+ return None
+
+ @staticmethod
+ def _write_json_atomically(path: Path, payload: dict[str, Any]) -> None:
+ path.parent.mkdir(parents=True, exist_ok=True, mode=0o700)
+ content = (json.dumps(payload, ensure_ascii=False, indent=2) + "\n").encode("utf-8")
+ fd, temporary = tempfile.mkstemp(dir=path.parent, prefix=f".{path.name}.", suffix=".tmp")
+ try:
+ try:
+ with os.fdopen(fd, "wb") as handle:
+ fd = -1
+ handle.write(content)
+ handle.flush()
+ if hasattr(os, "fchmod"):
+ os.fchmod(handle.fileno(), 0o600)
+ os.fsync(handle.fileno())
+ finally:
+ if fd >= 0:
+ os.close(fd)
+ os.replace(temporary, path)
+ except BaseException:
+ try:
+ os.unlink(temporary)
+ except OSError:
+ pass
+ raise
+ try:
+ os.chmod(path, 0o600)
+ except OSError:
+ pass
+
+
+ def record_from_task(task_id: str, task: dict, *, started_at: Optional[float] = None) -> TaskRecord:
+ """Project an in-memory task entry onto its durable shape."""
- started = task.get("started_at")
++ started = task.get("started_wall_time", task.get("started_at"))
+ return TaskRecord(
+ schema_version=SCHEMA_VERSION,
+ task_id=task_id,
+ owner_id=str(task.get("owner_id") or ""),
+ attempt_id=task.get("attempt_id"),
+ status=str(task.get("status") or "error"),
+ command_preview=_preview(task.get("command")),
+ result_preview=_preview(task.get("result")),
+ exit_code=task.get("exit_code") if isinstance(task.get("exit_code"), int) else None,
+ started_at=float(started) if isinstance(started, (int, float)) else started_at,
+ ended_at=task.get("ended_at"),
+ duration_seconds=task.get("duration_seconds"),
+ pid=task.get("pid") if isinstance(task.get("pid"), int) else None,
+ )
+
+
+ def reconcile_after_restart(
+ records: Iterable[TaskRecord],
+ ) -> tuple[list[TaskRecord], list[TaskRecord]]:
+ """Turn a previous process's live records into explicit outcomes.
+
+ Nothing is re-run and nothing is signalled. A record left ``running`` cannot
+ be distinguished from one whose output was never written, so it becomes
+ ``interrupted``; a record left ``cancelling`` cannot be distinguished from
+ one whose reaping never finished, so it becomes ``outcome_unknown``. Both
+ carry a recovery reason rather than being dropped.
+ """
+ recovered: list[TaskRecord] = []
+ unchanged: list[TaskRecord] = []
+ for record in records:
+ if record.status == "running":
+ recovered.append(
+ replace(
+ record,
+ status="interrupted",
+ recovery_reason="service_restart",
+ ended_at=time.time(),
+ )
+ )
+ elif record.status == "cancelling":
+ recovered.append(
+ replace(
+ record,
+ status="outcome_unknown",
+ recovery_reason="service_restart_before_reap_completed",
+ ended_at=time.time(),
+ )
+ )
+ else:
+ unchanged.append(record)
+ return recovered, unchanged
diff --cc agent/src/tools/background_tools.py
index 37c50732e,8d49aeefc..1d913f8d5
--- a/agent/src/tools/background_tools.py
+++ b/agent/src/tools/background_tools.py
@@@ -141,8 -145,10 +145,11 @@@ class BackgroundManager
scope: Optional[BackgroundScope] = None,
*,
on_completion: Optional[Callable[[str, dict], None]] = None,
+ store: Optional[Any] = None,
) -> None:
self.tasks: Dict[str, dict] = {}
+ self._store = store
++ self._persist_lock = threading.Lock()
# (owner_id, payload). Kept as pairs so the payload shape consumers and
# existing tests see stays exactly what it was; routing is internal.
self._notifications: List[tuple[str, dict]] = []
@@@ -170,6 -176,7 +177,8 @@@
other._lock = self._lock
other._condition = self._condition
other._accepting = self._accepting
+ other._store = self._store
++ other._persist_lock = self._persist_lock
other._on_completion = (
self._on_completion if on_completion is None else on_completion
)
@@@ -178,6 -185,43 +187,57 @@@
def _owns(self, task: dict) -> bool:
return task.get("owner_id") == self._scope.owner_id
+ def _persist(self, task_id: str) -> None:
+ """Best effort: a full or unwritable runtime root must not break the run."""
+ if self._store is None:
+ return
- task = self.tasks.get(task_id)
- if task is None:
- return
- try:
- from src.tools.background_store import record_from_task
-
- self._store.save(record_from_task(task_id, task))
- except Exception:
- logging.getLogger(__name__).exception("Could not persist background task %s", task_id)
++ with self._persist_lock:
++ with self._lock:
++ task = dict(self.tasks.get(task_id) or {})
++ if not task:
++ return
++ try:
++ from src.tools.background_store import record_from_task
++ self._store.save(record_from_task(task_id, task))
++ except Exception:
++ logging.getLogger(__name__).exception("Could not persist background task %s", task_id)
+
+ def recover_on_start(self) -> dict:
+ """Reconcile records left by a previous process, once and without reruns.
+
+ A task id this interpreter is already running is left alone: a second
+ service constructed in the same process must not declare the first
+ one's live work interrupted.
+ """
+ if self._store is None:
+ return {"recovered": 0, "unchanged": 0, "unreadable": []}
+ from src.tools.background_store import reconcile_after_restart
+
+ records, unreadable = self._store.load_all()
+ with _LIVE_TASK_IDS_LOCK:
+ live = set(_LIVE_TASK_IDS)
+ stale = [record for record in records if record.task_id not in live]
+ recovered, unchanged = reconcile_after_restart(stale)
+ for record in recovered:
+ try:
+ self._store.save(record)
+ except Exception:
+ logging.getLogger(__name__).exception("Could not persist recovered task %s", record.task_id)
++ with self._lock:
++ for record in recovered + unchanged:
++ if record.task_id in self.tasks:
++ continue
++ self.tasks[record.task_id] = {
++ "status": record.status, "command": record.command_preview,
++ "result": record.result_preview, "exit_code": record.exit_code,
++ "owner_id": record.owner_id, "attempt_id": record.attempt_id,
++ "process": None, "started_at": None,
++ "started_wall_time": record.started_at, "ended_at": record.ended_at,
++ "duration_seconds": record.duration_seconds,
++ "recovery_reason": record.recovery_reason,
++ }
+ return {"recovered": len(recovered), "unchanged": len(unchanged), "unreadable": unreadable}
+
def run(self, command: str) -> str:
"""Start a background task and return its task_id.
@@@ -207,6 -251,6 +267,7 @@@
"process": None,
"cancel_requested": False,
"started_at": started_at,
++ "started_wall_time": time.time(),
"owner_id": self._scope.owner_id,
"attempt_id": self._scope.attempt_id,
}
@@@ -227,7 -274,7 +291,9 @@@
with self._lock:
task = self.tasks[task_id]
task["process"] = process
++ task["pid"] = process.pid
cancel_requested = bool(task.get("cancel_requested"))
++ self._persist(task_id)
if cancel_requested:
_force_stop_process_tree(process)
@@@ -283,7 -330,12 +349,12 @@@
"exit_code": exit_code,
}
self._notifications.append((task["owner_id"], payload))
+ task["ended_at"] = time.time()
+ task["pid"] = getattr(process, "pid", None)
self._condition.notify_all()
++ self._persist(task_id)
+ with _LIVE_TASK_IDS_LOCK:
+ _LIVE_TASK_IDS.discard(task_id)
- self._persist(task_id)
if self._on_completion is not None:
try:
self._on_completion(task["owner_id"], {**payload, "attempt_id": task.get("attempt_id")})
@@@ -361,6 -413,6 +432,7 @@@
task["cancel_requested"] = True
task["status"] = "cancelling"
++ self._persist(task_id)
if process is not None:
_force_stop_process_tree(process)
return json.dumps({
@@@ -397,6 -449,6 +469,10 @@@
"exit_code": t.get("exit_code"),
"attempt_id": t.get("attempt_id"),
}
++ if t.get("recovery_reason"):
++ payload["recovery_reason"] = t["recovery_reason"]
++ if t.get("started_wall_time") is not None:
++ payload["started_at"] = t["started_wall_time"]
started_at = t.get("started_at")
if isinstance(started_at, (int, float)):
duration = t.get("duration_seconds")
@@@ -435,19 -487,38 +511,35 @@@
return mine
--_BG = BackgroundManager()
-_BG_STORE: Optional[Any] = None
++_STORE_MANAGERS: dict[Path, BackgroundManager] = {}
++_STORE_MANAGERS_LOCK = threading.Lock()
+
+
- def get_background_manager() -> BackgroundManager:
- """Return an unowned handle over the process-wide task table.
++def _manager_for_root(root: Path) -> BackgroundManager:
++ from src.tools.background_store import BackgroundTaskStore
+
- Every call binds its own isolated scope, so two callers each get a private
- view and cannot see, read or cancel each other's tasks. A host that wants a
- session or worker to receive its own completion notifications must build a
- scoped handle with :meth:`BackgroundManager.for_scope` and hand the same
- object to both the tool registry and ``AgentLoop``.
- """
- return _BG.for_scope(BackgroundScope(owner_id=f"isolated:{uuid.uuid4().hex}"))
++ key = Path(root).resolve()
++ with _STORE_MANAGERS_LOCK:
++ if key not in _STORE_MANAGERS:
++ manager = BackgroundManager(store=BackgroundTaskStore(key))
++ summary = manager.recover_on_start()
++ if summary["unreadable"]:
++ logging.getLogger(__name__).warning("Unreadable background records: %s", summary["unreadable"])
++ _STORE_MANAGERS[key] = manager
++ return _STORE_MANAGERS[key]
+
+
+ def init_background_store(root: Path) -> Any:
- """Attach durable task records to the process-wide table, once per process.
++ """Initialize and reconcile one runtime root exactly once in this process."""
++ return _manager_for_root(root)._store
+
- Recovery runs here and only here, so a second service constructed in the
- same interpreter cannot declare the first one's live work interrupted.
- """
- global _BG_STORE
- with _LIVE_TASK_IDS_LOCK:
- if _BG_STORE is not None:
- return _BG_STORE
- from src.tools.background_store import BackgroundTaskStore
-
- _BG_STORE = BackgroundTaskStore(Path(root))
- _BG._store = _BG_STORE
- _BG.recover_on_start()
- return _BG_STORE
-
-
-def get_background_manager() -> BackgroundManager:
- """Return an unowned handle over the process-wide task table.
-
- Every call binds its own isolated scope, so two callers each get a private
- view and cannot see, read or cancel each other's tasks. A host that wants a
- session or worker to receive its own completion notifications must build a
- scoped handle with :meth:`BackgroundManager.for_scope` and hand the same
- object to both the tool registry and ``AgentLoop``.
- """
- return _BG.for_scope(BackgroundScope(owner_id=f"isolated:{uuid.uuid4().hex}"))
++
++def get_background_manager(root: Path | None = None) -> BackgroundManager:
++ """Return a private handle over this runtime root's durable task table."""
++ if root is None:
++ from src.config.paths import get_runtime_root
++ root = get_runtime_root()
++ return _manager_for_root(root).for_scope(BackgroundScope(owner_id=f"isolated:{uuid.uuid4().hex}"))
class BackgroundRunTool(BaseTool):
diff --cc agent/tests/test_background_store_recovery.py
index 000000000,87d3aa9cc..6d55efde4
mode 000000,100644..100644
--- a/agent/tests/test_background_store_recovery.py
+++ b/agent/tests/test_background_store_recovery.py
@@@ -1,0 -1,250 +1,288 @@@
+ """Durable task records and the restart reconciliation that must never re-run."""
+
+ from __future__ import annotations
+
+ import json
+ import os
+ import stat
+ import time
+ from pathlib import Path
+
+ import pytest
+
+ from src.tools import background_store as store_mod
+ from src.tools.background_store import (
+ SCHEMA_VERSION,
+ BackgroundStoreError,
+ BackgroundTaskStore,
+ TaskRecord,
+ record_from_task,
+ reconcile_after_restart,
+ )
+
+
+ def make(**overrides) -> TaskRecord:
+ values = dict(
+ schema_version=SCHEMA_VERSION,
+ task_id="deadbeef",
+ owner_id="session:s1",
+ status="completed",
+ command_preview="echo hi",
+ exit_code=0,
+ result_preview="hi",
+ )
+ values.update(overrides)
+ return TaskRecord(**values)
+
+
+ class TestRecoveryIsConservative:
+ @pytest.fixture(autouse=True)
+ def forbid_side_effects(self, monkeypatch):
+ """Recovery must not spawn, signal, fetch or call a model."""
+
+ def explode(*args, **kwargs):
+ raise AssertionError(f"recovery must not call {kwargs or args}")
+
+ from tests.module_os_helpers import patch_module_os
+
+ patch_module_os(monkeypatch, store_mod, kill=explode, killpg=explode)
+ monkeypatch.setattr(store_mod.time, "sleep", explode)
+
+ def test_running_becomes_interrupted_without_rerun(self):
+ recovered, unchanged = reconcile_after_restart([make(status="running")])
+ assert unchanged == []
+ assert recovered[0].status == "interrupted"
+ assert recovered[0].recovery_reason == "service_restart"
+ assert recovered[0].command_preview == "echo hi"
+
+ def test_cancelling_stays_outcome_unknown(self):
+ recovered, _ = reconcile_after_restart([make(status="cancelling")])
+ assert recovered[0].status == "outcome_unknown"
+ assert recovered[0].recovery_reason == "service_restart_before_reap_completed"
+
+ @pytest.mark.parametrize("status", ["completed", "error", "timeout", "cancelled"])
+ def test_terminal_outcomes_are_kept_queryable(self, status):
+ recovered, unchanged = reconcile_after_restart([make(status=status, exit_code=3)])
+ assert recovered == []
+ assert unchanged[0].status == status
+ assert unchanged[0].exit_code == 3
+
+ def test_recovery_is_idempotent(self):
+ once, _ = reconcile_after_restart([make(status="running")])
+ twice, unchanged = reconcile_after_restart(once)
+ assert twice == []
+ assert unchanged[0].status == "interrupted"
+
+ def test_stored_pid_is_descriptive_only(self):
+ record = make(status="running", pid=os.getpid())
+ recovered, _ = reconcile_after_restart([record])
+ assert recovered[0].pid == os.getpid()
+ # No signal path reads it; nothing above touches os.kill.
+
+
+ class TestStore:
+ def test_round_trip_and_layout_is_stable_per_owner(self, tmp_path):
+ store = BackgroundTaskStore(tmp_path)
+ record = make()
+ path = store.save(record)
+ assert path.parent == tmp_path / "background-tasks"
+ assert path.name == path.name # deterministic
+ assert store.save(record).name == path.name
+ records, unreadable = store.load_all()
+ assert unreadable == []
+ assert records == [record]
+
+ def test_owner_id_is_not_a_filename(self, tmp_path):
+ store = BackgroundTaskStore(tmp_path)
+ path = store.save(make(owner_id="session:../escape"))
+ assert ".." not in path.name
+ assert path.parent == tmp_path / "background-tasks"
+
+ def test_unsafe_task_id_is_rejected(self, tmp_path):
+ store = BackgroundTaskStore(tmp_path)
+ with pytest.raises(BackgroundStoreError):
+ store.save(make(task_id="../../etc/passwd"))
+
+ def test_private_permissions(self, tmp_path):
+ store = BackgroundTaskStore(tmp_path)
+ path = store.save(make())
+ assert stat.S_IMODE(path.stat().st_mode) == 0o600
+ assert stat.S_IMODE(path.parent.stat().st_mode) == 0o700
+
+ def test_unknown_version_is_reported_not_overwritten(self, tmp_path):
+ store = BackgroundTaskStore(tmp_path)
+ store.save(make())
+ for path in (tmp_path / "background-tasks").glob("*.json"):
+ payload = json.loads(path.read_text())
+ payload["schema_version"] = SCHEMA_VERSION + 1
+ path.write_text(json.dumps(payload))
+ records, unreadable = store.load_all()
+ assert records == []
+ assert len(unreadable) == 1
+ assert json.loads(Path(unreadable[0]).read_text())["schema_version"] == SCHEMA_VERSION + 1
+
+ def test_half_written_file_is_reported_not_invented(self, tmp_path):
+ store = BackgroundTaskStore(tmp_path)
+ directory = tmp_path / "background-tasks"
+ directory.mkdir()
+ (directory / "deadbeef.json").write_text('{"schema_version": 1, "task_id"')
+ records, unreadable = store.load_all()
+ assert records == []
+ assert len(unreadable) == 1
+
+ def test_symlink_is_refused(self, tmp_path):
+ store = BackgroundTaskStore(tmp_path)
+ directory = tmp_path / "background-tasks"
+ directory.mkdir()
+ outside = tmp_path / "outside.json"
- outside.write_text(json.dumps({"schema_version": SCHEMA_VERSION}))
++ outside.write_text(json.dumps(__import__("dataclasses").asdict(make(task_id="deadbeef"))))
+ link = directory / f"{'0' * 16}-deadbeef.json"
+ link.symlink_to(outside)
+ _records, unreadable = store.load_all()
+ assert len(unreadable) == 1
+
+ def test_previews_are_bounded_and_redacted(self, tmp_path):
+ record = make(
+ command_preview=store_mod._preview(" ".join(["x" * 100] * 50)),
+ result_preview=store_mod._preview(f"token sk-{'a' * 40} end"),
+ )
+ assert len(record.command_preview) <= store_mod.MAX_PREVIEW_CHARS
+ assert len(record.result_preview) <= store_mod.MAX_PREVIEW_CHARS
+ assert f"sk-{'a' * 40}" not in record.result_preview
+
+ def test_two_local_processes_can_share_the_directory(self, tmp_path):
+ """One process's records stay readable after another writes beside them."""
+ first = BackgroundTaskStore(tmp_path)
+ second = BackgroundTaskStore(tmp_path)
+ first.save(make(task_id="aaaaaaaa", owner_id="session:a"))
+ second.save(make(task_id="bbbbbbbb", owner_id="session:b"))
+ records, unreadable = first.load_all()
+ assert unreadable == []
+ assert {r.task_id for r in records} == {"aaaaaaaa", "bbbbbbbb"}
+
+ def test_recovery_survives_a_process_that_never_wrote(self, tmp_path):
+ """A crash before the first save leaves no record, and none is invented."""
+ store = BackgroundTaskStore(tmp_path)
+ assert store.load_all() == ([], [])
+ recovered, unchanged = reconcile_after_restart([])
+ assert recovered == [] and unchanged == []
+
+ def test_record_from_task_projects_the_live_shape(self):
+ task = {
+ "status": "running",
+ "owner_id": "session:s1",
+ "attempt_id": "a1",
+ "command": "echo hi",
+ "result": None,
+ "exit_code": None,
+ "started_at": 10.0,
+ "duration_seconds": 2.5,
+ }
+ record = record_from_task("deadbeef", task)
+ assert record.owner_id == "session:s1"
+ assert record.attempt_id == "a1"
+ assert record.status == "running"
+ assert record.started_at == 10.0
+ assert record.duration_seconds == 2.5
+ assert record.is_live and not record.is_terminal
+
+
+ class TestManagerWiring:
+ def test_a_completed_run_leaves_a_queryable_record(self, tmp_path, monkeypatch):
+ import json as _json
+ import shlex
+ import sys as _sys
+
+ from src.tools import background_tools as bg
+
+ monkeypatch.setattr(bg, "WORKDIR", tmp_path)
+ manager = bg.BackgroundManager(bg.BackgroundScope("session:s1"), store=BackgroundTaskStore(tmp_path))
+ command = shlex.join([_sys.executable, "-c", "print('stored')"])
+ task_id = _json.loads(manager.run(command))["task_id"]
+ deadline = time.monotonic() + 15
+ while time.monotonic() < deadline and _json.loads(manager.check(task_id))["status"] == "running":
+ time.sleep(0.01)
+ # The in-memory status flips inside the lock while the best-effort write
+ # happens after it, so reading the store the moment check() moves on
+ # races the persist. Wait for the record to land.
+ records: list = []
+ unreadable: list = []
+ while time.monotonic() < deadline:
+ records, unreadable = manager._store.load_all()
+ if records and records[0].status != "running":
+ break
+ time.sleep(0.01)
+ assert unreadable == []
+ assert records[0].task_id == task_id
+ assert records[0].status == "completed"
+ assert records[0].attempt_id is None
+ assert records[0].command_preview
+
+ def test_restart_recovery_leaves_this_processes_live_tasks_alone(self, tmp_path):
+ from src.tools import background_tools as bg
+
+ store = BackgroundTaskStore(tmp_path)
+ live = make(task_id="aaaaaaaa", status="running", owner_id="session:live")
+ stale = make(task_id="bbbbbbbb", status="running", owner_id="session:stale")
+ store.save(live)
+ store.save(stale)
+ bg._LIVE_TASK_IDS.add("aaaaaaaa")
+ try:
+ first = BackgroundTaskStore(tmp_path)
+ manager = bg.BackgroundManager(bg.BackgroundScope("session:x"), store=first)
+ summary = manager.recover_on_start()
+ finally:
+ bg._LIVE_TASK_IDS.discard("aaaaaaaa")
+ assert summary["recovered"] == 1
+ records, _ = first.load_all()
+ by_id = {record.task_id: record for record in records}
+ assert by_id["aaaaaaaa"].status == "running"
+ assert by_id["bbbbbbbb"].status == "interrupted"
+
+ def test_recovery_does_not_rerun_a_previous_commands(self, tmp_path, monkeypatch):
+ from src.tools import background_tools as bg
+
+ monkeypatch.setattr(bg, "WORKDIR", tmp_path)
+ store = BackgroundTaskStore(tmp_path)
+ store.save(make(task_id="cccccccc", status="running", command_preview="touch should-not-exist"))
+ manager = bg.BackgroundManager(bg.BackgroundScope("session:x"), store=store)
+ assert manager.recover_on_start()["recovered"] == 1
+ assert not (tmp_path / "should-not-exist").exists()
++
++
++
++def test_recovered_task_is_queryable_by_owner_not_other_sessions(tmp_path):
++ from src.tools import background_tools as bg
++ store = BackgroundTaskStore(tmp_path)
++ store.save(make(task_id="abcdef12", owner_id="session:a", status="running"))
++ manager = bg.BackgroundManager(bg.BackgroundScope("session:a"), store=store)
++ manager.recover_on_start()
++ result = json.loads(manager.check("abcdef12"))
++ assert result["status"] == "interrupted"
++ assert result["recovery_reason"] == "service_restart"
++ assert "abcdef12" in manager.check()
++ sibling = manager.for_scope(bg.BackgroundScope("session:b"))
++ assert json.loads(sibling.check("abcdef12"))["status"] == "error"
++
++
++def test_runtime_roots_keep_their_records_and_tables_separate(tmp_path):
++ from src.tools import background_tools as bg
++ first = bg.get_background_manager(tmp_path / "a").for_scope(bg.BackgroundScope("session:a"))
++ second = bg.get_background_manager(tmp_path / "b").for_scope(bg.BackgroundScope("session:a"))
++ first.tasks["deadbeef"] = {"owner_id": "session:a", "status": "completed", "command": "echo done", "result": "done"}
++ first._persist("deadbeef")
++ assert json.loads(second.check("deadbeef"))["status"] == "error"
++ assert second._store.load_all() == ([], [])
++
++
++def test_cancel_intent_is_persisted_before_reaping(tmp_path, monkeypatch):
++ from src.tools import background_tools as bg
++ store = BackgroundTaskStore(tmp_path)
++ manager = bg.BackgroundManager(bg.BackgroundScope("session:a"), store=store)
++ manager.tasks["deadbeef"] = {"owner_id": "session:a", "status": "running", "command": "echo done", "process": None}
++ manager.cancel("deadbeef")
++ records, unreadable = store.load_all()
++ assert not unreadable
++ assert records[0].status == "cancelling"
++ recovered, _ = reconcile_after_restart(records)
++ assert recovered[0].status == "outcome_unknown"