"Vibe-Trading: Your Personal Trading Agent"
Python ai-agentalgorithmic-tradingbacktestingfintechllmmcpmulti-agentpythonquantitative-financetrading
fix(background): restore pending notifications without trusting corrupt records
Haozhe Wu697b6b2f3bdc
diff --git a/agent/src/tools/background_store.py b/agent/src/tools/background_store.pyindex 5a7b00510..0af20231d 100644--- a/agent/src/tools/background_store.py+++ b/agent/src/tools/background_store.py@@ -10,6 +10,11 @@ Recovery is deliberately one-way and never executes anything. A record left in``interrupted`` or ``outcome_unknown`` with a recovery reason. It is neverre-run, and a stored PID is descriptive only: it is not used to signal aprocess, because that pid may have been reused by an unrelated program.++Terminal notifications are restored until their owner drains them. Delivery is+at least once: a crash before the acknowledgement reaches disk can replay a+notice, never silently discard it. Version-1 terminal records have no delivery+marker, so they remain queryable without inventing a duplicate notification."""from __future__ import annotations@@ -27,7 +32,8 @@ from typing import Any, Iterable, Optionalfrom src.tools.redaction import redact_text-SCHEMA_VERSION = 1+SCHEMA_VERSION = 2+_READABLE_SCHEMA_VERSIONS = frozenset({1, SCHEMA_VERSION})#: Preview length kept per field. Enough to identify a command or outcome#: without becoming a second copy of the run's full output.MAX_PREVIEW_CHARS = 2_000@@ -61,6 +67,8 @@ class TaskRecord:recovery_reason: Optional[str] = None#: Descriptive only. Never used to signal a process after a restart.pid: Optional[int] = None+ #: None means a version-1 record whose delivery state is unknowable.+ notification_pending: Optional[bool] = Noneupdated_at: float = field(default_factory=time.time)@property@@ -133,13 +141,15 @@ class BackgroundTaskStore:return Noneif not isinstance(payload, dict):return None- if payload.get("schema_version") != SCHEMA_VERSION:+ version = payload.get("schema_version")+ if type(version) is not int or version not in _READABLE_SCHEMA_VERSIONS:return Nonetry:record = TaskRecord(**payload)if (not isinstance(record.task_id, str) or not _TASK_ID_RE.fullmatch(record.task_id)or not isinstance(record.owner_id, str) or not record.owner_id.strip()or record.status not in LIVE_STATUSES | TERMINAL_STATUSES | {"interrupted", "outcome_unknown"}+ or not isinstance(record.notification_pending, (bool, type(None)))or path.name != self._path_for(record).name):return Nonefor value in (record.started_at, record.ended_at, record.duration_seconds, record.updated_at):@@ -198,6 +208,11 @@ def record_from_task(task_id: str, task: dict, *, started_at: Optional[float] =ended_at=task.get("ended_at"),duration_seconds=task.get("duration_seconds"),pid=task.get("pid") if isinstance(task.get("pid"), int) else None,+ notification_pending=(+ task.get("notification_pending")+ if isinstance(task.get("notification_pending"), bool)+ else None+ ),)@@ -219,8 +234,10 @@ def reconcile_after_restart(recovered.append(replace(record,+ schema_version=SCHEMA_VERSION,status="interrupted",recovery_reason="service_restart",+ notification_pending=True,ended_at=time.time(),))@@ -228,8 +245,10 @@ def reconcile_after_restart(recovered.append(replace(record,+ schema_version=SCHEMA_VERSION,status="outcome_unknown",recovery_reason="service_restart_before_reap_completed",+ notification_pending=True,ended_at=time.time(),))diff --git a/agent/src/tools/background_tools.py b/agent/src/tools/background_tools.pyindex 1d913f8d5..6c4212626 100644--- a/agent/src/tools/background_tools.py+++ b/agent/src/tools/background_tools.py@@ -211,7 +211,7 @@ class BackgroundManager:"""if self._store is None:return {"recovered": 0, "unchanged": 0, "unreadable": []}- from src.tools.background_store import reconcile_after_restart+ from src.tools.background_store import TaskRecord, reconcile_after_restartrecords, unreadable = self._store.load_all()with _LIVE_TASK_IDS_LOCK:@@ -224,6 +224,7 @@ class BackgroundManager:except Exception:logging.getLogger(__name__).exception("Could not persist recovered task %s", record.task_id)with self._lock:+ restored: list[TaskRecord] = []for record in recovered + unchanged:if record.task_id in self.tasks:continue@@ -235,7 +236,20 @@ class BackgroundManager:"started_wall_time": record.started_at, "ended_at": record.ended_at,"duration_seconds": record.duration_seconds,"recovery_reason": record.recovery_reason,+ "notification_pending": record.notification_pending,}+ if record.notification_pending is True:+ restored.append(record)+ restored.sort(key=lambda record: (+ record.ended_at if record.ended_at is not None else record.updated_at,+ record.task_id,+ ))+ self._notifications[:0] = [(record.owner_id, {+ "task_id": record.task_id, "status": record.status,+ "command": record.command_preview[:80],+ "result": (record.result_preview or "(no output)")[:500],+ "exit_code": record.exit_code,+ }) for record in restored]return {"recovered": len(recovered), "unchanged": len(unchanged), "unreadable": unreadable}def run(self, command: str) -> str:@@ -270,6 +284,7 @@ class BackgroundManager:"started_wall_time": time.time(),"owner_id": self._scope.owner_id,"attempt_id": self._scope.attempt_id,+ "notification_pending": True,}with _LIVE_TASK_IDS_LOCK:_LIVE_TASK_IDS.add(task_id)@@ -501,13 +516,26 @@ class BackgroundManager:"""Return this owner's queued notifications, oldest first.Notifications belonging to other owners stay queued for them; draining- one session can never consume another's result.+ one session can never consume another's result. Durable delivery is+ acknowledged after the in-memory claim. If that write or the process+ fails, the next restart can replay a notice (at-least-once delivery)."""+ acknowledged: list[str] = []with self._lock:mine = [payload for owner, payload in self._notificationsif owner == self._scope.owner_id]self._notifications[:] = [(owner, payload) for owner, payload in self._notificationsif owner != self._scope.owner_id]+ for payload in mine:+ task_id = payload.get("task_id")+ task = self.tasks.get(task_id)+ if task is not None and task.get("notification_pending") is True:+ task["notification_pending"] = False+ acknowledged.append(task_id)+ # Do not hold the task lock while taking the persistence lock: _persist+ # takes them in the opposite order on completion and cancellation.+ for task_id in acknowledged:+ self._persist(task_id)return minediff --git a/agent/tests/test_background_store_recovery.py b/agent/tests/test_background_store_recovery.pyindex 3a81efe7d..706f35124 100644--- a/agent/tests/test_background_store_recovery.py+++ b/agent/tests/test_background_store_recovery.py@@ -219,6 +219,12 @@ class TestManagerWiring:assert records[0].status == "completed"assert records[0].attempt_id is Noneassert records[0].command_preview+ assert records[0].notification_pending is True++ assert [item["task_id"] for item in manager.drain_notifications()] == [task_id]+ records, unreadable = manager._store.load_all()+ assert unreadable == []+ assert records[0].notification_pending is Falsedef test_restart_recovery_leaves_this_processes_live_tasks_alone(self, tmp_path):from src.tools import background_tools as bg@@ -288,3 +294,150 @@ def test_cancel_intent_is_persisted_before_reaping(tmp_path, monkeypatch):assert records[0].status == "cancelling"recovered, _ = reconcile_after_restart(records)assert recovered[0].status == "outcome_unknown"+++def test_pending_terminal_notification_is_restored_once_and_acknowledged(tmp_path):+ from src.tools import background_tools as bg++ store = BackgroundTaskStore(tmp_path)+ store.save(make(task_id="eeeeeeee", owner_id="session:a", status="completed",+ result_preview="finished", notification_pending=True))+ first = bg.BackgroundManager(bg.BackgroundScope("session:a"), store=store)+ first.recover_on_start()++ other = first.for_scope(bg.BackgroundScope("session:b"))+ assert other.drain_notifications() == []+ assert first.drain_notifications() == [{+ "task_id": "eeeeeeee", "status": "completed", "command": "echo hi",+ "result": "finished", "exit_code": 0,+ }]+ records, unreadable = store.load_all()+ assert unreadable == [] and records[0].notification_pending is False++ restarted = bg.BackgroundManager(bg.BackgroundScope("session:a"), store=store)+ restarted.recover_on_start()+ assert restarted.drain_notifications() == []+++def test_interrupted_task_produces_a_recovered_notification(tmp_path):+ from src.tools import background_tools as bg++ store = BackgroundTaskStore(tmp_path)+ store.save(make(task_id="ffffffff", owner_id="session:a", status="running",+ notification_pending=True))+ manager = bg.BackgroundManager(bg.BackgroundScope("session:a"), store=store)+ manager.recover_on_start()++ notices = manager.drain_notifications()+ assert len(notices) == 1+ assert notices[0]["task_id"] == "ffffffff"+ assert notices[0]["status"] == "interrupted"+++def test_failed_delivery_ack_is_retried_after_another_restart(tmp_path):+ from src.tools import background_tools as bg++ class FailingAckStore(BackgroundTaskStore):+ def save(self, record):+ if record.notification_pending is False:+ raise OSError("disk full during delivery acknowledgement")+ return super().save(record)++ store = FailingAckStore(tmp_path)+ store.save(make(task_id="aaaaaaa1", owner_id="session:a", status="completed",+ result_preview="deliver me", notification_pending=True))+ first = bg.BackgroundManager(bg.BackgroundScope("session:a"), store=store)+ first.recover_on_start()+ assert len(first.drain_notifications()) == 1++ restarted = bg.BackgroundManager(bg.BackgroundScope("session:a"), store=store)+ restarted.recover_on_start()+ assert [item["result"] for item in restarted.drain_notifications()] == ["deliver me"]+++def test_v1_terminal_record_does_not_invent_a_duplicate_notification(tmp_path):+ from dataclasses import asdict+ from src.tools import background_tools as bg++ store = BackgroundTaskStore(tmp_path)+ legacy = make(task_id="bbbbbbb1", owner_id="session:a", status="completed")+ payload = asdict(legacy)+ payload["schema_version"] = 1+ payload.pop("notification_pending", None)+ store._write_json_atomically(store._path_for(legacy), payload)++ manager = bg.BackgroundManager(bg.BackgroundScope("session:a"), store=store)+ summary = manager.recover_on_start()+ assert summary["unchanged"] == 1+ assert json.loads(manager.check("bbbbbbb1"))["status"] == "completed"+ assert manager.drain_notifications() == []+++def test_recovered_notifications_precede_new_work_and_recovery_is_idempotent(tmp_path):+ from src.tools import background_tools as bg++ store = BackgroundTaskStore(tmp_path)+ store.save(make(task_id="ccccccc1", owner_id="session:a", status="completed",+ notification_pending=True, ended_at=1.0))+ manager = bg.BackgroundManager(bg.BackgroundScope("session:a"), store=store)+ manager._notifications.append(("session:a", {+ "task_id": "new-work", "status": "completed", "command": "new",+ "result": "new result", "exit_code": 0,+ }))++ manager.recover_on_start()+ manager.recover_on_start()++ notices = manager.drain_notifications()+ assert [item["task_id"] for item in notices] == ["ccccccc1", "new-work"]+++def test_v1_live_record_is_migrated_when_recovered(tmp_path):+ from dataclasses import asdict+ from src.tools import background_tools as bg++ store = BackgroundTaskStore(tmp_path)+ legacy = make(task_id="ddddddd1", owner_id="session:a", status="running")+ payload = asdict(legacy)+ payload["schema_version"] = 1+ payload.pop("notification_pending", None)+ store._write_json_atomically(store._path_for(legacy), payload)++ manager = bg.BackgroundManager(bg.BackgroundScope("session:a"), store=store)+ manager.recover_on_start()++ records, unreadable = store.load_all()+ assert unreadable == []+ assert records[0].schema_version == SCHEMA_VERSION+ assert records[0].status == "interrupted"+ assert records[0].notification_pending is True+ assert len(manager.drain_notifications()) == 1+++def test_recovered_notification_preserves_live_payload_limits(tmp_path):+ from src.tools import background_tools as bg++ store = BackgroundTaskStore(tmp_path)+ store.save(make(task_id="eeeeeee1", owner_id="session:a", status="completed",+ command_preview="c" * 200, result_preview="r" * 2000,+ notification_pending=True))+ manager = bg.BackgroundManager(bg.BackgroundScope("session:a"), store=store)+ manager.recover_on_start()++ notice, = manager.drain_notifications()+ assert notice["command"] == "c" * 80+ assert notice["result"] == "r" * 500++[email protected]("version", [[], {}, True, None, "2"])+def test_corrupt_version_does_not_abort_other_task_recovery(tmp_path, version):+ store = BackgroundTaskStore(tmp_path)+ store.save(make(task_id="aaaaaaaa", notification_pending=True))+ broken = store._directory / "broken.json"+ broken.write_text(json.dumps({"schema_version": version}))+ from src.tools.background_tools import BackgroundManager, BackgroundScope++ manager = BackgroundManager(BackgroundScope("session:s1"), store=store)+ report = manager.recover_on_start()+ assert report["unreadable"] == [str(broken)]+ assert [notice["task_id"] for notice in manager.drain_notifications()] == ["aaaaaaaa"]