Skip to content

"Vibe-Trading: Your Personal Trading Agent"

fix(background): restore pending notifications without trusting corrupt records

Haozhe Wu697b6b2f3bdc
diff --git a/agent/src/tools/background_store.py b/agent/src/tools/background_store.py
index 5a7b00510..0af20231d 100644
--- a/agent/src/tools/background_store.py
+++ b/agent/src/tools/background_store.py
@@ -10,6 +10,11 @@ Recovery is deliberately one-way and never executes anything. A record left in
``interrupted`` or ``outcome_unknown`` with a recovery reason. It is never
re-run, and a stored PID is descriptive only: it is not used to signal a
process, because that pid may have been reused by an unrelated program.
+
+Terminal notifications are restored until their owner drains them. Delivery is
+at least once: a crash before the acknowledgement reaches disk can replay a
+notice, never silently discard it. Version-1 terminal records have no delivery
+marker, so they remain queryable without inventing a duplicate notification.
"""
from __future__ import annotations
@@ -27,7 +32,8 @@ from typing import Any, Iterable, Optional
from src.tools.redaction import redact_text
-SCHEMA_VERSION = 1
+SCHEMA_VERSION = 2
+_READABLE_SCHEMA_VERSIONS = frozenset({1, SCHEMA_VERSION})
#: Preview length kept per field. Enough to identify a command or outcome
#: without becoming a second copy of the run's full output.
MAX_PREVIEW_CHARS = 2_000
@@ -61,6 +67,8 @@ class TaskRecord:
recovery_reason: Optional[str] = None
#: Descriptive only. Never used to signal a process after a restart.
pid: Optional[int] = None
+ #: None means a version-1 record whose delivery state is unknowable.
+ notification_pending: Optional[bool] = None
updated_at: float = field(default_factory=time.time)
@property
@@ -133,13 +141,15 @@ class BackgroundTaskStore:
return None
if not isinstance(payload, dict):
return None
- if payload.get("schema_version") != SCHEMA_VERSION:
+ version = payload.get("schema_version")
+ if type(version) is not int or version not in _READABLE_SCHEMA_VERSIONS:
return None
try:
record = TaskRecord(**payload)
if (not isinstance(record.task_id, str) or not _TASK_ID_RE.fullmatch(record.task_id)
or not isinstance(record.owner_id, str) or not record.owner_id.strip()
or record.status not in LIVE_STATUSES | TERMINAL_STATUSES | {"interrupted", "outcome_unknown"}
+ or not isinstance(record.notification_pending, (bool, type(None)))
or path.name != self._path_for(record).name):
return None
for value in (record.started_at, record.ended_at, record.duration_seconds, record.updated_at):
@@ -198,6 +208,11 @@ def record_from_task(task_id: str, task: dict, *, started_at: Optional[float] =
ended_at=task.get("ended_at"),
duration_seconds=task.get("duration_seconds"),
pid=task.get("pid") if isinstance(task.get("pid"), int) else None,
+ notification_pending=(
+ task.get("notification_pending")
+ if isinstance(task.get("notification_pending"), bool)
+ else None
+ ),
)
@@ -219,8 +234,10 @@ def reconcile_after_restart(
recovered.append(
replace(
record,
+ schema_version=SCHEMA_VERSION,
status="interrupted",
recovery_reason="service_restart",
+ notification_pending=True,
ended_at=time.time(),
)
)
@@ -228,8 +245,10 @@ def reconcile_after_restart(
recovered.append(
replace(
record,
+ schema_version=SCHEMA_VERSION,
status="outcome_unknown",
recovery_reason="service_restart_before_reap_completed",
+ notification_pending=True,
ended_at=time.time(),
)
)
diff --git a/agent/src/tools/background_tools.py b/agent/src/tools/background_tools.py
index 1d913f8d5..6c4212626 100644
--- a/agent/src/tools/background_tools.py
+++ b/agent/src/tools/background_tools.py
@@ -211,7 +211,7 @@ class BackgroundManager:
"""
if self._store is None:
return {"recovered": 0, "unchanged": 0, "unreadable": []}
- from src.tools.background_store import reconcile_after_restart
+ from src.tools.background_store import TaskRecord, reconcile_after_restart
records, unreadable = self._store.load_all()
with _LIVE_TASK_IDS_LOCK:
@@ -224,6 +224,7 @@ class BackgroundManager:
except Exception:
logging.getLogger(__name__).exception("Could not persist recovered task %s", record.task_id)
with self._lock:
+ restored: list[TaskRecord] = []
for record in recovered + unchanged:
if record.task_id in self.tasks:
continue
@@ -235,7 +236,20 @@ class BackgroundManager:
"started_wall_time": record.started_at, "ended_at": record.ended_at,
"duration_seconds": record.duration_seconds,
"recovery_reason": record.recovery_reason,
+ "notification_pending": record.notification_pending,
}
+ if record.notification_pending is True:
+ restored.append(record)
+ restored.sort(key=lambda record: (
+ record.ended_at if record.ended_at is not None else record.updated_at,
+ record.task_id,
+ ))
+ self._notifications[:0] = [(record.owner_id, {
+ "task_id": record.task_id, "status": record.status,
+ "command": record.command_preview[:80],
+ "result": (record.result_preview or "(no output)")[:500],
+ "exit_code": record.exit_code,
+ }) for record in restored]
return {"recovered": len(recovered), "unchanged": len(unchanged), "unreadable": unreadable}
def run(self, command: str) -> str:
@@ -270,6 +284,7 @@ class BackgroundManager:
"started_wall_time": time.time(),
"owner_id": self._scope.owner_id,
"attempt_id": self._scope.attempt_id,
+ "notification_pending": True,
}
with _LIVE_TASK_IDS_LOCK:
_LIVE_TASK_IDS.add(task_id)
@@ -501,13 +516,26 @@ class BackgroundManager:
"""Return this owner's queued notifications, oldest first.
Notifications belonging to other owners stay queued for them; draining
- one session can never consume another's result.
+ one session can never consume another's result. Durable delivery is
+ acknowledged after the in-memory claim. If that write or the process
+ fails, the next restart can replay a notice (at-least-once delivery).
"""
+ acknowledged: list[str] = []
with self._lock:
mine = [payload for owner, payload in self._notifications
if owner == self._scope.owner_id]
self._notifications[:] = [(owner, payload) for owner, payload in self._notifications
if owner != self._scope.owner_id]
+ for payload in mine:
+ task_id = payload.get("task_id")
+ task = self.tasks.get(task_id)
+ if task is not None and task.get("notification_pending") is True:
+ task["notification_pending"] = False
+ acknowledged.append(task_id)
+ # Do not hold the task lock while taking the persistence lock: _persist
+ # takes them in the opposite order on completion and cancellation.
+ for task_id in acknowledged:
+ self._persist(task_id)
return mine
diff --git a/agent/tests/test_background_store_recovery.py b/agent/tests/test_background_store_recovery.py
index 3a81efe7d..706f35124 100644
--- a/agent/tests/test_background_store_recovery.py
+++ b/agent/tests/test_background_store_recovery.py
@@ -219,6 +219,12 @@ class TestManagerWiring:
assert records[0].status == "completed"
assert records[0].attempt_id is None
assert records[0].command_preview
+ assert records[0].notification_pending is True
+
+ assert [item["task_id"] for item in manager.drain_notifications()] == [task_id]
+ records, unreadable = manager._store.load_all()
+ assert unreadable == []
+ assert records[0].notification_pending is False
def test_restart_recovery_leaves_this_processes_live_tasks_alone(self, tmp_path):
from src.tools import background_tools as bg
@@ -288,3 +294,150 @@ def test_cancel_intent_is_persisted_before_reaping(tmp_path, monkeypatch):
assert records[0].status == "cancelling"
recovered, _ = reconcile_after_restart(records)
assert recovered[0].status == "outcome_unknown"
+
+
+def test_pending_terminal_notification_is_restored_once_and_acknowledged(tmp_path):
+ from src.tools import background_tools as bg
+
+ store = BackgroundTaskStore(tmp_path)
+ store.save(make(task_id="eeeeeeee", owner_id="session:a", status="completed",
+ result_preview="finished", notification_pending=True))
+ first = bg.BackgroundManager(bg.BackgroundScope("session:a"), store=store)
+ first.recover_on_start()
+
+ other = first.for_scope(bg.BackgroundScope("session:b"))
+ assert other.drain_notifications() == []
+ assert first.drain_notifications() == [{
+ "task_id": "eeeeeeee", "status": "completed", "command": "echo hi",
+ "result": "finished", "exit_code": 0,
+ }]
+ records, unreadable = store.load_all()
+ assert unreadable == [] and records[0].notification_pending is False
+
+ restarted = bg.BackgroundManager(bg.BackgroundScope("session:a"), store=store)
+ restarted.recover_on_start()
+ assert restarted.drain_notifications() == []
+
+
+def test_interrupted_task_produces_a_recovered_notification(tmp_path):
+ from src.tools import background_tools as bg
+
+ store = BackgroundTaskStore(tmp_path)
+ store.save(make(task_id="ffffffff", owner_id="session:a", status="running",
+ notification_pending=True))
+ manager = bg.BackgroundManager(bg.BackgroundScope("session:a"), store=store)
+ manager.recover_on_start()
+
+ notices = manager.drain_notifications()
+ assert len(notices) == 1
+ assert notices[0]["task_id"] == "ffffffff"
+ assert notices[0]["status"] == "interrupted"
+
+
+def test_failed_delivery_ack_is_retried_after_another_restart(tmp_path):
+ from src.tools import background_tools as bg
+
+ class FailingAckStore(BackgroundTaskStore):
+ def save(self, record):
+ if record.notification_pending is False:
+ raise OSError("disk full during delivery acknowledgement")
+ return super().save(record)
+
+ store = FailingAckStore(tmp_path)
+ store.save(make(task_id="aaaaaaa1", owner_id="session:a", status="completed",
+ result_preview="deliver me", notification_pending=True))
+ first = bg.BackgroundManager(bg.BackgroundScope("session:a"), store=store)
+ first.recover_on_start()
+ assert len(first.drain_notifications()) == 1
+
+ restarted = bg.BackgroundManager(bg.BackgroundScope("session:a"), store=store)
+ restarted.recover_on_start()
+ assert [item["result"] for item in restarted.drain_notifications()] == ["deliver me"]
+
+
+def test_v1_terminal_record_does_not_invent_a_duplicate_notification(tmp_path):
+ from dataclasses import asdict
+ from src.tools import background_tools as bg
+
+ store = BackgroundTaskStore(tmp_path)
+ legacy = make(task_id="bbbbbbb1", owner_id="session:a", status="completed")
+ payload = asdict(legacy)
+ payload["schema_version"] = 1
+ payload.pop("notification_pending", None)
+ store._write_json_atomically(store._path_for(legacy), payload)
+
+ manager = bg.BackgroundManager(bg.BackgroundScope("session:a"), store=store)
+ summary = manager.recover_on_start()
+ assert summary["unchanged"] == 1
+ assert json.loads(manager.check("bbbbbbb1"))["status"] == "completed"
+ assert manager.drain_notifications() == []
+
+
+def test_recovered_notifications_precede_new_work_and_recovery_is_idempotent(tmp_path):
+ from src.tools import background_tools as bg
+
+ store = BackgroundTaskStore(tmp_path)
+ store.save(make(task_id="ccccccc1", owner_id="session:a", status="completed",
+ notification_pending=True, ended_at=1.0))
+ manager = bg.BackgroundManager(bg.BackgroundScope("session:a"), store=store)
+ manager._notifications.append(("session:a", {
+ "task_id": "new-work", "status": "completed", "command": "new",
+ "result": "new result", "exit_code": 0,
+ }))
+
+ manager.recover_on_start()
+ manager.recover_on_start()
+
+ notices = manager.drain_notifications()
+ assert [item["task_id"] for item in notices] == ["ccccccc1", "new-work"]
+
+
+def test_v1_live_record_is_migrated_when_recovered(tmp_path):
+ from dataclasses import asdict
+ from src.tools import background_tools as bg
+
+ store = BackgroundTaskStore(tmp_path)
+ legacy = make(task_id="ddddddd1", owner_id="session:a", status="running")
+ payload = asdict(legacy)
+ payload["schema_version"] = 1
+ payload.pop("notification_pending", None)
+ store._write_json_atomically(store._path_for(legacy), payload)
+
+ manager = bg.BackgroundManager(bg.BackgroundScope("session:a"), store=store)
+ manager.recover_on_start()
+
+ records, unreadable = store.load_all()
+ assert unreadable == []
+ assert records[0].schema_version == SCHEMA_VERSION
+ assert records[0].status == "interrupted"
+ assert records[0].notification_pending is True
+ assert len(manager.drain_notifications()) == 1
+
+
+def test_recovered_notification_preserves_live_payload_limits(tmp_path):
+ from src.tools import background_tools as bg
+
+ store = BackgroundTaskStore(tmp_path)
+ store.save(make(task_id="eeeeeee1", owner_id="session:a", status="completed",
+ command_preview="c" * 200, result_preview="r" * 2000,
+ notification_pending=True))
+ manager = bg.BackgroundManager(bg.BackgroundScope("session:a"), store=store)
+ manager.recover_on_start()
+
+ notice, = manager.drain_notifications()
+ assert notice["command"] == "c" * 80
+ assert notice["result"] == "r" * 500
+
+
[email protected]("version", [[], {}, True, None, "2"])
+def test_corrupt_version_does_not_abort_other_task_recovery(tmp_path, version):
+ store = BackgroundTaskStore(tmp_path)
+ store.save(make(task_id="aaaaaaaa", notification_pending=True))
+ broken = store._directory / "broken.json"
+ broken.write_text(json.dumps({"schema_version": version}))
+ from src.tools.background_tools import BackgroundManager, BackgroundScope
+
+ manager = BackgroundManager(BackgroundScope("session:s1"), store=store)
+ report = manager.recover_on_start()
+ assert report["unreadable"] == [str(broken)]
+ assert [notice["task_id"] for notice in manager.drain_notifications()] == ["aaaaaaaa"]