Skip to content

An always-on second brain you talk to. Voice notes in Telegram → typed, linked knowledge in your Obsidian vault. Runs 24/7 on the Claude subscription you already have.

fix(cron): crash-safety hardening from blind review

smixsc2f29a1682fd
diff --git a/src/d_brain/services/cron_runner.py b/src/d_brain/services/cron_runner.py
index 676ab77..945e596 100644
--- a/src/d_brain/services/cron_runner.py
+++ b/src/d_brain/services/cron_runner.py
@@ -14,6 +14,7 @@ Semantics:
"""
import asyncio
+import contextlib
import copy
import logging
from collections.abc import Awaitable, Callable
@@ -75,8 +76,10 @@ class CronRunner:
"""Snapshot due jobs and advance their next_run under the lock.
Persisting the advance BEFORE execution gives at-most-once: a
- crash mid-ask never refires the slot. One-shot ('at') jobs get
- next_run=None here; success deletes them, failure re-arms a retry.
+ crash mid-ask never refires the slot. One-shot ('at') jobs are
+ re-armed to now+retry (not None) — so a crash between claim and
+ record leaves a job that fires again instead of a bricked one;
+ success then deletes it or clears next_run.
"""
claimed: list[CronJob] = []
@@ -84,11 +87,24 @@ class CronRunner:
for job in jobs:
if not job.enabled or not job.state.next_run:
continue
- if datetime.fromisoformat(job.state.next_run) > now:
+ try:
+ due_at = datetime.fromisoformat(job.state.next_run)
+ except ValueError:
+ # One malformed entry must not stall the whole schedule.
+ logger.error(
+ "job %s has malformed next_run %r — skipping",
+ job.id,
+ job.state.next_run,
+ )
+ continue
+ if due_at.tzinfo is None:
+ due_at = due_at.replace(tzinfo=UTC)
+ if due_at > now:
continue
claimed.append(copy.deepcopy(job))
if job.schedule.kind == "at":
- job.state.next_run = None
+ retry_at = now + timedelta(seconds=self.retry_seconds)
+ job.state.next_run = retry_at.isoformat()
else:
nxt = compute_next_run(job.schedule, now=now)
job.state.next_run = nxt.isoformat() if nxt else None
@@ -115,14 +131,28 @@ class CronRunner:
if res.ok:
reply = (res.reply or "").strip()
silent = reply.startswith(SILENT_MARKER)
+ delivery_error: str | None = None
if reply and not silent:
chat = job.chat_id or self.default_chat_id
if chat is not None:
- await self.deliver(chat, reply)
+ try:
+ await self.deliver(chat, reply)
+ except Exception as exc: # noqa: BLE001 — Telegram hiccup
+ logger.exception("cron job %s delivery failed", job.id)
+ delivery_error = str(exc)[:200]
# Jobs are stateless by contract; drop the turn's context so
- # the next job starts clean and the window never grows.
- await asyncio.to_thread(self.session.send_control, "/clear")
- self._record_success(job, now, silent=silent)
+ # the next job starts clean and the window never grows. Best
+ # effort — a failed /clear must not block the state update.
+ with contextlib.suppress(Exception):
+ await asyncio.to_thread(self.session.send_control, "/clear")
+ if delivery_error is not None:
+ # The work happened but the user never saw it: keep the job
+ # (one-shots retry) and count the failure.
+ self._record_failure(
+ job, now, status="deliver_error", detail=delivery_error, count=True
+ )
+ else:
+ self._record_success(job, now, silent=silent)
return
logger.error("cron job %s failed: %s %s", job.id, res.status, res.detail)
@@ -152,6 +182,9 @@ class CronRunner:
current.state.last_status = "ok-silent" if silent else "ok"
current.state.last_error = None
current.state.consecutive_errors = 0
+ if current.schedule.kind == "at":
+ # Claim re-armed a retry slot; the one-shot has now fired.
+ current.state.next_run = None
self.store.mutate(fn)
@@ -193,7 +226,18 @@ class CronRunner:
async def tick(self) -> None:
for job in self.claim_due(self.clock()):
- await self.run_job(job)
+ try:
+ await self.run_job(job)
+ except Exception as exc: # noqa: BLE001 — one job must not kill the batch
+ logger.exception("cron job %s crashed", job.id)
+ with contextlib.suppress(Exception):
+ self._record_failure(
+ job,
+ self.clock(),
+ status="crash",
+ detail=str(exc)[:200],
+ count=True,
+ )
async def run(self, tick_seconds: float) -> None:
logger.info("cron runner started (tick %.0fs)", tick_seconds)
diff --git a/src/d_brain/services/cron_store.py b/src/d_brain/services/cron_store.py
index d462347..8455353 100644
--- a/src/d_brain/services/cron_store.py
+++ b/src/d_brain/services/cron_store.py
@@ -140,17 +140,21 @@ class CronStore:
self.lock_file = self.cron_dir / "jobs.lock"
def load(self) -> list[CronJob]:
- if not self.jobs_file.exists():
- return []
try:
raw = json.loads(self.jobs_file.read_text())
return [_job_from_dict(j) for j in raw.get("jobs", [])]
+ except FileNotFoundError:
+ # Also covers a concurrent quarantine-rename by another process.
+ return []
except (json.JSONDecodeError, KeyError, TypeError) as exc:
# Keep the evidence, never wipe it: rename and start empty.
quarantine = self.jobs_file.with_name(
f"jobs.json.corrupt-{int(time.time())}"
)
- self.jobs_file.rename(quarantine)
+ try:
+ self.jobs_file.rename(quarantine)
+ except FileNotFoundError:
+ return [] # the other reader quarantined it first
logger.error("corrupt jobs.json moved to %s: %s", quarantine, exc)
return []
diff --git a/tests/test_cron_runner.py b/tests/test_cron_runner.py
index 57924f5..b9554f5 100644
--- a/tests/test_cron_runner.py
+++ b/tests/test_cron_runner.py
@@ -4,13 +4,13 @@ FakeSession pattern as in test_claude_session.py — no tmux, scripted
AskResults, recorder callables for deliver/alert, a manual clock.
"""
-from datetime import datetime, timedelta, timezone
+from datetime import UTC, datetime, timedelta
from d_brain.services.claude_session import AskResult
from d_brain.services.cron_runner import SILENT_MARKER, CronRunner, wrap_job_prompt
from d_brain.services.cron_store import CronJob, CronStore, Schedule
-NOW = datetime(2026, 6, 10, 12, 0, tzinfo=timezone.utc)
+NOW = datetime(2026, 6, 10, 12, 0, tzinfo=UTC)
class FakeSession:
@@ -244,6 +244,110 @@ async def test_rate_limited_skips_without_recover_or_count(tmp_path):
assert "/clear" not in session.controls
+# ── crash-safety (blind-review findings) ─────────────────────────────
+
+
+async def test_at_claim_rearms_retry_instead_of_none(tmp_path):
+ """A bot restart between claim and record must not brick a one-shot:
+ claim leaves a retry next_run, success/record clears or deletes it."""
+ store = _store(tmp_path)
+ _add_job(store, "once", kind="at", delete_after_run=True)
+ runner = _runner(store, FakeSession())
+ claimed = runner.claim_due(NOW)
+ assert len(claimed) == 1
+ persisted = store.load()[0]
+ assert persisted.state.next_run == (NOW + timedelta(seconds=300)).isoformat()
+
+
+async def test_oneshot_success_clears_next_run_when_kept(tmp_path):
+ """kind=at without delete_after_run must not refire after success."""
+ store = _store(tmp_path)
+ _add_job(store, "once", kind="at", delete_after_run=False)
+ await _runner(store, FakeSession([AskResult("ok", reply="x")])).tick()
+ job = store.load()[0]
+ assert job.state.next_run is None
+ assert job.state.last_status == "ok"
+
+
+async def test_deliver_crash_counts_as_failure_and_clears(tmp_path):
+ store = _store(tmp_path)
+ _add_job(store, "once", kind="at", delete_after_run=True)
+
+ async def bad_deliver(*args):
+ raise RuntimeError("telegram down")
+
+ session = FakeSession([AskResult("ok", reply="x")])
+ await _runner(store, session, deliver=bad_deliver).tick()
+ jobs = store.load()
+ assert len(jobs) == 1 # NOT deleted — will retry
+ assert jobs[0].state.last_status == "deliver_error"
+ assert jobs[0].state.consecutive_errors == 1
+ assert "/clear" in session.controls # context still dropped
+
+
+async def test_tick_survives_job_crash_and_runs_rest_of_batch(tmp_path):
+ store = _store(tmp_path)
+ _add_job(store, "boom")
+ _add_job(store, "fine")
+
+ class ExplodingRecover(FakeSession):
+ def force_recover(self):
+ raise RuntimeError("session not ready")
+
+ session = ExplodingRecover(
+ [AskResult("error", detail="x"), AskResult("ok", reply="y")]
+ )
+ deliver = Recorder()
+ await _runner(store, session, deliver=deliver).tick()
+ assert len(session.asked) == 2 # second job still ran
+ boom = next(j for j in store.load() if j.id == "boom")
+ assert boom.state.last_status == "crash"
+ assert boom.state.consecutive_errors == 1
+
+
+async def test_claim_skips_malformed_next_run_without_killing_tick(tmp_path):
+ store = _store(tmp_path)
+ _add_job(store, "bad")
+ _add_job(store, "good")
+
+ def corrupt(jobs):
+ jobs[0].state.next_run = "definitely not a date"
+
+ store.mutate(corrupt)
+ session = FakeSession([AskResult("ok", reply="x")])
+ await _runner(store, session).tick()
+ assert len(session.asked) == 1 # good ran, bad skipped, no exception
+
+
+async def test_claim_treats_naive_next_run_as_utc(tmp_path):
+ store = _store(tmp_path)
+ _add_job(store, "naive")
+
+ def make_naive(jobs):
+ jobs[0].state.next_run = NOW.replace(tzinfo=None).isoformat()
+
+ store.mutate(make_naive)
+ session = FakeSession([AskResult("ok", reply="x")])
+ await _runner(store, session).tick()
+ assert len(session.asked) == 1
+
+
+def test_load_tolerates_concurrent_rename(tmp_path):
+ """FileNotFoundError between exists() and read must not escape load()."""
+ from d_brain.services.cron_store import CronStore as CS
+
+ store = CS(tmp_path / "cron")
+ store.save([])
+ # Simulate the race directly: file vanishes after exists() check
+ store.jobs_file.unlink()
+ real_exists = type(store.jobs_file).exists
+ try:
+ type(store.jobs_file).exists = lambda self: True # type: ignore[method-assign]
+ assert store.load() == []
+ finally:
+ type(store.jobs_file).exists = real_exists # type: ignore[method-assign]
+
+
# ── prompt wrapper ───────────────────────────────────────────────────