An always-on second brain you talk to. Voice notes in Telegram → typed, linked knowledge in your Obsidian vault. Runs 24/7 on the Claude subscription you already have.
Python ai-agentsai-automationclaudeclaude-codeebbinghausknowledge-graphllmmemory-systemobsidianpersonal-assistantpkmproductivitypythonsecond-braintelegram-botvoice-assistant
fix(watchdog): back off persistent-fault alerts instead of hourly spam
smixse0d38f15fc4b
diff --git a/src/d_brain/services/watchdog.py b/src/d_brain/services/watchdog.pyindex 8549fa5..89e804e 100644--- a/src/d_brain/services/watchdog.py+++ b/src/d_brain/services/watchdog.py@@ -38,7 +38,8 @@ logger = logging.getLogger(__name__)DEFAULT_TICK = 15.0DEFAULT_STALL_THRESHOLD = 300.0 # 5 min stuck without visible work ⇒ wedgedDEFAULT_MIN_DISK = 500_000_000 # 500 MB-DEFAULT_ALERT_COOLDOWN = 3600.0 # re-alert a persistent fault at most hourly+DEFAULT_ALERT_COOLDOWN = 3600.0 # first re-alert of a persistent fault: 1h+DEFAULT_ALERT_COOLDOWN_MAX = 12 * 3600.0 # back-off cap (doubles 1h→2h→…→12h)_SERVICEABLE = {PaneState.READY,@@ -61,6 +62,7 @@ class Watchdog:stall_threshold: float = DEFAULT_STALL_THRESHOLD,min_disk_bytes: int = DEFAULT_MIN_DISK,alert_cooldown: float = DEFAULT_ALERT_COOLDOWN,+ alert_cooldown_max: float = DEFAULT_ALERT_COOLDOWN_MAX,) -> None:self.session = sessionself.runtime_dir = Path(runtime_dir)@@ -74,10 +76,12 @@ class Watchdog:self._stall_threshold = stall_thresholdself._min_disk = min_disk_bytesself._alert_cooldown = alert_cooldown+ self._alert_cooldown_max = alert_cooldown_maxself._inflight = self.runtime_dir / "inflight"self._status = self.runtime_dir / "STATUS.md"self._last_alert_key: str | None = Noneself._last_alert_ts = 0.0+ self._alert_repeats = 0self._stuck_since: float | None = Nonedef _is_hung(self, state: PaneState) -> bool:@@ -95,11 +99,19 @@ class Watchdog:def _maybe_alert(self, key: str, msg: str) -> None:now = self._clock()- if (- self._last_alert_key == key- and now - self._last_alert_ts < self._alert_cooldown- ):- return+ if self._last_alert_key == key:+ # Same persistent fault: back off exponentially (1h, 2h, 4h … up to+ # the cap) so an overnight outage sends a few escalating reminders+ # instead of one identical message every hour.+ cooldown = min(+ self._alert_cooldown * (2**self._alert_repeats),+ self._alert_cooldown_max,+ )+ if now - self._last_alert_ts < cooldown:+ return+ self._alert_repeats += 1+ else:+ self._alert_repeats = 0self._alert_fn(msg)self._last_alert_key = keyself._last_alert_ts = now@@ -107,6 +119,7 @@ class Watchdog:def _note_good(self) -> None:# Returning to a good state re-arms alerts for the next incident.self._last_alert_key = None+ self._alert_repeats = 0def _write_status(self, state: str) -> None:try:diff --git a/tests/test_watchdog.py b/tests/test_watchdog.pyindex 9f3cb91..8db0a91 100644--- a/tests/test_watchdog.py+++ b/tests/test_watchdog.py@@ -184,6 +184,22 @@ def test_alert_refires_after_recovery(tmp_path):assert len(alerts) == 2+def test_persistent_fault_alert_backs_off(tmp_path):+ """A persistent fault must not spam at a fixed hourly rate — the re-alert+ interval doubles (1h, 2h, 4h…), so an 8-hour overnight outage sends a few+ escalating reminders, not one every hour."""+ sess = FakeSession(state=PaneState.LOGGED_OUT)+ alerts = []+ clock = {"now": 0.0}+ wd = make_wd(tmp_path, sess, alerts=alerts, clock=clock)+ # Tick every 30 min for 8 hours (16 ticks).+ for _ in range(16):+ wd.check_once()+ clock["now"] += 1800.0+ # Fixed-hourly would fire ~8 times; back-off fires at 0h,1h,3h,7h = 4.+ assert len(alerts) == 4++# ── disk + status ─────────────────────────────────────────────────────────