An always-on second brain you talk to. Voice notes in Telegram → typed, linked knowledge in your Obsidian vault. Runs 24/7 on the Claude subscription you already have.
Python ai-agentsai-automationclaudeclaude-codeebbinghausknowledge-graphllmmemory-systemobsidianpersonal-assistantpkmproductivitypythonsecond-braintelegram-botvoice-assistant
refactor(liveness)!: hung == stuck without visible work, not silence
smixsb4bd54388d51
diff --git a/implementation-notes.md b/implementation-notes.mdindex 8ff85ea..e9d64b4 100644--- a/implementation-notes.md+++ b/implementation-notes.md@@ -103,3 +103,15 @@ TODO: ответы >~экрана читать из pane.log (сейчас captu(runtime.py guards with mcp.exists()).- Gotcha: goal-aligner agent's sole purpose was todoist↔goals alignment → deleted,not trimmed.++## 2026-06-10 — v3.0 T8: liveness pair rewrite++- Decision: stall/hung redefined from "no pane.log bytes" to "no visible turn"+ (working spinner '(esc to interrupt)' absent). ask() and watchdog share the+ is_working() predicate; watchdog hung is persistence-based (_stuck_since),+ not instant — one observation never kills.+- VERIFY-LIVE: the '(esc to interrupt)' anchor must be confirmed against the+ live CLI version on ovh1 (like the pane-geometry notes). If the TUI renames+ the hint, is_working() goes blind — check during smoke.+- Gotcha: removed liveness_fn/_pane_log_mtime; pane.log stays (pipe-pane log)+ but is no longer a liveness signal.diff --git a/src/d_brain/services/claude_session.py b/src/d_brain/services/claude_session.pyindex 5efc343..0a74300 100644--- a/src/d_brain/services/claude_session.py+++ b/src/d_brain/services/claude_session.py@@ -9,8 +9,9 @@ All access to the pane goes through a single cross-process file lock(`pane.lock`) so the bot, the daily pipeline and the watchdog never talk tothe pane at once. The watchdog recovers a wedged session via``force_recover()`` (non-blocking lock), and ``ask()`` self-detects a stall-(no new bytes in pane.log) so it releases the lock quickly instead of-holding it for the full timeout.+(no visible turn — the working spinner gone without completion) so it+releases the lock quickly instead of holding it for the full timeout.+Silence is NOT a hang signal: a long quiet task still shows the spinner.Pure text parsing lives in tmux_parse; this module only orchestrates tmuxand timing, so it is tested with a fake runner + injected clock/sleep/rid.@@ -37,6 +38,7 @@ from d_brain.services.tmux_parse import (extract_reply,is_complete,is_idle,+ is_working,strip_chrome,)@@ -88,7 +90,6 @@ class ClaudeSession:sleep_fn: Callable[[float], None] = time.sleep,clock_fn: Callable[[], float] = time.monotonic,rid_factory: Callable[[], str] | None = None,- liveness_fn: Callable[[], float] | None = None,poll_interval: float = 1.0,paste_settle: float = 0.3,startup_timeout: float = 90.0,@@ -108,7 +109,6 @@ class ClaudeSession:self._sleep = sleep_fnself._clock = clock_fnself._rid_factory = rid_factory or (lambda: uuid.uuid4().hex[:8])- self._liveness_fn = liveness_fn or self._pane_log_mtimeself._poll_interval = poll_intervalself._paste_settle = paste_settleself._startup_timeout = startup_timeout@@ -157,12 +157,6 @@ class ClaudeSession:def _interrupt(self) -> None:self._tmux("send-keys", "-t", self._target, "C-c")- def _pane_log_mtime(self) -> float:- try:- return self._pane_log.stat().st_mtime- except OSError:- return 0.0-# ── file lock (single lock; see module docstring) ────────────────@contextmanager@@ -258,6 +252,10 @@ class ClaudeSession:"""Classify the live pane (for the watchdog / doctor)."""return classify_state(self._capture())+ def is_working(self) -> bool:+ """True iff the pane shows an active turn (for the watchdog)."""+ return is_working(self._capture())+def force_recover(self) -> bool:"""Watchdog entry point: take the lock non-blocking; if free, kill andrecreate. Returns False if a live ask() currently holds the lock."""@@ -365,8 +363,7 @@ class ClaudeSession:self._inflight.write_text(f"{log_id}\n{self._clock()}\n")self._send_prompt(prompt, rid, wrap=wrap)- last_live = self._liveness_fn()- last_change = self._clock()+ last_active = self._clock()deadline = self._clock() + timeoutidle_streak = 0while self._clock() < deadline:@@ -390,13 +387,15 @@ class ClaudeSession:else:idle_streak = 0- live = self._liveness_fn()- if live != last_live:- last_live, last_change = live, self._clock()- elif self._clock() - last_change > self._stall_timeout:+ # Stall model: silence is NOT a hang signal — a quiet task+ # still shows the working spinner. Stuck == no visible turn+ # (and no completion) for longer than stall_timeout.+ if is_working(cap):+ last_active = self._clock()+ elif self._clock() - last_active > self._stall_timeout:self._interrupt()# leave inflight as an orphan/stuck signal for the watchdog- return AskResult("error", detail="session stalled (no output)")+ return AskResult("error", detail="session stalled (no active turn)")self._sleep(self._poll_interval)# timed out: prompt is still physically in the pane → keep inflightdiff --git a/src/d_brain/services/tmux_parse.py b/src/d_brain/services/tmux_parse.pyindex c5829f5..80213c3 100644--- a/src/d_brain/services/tmux_parse.py+++ b/src/d_brain/services/tmux_parse.py@@ -150,6 +150,16 @@ def strip_chrome(text: str) -> str:return "\n".join(kept).strip()+def is_working(text: str) -> bool:+ """True iff the pane shows an ACTIVE turn (the working spinner).++ The shared liveness predicate for ask()'s stall detector and the+ watchdog: silence is not a hang signal — a long task that prints nothing+ still shows '(esc to interrupt)'. Hung == stuck WITHOUT this marker.+ """+ return bool(_WORKING_RE.search(_chrome(text)))++def is_idle(text: str) -> bool:"""True iff the session sits at an idle input prompt (no active turn).diff --git a/src/d_brain/services/watchdog.py b/src/d_brain/services/watchdog.pyindex 498ce86..8549fa5 100644--- a/src/d_brain/services/watchdog.py+++ b/src/d_brain/services/watchdog.py@@ -36,7 +36,7 @@ from d_brain.services.tmux_parse import PaneStatelogger = logging.getLogger(__name__)DEFAULT_TICK = 15.0-DEFAULT_STALL_THRESHOLD = 300.0 # 5 min of no pane output ⇒ wedged+DEFAULT_STALL_THRESHOLD = 300.0 # 5 min stuck without visible work ⇒ wedgedDEFAULT_MIN_DISK = 500_000_000 # 500 MBDEFAULT_ALERT_COOLDOWN = 3600.0 # re-alert a persistent fault at most hourly@@ -75,23 +75,23 @@ class Watchdog:self._min_disk = min_disk_bytesself._alert_cooldown = alert_cooldownself._inflight = self.runtime_dir / "inflight"- self._pane_log = self.runtime_dir / "pane.log"self._status = self.runtime_dir / "STATUS.md"self._last_alert_key: str | None = Noneself._last_alert_ts = 0.0-- def _mtime_age(self, path: Path) -> float:- try:- return max(0.0, self._clock() - path.stat().st_mtime)- except OSError:- return float("inf")+ self._stuck_since: float | None = Nonedef _is_hung(self, state: PaneState) -> bool:- # Serviceable states are never hung. Otherwise hung only if no bytes- # are flowing (a long-but-live task keeps pane.log growing).- if state in _SERVICEABLE:+ # Hang model (paired with ask()'s stall detector): silence is NOT a+ # signal — a long quiet task still shows the working spinner. Hung ==+ # non-serviceable AND no visible work, PERSISTING past the threshold.+ if state in _SERVICEABLE or self.session.is_working():+ self._stuck_since = None+ return False+ now = self._clock()+ if self._stuck_since is None:+ self._stuck_since = nowreturn False- return self._mtime_age(self._pane_log) >= self._stall_threshold+ return now - self._stuck_since >= self._stall_thresholddef _maybe_alert(self, key: str, msg: str) -> None:now = self._clock()diff --git a/tests/test_claude_session.py b/tests/test_claude_session.pyindex 1c1196f..a2f2648 100644--- a/tests/test_claude_session.py+++ b/tests/test_claude_session.py@@ -2,7 +2,7 @@The real tmux/subprocess layer is replaced by FakeTmux, which models thepane as a small state machine and returns scripted capture-pane output.-Clock, sleep, rid generation and liveness are injected so polling, timeout+Clock, sleep and rid generation are injected so polling, timeoutand stall detection are deterministic and fast."""@@ -81,7 +81,6 @@ def make_session(clock: dict,*,rid: str = "rid00001",- liveness=None,) -> ClaudeSession:def sleep_fn(seconds: float) -> None:clock["now"] += seconds@@ -94,7 +93,6 @@ def make_session(sleep_fn=sleep_fn,clock_fn=lambda: clock["now"],rid_factory=lambda: rid,- liveness_fn=liveness or (lambda: clock["now"]), # default: always "fresh"poll_interval=1.0,startup_timeout=30.0,stall_timeout=10.0,@@ -227,23 +225,34 @@ def test_ask_detects_logged_out(tmp_path, clock):def test_ask_times_out_when_no_reply(tmp_path, clock):fake = FakeTmux([READY, THINKING], exists=True)- # liveness keeps advancing (output flowing) so it's a timeout, not a stall- s = make_session(tmp_path, fake, clock, liveness=lambda: clock["now"])+ # the working spinner is visible, so it's a timeout, not a stall+ s = make_session(tmp_path, fake, clock)res = s.ask("ping", timeout=3)assert res.status == "timeout"def test_ask_detects_stall_and_interrupts(tmp_path, clock):- """No new bytes (frozen liveness) longer than stall_timeout → stall, not- a 20-min hang. Must interrupt and return promptly (C1)."""- fake = FakeTmux([READY, THINKING], exists=True)- s = make_session(tmp_path, fake, clock, liveness=lambda: 42.0) # frozen+ """v3.0 stall model: a turn with NO visible work (no 'esc to interrupt'+ spinner, no markers) longer than stall_timeout is stuck — the prompt got+ eaten or the turn is waiting on something undrivable. Interrupt promptly."""+ fake = FakeTmux([READY, READY], exists=True) # never shows work+ s = make_session(tmp_path, fake, clock)res = s.ask("ping", timeout=600)assert res.status == "error"assert "stall" in (res.detail or "").lower()assert any(c[-1] == "C-c" for c in fake.sent_keys())+def test_ask_long_silent_work_not_interrupted(tmp_path, clock):+ """The working spinner is visible → the turn is ALIVE however quiet it+ is. No C-c; the hard timeout (not a stall) ends the wait."""+ fake = FakeTmux([READY, THINKING], exists=True)+ s = make_session(tmp_path, fake, clock)+ res = s.ask("ping", timeout=30)+ assert res.status == "timeout"+ assert not any(c[-1] == "C-c" for c in fake.sent_keys())++def test_ask_returns_error_when_ensure_fails(tmp_path, clock):"""ensure_session failing must surface as AskResult('error'), never anexception out of ask() (C2)."""@@ -257,7 +266,7 @@ def test_ask_leaves_inflight_orphan_on_timeout(tmp_path, clock):"""A timed-out prompt is still physically in the pane; inflight markermust persist as a stuck-signal (M6)."""fake = FakeTmux([READY, THINKING], exists=True)- s = make_session(tmp_path, fake, clock, liveness=lambda: clock["now"])+ s = make_session(tmp_path, fake, clock)s.ask("ping", timeout=3)assert (tmp_path / ".dbrain" / "inflight").exists()diff --git a/tests/test_watchdog.py b/tests/test_watchdog.pyindex 73ea6bf..9f3cb91 100644--- a/tests/test_watchdog.py+++ b/tests/test_watchdog.py@@ -1,23 +1,24 @@"""Tests for the liveness watchdog.-Hang model (after review): a session is hung when its pane state is NOT-serviceable (not READY/RATE_LIMITED/LOGGED_OUT) AND no new bytes have flowed-to pane.log for stall_threshold. This catches a wedged request AND a stuck-startup, never kills a long-but-live task (pane.log keeps growing), and never-kills a healthy idle READY session that merely has an orphan inflight marker.+Hang model (v3.0): a session is hung when its pane state is NOT serviceable+(not READY/RATE_LIMITED/LOGGED_OUT) AND the pane shows no active work (no+'esc to interrupt' spinner) PERSISTENTLY for stall_threshold. Silence is not+a signal: a long task that prints nothing still shows the working spinner and+must never be killed."""-import os-from d_brain.services.tmux_parse import PaneStatefrom d_brain.services.watchdog import Watchdogclass FakeSession:- def __init__(self, *, healthy=True, state=PaneState.READY, recover_ok=True):+ def __init__(+ self, *, healthy=True, state=PaneState.READY, recover_ok=True, working=False+ ):self._healthy = healthyself.state = stateself._recover_ok = recover_ok+ self.working = workingself.recovered = 0def is_healthy(self) -> bool:@@ -26,6 +27,9 @@ class FakeSession:def current_state(self) -> PaneState:return self.state+ def is_working(self) -> bool:+ return self.working+def force_recover(self) -> bool:self.recovered += 1if self._recover_ok:@@ -33,12 +37,13 @@ class FakeSession:return self._recover_ok-def make_wd(tmp_path, session, *, disk_free=10_000_000_000, now=1000.0, alerts=None):+def make_wd(tmp_path, session, *, disk_free=10_000_000_000, clock=None, alerts=None):+ clock = clock if clock is not None else {"now": 1000.0}return Watchdog(session,runtime_dir=tmp_path,disk_free_fn=lambda: disk_free,- clock_fn=lambda: now,+ clock_fn=lambda: clock["now"],alert_fn=(alerts.append if alerts is not None else (lambda m: None)),min_disk_bytes=500_000_000,stall_threshold=300.0,@@ -46,18 +51,6 @@ def make_wd(tmp_path, session, *, disk_free=10_000_000_000, now=1000.0, alerts=N)-def _stale_log(tmp_path, age_from=1000.0, age=400.0):- p = tmp_path / "pane.log"- p.write_text("...")- os.utime(p, (age_from - age, age_from - age))---def _fresh_log(tmp_path, now=1000.0):- p = tmp_path / "pane.log"- p.write_text("...")- os.utime(p, (now - 1, now - 1))--# ── basic states ─────────────────────────────────────────────────────────@@ -93,39 +86,62 @@ def test_logged_out_alerts_without_restart(tmp_path):assert alerts-# ── hang detection (state-based) ──────────────────────────────────────────+# ── hang detection (stuck-without-work, persistence-based) ────────────────-def test_hung_when_nonserviceable_and_log_stale(tmp_path):- sess = FakeSession(state=PaneState.UNKNOWN)- _stale_log(tmp_path)- wd = make_wd(tmp_path, sess)+def test_hung_when_stuck_without_work(tmp_path):+ """Non-serviceable + no working spinner, persisting past the threshold →+ recover. A single observation never kills."""+ clock = {"now": 1000.0}+ sess = FakeSession(state=PaneState.UNKNOWN, working=False)+ wd = make_wd(tmp_path, sess, clock=clock)+ assert wd.check_once() == "healthy" # arms the stuck timer only+ assert sess.recovered == 0+ clock["now"] += 301.0assert wd.check_once() == "recovered_hung"assert sess.recovered == 1def test_stuck_startup_is_recovered(tmp_path):- sess = FakeSession(state=PaneState.STARTING)- _stale_log(tmp_path)- wd = make_wd(tmp_path, sess)+ clock = {"now": 1000.0}+ sess = FakeSession(state=PaneState.STARTING, working=False)+ wd = make_wd(tmp_path, sess, clock=clock)+ wd.check_once()+ clock["now"] += 301.0assert wd.check_once() == "recovered_hung"-def test_long_live_task_not_killed(tmp_path):- """Non-serviceable (working) but bytes still flowing → alive, not hung."""- sess = FakeSession(state=PaneState.UNKNOWN)- _fresh_log(tmp_path)- wd = make_wd(tmp_path, sess)+def test_long_silent_task_not_killed(tmp_path):+ """The working spinner is visible → alive, however long and however quiet+ pane.log is. Silence is NOT a hang signal."""+ clock = {"now": 1000.0}+ sess = FakeSession(state=PaneState.UNKNOWN, working=True)+ wd = make_wd(tmp_path, sess, clock=clock)+ wd.check_once()+ clock["now"] += 10_000.0assert wd.check_once() == "healthy"assert sess.recovered == 0+def test_work_resumes_resets_stuck_timer(tmp_path):+ clock = {"now": 1000.0}+ sess = FakeSession(state=PaneState.UNKNOWN, working=False)+ wd = make_wd(tmp_path, sess, clock=clock)+ wd.check_once() # arms timer+ clock["now"] += 200.0+ sess.working = True+ wd.check_once() # work visible → timer reset+ sess.working = False+ clock["now"] += 200.0+ assert wd.check_once() == "healthy" # only 200s into a NEW stuck window+ assert sess.recovered == 0++def test_idle_ready_with_orphan_inflight_not_killed(tmp_path):"""CRITICAL regression: a healthy idle READY session with a leftover- orphan inflight + frozen pane.log must NOT be recovered."""+ orphan inflight marker must NOT be recovered."""sess = FakeSession(state=PaneState.READY)(tmp_path / "inflight").write_text("orphan\n")- _stale_log(tmp_path)wd = make_wd(tmp_path, sess)assert wd.check_once() == "healthy"assert sess.recovered == 0@@ -135,9 +151,11 @@ def test_idle_ready_with_orphan_inflight_not_killed(tmp_path):def test_recover_deferred_when_lock_busy(tmp_path):"""force_recover returns False (a live request holds the lock) → reportdeferred, do not claim a restart happened."""- sess = FakeSession(state=PaneState.UNKNOWN, recover_ok=False)- _stale_log(tmp_path)- wd = make_wd(tmp_path, sess)+ clock = {"now": 1000.0}+ sess = FakeSession(state=PaneState.UNKNOWN, working=False, recover_ok=False)+ wd = make_wd(tmp_path, sess, clock=clock)+ wd.check_once()+ clock["now"] += 301.0assert wd.check_once() == "recover_deferred"