Skip to content

An always-on second brain you talk to. Voice notes in Telegram → typed, linked knowledge in your Obsidian vault. Runs 24/7 on the Claude subscription you already have.

refactor(liveness)!: hung == stuck without visible work, not silence

smixsb4bd54388d51
diff --git a/implementation-notes.md b/implementation-notes.md
index 8ff85ea..e9d64b4 100644
--- a/implementation-notes.md
+++ b/implementation-notes.md
@@ -103,3 +103,15 @@ TODO: ответы >~экрана читать из pane.log (сейчас captu
(runtime.py guards with mcp.exists()).
- Gotcha: goal-aligner agent's sole purpose was todoist↔goals alignment → deleted,
not trimmed.
+
+## 2026-06-10 — v3.0 T8: liveness pair rewrite
+
+- Decision: stall/hung redefined from "no pane.log bytes" to "no visible turn"
+ (working spinner '(esc to interrupt)' absent). ask() and watchdog share the
+ is_working() predicate; watchdog hung is persistence-based (_stuck_since),
+ not instant — one observation never kills.
+- VERIFY-LIVE: the '(esc to interrupt)' anchor must be confirmed against the
+ live CLI version on ovh1 (like the pane-geometry notes). If the TUI renames
+ the hint, is_working() goes blind — check during smoke.
+- Gotcha: removed liveness_fn/_pane_log_mtime; pane.log stays (pipe-pane log)
+ but is no longer a liveness signal.
diff --git a/src/d_brain/services/claude_session.py b/src/d_brain/services/claude_session.py
index 5efc343..0a74300 100644
--- a/src/d_brain/services/claude_session.py
+++ b/src/d_brain/services/claude_session.py
@@ -9,8 +9,9 @@ All access to the pane goes through a single cross-process file lock
(`pane.lock`) so the bot, the daily pipeline and the watchdog never talk to
the pane at once. The watchdog recovers a wedged session via
``force_recover()`` (non-blocking lock), and ``ask()`` self-detects a stall
-(no new bytes in pane.log) so it releases the lock quickly instead of
-holding it for the full timeout.
+(no visible turn — the working spinner gone without completion) so it
+releases the lock quickly instead of holding it for the full timeout.
+Silence is NOT a hang signal: a long quiet task still shows the spinner.
Pure text parsing lives in tmux_parse; this module only orchestrates tmux
and timing, so it is tested with a fake runner + injected clock/sleep/rid.
@@ -37,6 +38,7 @@ from d_brain.services.tmux_parse import (
extract_reply,
is_complete,
is_idle,
+ is_working,
strip_chrome,
)
@@ -88,7 +90,6 @@ class ClaudeSession:
sleep_fn: Callable[[float], None] = time.sleep,
clock_fn: Callable[[], float] = time.monotonic,
rid_factory: Callable[[], str] | None = None,
- liveness_fn: Callable[[], float] | None = None,
poll_interval: float = 1.0,
paste_settle: float = 0.3,
startup_timeout: float = 90.0,
@@ -108,7 +109,6 @@ class ClaudeSession:
self._sleep = sleep_fn
self._clock = clock_fn
self._rid_factory = rid_factory or (lambda: uuid.uuid4().hex[:8])
- self._liveness_fn = liveness_fn or self._pane_log_mtime
self._poll_interval = poll_interval
self._paste_settle = paste_settle
self._startup_timeout = startup_timeout
@@ -157,12 +157,6 @@ class ClaudeSession:
def _interrupt(self) -> None:
self._tmux("send-keys", "-t", self._target, "C-c")
- def _pane_log_mtime(self) -> float:
- try:
- return self._pane_log.stat().st_mtime
- except OSError:
- return 0.0
-
# ── file lock (single lock; see module docstring) ────────────────
@contextmanager
@@ -258,6 +252,10 @@ class ClaudeSession:
"""Classify the live pane (for the watchdog / doctor)."""
return classify_state(self._capture())
+ def is_working(self) -> bool:
+ """True iff the pane shows an active turn (for the watchdog)."""
+ return is_working(self._capture())
+
def force_recover(self) -> bool:
"""Watchdog entry point: take the lock non-blocking; if free, kill and
recreate. Returns False if a live ask() currently holds the lock."""
@@ -365,8 +363,7 @@ class ClaudeSession:
self._inflight.write_text(f"{log_id}\n{self._clock()}\n")
self._send_prompt(prompt, rid, wrap=wrap)
- last_live = self._liveness_fn()
- last_change = self._clock()
+ last_active = self._clock()
deadline = self._clock() + timeout
idle_streak = 0
while self._clock() < deadline:
@@ -390,13 +387,15 @@ class ClaudeSession:
else:
idle_streak = 0
- live = self._liveness_fn()
- if live != last_live:
- last_live, last_change = live, self._clock()
- elif self._clock() - last_change > self._stall_timeout:
+ # Stall model: silence is NOT a hang signal — a quiet task
+ # still shows the working spinner. Stuck == no visible turn
+ # (and no completion) for longer than stall_timeout.
+ if is_working(cap):
+ last_active = self._clock()
+ elif self._clock() - last_active > self._stall_timeout:
self._interrupt()
# leave inflight as an orphan/stuck signal for the watchdog
- return AskResult("error", detail="session stalled (no output)")
+ return AskResult("error", detail="session stalled (no active turn)")
self._sleep(self._poll_interval)
# timed out: prompt is still physically in the pane → keep inflight
diff --git a/src/d_brain/services/tmux_parse.py b/src/d_brain/services/tmux_parse.py
index c5829f5..80213c3 100644
--- a/src/d_brain/services/tmux_parse.py
+++ b/src/d_brain/services/tmux_parse.py
@@ -150,6 +150,16 @@ def strip_chrome(text: str) -> str:
return "\n".join(kept).strip()
+def is_working(text: str) -> bool:
+ """True iff the pane shows an ACTIVE turn (the working spinner).
+
+ The shared liveness predicate for ask()'s stall detector and the
+ watchdog: silence is not a hang signal — a long task that prints nothing
+ still shows '(esc to interrupt)'. Hung == stuck WITHOUT this marker.
+ """
+ return bool(_WORKING_RE.search(_chrome(text)))
+
+
def is_idle(text: str) -> bool:
"""True iff the session sits at an idle input prompt (no active turn).
diff --git a/src/d_brain/services/watchdog.py b/src/d_brain/services/watchdog.py
index 498ce86..8549fa5 100644
--- a/src/d_brain/services/watchdog.py
+++ b/src/d_brain/services/watchdog.py
@@ -36,7 +36,7 @@ from d_brain.services.tmux_parse import PaneState
logger = logging.getLogger(__name__)
DEFAULT_TICK = 15.0
-DEFAULT_STALL_THRESHOLD = 300.0 # 5 min of no pane output ⇒ wedged
+DEFAULT_STALL_THRESHOLD = 300.0 # 5 min stuck without visible work ⇒ wedged
DEFAULT_MIN_DISK = 500_000_000 # 500 MB
DEFAULT_ALERT_COOLDOWN = 3600.0 # re-alert a persistent fault at most hourly
@@ -75,23 +75,23 @@ class Watchdog:
self._min_disk = min_disk_bytes
self._alert_cooldown = alert_cooldown
self._inflight = self.runtime_dir / "inflight"
- self._pane_log = self.runtime_dir / "pane.log"
self._status = self.runtime_dir / "STATUS.md"
self._last_alert_key: str | None = None
self._last_alert_ts = 0.0
-
- def _mtime_age(self, path: Path) -> float:
- try:
- return max(0.0, self._clock() - path.stat().st_mtime)
- except OSError:
- return float("inf")
+ self._stuck_since: float | None = None
def _is_hung(self, state: PaneState) -> bool:
- # Serviceable states are never hung. Otherwise hung only if no bytes
- # are flowing (a long-but-live task keeps pane.log growing).
- if state in _SERVICEABLE:
+ # Hang model (paired with ask()'s stall detector): silence is NOT a
+ # signal — a long quiet task still shows the working spinner. Hung ==
+ # non-serviceable AND no visible work, PERSISTING past the threshold.
+ if state in _SERVICEABLE or self.session.is_working():
+ self._stuck_since = None
+ return False
+ now = self._clock()
+ if self._stuck_since is None:
+ self._stuck_since = now
return False
- return self._mtime_age(self._pane_log) >= self._stall_threshold
+ return now - self._stuck_since >= self._stall_threshold
def _maybe_alert(self, key: str, msg: str) -> None:
now = self._clock()
diff --git a/tests/test_claude_session.py b/tests/test_claude_session.py
index 1c1196f..a2f2648 100644
--- a/tests/test_claude_session.py
+++ b/tests/test_claude_session.py
@@ -2,7 +2,7 @@
The real tmux/subprocess layer is replaced by FakeTmux, which models the
pane as a small state machine and returns scripted capture-pane output.
-Clock, sleep, rid generation and liveness are injected so polling, timeout
+Clock, sleep and rid generation are injected so polling, timeout
and stall detection are deterministic and fast.
"""
@@ -81,7 +81,6 @@ def make_session(
clock: dict,
*,
rid: str = "rid00001",
- liveness=None,
) -> ClaudeSession:
def sleep_fn(seconds: float) -> None:
clock["now"] += seconds
@@ -94,7 +93,6 @@ def make_session(
sleep_fn=sleep_fn,
clock_fn=lambda: clock["now"],
rid_factory=lambda: rid,
- liveness_fn=liveness or (lambda: clock["now"]), # default: always "fresh"
poll_interval=1.0,
startup_timeout=30.0,
stall_timeout=10.0,
@@ -227,23 +225,34 @@ def test_ask_detects_logged_out(tmp_path, clock):
def test_ask_times_out_when_no_reply(tmp_path, clock):
fake = FakeTmux([READY, THINKING], exists=True)
- # liveness keeps advancing (output flowing) so it's a timeout, not a stall
- s = make_session(tmp_path, fake, clock, liveness=lambda: clock["now"])
+ # the working spinner is visible, so it's a timeout, not a stall
+ s = make_session(tmp_path, fake, clock)
res = s.ask("ping", timeout=3)
assert res.status == "timeout"
def test_ask_detects_stall_and_interrupts(tmp_path, clock):
- """No new bytes (frozen liveness) longer than stall_timeout → stall, not
- a 20-min hang. Must interrupt and return promptly (C1)."""
- fake = FakeTmux([READY, THINKING], exists=True)
- s = make_session(tmp_path, fake, clock, liveness=lambda: 42.0) # frozen
+ """v3.0 stall model: a turn with NO visible work (no 'esc to interrupt'
+ spinner, no markers) longer than stall_timeout is stuck — the prompt got
+ eaten or the turn is waiting on something undrivable. Interrupt promptly."""
+ fake = FakeTmux([READY, READY], exists=True) # never shows work
+ s = make_session(tmp_path, fake, clock)
res = s.ask("ping", timeout=600)
assert res.status == "error"
assert "stall" in (res.detail or "").lower()
assert any(c[-1] == "C-c" for c in fake.sent_keys())
+def test_ask_long_silent_work_not_interrupted(tmp_path, clock):
+ """The working spinner is visible → the turn is ALIVE however quiet it
+ is. No C-c; the hard timeout (not a stall) ends the wait."""
+ fake = FakeTmux([READY, THINKING], exists=True)
+ s = make_session(tmp_path, fake, clock)
+ res = s.ask("ping", timeout=30)
+ assert res.status == "timeout"
+ assert not any(c[-1] == "C-c" for c in fake.sent_keys())
+
+
def test_ask_returns_error_when_ensure_fails(tmp_path, clock):
"""ensure_session failing must surface as AskResult('error'), never an
exception out of ask() (C2)."""
@@ -257,7 +266,7 @@ def test_ask_leaves_inflight_orphan_on_timeout(tmp_path, clock):
"""A timed-out prompt is still physically in the pane; inflight marker
must persist as a stuck-signal (M6)."""
fake = FakeTmux([READY, THINKING], exists=True)
- s = make_session(tmp_path, fake, clock, liveness=lambda: clock["now"])
+ s = make_session(tmp_path, fake, clock)
s.ask("ping", timeout=3)
assert (tmp_path / ".dbrain" / "inflight").exists()
diff --git a/tests/test_watchdog.py b/tests/test_watchdog.py
index 73ea6bf..9f3cb91 100644
--- a/tests/test_watchdog.py
+++ b/tests/test_watchdog.py
@@ -1,23 +1,24 @@
"""Tests for the liveness watchdog.
-Hang model (after review): a session is hung when its pane state is NOT
-serviceable (not READY/RATE_LIMITED/LOGGED_OUT) AND no new bytes have flowed
-to pane.log for stall_threshold. This catches a wedged request AND a stuck
-startup, never kills a long-but-live task (pane.log keeps growing), and never
-kills a healthy idle READY session that merely has an orphan inflight marker.
+Hang model (v3.0): a session is hung when its pane state is NOT serviceable
+(not READY/RATE_LIMITED/LOGGED_OUT) AND the pane shows no active work (no
+'esc to interrupt' spinner) PERSISTENTLY for stall_threshold. Silence is not
+a signal: a long task that prints nothing still shows the working spinner and
+must never be killed.
"""
-import os
-
from d_brain.services.tmux_parse import PaneState
from d_brain.services.watchdog import Watchdog
class FakeSession:
- def __init__(self, *, healthy=True, state=PaneState.READY, recover_ok=True):
+ def __init__(
+ self, *, healthy=True, state=PaneState.READY, recover_ok=True, working=False
+ ):
self._healthy = healthy
self.state = state
self._recover_ok = recover_ok
+ self.working = working
self.recovered = 0
def is_healthy(self) -> bool:
@@ -26,6 +27,9 @@ class FakeSession:
def current_state(self) -> PaneState:
return self.state
+ def is_working(self) -> bool:
+ return self.working
+
def force_recover(self) -> bool:
self.recovered += 1
if self._recover_ok:
@@ -33,12 +37,13 @@ class FakeSession:
return self._recover_ok
-def make_wd(tmp_path, session, *, disk_free=10_000_000_000, now=1000.0, alerts=None):
+def make_wd(tmp_path, session, *, disk_free=10_000_000_000, clock=None, alerts=None):
+ clock = clock if clock is not None else {"now": 1000.0}
return Watchdog(
session,
runtime_dir=tmp_path,
disk_free_fn=lambda: disk_free,
- clock_fn=lambda: now,
+ clock_fn=lambda: clock["now"],
alert_fn=(alerts.append if alerts is not None else (lambda m: None)),
min_disk_bytes=500_000_000,
stall_threshold=300.0,
@@ -46,18 +51,6 @@ def make_wd(tmp_path, session, *, disk_free=10_000_000_000, now=1000.0, alerts=N
)
-def _stale_log(tmp_path, age_from=1000.0, age=400.0):
- p = tmp_path / "pane.log"
- p.write_text("...")
- os.utime(p, (age_from - age, age_from - age))
-
-
-def _fresh_log(tmp_path, now=1000.0):
- p = tmp_path / "pane.log"
- p.write_text("...")
- os.utime(p, (now - 1, now - 1))
-
-
# ── basic states ─────────────────────────────────────────────────────────
@@ -93,39 +86,62 @@ def test_logged_out_alerts_without_restart(tmp_path):
assert alerts
-# ── hang detection (state-based) ──────────────────────────────────────────
+# ── hang detection (stuck-without-work, persistence-based) ────────────────
-def test_hung_when_nonserviceable_and_log_stale(tmp_path):
- sess = FakeSession(state=PaneState.UNKNOWN)
- _stale_log(tmp_path)
- wd = make_wd(tmp_path, sess)
+def test_hung_when_stuck_without_work(tmp_path):
+ """Non-serviceable + no working spinner, persisting past the threshold →
+ recover. A single observation never kills."""
+ clock = {"now": 1000.0}
+ sess = FakeSession(state=PaneState.UNKNOWN, working=False)
+ wd = make_wd(tmp_path, sess, clock=clock)
+ assert wd.check_once() == "healthy" # arms the stuck timer only
+ assert sess.recovered == 0
+ clock["now"] += 301.0
assert wd.check_once() == "recovered_hung"
assert sess.recovered == 1
def test_stuck_startup_is_recovered(tmp_path):
- sess = FakeSession(state=PaneState.STARTING)
- _stale_log(tmp_path)
- wd = make_wd(tmp_path, sess)
+ clock = {"now": 1000.0}
+ sess = FakeSession(state=PaneState.STARTING, working=False)
+ wd = make_wd(tmp_path, sess, clock=clock)
+ wd.check_once()
+ clock["now"] += 301.0
assert wd.check_once() == "recovered_hung"
-def test_long_live_task_not_killed(tmp_path):
- """Non-serviceable (working) but bytes still flowing → alive, not hung."""
- sess = FakeSession(state=PaneState.UNKNOWN)
- _fresh_log(tmp_path)
- wd = make_wd(tmp_path, sess)
+def test_long_silent_task_not_killed(tmp_path):
+ """The working spinner is visible → alive, however long and however quiet
+ pane.log is. Silence is NOT a hang signal."""
+ clock = {"now": 1000.0}
+ sess = FakeSession(state=PaneState.UNKNOWN, working=True)
+ wd = make_wd(tmp_path, sess, clock=clock)
+ wd.check_once()
+ clock["now"] += 10_000.0
assert wd.check_once() == "healthy"
assert sess.recovered == 0
+def test_work_resumes_resets_stuck_timer(tmp_path):
+ clock = {"now": 1000.0}
+ sess = FakeSession(state=PaneState.UNKNOWN, working=False)
+ wd = make_wd(tmp_path, sess, clock=clock)
+ wd.check_once() # arms timer
+ clock["now"] += 200.0
+ sess.working = True
+ wd.check_once() # work visible → timer reset
+ sess.working = False
+ clock["now"] += 200.0
+ assert wd.check_once() == "healthy" # only 200s into a NEW stuck window
+ assert sess.recovered == 0
+
+
def test_idle_ready_with_orphan_inflight_not_killed(tmp_path):
"""CRITICAL regression: a healthy idle READY session with a leftover
- orphan inflight + frozen pane.log must NOT be recovered."""
+ orphan inflight marker must NOT be recovered."""
sess = FakeSession(state=PaneState.READY)
(tmp_path / "inflight").write_text("orphan\n")
- _stale_log(tmp_path)
wd = make_wd(tmp_path, sess)
assert wd.check_once() == "healthy"
assert sess.recovered == 0
@@ -135,9 +151,11 @@ def test_idle_ready_with_orphan_inflight_not_killed(tmp_path):
def test_recover_deferred_when_lock_busy(tmp_path):
"""force_recover returns False (a live request holds the lock) → report
deferred, do not claim a restart happened."""
- sess = FakeSession(state=PaneState.UNKNOWN, recover_ok=False)
- _stale_log(tmp_path)
- wd = make_wd(tmp_path, sess)
+ clock = {"now": 1000.0}
+ sess = FakeSession(state=PaneState.UNKNOWN, working=False, recover_ok=False)
+ wd = make_wd(tmp_path, sess, clock=clock)
+ wd.check_once()
+ clock["now"] += 301.0
assert wd.check_once() == "recover_deferred"