Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

Don't let a host suspend kill an in-flight Persistent Mind turn

atomanticagent1cd164a13b30
diff --git a/server/services/persistentMindSupervisor.js b/server/services/persistentMindSupervisor.js
index 2903cea1fe..0bbd1e9090 100644
--- a/server/services/persistentMindSupervisor.js
+++ b/server/services/persistentMindSupervisor.js
@@ -88,6 +88,11 @@ export const PERSISTENT_MIND_WAKE_EVENT_ID = 'cos-persistent-mind-wake';
export const PERSISTENT_MIND_WATCHDOG_EVENT_ID = 'cos-persistent-mind-watchdog';
export const PERSISTENT_MIND_USAGE_LIMIT_PROBE_EVENT_ID = 'cos-persistent-mind-usage-limit-probe';
export const PERSISTENT_MIND_WATCHDOG_INTERVAL_MS = 30_000;
+// A watchdog tick that arrives this long after the previous one means this
+// process itself did not run in between (host sleep, VM suspend/snapshot, or a
+// frozen event loop). The turn's heartbeat timer was frozen for the same span,
+// so the heartbeat's age measures the pause rather than a dead turn.
+export const PERSISTENT_MIND_WATCHDOG_SUSPEND_GAP_MS = PERSISTENT_MIND_LIMITS.HEARTBEAT_STALE_WARNING_MS;
let turnAdapter = null;
let activeRun = null;
@@ -96,6 +101,10 @@ let runtimeGeneration = 0;
let supervisorStopping = false;
/** Zero-based attempt count for the in-flight usage-limit readiness probe. */
let usageLimitProbeAttempt = 0;
+/** Wall time of the previous watchdog tick, to detect host suspend gaps. */
+let lastWatchdogCheckAt = null;
+/** Wall time this process was last seen resuming from such a gap. */
+let watchdogResumedAt = null;
const nowIso = () => new Date().toISOString();
const errorMessage = (error) => String(error?.message || error || 'Persistent mind turn failed')
@@ -178,6 +187,8 @@ function emitMindStatus(state) {
function armWatchdog() {
if (supervisorStopping || !isDaemonRunning()) return;
+ // A fresh interval has no prior tick; a gap spent disarmed is not a suspend.
+ lastWatchdogCheckAt = null;
schedule({
id: PERSISTENT_MIND_WATCHDOG_EVENT_ID,
type: 'interval',
@@ -1594,9 +1605,29 @@ export async function drainPersistentMind({ rearm = true } = {}) {
return activeRun;
}
-export async function checkPersistentMindWatchdog() {
+/**
+ * Track the watchdog's own cadence. When a tick lands long after the previous
+ * one, the whole process (and with it the turn's heartbeat pulse and, on a
+ * suspended host, the local model server) was paused, so the stale window is
+ * restarted from the resume instead of killing an in-flight call that will
+ * continue as soon as the host runs again.
+ */
+function noteWatchdogTick(now) {
+ const previous = lastWatchdogCheckAt;
+ lastWatchdogCheckAt = now;
+ if (previous != null && now - previous >= PERSISTENT_MIND_WATCHDOG_SUSPEND_GAP_MS) {
+ watchdogResumedAt = now;
+ console.log(`⏸️ Persistent mind watchdog resumed after a ${Math.round((now - previous) / 1000)}s pause; restarting the heartbeat window`);
+ }
+}
+
+export async function checkPersistentMindWatchdog(now = Date.now()) {
+ noteWatchdogTick(now);
const state = await getPersistentMindState();
- if (!persistentMindTurnIsStale(state)) return { interrupted: false };
+ if (!persistentMindTurnIsStale(state, now)) return { interrupted: false };
+ if (watchdogResumedAt != null && now - watchdogResumedAt < PERSISTENT_MIND_LIMITS.WATCHDOG_STALE_MS) {
+ return { interrupted: false, resumeGrace: true };
+ }
const result = await interruptActiveTurn('Persistent mind turn heartbeat expired', 'interrupted', {
retry: true,
expectedTurnId: state.activeTurn.id,
@@ -1693,6 +1724,8 @@ export function __resetPersistentMindSupervisorForTests() {
runtimeGeneration = 0;
supervisorStopping = false;
usageLimitProbeAttempt = 0;
+ lastWatchdogCheckAt = null;
+ watchdogResumedAt = null;
cancel(PERSISTENT_MIND_WAKE_EVENT_ID);
cancel(PERSISTENT_MIND_WATCHDOG_EVENT_ID);
cancel(PERSISTENT_MIND_USAGE_LIMIT_PROBE_EVENT_ID);
diff --git a/server/services/persistentMindSupervisor.test.js b/server/services/persistentMindSupervisor.test.js
index 57f5bd6abb..18937dd99b 100644
--- a/server/services/persistentMindSupervisor.test.js
+++ b/server/services/persistentMindSupervisor.test.js
@@ -916,6 +916,46 @@ describe('persistent mind supervisor', () => {
expect(mock.root.persistentMind.status).toBe('interrupted');
});
+ it('watchdog restarts the stale window after a host suspend instead of killing the in-flight turn', async () => {
+ const run = vi.fn(({ signal }) => new Promise((resolve, reject) => {
+ signal.addEventListener('abort', () => reject(new Error('aborted')), { once: true });
+ }));
+ await supervisor.registerPersistentMindTurnAdapter({
+ prepare: vi.fn(async () => ({ ok: true, provider: { id: 'example-cloud' } })),
+ run,
+ });
+ await supervisor.setPersistentMindEnabled(true);
+ await supervisor.startPersistentMind();
+ await supervisor.enqueuePersistentMindMessage({ id: 'message-1', text: 'Survive a host suspend.' });
+ const drain = supervisor.drainPersistentMind();
+ await vi.waitFor(() => expect(run).toHaveBeenCalledTimes(1));
+
+ const beforeSuspend = Date.now();
+ await expect(supervisor.checkPersistentMindWatchdog(beforeSuspend)).resolves.toEqual({ interrupted: false });
+ // The host (and this process with it) froze for 20 minutes mid-call: the
+ // heartbeat is far past the stale window only because nothing ran.
+ const resumedAt = beforeSuspend + 20 * 60_000;
+ mock.root.persistentMind.activeTurn.heartbeatAt = new Date(beforeSuspend).toISOString();
+ await expect(supervisor.checkPersistentMindWatchdog(resumedAt)).resolves.toEqual({ interrupted: false, resumeGrace: true });
+ expect(mock.root.persistentMind.activeTurn).not.toBeNull();
+ // Still inside the grace window on the next regular tick.
+ await expect(supervisor.checkPersistentMindWatchdog(resumedAt + supervisor.PERSISTENT_MIND_WATCHDOG_INTERVAL_MS))
+ .resolves.toEqual({ interrupted: false, resumeGrace: true });
+
+ // A turn that truly stops beating after the resume is still interrupted
+ // once a full stale window of running time has passed.
+ let tick = resumedAt;
+ while (tick < resumedAt + PERSISTENT_MIND_LIMITS.WATCHDOG_STALE_MS - supervisor.PERSISTENT_MIND_WATCHDOG_INTERVAL_MS) {
+ tick += supervisor.PERSISTENT_MIND_WATCHDOG_INTERVAL_MS;
+ await supervisor.checkPersistentMindWatchdog(tick);
+ }
+ await expect(supervisor.checkPersistentMindWatchdog(resumedAt + PERSISTENT_MIND_LIMITS.WATCHDOG_STALE_MS))
+ .resolves.toEqual({ interrupted: true });
+ await drain;
+ expect(mock.root.persistentMind.activeTurn).toBeNull();
+ expect(mock.root.persistentMind.status).toBe('interrupted');
+ });
+
it('does not interrupt a replacement turn when a stale watchdog snapshot loses the race', async () => {
mock.root.persistentMind = {
...createDefaultPersistentMindState(),