Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

fix: recover agent completion when filesystem watching fails

Adam Eivy31f78b9c82ab
diff --git a/server/lib/fileCore.js b/server/lib/fileCore.js
index 379b54b9f1..3d7a9e9430 100644
--- a/server/lib/fileCore.js
+++ b/server/lib/fileCore.js
@@ -85,19 +85,28 @@ export function watchForFile(
let detected = false;
let settleTimer = null;
let pollTimer = null;
+ let watcher = null;
+ let onWatcherClosed = null;
const notify = () => {
- Promise.resolve(onDetected()).catch((err) => {
+ Promise.resolve().then(onDetected).catch((err) => {
console.error(`❌ File watcher callback failed: ${err.message}`);
});
};
+ const watcherClosed = () => {
+ watcher = null;
+ const callback = onWatcherClosed;
+ onWatcherClosed = null;
+ callback?.();
+ };
const close = (onClosed) => {
if (closed) return;
closed = true;
if (settleTimer) clearTimeout(settleTimer);
if (pollTimer) clearInterval(pollTimer);
- if (onClosed) watcher.once('close', onClosed);
- watcher.close();
+ onWatcherClosed = onClosed;
+ if (watcher) watcher.close();
+ else watcherClosed();
};
const detect = (settle = true) => {
if (closed || detected || !existsSync(filePath) || !shouldDetect(filePath)) return;
@@ -112,9 +121,24 @@ export function watchForFile(
close(notify);
}, settleMs);
};
- const watcher = watchFileSystem(targetDir, (_eventType, changedName) => {
- if (changedName == null || changedName.toString() === targetName) detect();
- });
+ const watchFailed = (err) => {
+ // Native errors can close the handle without emitting `close`. Drop it so
+ // fallback detection never waits for an event that can no longer arrive.
+ const failedWatcher = watcher;
+ watcher = null;
+ failedWatcher?.close();
+ watcherClosed();
+ console.warn(`⚠️ File watcher unavailable (${err.code || 'UNKNOWN'}); using fallback polling`);
+ };
+ try {
+ watcher = watchFileSystem(targetDir, (_eventType, changedName) => {
+ if (changedName == null || changedName.toString() === targetName) detect();
+ });
+ watcher.once('close', watcherClosed);
+ watcher.on('error', watchFailed);
+ } catch (err) {
+ watchFailed(err);
+ }
// fs.watch is low-latency but best-effort: native backends can miss a file
// creation or atomic rename. Keep a deliberately relaxed fallback so a
// missed event cannot strand an agent indefinitely.
diff --git a/server/lib/fileCore.watch.test.js b/server/lib/fileCore.watch.test.js
new file mode 100644
index 0000000000..883676fb91
--- /dev/null
+++ b/server/lib/fileCore.watch.test.js
@@ -0,0 +1,104 @@
+import { EventEmitter } from 'node:events';
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
+import { existsSync, watch } from 'fs';
+import { watchForFile } from './fileCore.js';
+
+vi.mock('fs', async (importOriginal) => ({
+ ...await importOriginal(),
+ existsSync: vi.fn(),
+ watch: vi.fn(),
+}));
+
+describe('completion watcher failure recovery', () => {
+ let watcher;
+ let nativeClosed;
+ let stop;
+
+ beforeEach(() => {
+ vi.useFakeTimers();
+ vi.spyOn(console, 'warn').mockImplementation(() => {});
+ existsSync.mockReturnValue(false);
+ nativeClosed = false;
+ watcher = new EventEmitter();
+ watcher.close = vi.fn(() => {
+ if (nativeClosed) return;
+ nativeClosed = true;
+ queueMicrotask(() => watcher.emit('close'));
+ });
+ watch.mockReturnValue(watcher);
+ });
+
+ afterEach(() => {
+ stop?.();
+ stop = null;
+ vi.useRealTimers();
+ vi.restoreAllMocks();
+ });
+
+ const failNativeWatcher = () => {
+ // Node closes the native handle before emitting an error. A later close()
+ // is a no-op and does NOT emit the close event completion used to await.
+ nativeClosed = true;
+ watcher.emit('error', Object.assign(new Error('synthetic watcher failure'), { code: 'EIO' }));
+ };
+
+ it.each(['startup', 'asynchronous'])('detects completion once after %s watcher failure', async (failure) => {
+ if (failure === 'startup') watch.mockImplementationOnce(() => {
+ throw Object.assign(new Error('synthetic watch limit'), { code: 'ENOSPC' });
+ });
+ const onDetected = vi.fn();
+ const shouldDetect = vi.fn(() => false);
+ stop = watchForFile('done.sentinel', onDetected, { shouldDetect });
+ if (failure === 'asynchronous') failNativeWatcher();
+
+ existsSync.mockReturnValue(true);
+ await vi.advanceTimersByTimeAsync(5050);
+ expect(onDetected).not.toHaveBeenCalled();
+ shouldDetect.mockReturnValue(true);
+ await vi.advanceTimersByTimeAsync(5000);
+ expect(onDetected).toHaveBeenCalledTimes(1);
+ expect(vi.getTimerCount()).toBe(0);
+ expect(console.warn).toHaveBeenCalledWith(expect.stringContaining('using fallback polling'));
+ await vi.advanceTimersByTimeAsync(10000);
+ expect(onDetected).toHaveBeenCalledTimes(1);
+ });
+
+ it('finishes a settling completion after the native watcher fails', async () => {
+ const onDetected = vi.fn();
+ stop = watchForFile('done.sentinel', onDetected);
+ existsSync.mockReturnValue(true);
+ watch.mock.calls[0][1]('rename', 'done.sentinel');
+ failNativeWatcher();
+
+ await vi.advanceTimersByTimeAsync(50);
+ expect(onDetected).toHaveBeenCalledTimes(1);
+ expect(vi.getTimerCount()).toBe(0);
+ });
+
+ it('delivers an existing completion asynchronously when watcher startup fails', async () => {
+ watch.mockImplementationOnce(() => {
+ throw Object.assign(new Error('synthetic watch limit'), { code: 'ENOSPC' });
+ });
+ existsSync.mockReturnValue(true);
+ const onDetected = vi.fn();
+
+ stop = watchForFile('done.sentinel', onDetected);
+ expect(onDetected).not.toHaveBeenCalled();
+ await vi.advanceTimersByTimeAsync(0);
+ expect(onDetected).toHaveBeenCalledTimes(1);
+ expect(vi.getTimerCount()).toBe(0);
+ });
+
+ it('cancels fallback detection and releases timers after a watcher failure', async () => {
+ const onDetected = vi.fn();
+ stop = watchForFile('done.sentinel', onDetected);
+ failNativeWatcher();
+ stop();
+ stop();
+ existsSync.mockReturnValue(true);
+
+ await vi.advanceTimersByTimeAsync(10000);
+ expect(onDetected).not.toHaveBeenCalled();
+ expect(vi.getTimerCount()).toBe(0);
+ });
+});