Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
fix: stop TUI raw.txt spool reporting ENOENT after a run's final drain (#12876)
Adam Eivybb63de1384ea
diff --git a/server/services/agentTuiSpawning/outputSpooler.js b/server/services/agentTuiSpawning/outputSpooler.jsindex 79201f8d9a..8768616bcb 100644--- a/server/services/agentTuiSpawning/outputSpooler.js+++ b/server/services/agentTuiSpawning/outputSpooler.js@@ -32,9 +32,16 @@ const RAW_FLUSH_INTERVAL_MS = 1000;** @param {object} opts* @param {string} opts.agentId Agent id (used in logs and metadata updates).+ * @param {(fileLabel: string) => boolean} [opts.isDrained] True once the+ * final drain for that file's pipeline finished. An ENOENT after that point+ * is a straggler flush landing in a run directory the completion path has+ * already archived or removed — the transcript was fully written by the+ * drain, so it is neither flagged nor sent to stderr. ENOENT on a pipeline+ * that has not drained is still a fault: the directory vanished under a+ * live run.* @returns {object} Reporter with `report(fileLabel, err)` method.*/-export function createTranscriptWriteReporter({ agentId }) {+export function createTranscriptWriteReporter({ agentId, isDrained = () => false }) {const reportedFailures = new Set();return {@@ -46,6 +53,11 @@ export function createTranscriptWriteReporter({ agentId }) {}reportedFailures.add(failureKey);+ if (err.code === 'ENOENT' && isDrained(fileLabel)) {+ console.log(`⏭️ agent ${agentId} ${fileLabel} late write skipped after the final drain (ENOENT) — run directory already archived`);+ return;+ }+// Log the first failure with error code and messageconsole.error(`❌ agent ${agentId} ${fileLabel} write failed (${err.code || 'error'}): ${err.message}`);@@ -94,8 +106,20 @@ export function createOutputSpooler({ agentId, outputFile, rawFile }) {let rawFlushing = null;let rawBytesWritten = 0;let rawSpoolTruncationWarned = false;+ // A drain owns its pipeline until it returns: the debounce timer must not+ // launch a second concurrent flush, or the drain can see an empty queue and+ // resolve while that other flush's append is still in flight — the completion+ // path would then archive the run directory under the pending write.+ let linesDraining = false;+ let rawDraining = false;+ // Set once a drain has written everything queued; see the reporter's isDrained.+ let linesDrained = false;+ let rawDrained = false;- const reporter = createTranscriptWriteReporter({ agentId });+ const reporter = createTranscriptWriteReporter({+ agentId,+ isDrained: (fileLabel) => (fileLabel === 'raw.txt' ? rawDrained : linesDrained),+ });const flushPendingLines = () => withBackupAssetPublication(async () => {if (flushTimer) { clearTimeout(flushTimer); flushTimer = null; }@@ -109,7 +133,7 @@ export function createOutputSpooler({ agentId, outputFile, rawFile }) {});const scheduleFlush = () => {- if (flushTimer || flushing) return;+ if (flushTimer || flushing || linesDraining) return;flushTimer = setTimeout(() => {flushTimer = null;flushing = flushPendingLines().catch(err => reporter.report('state', err)).finally(() => {@@ -192,7 +216,7 @@ export function createOutputSpooler({ agentId, outputFile, rawFile }) {});const scheduleRawFlush = () => {- if (rawFlushTimer || rawFlushing) return;+ if (rawFlushTimer || rawFlushing || rawDraining) return;rawFlushTimer = setTimeout(() => {rawFlushTimer = null;rawFlushing = flushPendingRawChunks().catch(err => reporter.report('raw.txt', err)).finally(() => {@@ -233,18 +257,51 @@ export function createOutputSpooler({ agentId, outputFile, rawFile }) {scheduleRawFlush();};- // Drain the parsed-line pipeline: await any in-flight flush, then flush- // whatever is still pending. Called at finalize (and mirrored order matters —- // see drainRaw) so completion events don't beat the last batch to disk.+ // Drain a pipeline to empty: stop its timer, wait out the in-flight flush,+ // then flush until nothing is queued. The drain is the only writer while it+ // runs (scheduleFlush/scheduleRawFlush stand down on the draining flag), and+ // the flag drops in `finally` so a rejected drain (e.g. BACKUP_SNAPSHOT_BUSY)+ // does not wedge later flushes.+ const drainPipeline = async ({ setDraining, getTimer, clearTimer, getInFlight, flush, hasPending }) => {+ setDraining(true);+ try {+ for (;;) {+ if (getTimer()) clearTimer();+ const inFlight = getInFlight();+ if (!inFlight) break;+ await inFlight.catch(() => {});+ }+ do { await flush(); } while (hasPending());+ } finally {+ setDraining(false);+ }+ };++ // Drain the parsed-line pipeline. Called at finalize (order matters — see+ // drainRaw) so completion events don't beat the last batch to disk.const drainLines = async () => {- if (flushing) await flushing.catch(() => {});- await flushPendingLines();+ await drainPipeline({+ setDraining: (v) => { linesDraining = v; },+ getTimer: () => flushTimer,+ clearTimer: () => { clearTimeout(flushTimer); flushTimer = null; },+ getInFlight: () => flushing,+ flush: flushPendingLines,+ hasPending: () => pendingLines.length > 0,+ });+ linesDrained = true;};// Drain the raw-chunk pipeline. Same shape as drainLines.const drainRaw = async () => {- if (rawFlushing) await rawFlushing.catch(() => {});- await flushPendingRawChunks();+ await drainPipeline({+ setDraining: (v) => { rawDraining = v; },+ getTimer: () => rawFlushTimer,+ clearTimer: () => { clearTimeout(rawFlushTimer); rawFlushTimer = null; },+ getInFlight: () => rawFlushing,+ flush: flushPendingRawChunks,+ hasPending: () => pendingRawChunks.length > 0,+ });+ rawDrained = true;};return {diff --git a/server/services/agentTuiSpawning/outputSpooler.test.js b/server/services/agentTuiSpawning/outputSpooler.test.jsindex 661039022c..1a9718051f 100644--- a/server/services/agentTuiSpawning/outputSpooler.test.js+++ b/server/services/agentTuiSpawning/outputSpooler.test.js@@ -185,6 +185,73 @@ describe('createOutputSpooler', () => {});});+describe('createOutputSpooler — run directory removed under the spool', () => {+ let errorSpy;+ let logSpy;+ const enoent = () => Object.assign(new Error("ENOENT: no such file or directory, open '/tmp/agent/raw.txt'"), { code: 'ENOENT' });+ const transcriptFlags = () => vi.mocked(updateAgent).mock.calls.filter(([, p]) => p?.metadata?.transcriptWriteFailed);++ beforeEach(() => {+ vi.useFakeTimers();+ vi.clearAllMocks();+ errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {});+ logSpy = vi.spyOn(console, 'log').mockImplementation(() => {});+ });++ afterEach(() => {+ vi.useRealTimers();+ errorSpy.mockRestore();+ logSpy.mockRestore();+ });++ it('still logs and flags ENOENT while the run is live (no final drain yet)', async () => {+ const s = makeSpooler();+ vi.mocked(appendFile).mockRejectedValueOnce(enoent());++ s.pushRaw('chunk');+ await vi.advanceTimersByTimeAsync(1000);++ expect(errorSpy.mock.calls.filter(([m]) => String(m).includes('raw.txt write failed (ENOENT)'))).toHaveLength(1);+ expect(transcriptFlags()).toHaveLength(1);+ });++ it('treats a straggler flush that hits a removed directory after the final drain as benign', async () => {+ const s = makeSpooler();+ s.pushRaw('before');+ await s.drainRaw();+ expect(appendFile).toHaveBeenCalledWith(RAW_FILE, 'before');++ // The completion path archived the directory; a late chunk now cannot land.+ vi.mocked(appendFile).mockRejectedValueOnce(enoent());+ s.pushRaw('late');+ await vi.advanceTimersByTimeAsync(1000);++ expect(errorSpy).not.toHaveBeenCalled();+ expect(transcriptFlags()).toHaveLength(0);+ });++ it('does not let the debounce timer start a second flush while a drain is writing', async () => {+ const s = makeSpooler();+ let finishAppend;+ vi.mocked(appendFile).mockImplementationOnce(() => new Promise((resolve) => { finishAppend = resolve; }));++ s.pushRaw('a');+ let drained = false;+ const drain = s.drainRaw().then(() => { drained = true; });+ await vi.advanceTimersByTimeAsync(1000);+ s.pushRaw('b'); // arrives mid-drain; the drain, not the timer, must write it+ await vi.advanceTimersByTimeAsync(1000);++ expect(appendFile).toHaveBeenCalledTimes(1);+ expect(drained).toBe(false);++ finishAppend();+ await drain;+ expect(appendFile).toHaveBeenCalledTimes(2);+ expect(appendFile).toHaveBeenLastCalledWith(RAW_FILE, 'b');+ });+});+describe('createTranscriptWriteReporter', () => {let errorSpy;diff --git a/server/services/agentTuiSpawning/sessionController.test.js b/server/services/agentTuiSpawning/sessionController.test.jsindex d3c1d0e579..4b4d78b6c4 100644--- a/server/services/agentTuiSpawning/sessionController.test.js+++ b/server/services/agentTuiSpawning/sessionController.test.js@@ -176,6 +176,26 @@ describe('TUI session controller — teardown owns its own machinery (#8021)', (expect(emitLog).toHaveBeenCalledWith('warn', expect.stringContaining(`${label} drain failed`), { agentId: 'agent-8021' });});+ it('holds error analysis and completion until the raw PTY spool drain settles', async () => {+ let releaseDrain;+ const drainRaw = vi.fn(() => new Promise((resolve) => { releaseDrain = resolve; }));+ const { controller, finalizeAgent, resolveErrorAnalysis } = makeController({ spoolerOverrides: { drainRaw } });++ controller.attachSession({ sessionId: 'session-abcdef12', pid: 4242 });+ const exited = controller.handleExit({ exitCode: 0, killed: false });+ await vi.advanceTimersByTimeAsync(0);++ // completeAgent (inside finalizeAgent) archives the run directory; it must+ // not start while raw.txt still has queued bytes.+ expect(drainRaw).toHaveBeenCalledTimes(1);+ expect(resolveErrorAnalysis).not.toHaveBeenCalled();+ expect(finalizeAgent).not.toHaveBeenCalled();++ releaseDrain();+ await exited;+ expect(finalizeAgent).toHaveBeenCalledTimes(1);+ });+it('keeps the run alive when a completion sentinel read fails, then retries it', async () => {let reads = 0;const { controller, closers, finalizeAgent, watch } = makeController({