Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

fix: stop TUI raw.txt spool reporting ENOENT after a run's final drain (#12876)

Adam Eivybb63de1384ea
diff --git a/server/services/agentTuiSpawning/outputSpooler.js b/server/services/agentTuiSpawning/outputSpooler.js
index 79201f8d9a..8768616bcb 100644
--- a/server/services/agentTuiSpawning/outputSpooler.js
+++ b/server/services/agentTuiSpawning/outputSpooler.js
@@ -32,9 +32,16 @@ const RAW_FLUSH_INTERVAL_MS = 1000;
*
* @param {object} opts
* @param {string} opts.agentId Agent id (used in logs and metadata updates).
+ * @param {(fileLabel: string) => boolean} [opts.isDrained] True once the
+ * final drain for that file's pipeline finished. An ENOENT after that point
+ * is a straggler flush landing in a run directory the completion path has
+ * already archived or removed — the transcript was fully written by the
+ * drain, so it is neither flagged nor sent to stderr. ENOENT on a pipeline
+ * that has not drained is still a fault: the directory vanished under a
+ * live run.
* @returns {object} Reporter with `report(fileLabel, err)` method.
*/
-export function createTranscriptWriteReporter({ agentId }) {
+export function createTranscriptWriteReporter({ agentId, isDrained = () => false }) {
const reportedFailures = new Set();
return {
@@ -46,6 +53,11 @@ export function createTranscriptWriteReporter({ agentId }) {
}
reportedFailures.add(failureKey);
+ if (err.code === 'ENOENT' && isDrained(fileLabel)) {
+ console.log(`⏭️ agent ${agentId} ${fileLabel} late write skipped after the final drain (ENOENT) — run directory already archived`);
+ return;
+ }
+
// Log the first failure with error code and message
console.error(`❌ agent ${agentId} ${fileLabel} write failed (${err.code || 'error'}): ${err.message}`);
@@ -94,8 +106,20 @@ export function createOutputSpooler({ agentId, outputFile, rawFile }) {
let rawFlushing = null;
let rawBytesWritten = 0;
let rawSpoolTruncationWarned = false;
+ // A drain owns its pipeline until it returns: the debounce timer must not
+ // launch a second concurrent flush, or the drain can see an empty queue and
+ // resolve while that other flush's append is still in flight — the completion
+ // path would then archive the run directory under the pending write.
+ let linesDraining = false;
+ let rawDraining = false;
+ // Set once a drain has written everything queued; see the reporter's isDrained.
+ let linesDrained = false;
+ let rawDrained = false;
- const reporter = createTranscriptWriteReporter({ agentId });
+ const reporter = createTranscriptWriteReporter({
+ agentId,
+ isDrained: (fileLabel) => (fileLabel === 'raw.txt' ? rawDrained : linesDrained),
+ });
const flushPendingLines = () => withBackupAssetPublication(async () => {
if (flushTimer) { clearTimeout(flushTimer); flushTimer = null; }
@@ -109,7 +133,7 @@ export function createOutputSpooler({ agentId, outputFile, rawFile }) {
});
const scheduleFlush = () => {
- if (flushTimer || flushing) return;
+ if (flushTimer || flushing || linesDraining) return;
flushTimer = setTimeout(() => {
flushTimer = null;
flushing = flushPendingLines().catch(err => reporter.report('state', err)).finally(() => {
@@ -192,7 +216,7 @@ export function createOutputSpooler({ agentId, outputFile, rawFile }) {
});
const scheduleRawFlush = () => {
- if (rawFlushTimer || rawFlushing) return;
+ if (rawFlushTimer || rawFlushing || rawDraining) return;
rawFlushTimer = setTimeout(() => {
rawFlushTimer = null;
rawFlushing = flushPendingRawChunks().catch(err => reporter.report('raw.txt', err)).finally(() => {
@@ -233,18 +257,51 @@ export function createOutputSpooler({ agentId, outputFile, rawFile }) {
scheduleRawFlush();
};
- // Drain the parsed-line pipeline: await any in-flight flush, then flush
- // whatever is still pending. Called at finalize (and mirrored order matters —
- // see drainRaw) so completion events don't beat the last batch to disk.
+ // Drain a pipeline to empty: stop its timer, wait out the in-flight flush,
+ // then flush until nothing is queued. The drain is the only writer while it
+ // runs (scheduleFlush/scheduleRawFlush stand down on the draining flag), and
+ // the flag drops in `finally` so a rejected drain (e.g. BACKUP_SNAPSHOT_BUSY)
+ // does not wedge later flushes.
+ const drainPipeline = async ({ setDraining, getTimer, clearTimer, getInFlight, flush, hasPending }) => {
+ setDraining(true);
+ try {
+ for (;;) {
+ if (getTimer()) clearTimer();
+ const inFlight = getInFlight();
+ if (!inFlight) break;
+ await inFlight.catch(() => {});
+ }
+ do { await flush(); } while (hasPending());
+ } finally {
+ setDraining(false);
+ }
+ };
+
+ // Drain the parsed-line pipeline. Called at finalize (order matters — see
+ // drainRaw) so completion events don't beat the last batch to disk.
const drainLines = async () => {
- if (flushing) await flushing.catch(() => {});
- await flushPendingLines();
+ await drainPipeline({
+ setDraining: (v) => { linesDraining = v; },
+ getTimer: () => flushTimer,
+ clearTimer: () => { clearTimeout(flushTimer); flushTimer = null; },
+ getInFlight: () => flushing,
+ flush: flushPendingLines,
+ hasPending: () => pendingLines.length > 0,
+ });
+ linesDrained = true;
};
// Drain the raw-chunk pipeline. Same shape as drainLines.
const drainRaw = async () => {
- if (rawFlushing) await rawFlushing.catch(() => {});
- await flushPendingRawChunks();
+ await drainPipeline({
+ setDraining: (v) => { rawDraining = v; },
+ getTimer: () => rawFlushTimer,
+ clearTimer: () => { clearTimeout(rawFlushTimer); rawFlushTimer = null; },
+ getInFlight: () => rawFlushing,
+ flush: flushPendingRawChunks,
+ hasPending: () => pendingRawChunks.length > 0,
+ });
+ rawDrained = true;
};
return {
diff --git a/server/services/agentTuiSpawning/outputSpooler.test.js b/server/services/agentTuiSpawning/outputSpooler.test.js
index 661039022c..1a9718051f 100644
--- a/server/services/agentTuiSpawning/outputSpooler.test.js
+++ b/server/services/agentTuiSpawning/outputSpooler.test.js
@@ -185,6 +185,73 @@ describe('createOutputSpooler', () => {
});
});
+describe('createOutputSpooler — run directory removed under the spool', () => {
+ let errorSpy;
+ let logSpy;
+ const enoent = () => Object.assign(new Error("ENOENT: no such file or directory, open '/tmp/agent/raw.txt'"), { code: 'ENOENT' });
+ const transcriptFlags = () => vi.mocked(updateAgent).mock.calls.filter(([, p]) => p?.metadata?.transcriptWriteFailed);
+
+ beforeEach(() => {
+ vi.useFakeTimers();
+ vi.clearAllMocks();
+ errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {});
+ logSpy = vi.spyOn(console, 'log').mockImplementation(() => {});
+ });
+
+ afterEach(() => {
+ vi.useRealTimers();
+ errorSpy.mockRestore();
+ logSpy.mockRestore();
+ });
+
+ it('still logs and flags ENOENT while the run is live (no final drain yet)', async () => {
+ const s = makeSpooler();
+ vi.mocked(appendFile).mockRejectedValueOnce(enoent());
+
+ s.pushRaw('chunk');
+ await vi.advanceTimersByTimeAsync(1000);
+
+ expect(errorSpy.mock.calls.filter(([m]) => String(m).includes('raw.txt write failed (ENOENT)'))).toHaveLength(1);
+ expect(transcriptFlags()).toHaveLength(1);
+ });
+
+ it('treats a straggler flush that hits a removed directory after the final drain as benign', async () => {
+ const s = makeSpooler();
+ s.pushRaw('before');
+ await s.drainRaw();
+ expect(appendFile).toHaveBeenCalledWith(RAW_FILE, 'before');
+
+ // The completion path archived the directory; a late chunk now cannot land.
+ vi.mocked(appendFile).mockRejectedValueOnce(enoent());
+ s.pushRaw('late');
+ await vi.advanceTimersByTimeAsync(1000);
+
+ expect(errorSpy).not.toHaveBeenCalled();
+ expect(transcriptFlags()).toHaveLength(0);
+ });
+
+ it('does not let the debounce timer start a second flush while a drain is writing', async () => {
+ const s = makeSpooler();
+ let finishAppend;
+ vi.mocked(appendFile).mockImplementationOnce(() => new Promise((resolve) => { finishAppend = resolve; }));
+
+ s.pushRaw('a');
+ let drained = false;
+ const drain = s.drainRaw().then(() => { drained = true; });
+ await vi.advanceTimersByTimeAsync(1000);
+ s.pushRaw('b'); // arrives mid-drain; the drain, not the timer, must write it
+ await vi.advanceTimersByTimeAsync(1000);
+
+ expect(appendFile).toHaveBeenCalledTimes(1);
+ expect(drained).toBe(false);
+
+ finishAppend();
+ await drain;
+ expect(appendFile).toHaveBeenCalledTimes(2);
+ expect(appendFile).toHaveBeenLastCalledWith(RAW_FILE, 'b');
+ });
+});
+
describe('createTranscriptWriteReporter', () => {
let errorSpy;
diff --git a/server/services/agentTuiSpawning/sessionController.test.js b/server/services/agentTuiSpawning/sessionController.test.js
index d3c1d0e579..4b4d78b6c4 100644
--- a/server/services/agentTuiSpawning/sessionController.test.js
+++ b/server/services/agentTuiSpawning/sessionController.test.js
@@ -176,6 +176,26 @@ describe('TUI session controller — teardown owns its own machinery (#8021)', (
expect(emitLog).toHaveBeenCalledWith('warn', expect.stringContaining(`${label} drain failed`), { agentId: 'agent-8021' });
});
+ it('holds error analysis and completion until the raw PTY spool drain settles', async () => {
+ let releaseDrain;
+ const drainRaw = vi.fn(() => new Promise((resolve) => { releaseDrain = resolve; }));
+ const { controller, finalizeAgent, resolveErrorAnalysis } = makeController({ spoolerOverrides: { drainRaw } });
+
+ controller.attachSession({ sessionId: 'session-abcdef12', pid: 4242 });
+ const exited = controller.handleExit({ exitCode: 0, killed: false });
+ await vi.advanceTimersByTimeAsync(0);
+
+ // completeAgent (inside finalizeAgent) archives the run directory; it must
+ // not start while raw.txt still has queued bytes.
+ expect(drainRaw).toHaveBeenCalledTimes(1);
+ expect(resolveErrorAnalysis).not.toHaveBeenCalled();
+ expect(finalizeAgent).not.toHaveBeenCalled();
+
+ releaseDrain();
+ await exited;
+ expect(finalizeAgent).toHaveBeenCalledTimes(1);
+ });
+
it('keeps the run alive when a completion sentinel read fails, then retries it', async () => {
let reads = 0;
const { controller, closers, finalizeAgent, watch } = makeController({