Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
fix: finalize a TUI agent even when its output drain is blocked by a backup (#11857)
Adam Eivy33b59e86c4f2
diff --git a/server/services/agentTuiSpawning/outputSpooler.js b/server/services/agentTuiSpawning/outputSpooler.jsindex 68aee6cb21..2abd92e570 100644--- a/server/services/agentTuiSpawning/outputSpooler.js+++ b/server/services/agentTuiSpawning/outputSpooler.js@@ -162,8 +162,16 @@ export function createOutputSpooler({ agentId, outputFile, rawFile }) {if (!rawSpoolTruncationWarned) {console.warn(`⚠️ TUI agent ${agentId} raw PTY spool reached ${Math.round(RAW_SPOOL_MAX_BYTES / 1024 / 1024)}MB — truncating spool (oldest bytes dropped; tail-read still reflects most recent)`);// Publish the warning before dropping bytes; if this fails, preserve- // the previous spool and retry the warning on the next flush.- await updateAgent(agentId, { metadata: { rawSpoolTruncated: true } });+ // the previous spool and retry the warning on the next flush. Reported,+ // never thrown: drainRaw runs inside finish(), where a rejection would+ // abandon the run's teardown.+ const published = await updateAgent(agentId, { metadata: { rawSpoolTruncated: true } })+ .then(() => true)+ .catch(async (err) => {+ await reporter.report('state', err);+ return false;+ });+ if (!published) return;rawSpoolTruncationWarned = true;}// Only update the byte counter on successful write — a failed writediff --git a/server/services/agentTuiSpawning/outputSpooler.test.js b/server/services/agentTuiSpawning/outputSpooler.test.jsindex 1860d54ea5..3a01372100 100644--- a/server/services/agentTuiSpawning/outputSpooler.test.js+++ b/server/services/agentTuiSpawning/outputSpooler.test.js@@ -100,6 +100,29 @@ describe('createOutputSpooler', () => {expect(metaCalls).toHaveLength(1);});+ it('does not reject drainRaw when the truncation warning cannot be published, and retries it next flush', async () => {+ const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {});+ const s = makeSpooler();+ s.pushRaw('a'.repeat(80));+ await s.drainRaw();+ vi.mocked(writeFile).mockClear();+ vi.mocked(updateAgent).mockRejectedValueOnce(new Error('BACKUP_SNAPSHOT_BUSY'));++ s.pushRaw('b'.repeat(80)); // crosses the cap, but the metadata write fails+ await expect(s.drainRaw()).resolves.toBeUndefined();+ // The previous spool is preserved: nothing was dropped without the warning.+ expect(vi.mocked(writeFile).mock.calls.filter(([p]) => p === RAW_FILE)).toHaveLength(0);+ expect(errorSpy.mock.calls.some(([msg]) => String(msg).includes('state write failed'))).toBe(true);++ s.pushRaw('c'.repeat(80)); // next flush retries the warning and then truncates+ await s.drainRaw();+ expect(vi.mocked(writeFile).mock.calls.filter(([p]) => p === RAW_FILE)).toHaveLength(1);+ const metaCalls = vi.mocked(updateAgent).mock.calls.filter(([, p]) => p?.metadata?.rawSpoolTruncated === true);+ expect(metaCalls).toHaveLength(2);++ errorSpy.mockRestore();+ });+it('drainLines / drainRaw are no-ops when nothing is pending', async () => {const s = makeSpooler();await s.drainLines();diff --git a/server/services/agentTuiSpawning/sessionController.js b/server/services/agentTuiSpawning/sessionController.jsindex 16cf3bca99..b07ffd394c 100644--- a/server/services/agentTuiSpawning/sessionController.js+++ b/server/services/agentTuiSpawning/sessionController.js@@ -1073,8 +1073,16 @@ export function createTuiSessionController({// Drain pending parsed lines AND raw chunks before the final state// writes so completion events don't beat the last output batch to disk.- await drainLines();- await drainRaw();+ // Non-fatal: the phase is already 'finalized', so a rejected drain (e.g.+ // BACKUP_SNAPSHOT_BUSY while a long backup holds the snapshot cut) must not+ // skip the teardown below — a later exit/Kill would find the run terminal+ // and no-op, stranding the lane, the PTY and the worktree until restart.+ for (const [drain, label] of [[drainLines, 'parsed output'], [drainRaw, 'raw PTY spool']]) {+ await drain().catch(err => {+ persistence.markUnsettled?.();+ emitLog('warn', `TUI ${label} drain failed for ${agentId}: ${err.message}`, { agentId });+ });+ }// The teardown both in-process spawners share: paused early return, then// the host-shutdown abandon gate, the user-termination consume, the finaldiff --git a/server/services/agentTuiSpawning/sessionController.test.js b/server/services/agentTuiSpawning/sessionController.test.jsindex 033e0ff59a..ed4b5ebd38 100644--- a/server/services/agentTuiSpawning/sessionController.test.js+++ b/server/services/agentTuiSpawning/sessionController.test.js@@ -22,6 +22,8 @@ import { STALL_NUDGE_IDLE_MS, STALL_NUDGE_TEXT } from '../../lib/tuiHandshake.jsvi.mock('../cosEvents.js', () => ({ emitLog: vi.fn() }));+import { emitLog } from '../cosEvents.js';+const { createTuiSessionController } = await import('./sessionController.js');const TASK = { id: 'task-8021', description: 'extract the session controller' };@@ -47,6 +49,7 @@ function makeController({prompt = 'do the work',securityScanGrant = null,sentinelRead = async () => sentinelSummary ?? '',+ spoolerOverrides = {},} = {}) {// A DISTINCT closer per arm: the merge-gate case below has to tell the// re-armed watcher apart from the one it replaced, which one shared spy@@ -68,6 +71,7 @@ function makeController({write: vi.fn(),paste: vi.fn(() => ({ /* a live submit-Enter interval handle */ })),kill: vi.fn(),+ markUnsettled: vi.fn(),resolveErrorAnalysis: vi.fn(async ({ immediateFallbackAnalysis }) => immediateFallbackAnalysis),};const controller = createTuiSessionController({@@ -93,6 +97,7 @@ function makeController({drainLines: vi.fn().mockResolvedValue(undefined),drainRaw: vi.fn().mockResolvedValue(undefined),getOutputBuffer: () => '',+ ...spoolerOverrides,},session: {write: seams.write,@@ -102,6 +107,7 @@ function makeController({hasLiveChild: async () => true,},persistence: {+ markUnsettled: seams.markUnsettled,updateAgent: seams.updateAgent,appendRunEvent: vi.fn(),// THE CASE UNDER TEST: no active-run entry for this agent.@@ -149,6 +155,27 @@ describe('TUI session controller — teardown owns its own machinery (#8021)', (expect(releaseRunRecord).toHaveBeenCalledWith(null);});+ it.each([+ ['drainLines', 'parsed output'],+ ['drainRaw', 'raw PTY spool'],+ ])('still finalizes, releases the lane and kills the session when %s rejects (BACKUP_SNAPSHOT_BUSY)', async (drainName, label) => {+ const busy = Object.assign(new Error('backup snapshot busy'), { code: 'BACKUP_SNAPSHOT_BUSY' });+ const { controller, finalizeAgent, releaseRunRecord, kill, markUnsettled } = makeController({+ spoolerOverrides: { [drainName]: vi.fn().mockRejectedValue(busy) },+ });++ controller.attachSession({ sessionId: 'session-abcdef12', pid: 4242 });+ await controller.handleExit({ exitCode: 0, killed: false });++ expect(controller.isTerminal()).toBe(true);+ expect(finalizeAgent).toHaveBeenCalledTimes(1);+ expect(finalizeAgent).toHaveBeenCalledWith(expect.objectContaining({ agentId: 'agent-8021', success: true }));+ expect(releaseRunRecord).toHaveBeenCalledTimes(1);+ expect(kill).toHaveBeenCalledWith('session-abcdef12');+ expect(markUnsettled).toHaveBeenCalled();+ expect(emitLog).toHaveBeenCalledWith('warn', expect.stringContaining(`${label} drain failed`), { agentId: 'agent-8021' });+ });+it('keeps the run alive when a completion sentinel read fails, then retries it', async () => {let reads = 0;const { controller, closers, finalizeAgent, watch } = makeController({