Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

fix: finalize a TUI agent even when its output drain is blocked by a backup (#11857)

Adam Eivy33b59e86c4f2
diff --git a/server/services/agentTuiSpawning/outputSpooler.js b/server/services/agentTuiSpawning/outputSpooler.js
index 68aee6cb21..2abd92e570 100644
--- a/server/services/agentTuiSpawning/outputSpooler.js
+++ b/server/services/agentTuiSpawning/outputSpooler.js
@@ -162,8 +162,16 @@ export function createOutputSpooler({ agentId, outputFile, rawFile }) {
if (!rawSpoolTruncationWarned) {
console.warn(`⚠️ TUI agent ${agentId} raw PTY spool reached ${Math.round(RAW_SPOOL_MAX_BYTES / 1024 / 1024)}MB — truncating spool (oldest bytes dropped; tail-read still reflects most recent)`);
// Publish the warning before dropping bytes; if this fails, preserve
- // the previous spool and retry the warning on the next flush.
- await updateAgent(agentId, { metadata: { rawSpoolTruncated: true } });
+ // the previous spool and retry the warning on the next flush. Reported,
+ // never thrown: drainRaw runs inside finish(), where a rejection would
+ // abandon the run's teardown.
+ const published = await updateAgent(agentId, { metadata: { rawSpoolTruncated: true } })
+ .then(() => true)
+ .catch(async (err) => {
+ await reporter.report('state', err);
+ return false;
+ });
+ if (!published) return;
rawSpoolTruncationWarned = true;
}
// Only update the byte counter on successful write — a failed write
diff --git a/server/services/agentTuiSpawning/outputSpooler.test.js b/server/services/agentTuiSpawning/outputSpooler.test.js
index 1860d54ea5..3a01372100 100644
--- a/server/services/agentTuiSpawning/outputSpooler.test.js
+++ b/server/services/agentTuiSpawning/outputSpooler.test.js
@@ -100,6 +100,29 @@ describe('createOutputSpooler', () => {
expect(metaCalls).toHaveLength(1);
});
+ it('does not reject drainRaw when the truncation warning cannot be published, and retries it next flush', async () => {
+ const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {});
+ const s = makeSpooler();
+ s.pushRaw('a'.repeat(80));
+ await s.drainRaw();
+ vi.mocked(writeFile).mockClear();
+ vi.mocked(updateAgent).mockRejectedValueOnce(new Error('BACKUP_SNAPSHOT_BUSY'));
+
+ s.pushRaw('b'.repeat(80)); // crosses the cap, but the metadata write fails
+ await expect(s.drainRaw()).resolves.toBeUndefined();
+ // The previous spool is preserved: nothing was dropped without the warning.
+ expect(vi.mocked(writeFile).mock.calls.filter(([p]) => p === RAW_FILE)).toHaveLength(0);
+ expect(errorSpy.mock.calls.some(([msg]) => String(msg).includes('state write failed'))).toBe(true);
+
+ s.pushRaw('c'.repeat(80)); // next flush retries the warning and then truncates
+ await s.drainRaw();
+ expect(vi.mocked(writeFile).mock.calls.filter(([p]) => p === RAW_FILE)).toHaveLength(1);
+ const metaCalls = vi.mocked(updateAgent).mock.calls.filter(([, p]) => p?.metadata?.rawSpoolTruncated === true);
+ expect(metaCalls).toHaveLength(2);
+
+ errorSpy.mockRestore();
+ });
+
it('drainLines / drainRaw are no-ops when nothing is pending', async () => {
const s = makeSpooler();
await s.drainLines();
diff --git a/server/services/agentTuiSpawning/sessionController.js b/server/services/agentTuiSpawning/sessionController.js
index 16cf3bca99..b07ffd394c 100644
--- a/server/services/agentTuiSpawning/sessionController.js
+++ b/server/services/agentTuiSpawning/sessionController.js
@@ -1073,8 +1073,16 @@ export function createTuiSessionController({
// Drain pending parsed lines AND raw chunks before the final state
// writes so completion events don't beat the last output batch to disk.
- await drainLines();
- await drainRaw();
+ // Non-fatal: the phase is already 'finalized', so a rejected drain (e.g.
+ // BACKUP_SNAPSHOT_BUSY while a long backup holds the snapshot cut) must not
+ // skip the teardown below — a later exit/Kill would find the run terminal
+ // and no-op, stranding the lane, the PTY and the worktree until restart.
+ for (const [drain, label] of [[drainLines, 'parsed output'], [drainRaw, 'raw PTY spool']]) {
+ await drain().catch(err => {
+ persistence.markUnsettled?.();
+ emitLog('warn', `TUI ${label} drain failed for ${agentId}: ${err.message}`, { agentId });
+ });
+ }
// The teardown both in-process spawners share: paused early return, then
// the host-shutdown abandon gate, the user-termination consume, the final
diff --git a/server/services/agentTuiSpawning/sessionController.test.js b/server/services/agentTuiSpawning/sessionController.test.js
index 033e0ff59a..ed4b5ebd38 100644
--- a/server/services/agentTuiSpawning/sessionController.test.js
+++ b/server/services/agentTuiSpawning/sessionController.test.js
@@ -22,6 +22,8 @@ import { STALL_NUDGE_IDLE_MS, STALL_NUDGE_TEXT } from '../../lib/tuiHandshake.js
vi.mock('../cosEvents.js', () => ({ emitLog: vi.fn() }));
+import { emitLog } from '../cosEvents.js';
+
const { createTuiSessionController } = await import('./sessionController.js');
const TASK = { id: 'task-8021', description: 'extract the session controller' };
@@ -47,6 +49,7 @@ function makeController({
prompt = 'do the work',
securityScanGrant = null,
sentinelRead = async () => sentinelSummary ?? '',
+ spoolerOverrides = {},
} = {}) {
// A DISTINCT closer per arm: the merge-gate case below has to tell the
// re-armed watcher apart from the one it replaced, which one shared spy
@@ -68,6 +71,7 @@ function makeController({
write: vi.fn(),
paste: vi.fn(() => ({ /* a live submit-Enter interval handle */ })),
kill: vi.fn(),
+ markUnsettled: vi.fn(),
resolveErrorAnalysis: vi.fn(async ({ immediateFallbackAnalysis }) => immediateFallbackAnalysis),
};
const controller = createTuiSessionController({
@@ -93,6 +97,7 @@ function makeController({
drainLines: vi.fn().mockResolvedValue(undefined),
drainRaw: vi.fn().mockResolvedValue(undefined),
getOutputBuffer: () => '',
+ ...spoolerOverrides,
},
session: {
write: seams.write,
@@ -102,6 +107,7 @@ function makeController({
hasLiveChild: async () => true,
},
persistence: {
+ markUnsettled: seams.markUnsettled,
updateAgent: seams.updateAgent,
appendRunEvent: vi.fn(),
// THE CASE UNDER TEST: no active-run entry for this agent.
@@ -149,6 +155,27 @@ describe('TUI session controller — teardown owns its own machinery (#8021)', (
expect(releaseRunRecord).toHaveBeenCalledWith(null);
});
+ it.each([
+ ['drainLines', 'parsed output'],
+ ['drainRaw', 'raw PTY spool'],
+ ])('still finalizes, releases the lane and kills the session when %s rejects (BACKUP_SNAPSHOT_BUSY)', async (drainName, label) => {
+ const busy = Object.assign(new Error('backup snapshot busy'), { code: 'BACKUP_SNAPSHOT_BUSY' });
+ const { controller, finalizeAgent, releaseRunRecord, kill, markUnsettled } = makeController({
+ spoolerOverrides: { [drainName]: vi.fn().mockRejectedValue(busy) },
+ });
+
+ controller.attachSession({ sessionId: 'session-abcdef12', pid: 4242 });
+ await controller.handleExit({ exitCode: 0, killed: false });
+
+ expect(controller.isTerminal()).toBe(true);
+ expect(finalizeAgent).toHaveBeenCalledTimes(1);
+ expect(finalizeAgent).toHaveBeenCalledWith(expect.objectContaining({ agentId: 'agent-8021', success: true }));
+ expect(releaseRunRecord).toHaveBeenCalledTimes(1);
+ expect(kill).toHaveBeenCalledWith('session-abcdef12');
+ expect(markUnsettled).toHaveBeenCalled();
+ expect(emitLog).toHaveBeenCalledWith('warn', expect.stringContaining(`${label} drain failed`), { agentId: 'agent-8021' });
+ });
+
it('keeps the run alive when a completion sentinel read fails, then retries it', async () => {
let reads = 0;
const { controller, closers, finalizeAgent, watch } = makeController({