Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
perf([issue-12333]): batch raw transcripts to reduce admission journal stalls
[._.]/ Adam Eivy6e1521a62c5f
diff --git a/server/services/agentTuiSpawning/outputSpooler.js b/server/services/agentTuiSpawning/outputSpooler.jsindex 2abd92e570..79201f8d9a 100644--- a/server/services/agentTuiSpawning/outputSpooler.js+++ b/server/services/agentTuiSpawning/outputSpooler.js@@ -9,21 +9,21 @@ import { withBackupAssetPublication } from '../../lib/backupSnapshotBoundary.js'* 2. Raw PTY bytes → the append-only `raw.txt` disk spool (no ANSI strip, no* line semantics) that `analyzeAgentFailure` reads on failure.*- * Both pipelines debounce writes on a 250ms window (see AGENTS.md "High-- * frequency state writes must batch") — a chatty TUI emits hundreds of chunks- * /sec, and per-chunk state writes would thrash the filesystem and slow the PTY- * event loop. Extracted from spawnTuiAgent so the buffering/spooling concern is- * self-contained and independently testable; the spawner keeps only the- * orchestration.+ * Parsed status uses a 250ms window; raw recovery output uses a 1s window.+ * Each publication durably registers and releases admission journals, whose+ * synchronous flushes block the server loop even though the append is async.+ * Raw output is not the live shell stream; batching it more coarsely reduces+ * that overhead, while finalization still drains immediately.*/import { appendFile, writeFile } from 'fs/promises';import { appendAgentOutputLines, updateAgent } from '../cosAgentLifecycle.js';import { OUTPUT_BUFFER_CAP, OUTPUT_BUFFER_HEADROOM, RAW_SPOOL_MAX_BYTES } from '../../lib/tuiHandshake.js';-// Debounce window for batching parsed output AND raw chunks to disk + state.-// 250ms is invisible to the live tail but cuts I/O by 1-2 orders of magnitude.+// Status updates stay responsive; raw recovery output can tolerate a longer+// batching window. Neither timer delays an explicit final drain.const OUTPUT_FLUSH_INTERVAL_MS = 250;+const RAW_FLUSH_INTERVAL_MS = 1000;/*** Create a per-run transcript write failure reporter. Logs the first failure@@ -128,7 +128,7 @@ export function createOutputSpooler({ agentId, outputFile, rawFile }) {// (node-pty's internal StringDecoder handles multi-byte boundaries before// we see chunks), so queueing strings here is sufficient — no Buffer// bookkeeping needed. pendingRawChunks holds whatever arrives during the- // 250ms debounce window AND while an appendFile is in-flight (the next+ // 1s debounce window AND while an appendFile is in-flight (the next// scheduleRawFlush is gated by rawFlushing); join() runs once per flush// tick, so peak in-memory raw data is bounded by one debounce-plus-IO// window of TUI output (typically hundreds of KB on a chatty agent).@@ -202,7 +202,7 @@ export function createOutputSpooler({ agentId, outputFile, rawFile }) {// finalize if the producer goes quiet immediately after.if (pendingRawChunks.length > 0) scheduleRawFlush();});- }, OUTPUT_FLUSH_INTERVAL_MS);+ }, RAW_FLUSH_INTERVAL_MS);};// TUI agents only emit a handful of internal status lines (session-started,diff --git a/server/services/agentTuiSpawning/outputSpooler.test.js b/server/services/agentTuiSpawning/outputSpooler.test.jsindex 1860d54ea5..661039022c 100644--- a/server/services/agentTuiSpawning/outputSpooler.test.js+++ b/server/services/agentTuiSpawning/outputSpooler.test.js@@ -84,6 +84,40 @@ describe('createOutputSpooler', () => {expect(appendFile).toHaveBeenCalledWith(RAW_FILE, 'abc');});+ it('batches a sustained raw stream once per second while status stays responsive', async () => {+ const s = makeSpooler();+ for (let tick = 0; tick < 8; tick++) {+ s.pushRaw(`${tick}`);+ if (tick === 0) s.appendLine('running');+ await vi.advanceTimersByTimeAsync(250);+ if (tick === 0) {+ expect(appendAgentOutputLines).toHaveBeenCalledWith('agent-1', ['running']);+ expect(appendFile).not.toHaveBeenCalledWith(RAW_FILE, expect.anything());+ }+ }+ expect(vi.mocked(appendFile).mock.calls.filter(([path]) => path === RAW_FILE))+ .toEqual([[RAW_FILE, '0123'], [RAW_FILE, '4567']]);+ });++ it('drains raw output immediately and includes chunks arriving during an in-flight append', async () => {+ const s = makeSpooler();+ let finishAppend;+ vi.mocked(appendFile).mockImplementationOnce(() => new Promise(resolve => { finishAppend = resolve; }));+ s.pushRaw('first');+ await vi.advanceTimersByTimeAsync(1000);+ s.pushRaw('tail');+ let drained = false;+ const drain = s.drainRaw().then(() => { drained = true; });+ await Promise.resolve();+ expect(drained).toBe(false);+ finishAppend();+ await drain;+ expect(vi.mocked(appendFile).mock.calls.filter(([path]) => path === RAW_FILE))+ .toEqual([[RAW_FILE, 'first'], [RAW_FILE, 'tail']]);+ await vi.advanceTimersByTimeAsync(1000);+ expect(vi.mocked(appendFile).mock.calls.filter(([path]) => path === RAW_FILE)).toHaveLength(2);+ });+it('truncates the raw spool (writeFile) once it crosses the cap, warning + flagging once', async () => {const s = makeSpooler();s.pushRaw('a'.repeat(80)); // under cap (100) → appendFile