Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
fix: append CLI agent transcripts between flushes
Adam Eivya9982ed36fad
diff --git a/server/services/agentCliSpawning.js b/server/services/agentCliSpawning.jsindex b6150c5cf9..ef3bb23659 100644--- a/server/services/agentCliSpawning.js+++ b/server/services/agentCliSpawning.js@@ -366,6 +366,7 @@ export async function spawnDirectly({}let outputBuffer = '';+ let outputRevision = 0;let rawStreamBuffer = ''; // Raw stdout for stream-json (used for error analysis)let hasStartedWorking = false;// Deliberately NOT `hasStartedWorking`: that flag also flips on a 3s@@ -417,6 +418,30 @@ export async function spawnDirectly({let immediateFallbackAnalysis = null;const detectImmediateFallbackSignal = createImmediateFallbackSignalDetector();+ let persistedOutputRevision = -1;+ let persistedOutputLength = 0;+ let transcriptFileWriteTail = Promise.resolve();+ const persistTranscript = (bytes, revision) => {+ // Serialize timer/manual drains. Only a successful write establishes the+ // prefix that the next snapshot can extend; an append may fail partway.+ const write = transcriptFileWriteTail.then(async () => {+ const append = persistedOutputRevision === revision && persistedOutputLength <= bytes.length;+ const delta = append ? bytes.slice(persistedOutputLength) : bytes;+ if (append && !delta) return;+ await writeFileGuarded(outputFile, delta, { flag: append ? 'a' : 'w' });+ persistedOutputRevision = revision;+ persistedOutputLength = bytes.length;+ }).catch(err => {+ // Rewrite the next snapshot after any failure: retrying an uncertain+ // append could duplicate bytes, while skipping it could lose output.+ persistedOutputRevision = -1;+ maintenance.markResourceUnsettled('agent', agentId);+ return transcriptWriteReporter.report('output.txt', err);+ });+ transcriptFileWriteTail = write;+ return write;+ };+// Debounced state-write batcher for streamed output. stdout/stderr `data`// events fire per chunk (and stream-json yields many lines per chunk), so a// per-line/per-chunk appendAgentOutput would round-trip a full state@@ -428,11 +453,9 @@ export async function spawnDirectly({// Capture bytes at the same instant as this batch, before waiting for the// cut. Chunks received during I/O belong to the next flush, not this pair.const bytes = outputBuffer;+ const revision = outputRevision;return withBackupAssetPublication(() => Promise.all([- writeFileGuarded(outputFile, bytes).catch(err => {- maintenance.markResourceUnsettled('agent', agentId);- return transcriptWriteReporter.report('output.txt', err);- }),+ persistTranscript(bytes, revision),isPrivateSecurityTask(task) ? undefined : appendAgentOutputLines(agentId, batch).catch(err => transcriptWriteReporter.report('state', err)),]));@@ -451,6 +474,7 @@ export async function spawnDirectly({if (cliAgentEntry) cliAgentEntry.flushOutput = async () => {await drainTranscriptWrites();await outputBatcher.flush();+ await transcriptFileWriteTail;};// Serialize stdout/stderr buffer updates before their shared debounced file@@ -603,6 +627,7 @@ export async function spawnDirectly({// the run so outputBuffer reflects everything that streamed (#2384).await drainTranscriptWrites();await outputBatcher.flush();+ await transcriptFileWriteTail;await completeAgent(agentId, { success: false, error: err.message });await completeAgentRun(runId, isPrivateSecurityTask(task) ? 'Private security assessment failed; inspect its local assessment archive.' : outputBuffer, 1, 0, { message: err.message, category: 'spawn-error' });unregisterSpawnedAgent(claudeProcess.pid);@@ -715,6 +740,7 @@ export async function spawnDirectly({const finalResult = streamParser.getFinalResult();if (finalResult) {outputBuffer = finalResult;+ outputRevision++;}}if (codexStderrFormatter) {@@ -729,6 +755,7 @@ export async function spawnDirectly({// below too, since output.txt is written next).await withBackupAssetPublication(async () => {await outputBatcher.flush();+ await transcriptFileWriteTail;await writeFileGuarded(outputFile, outputBuffer).catch((err) => {maintenance.markResourceUnsettled('agent', agentId);return transcriptWriteReporter.report('output.txt', err);diff --git a/server/services/agentCliSpawning.test.js b/server/services/agentCliSpawning.test.jsindex 7cdcacceb7..92cde941fb 100644--- a/server/services/agentCliSpawning.test.js+++ b/server/services/agentCliSpawning.test.js@@ -878,7 +878,7 @@ describe('stream error containment', () => {let reached; let finish;const reachedPromise = new Promise(resolve => { reached = resolve; });const finishPromise = new Promise(resolve => { finish = resolve; });- writeFileGuarded.mockImplementation((path, bytes) => fs.writeFile(path, bytes));+ writeFileGuarded.mockImplementation((path, bytes, options) => fs.writeFile(path, bytes, options));const persistedLines = [];agentStateMocks.appendAgentOutputLines.mockImplementation(async (_id, batch) => {reached(); await finishPromise;@@ -918,6 +918,123 @@ describe('stream error containment', () => {}});+ describe('incremental transcript persistence', () => {+ let fs;+ let dir;+ let writeFileGuarded;+ let activeAgents;+ beforeEach(async () => {+ fs = await vi.importActual('node:fs/promises');+ const { tmpdir } = await import('node:os');+ ({ writeFileGuarded } = await import('../lib/fileUtils.js'));+ ({ activeAgents } = await import('./agentState.js'));+ dir = await fs.mkdtemp(join(tmpdir(), 'cli-transcript-'));+ writeFileGuarded.mockReset();+ writeFileGuarded.mockImplementation((path, bytes, options) => fs.writeFile(path, bytes, options));+ });+ afterEach(async () => {+ writeFileGuarded.mockResolvedValue(undefined);+ await fs.rm(dir, { recursive: true, force: true });+ });+ const finish = async () => {+ fakeProcess.emit('close', 0);+ await vi.waitFor(() => expect(runSpawnerCompletionCleanup).toHaveBeenCalled());+ };+ const emitTextAndFlush = async text => {+ fakeProcess.stdout.emit('data', Buffer.from(text));+ await activeAgents.get('agent-test').flushOutput();+ };++ it('writes a growing transcript once per byte plus the terminal rewrite, preserving mixed UTF-8 output', async () => {+ // 240 flushes model one minute at the 250ms batch cadence without sleeps.+ // Before the stderr suffix, full-prefix writes submitted 238,878,720+ // bytes for 1,966,080 bytes of output; deltas plus final rewrite need 3,932,160.+ await fs.writeFile(join(dir, 'output.txt'), 'stale previous transcript');+ await spawnDirectly({ ...minimalArgs, agentDir: dir,+ cliConfig: { command: 'claude', args: [], stdinMode: 'prompt', streamFormat: 'text' } });+ const chunk = `é${'x'.repeat(8189)}\n`;+ for (let i = 0; i < 240; i++) await emitTextAndFlush(chunk);+ fakeProcess.stderr.emit('data', Buffer.from('尾\n'));+ await activeAgents.get('agent-test').flushOutput();+ const expected = `${chunk.repeat(240)}[stderr] 尾\n`;+ expect(await fs.readFile(join(dir, 'output.txt'), 'utf8')).toBe(expected);+ await finish();+ expect(await fs.readFile(join(dir, 'output.txt'), 'utf8')).toBe(expected);+ const submittedBytes = writeFileGuarded.mock.calls.reduce((sum, [, bytes]) => sum + Buffer.byteLength(bytes), 0);+ expect(submittedBytes).toBe(Buffer.byteLength(expected) * 2);+ });++ it('rewrites after an uncertain partial append instead of duplicating or dropping its bytes', async () => {+ vi.spyOn(console, 'error').mockImplementation(() => {});+ await spawnDirectly({ ...minimalArgs, agentDir: dir,+ cliConfig: { command: 'claude', args: [], stdinMode: 'prompt', streamFormat: 'text' } });+ await emitTextAndFlush('first\n');+ writeFileGuarded.mockImplementationOnce(async (path, bytes, options) => {+ await fs.writeFile(path, bytes.slice(0, 3), options);+ throw Object.assign(new Error('synthetic partial write'), { code: 'EIO' });+ });+ await emitTextAndFlush('second\n');+ await emitTextAndFlush('third\n');+ expect(await fs.readFile(join(dir, 'output.txt'), 'utf8')).toBe('first\nsecond\nthird\n');+ expect(agentStateMocks.updateAgent).toHaveBeenCalledWith('agent-test', {+ metadata: { transcriptWriteFailed: { file: 'output.txt', code: 'EIO' } },+ });+ await finish();+ });++ it('waits for an already draining transcript before a second manual drain can finish', async () => {+ await spawnDirectly({ ...minimalArgs, agentDir: dir,+ cliConfig: { command: 'claude', args: [], stdinMode: 'prompt', streamFormat: 'text' } });+ let reached;+ let release;+ const reachedPromise = new Promise(resolve => { reached = resolve; });+ const blocked = new Promise(resolve => { release = resolve; });+ writeFileGuarded.mockImplementationOnce(async (path, bytes, options) => {+ reached();+ await blocked;+ await fs.writeFile(path, bytes, options);+ });+ const first = emitTextAndFlush('complete before termination\n');+ await reachedPromise;+ let drained = false;+ const second = activeAgents.get('agent-test').flushOutput().then(() => { drained = true; });+ try {+ await new Promise(resolve => setImmediate(resolve));+ expect(drained).toBe(false);+ } finally { release(); }+ await Promise.all([first, second]);+ expect(await fs.readFile(join(dir, 'output.txt'), 'utf8')).toBe('complete before termination\n');+ await finish();+ });++ it('replaces streamed progress with the authoritative result when close overlaps a write', async () => {+ await spawnDirectly({ ...minimalArgs, agentDir: dir });+ const delta = text => JSON.stringify({ type: 'stream_event', event: {+ type: 'content_block_delta', delta: { type: 'text_delta', text },+ } }) + '\n';+ await emitTextAndFlush(delta('initial progress\n'));+ let reached;+ let release;+ const reachedPromise = new Promise(resolve => { reached = resolve; });+ const blocked = new Promise(resolve => { release = resolve; });+ writeFileGuarded.mockImplementationOnce(async (path, bytes, options) => {+ reached();+ await blocked;+ await fs.writeFile(path, bytes, options);+ });+ const flushing = emitTextAndFlush(delta('later progress\n'));+ await reachedPromise;+ fakeProcess.stdout.emit('data', Buffer.from(delta('last fragment') + JSON.stringify({+ type: 'result', result: 'Final result that differs from the streamed progress.',+ }) + '\n'));+ const finishing = finish();+ release();+ await Promise.all([flushing, finishing]);+ expect(await fs.readFile(join(dir, 'output.txt'), 'utf8'))+ .toBe('Final result that differs from the streamed progress.');+ });+ });+it('detects a fallback signal and terminates immediately even while an earlier transcript write is blocked, keeping output ordered (#2384)', async () => {killProcessTree.mockClear();