Skip to content

⭐️⭐️⭐️ Open-source, self-hosted alternative to Linear and Plane. Project management and issue tracking where teams and AI agents work side by side to plan and ship products.

fix(runner): preserve command results when final trace upload fails (#648)

I'm Groot 🌳86da012f092e
diff --git a/packages/runner/README.md b/packages/runner/README.md
index 51c3016d..ff5b179e 100644
--- a/packages/runner/README.md
+++ b/packages/runner/README.md
@@ -384,6 +384,10 @@ and tool calls at `/agent-runs/:runId/events`. The runner reports numbered batch
for the claimed attempt before reporting its result. Retries of a batch use the
same offsets. A server without this capability still receives the final result.
+A transient failure of the final event batch is retried once. If it still fails,
+the runner logs a trace-upload warning and reports the command's actual result.
+Authorization, stale-attempt and other permanent rejections are not retried or ignored.
+
The `codex-jsonl` format records the session UUID from `thread.started`, shell
commands, MCP calls and file changes. Plan shows an **Open in Codex** link for that
attempt. The session must be available in the viewer's local Codex app or a
diff --git a/packages/runner/src/__tests__/run.test.ts b/packages/runner/src/__tests__/run.test.ts
index cb02d774..e86019b3 100644
--- a/packages/runner/src/__tests__/run.test.ts
+++ b/packages/runner/src/__tests__/run.test.ts
@@ -1,4 +1,4 @@
-import { expect, it } from 'bun:test';
+import { expect, it, spyOn } from 'bun:test';
import { AnswerStream, type AgUiEvent } from '../agui';
import { Client, type Run } from '../client';
import type { RunnerConfig } from '../config';
@@ -171,3 +171,110 @@ it('retries a partially accepted stream batch with its original offsets', async
globalThis.fetch = original;
}
}, 15_000);
+
+const shortTaskConfig: RunnerConfig = {
+ name: '',
+ url: 'http://plan.test',
+ apiKey: 'test',
+ command: "printf 'Done'",
+ args: [],
+ env: {},
+ concurrency: 1,
+ pollIntervalMs: 1000,
+ timeoutMs: 5000,
+ outputFormat: 'text',
+};
+const shortRun: Run = {
+ id: 1,
+ attempts: 2,
+ traceEvents: true,
+ trigger: 'manual',
+ prompt: '',
+ systemPrompt: '',
+ issueId: null,
+ issueIdentifier: null,
+};
+
+function isFinalBatch(body: Record<string, unknown>): boolean {
+ const events = body.events as AgUiEvent[];
+ return events.some((event) => event.type === 'RUN_FINISHED' || event.type === 'RUN_ERROR');
+}
+
+it('retries a transient final event failure before reporting the successful command once', async () => {
+ const original = globalThis.fetch;
+ const reports: Record<string, unknown>[] = [];
+ const finalOffsets: number[] = [];
+ globalThis.fetch = (async (input, init) => {
+ const body = JSON.parse(String(init?.body));
+ if (String(input).endsWith('/events') && isFinalBatch(body)) {
+ finalOffsets.push(body.offset);
+ if (finalOffsets.length === 1) return new Response('Temporary failure', { status: 503 });
+ }
+ if (String(input).endsWith('/result')) reports.push(body);
+ return Response.json({ nextOffset: 100, truncated: false });
+ }) as typeof fetch;
+ try {
+ await handleRun(shortTaskConfig, new Client(shortTaskConfig), () => {}, shortRun);
+ expect(finalOffsets).toHaveLength(2);
+ expect(finalOffsets[1]).toBe(finalOffsets[0]);
+ expect(reports).toEqual([
+ expect.objectContaining({ status: 'success', output: 'Done', attempt: 2 }),
+ ]);
+ } finally {
+ globalThis.fetch = original;
+ }
+});
+
+it('warns about an unavailable final trace without replacing the command outcome', async () => {
+ const original = globalThis.fetch;
+ const warning = spyOn(console, 'error').mockImplementation(() => {});
+ const reports: Record<string, unknown>[] = [];
+ let finalAttempts = 0;
+ globalThis.fetch = (async (input, init) => {
+ const body = JSON.parse(String(init?.body));
+ if (String(input).endsWith('/events') && isFinalBatch(body)) {
+ finalAttempts++;
+ return new Response('Temporary failure', { status: 503 });
+ }
+ if (String(input).endsWith('/result')) reports.push(body);
+ return Response.json({ nextOffset: 100, truncated: false });
+ }) as typeof fetch;
+ try {
+ await handleRun(shortTaskConfig, new Client(shortTaskConfig), () => {}, shortRun);
+ expect(finalAttempts).toBe(2);
+ expect(reports).toEqual([
+ expect.objectContaining({ status: 'success', output: 'Done', attempt: 2 }),
+ ]);
+ expect(warning).toHaveBeenCalledTimes(1);
+ expect(warning.mock.calls[0]?.[0]).toContain('trace upload failed');
+ } finally {
+ globalThis.fetch = original;
+ warning.mockRestore();
+ }
+});
+
+it('does not retry or ignore a final event rejection for a stale attempt', async () => {
+ const original = globalThis.fetch;
+ let finalAttempts = 0;
+ let reports = 0;
+ globalThis.fetch = (async (input, init) => {
+ const body = JSON.parse(String(init?.body));
+ if (String(input).endsWith('/events') && isFinalBatch(body)) {
+ finalAttempts++;
+ return new Response('Active run attempt not found', { status: 404 });
+ }
+ if (String(input).endsWith('/result')) reports++;
+ return Response.json({ nextOffset: 100, truncated: false });
+ }) as typeof fetch;
+ try {
+ await expect(
+ runTask(shortTaskConfig, new Client(shortTaskConfig), shortRun),
+ ).rejects.toMatchObject({
+ status: 404,
+ });
+ expect(finalAttempts).toBe(1);
+ expect(reports).toBe(0);
+ } finally {
+ globalThis.fetch = original;
+ }
+});
diff --git a/packages/runner/src/run.ts b/packages/runner/src/run.ts
index 74093568..8a9e6749 100644
--- a/packages/runner/src/run.ts
+++ b/packages/runner/src/run.ts
@@ -1,5 +1,5 @@
import { AnswerStream, UsageReader } from './agui';
-import type { Client, Run } from './client';
+import { RequestError, type Client, type Run } from './client';
import type { RunnerConfig } from './config';
import { execute } from './execute';
@@ -50,8 +50,20 @@ export async function runTask(config: RunnerConfig, client: Client, run: Run) {
);
clearInterval(timer);
usage.end();
- if (outcome.status === 'success') await stream.finish(outcome.output);
- else await stream.fail(outcome.error ?? 'The command failed', outcome.output);
+ try {
+ if (outcome.status === 'success') await stream.finish(outcome.output);
+ else await stream.fail(outcome.error ?? 'The command failed', outcome.output);
+ } catch (error) {
+ if (!isTransientEventError(error)) throw error;
+ try {
+ await stream.flush();
+ } catch (retryError) {
+ if (!isTransientEventError(retryError)) throw retryError;
+ console.error(
+ `Run ${run.id}: trace upload failed; reporting the command result: ${String(retryError)}`,
+ );
+ }
+ }
// A stream format's stdout is its raw event lines; the run reports the answer read out
// of them. Plain text is the answer as printed.
const output = config.outputFormat === 'text' ? outcome.output : stream.answer();
@@ -66,3 +78,13 @@ export async function runTask(config: RunnerConfig, client: Client, run: Run) {
clearInterval(timer);
}
}
+
+function isTransientEventError(error: unknown): boolean {
+ if (error instanceof RequestError) {
+ return error.status === 408 || error.status === 429 || error.status >= 500;
+ }
+ return (
+ error instanceof TypeError ||
+ (error instanceof Error && ['AbortError', 'TimeoutError'].includes(error.name))
+ );
+}