Skip to content

Open-source SEO, GEO, and marketing skills for AI agents.

fix(notfair-cmo): surface the real cron error, not mid-retry chatter

Yuting Zhong2f1cbbd4ceab
diff --git a/notfair-cmo/src/server/adapters/codex-local/execute.ts b/notfair-cmo/src/server/adapters/codex-local/execute.ts
index 5566c0e3..52fd1659 100644
--- a/notfair-cmo/src/server/adapters/codex-local/execute.ts
+++ b/notfair-cmo/src/server/adapters/codex-local/execute.ts
@@ -130,7 +130,14 @@ export async function* executeCodexLocal(
for (const evt of out) events.push(evt);
stdoutBuf = "";
}
- if (code !== 0 && !state.finalized) {
+ if (code !== 0) {
+ // Always push the exit-code+stderr-tail error on non-zero exit,
+ // even when state.finalized is set. `turn.failed` events from
+ // Codex's MCP reconnect loop are tagged transient by parse.ts and
+ // intentionally do NOT finalize the turn so we still get this
+ // richer post-exit message — but on a legitimately-finalized turn
+ // (turn.completed followed by a non-zero exit) the exit info is
+ // still the most actionable signal, so capture it unconditionally.
const tail = stderrBuf.trim().split("\n").slice(-5).join("\n");
events.push({
kind: "error",
diff --git a/notfair-cmo/src/server/adapters/codex-local/parse.test.ts b/notfair-cmo/src/server/adapters/codex-local/parse.test.ts
index 0adbecef..c55d0303 100644
--- a/notfair-cmo/src/server/adapters/codex-local/parse.test.ts
+++ b/notfair-cmo/src/server/adapters/codex-local/parse.test.ts
@@ -161,7 +161,46 @@ describe("parseCodexLine", () => {
}),
state,
);
- expect(out).toEqual([{ kind: "error", message: "rate limit hit" }]);
+ expect(out).toEqual([
+ { kind: "error", message: "rate limit hit", transient: false },
+ ]);
expect(state.finalized).toBe(true);
});
+
+ it("tags Codex MCP reconnect chatter as transient and leaves the turn un-finalized", () => {
+ const state = makeCodexStreamState();
+ const out = parseCodexLine(
+ JSON.stringify({
+ type: "turn.failed",
+ error: {
+ message: "Reconnecting... 2/5 (timeout waiting for child process to exit)",
+ },
+ }),
+ state,
+ );
+ expect(out).toEqual([
+ {
+ kind: "error",
+ message: "Reconnecting... 2/5 (timeout waiting for child process to exit)",
+ transient: true,
+ },
+ ]);
+ // Critically: a transient error must NOT finalize the turn, otherwise
+ // execute.ts's close handler suppresses the richer post-exit error.
+ expect(state.finalized).toBe(false);
+ });
+
+ it("tags `type: error` lines that match the reconnect pattern as transient", () => {
+ const state = makeCodexStreamState();
+ const out = parseCodexLine(
+ JSON.stringify({
+ type: "error",
+ message: "Reconnecting... 1/5",
+ }),
+ state,
+ );
+ expect(out).toEqual([
+ { kind: "error", message: "Reconnecting... 1/5", transient: true },
+ ]);
+ });
});
diff --git a/notfair-cmo/src/server/adapters/codex-local/parse.ts b/notfair-cmo/src/server/adapters/codex-local/parse.ts
index a2a59a58..c8fec471 100644
--- a/notfair-cmo/src/server/adapters/codex-local/parse.ts
+++ b/notfair-cmo/src/server/adapters/codex-local/parse.ts
@@ -130,22 +130,50 @@ export function parseCodexLine(
}
if (event.type === "turn.failed") {
- state.finalized = true;
- events.push({
- kind: "error",
- message: event.error?.message ?? "codex turn failed",
- });
+ const message = event.error?.message ?? "codex turn failed";
+ const transient = isTransientCodexError(message);
+ // Only flip `finalized` on a *terminal* failure. Transient retry
+ // chatter (Codex's MCP reconnect loop: "Reconnecting... N/5 ...")
+ // arrives on `turn.failed` too — if we marked the turn finalized
+ // there, the close handler in execute.ts would suppress the richer
+ // "codex exited with code N: <stderr tail>" error that lands after
+ // the process actually gives up.
+ if (!transient) state.finalized = true;
+ events.push({ kind: "error", message, transient });
return events;
}
if (event.type === "error") {
- events.push({ kind: "error", message: event.message ?? "codex error" });
+ const message = event.message ?? "codex error";
+ events.push({
+ kind: "error",
+ message,
+ transient: isTransientCodexError(message),
+ });
return events;
}
return events;
}
+/**
+ * Codex's MCP reconnect loop prints `Reconnecting... N/5 (...)` whenever
+ * a streamable-HTTP MCP server connection drops mid-turn. These are
+ * intermediate retry-state snapshots, not the terminal error — the real
+ * cause (network blip, expired token, etc.) is what eventually shows up
+ * on stderr when codex finally gives up. Surfacing the retry message as
+ * the run's error makes failures look opaque ("2/5 — why not 5/5?");
+ * tagging them lets the scheduler prefer the post-exit message instead.
+ */
+const TRANSIENT_CODEX_ERROR_PATTERNS: RegExp[] = [
+ /^Reconnecting\.\.\.\s+\d+\/\d+/,
+];
+
+export function isTransientCodexError(message: string): boolean {
+ const trimmed = message.trim();
+ return TRANSIENT_CODEX_ERROR_PATTERNS.some((re) => re.test(trimmed));
+}
+
/**
* Derive the canonical tool `name` field the UI will see for any codex
* item type we route through as a tool event. The shape matters: the
diff --git a/notfair-cmo/src/server/adapters/types.ts b/notfair-cmo/src/server/adapters/types.ts
index 3d3b1a8e..df22e055 100644
--- a/notfair-cmo/src/server/adapters/types.ts
+++ b/notfair-cmo/src/server/adapters/types.ts
@@ -99,7 +99,17 @@ export type HarnessEvent =
}
| { kind: "lifecycle"; phase: string }
| { kind: "final"; text: string }
- | { kind: "error"; message: string }
+ | {
+ kind: "error";
+ message: string;
+ /**
+ * Set true when the message is mid-stream retry chatter (e.g. Codex's
+ * MCP reconnect loop printing "Reconnecting... 2/5"), not a terminal
+ * failure. Callers that need a single error to surface should prefer
+ * the most recent non-transient one.
+ */
+ transient?: boolean;
+ }
/**
* Adapter has learned the harness's own session id for this thread. The
* caller should persist it on the `sessions.harness_session_id` column
diff --git a/notfair-cmo/src/server/scheduler/tick.ts b/notfair-cmo/src/server/scheduler/tick.ts
index f578199e..b49e67fa 100644
--- a/notfair-cmo/src/server/scheduler/tick.ts
+++ b/notfair-cmo/src/server/scheduler/tick.ts
@@ -82,6 +82,7 @@ async function dispatchJob(job: ScheduledJob): Promise<string | null> {
let finalText: string | null = null;
let deltaBuffer = "";
const MAX_SUMMARY = 4000;
+ const errors: { message: string; transient: boolean }[] = [];
for await (const evt of adapter.execute({
projectSlug: project.slug,
@@ -101,11 +102,26 @@ async function dispatchJob(job: ScheduledJob): Promise<string | null> {
} else if (evt.kind === "delta" && deltaBuffer.length < MAX_SUMMARY) {
deltaBuffer += evt.text;
} else if (evt.kind === "error") {
- throw new Error(evt.message);
+ // Buffer rather than throw. Adapters (notably codex-local) can emit
+ // several error events during a single retry burst — the last one is
+ // usually the richest (exit code + stderr tail from execute.ts's
+ // close handler). Throwing on the first event surfaced opaque retry
+ // chatter like "Reconnecting... 2/5 (...)" and discarded the
+ // actionable post-exit message that arrived a few hundred ms later.
+ errors.push({ message: evt.message, transient: evt.transient ?? false });
}
}
touchSession(session.id);
+ // No `final` and we have errors → the turn failed. Prefer the most recent
+ // non-transient error (the terminal exit-code message). Fall back to the
+ // last entry when everything was tagged transient (we ran out of richer
+ // signal; the retry chatter is all we have).
+ if (finalText === null && errors.length > 0) {
+ const terminal = [...errors].reverse().find((e) => !e.transient);
+ throw new Error((terminal ?? errors[errors.length - 1]!).message);
+ }
+
const raw = (finalText ?? deltaBuffer).trim();
if (!raw) return null;
return raw.length > MAX_SUMMARY ? `${raw.slice(0, MAX_SUMMARY)}…` : raw;