The open-source app everyone uses to manage agents at work
TypeScript acpxagent-orchestrationagent-skillsagentic-aiai-agentsai-engineeringai-governanceai-toolsautonomous-agentsclaudeclaude-codeclaude-skillscodexhermesllmmcpmulti-agentopenclawproject-managementworkflow-automation
fix(codex): preserve silent active builds (#10153)
Dottab3b07ae44e06
diff --git a/packages/adapters/codex-local/src/index.ts b/packages/adapters/codex-local/src/index.tsindex 77db7a8af2..709b2de09f 100644--- a/packages/adapters/codex-local/src/index.ts+++ b/packages/adapters/codex-local/src/index.ts@@ -95,7 +95,7 @@ Core fields:Operational fields:- timeoutSec (number, optional): run timeout in seconds- graceSec (number, optional): SIGTERM grace period in seconds-- outputInactivityTimeoutMs (number | null, optional): inactivity monitor around the codex child. Resets whenever the child emits stdout or stderr bytes, including non-JSON progress from long-running verification commands. Defaults to 30 * 60_000 ms when unset or non-positive. Set to \`null\` to disable the monitor entirely (only do this for known-slow tasks; the platform-level 1h silent-run safety net still applies). On fire, the adapter sends SIGTERM to the process group, waits 5s, then SIGKILL, and surfaces the run as failed with errorMessage "monitor: no codex output for {N}m {S}s".+- outputInactivityTimeoutMs (number | null, optional): inactivity monitor around the codex child. Resets whenever the child emits stdout/stderr bytes or, on Linux, its process group shows meaningful CPU, disk I/O, or child-process churn during a silent build. Defaults to 30 * 60_000 ms when unset or non-positive. Set to \`null\` to disable the monitor entirely (only do this for known-slow tasks; the platform-level 1h silent-run safety net still applies). On fire, the adapter sends SIGTERM to the process group, waits 5s, then SIGKILL, and surfaces the run as failed with errorMessage "monitor: no codex activity (output or process) for {N}m {S}s".- agentCommand (string, optional): ACP server command override used only when engine="acp"; defaults to the package-local codex-acp binary- mode (string, optional): ACP session mode when engine="acp"; persistent or oneshot- nonInteractivePermissions (string, optional): ACP non-interactive permission fallback when engine="acp"; deny or faildiff --git a/packages/adapters/codex-local/src/server/execute.ts b/packages/adapters/codex-local/src/server/execute.tsindex 2e71c0a2b5..b2723ca0bd 100644--- a/packages/adapters/codex-local/src/server/execute.ts+++ b/packages/adapters/codex-local/src/server/execute.ts@@ -87,6 +87,11 @@ import {formatOutputInactivityMonitorErrorMessage,resolveCodexInactivityTimeout,} from "./output-inactivity-monitor.js";+import {+ CODEX_PROCESS_ACTIVITY_POLL_INTERVAL_MS,+ createCodexProcessActivityMonitor,+ type CodexProcessActivityMonitorHandle,+} from "./process-activity-monitor.js";import {createCodexAcpExecutor,formatCodexAcpFallbackMessage,@@ -1051,6 +1056,8 @@ export async function execute(ctx: AdapterExecutionContext): Promise<AdapterExeclet killTarget: { pid: number | null; processGroupId: number | null } | null = null;let sigkillTimer: ReturnType<typeof setTimeout> | null = null;let monitorLogPromise: Promise<unknown> | null = null;+ const processActivityMonitor: { current: CodexProcessActivityMonitorHandle | null } = { current: null };+ const resolvedMonitorTimeoutMs = monitorResolution.mode === "disabled" ? null : monitorResolution.timeoutMs;const monitor =monitorResolution.mode === "disabled"@@ -1068,7 +1075,8 @@ export async function execute(ctx: AdapterExecutionContext): Promise<AdapterExec`[paperclip] adapter.invoke ${message}; ` +`timeoutMs=${monitorResolution.timeoutMs} elapsedSinceLastEventMs=${monitorElapsedMs} ` +`outputChunkCount=${state.outputChunkCount} outputBytes=${state.outputBytes} ` +- `parsedEvents=${state.parsedEventCount} (timeout=${timeoutSecLabel}s elapsed=${elapsedSec}s); ` ++ `parsedEvents=${state.parsedEventCount} processActivityCount=${state.processActivityCount} ` ++ `(timeout=${timeoutSecLabel}s elapsed=${elapsedSec}s); ` +`terminating codex child via SIGTERM (5s grace, then SIGKILL).\n`;// Issue the log without awaiting on the kill hot path, but capture// the promise so the surrounding try/finally can await flush before@@ -1094,6 +1102,17 @@ export async function execute(ctx: AdapterExecutionContext): Promise<AdapterExecconst wrappedOnSpawn = async (meta: { pid: number; processGroupId: number | null; startedAt: string }) => {killTarget = { pid: meta.pid ?? null, processGroupId: meta.processGroupId };+ if (monitor && resolvedMonitorTimeoutMs !== null && !executionTargetIsRemote) {+ processActivityMonitor.current = createCodexProcessActivityMonitor({+ pid: meta.pid,+ processGroupId: meta.processGroupId,+ intervalMs: Math.min(+ CODEX_PROCESS_ACTIVITY_POLL_INTERVAL_MS,+ Math.max(1_000, Math.floor(resolvedMonitorTimeoutMs / 4)),+ ),+ onActivity: () => monitor.noteProcessActivity(),+ });+ }if (onSpawn) {await onSpawn(meta);}@@ -1139,6 +1158,7 @@ export async function execute(ctx: AdapterExecutionContext): Promise<AdapterExec: { fired: false as const },};} finally {+ processActivityMonitor.current?.stop();monitor?.stop();if (sigkillTimer) {clearTimeout(sigkillTimer);diff --git a/packages/adapters/codex-local/src/server/output-inactivity-monitor.integration.test.ts b/packages/adapters/codex-local/src/server/output-inactivity-monitor.integration.test.tsindex fda2aded6a..0c1f03bc74 100644--- a/packages/adapters/codex-local/src/server/output-inactivity-monitor.integration.test.ts+++ b/packages/adapters/codex-local/src/server/output-inactivity-monitor.integration.test.ts@@ -5,6 +5,7 @@ import {createCodexOutputInactivityMonitor,formatOutputInactivityMonitorErrorMessage,} from "./output-inactivity-monitor.js";+import { createCodexProcessActivityMonitor } from "./process-activity-monitor.js";const FAKE_CODEX_SCRIPT = `process.stdout.write(JSON.stringify({ type: "thread.started", thread_id: "abc" }) + "\\n");@@ -15,6 +16,56 @@ setInterval(() => {}, 60_000);`;describe("codex inactivity monitor (integration: real subprocess)", () => {+ it.skipIf(process.platform !== "linux")(+ "allows a long silent build while the child process group is consuming CPU",+ async () => {+ const runId = `monitor-active-build-${Date.now()}`;+ const timeoutMs = 500;+ const processActivityMonitor: {+ current: ReturnType<typeof createCodexProcessActivityMonitor> | null;+ } = { current: null };+ let monitorFired = false;+ const monitor = createCodexOutputInactivityMonitor({+ timeoutMs,+ onFire: () => {+ monitorFired = true;+ },+ });++ try {+ const proc = await runChildProcess(+ runId,+ process.execPath,+ ["-e", "const end = Date.now() + 2_000; while (Date.now() < end) {}"],+ {+ cwd: process.cwd(),+ env: process.env as Record<string, string>,+ timeoutSec: 5,+ graceSec: 1,+ onSpawn: async (meta) => {+ processActivityMonitor.current = createCodexProcessActivityMonitor({+ pid: meta.pid,+ processGroupId: meta.processGroupId,+ intervalMs: 50,+ onActivity: () => monitor.noteProcessActivity(),+ });+ },+ onLog: async (stream, chunk) => monitor.noteOutputChunk(stream, chunk),+ },+ );++ expect(proc.exitCode).toBe(0);+ expect(proc.timedOut).toBe(false);+ expect(monitorFired).toBe(false);+ expect(monitor.state().processActivityCount).toBeGreaterThan(0);+ } finally {+ processActivityMonitor.current?.stop();+ monitor.stop();+ }+ },+ 10_000,+ );+it("kills a codex child that goes silent after one event and surfaces a monitor failure",async () => {@@ -25,6 +76,9 @@ describe("codex inactivity monitor (integration: real subprocess)", () => {let monitorFired = false;let terminationSignal: NodeJS.Signals | null = null;let sigkillTimer: ReturnType<typeof setTimeout> | null = null;+ const processActivityMonitor: {+ current: ReturnType<typeof createCodexProcessActivityMonitor> | null;+ } = { current: null };let elapsedMs = 0;const kill = (signal: NodeJS.Signals) => {@@ -69,6 +123,12 @@ describe("codex inactivity monitor (integration: real subprocess)", () => {graceSec: 1,onSpawn: async (meta) => {killTarget = { pid: meta.pid, processGroupId: meta.processGroupId };+ processActivityMonitor.current = createCodexProcessActivityMonitor({+ pid: meta.pid,+ processGroupId: meta.processGroupId,+ intervalMs: 25,+ onActivity: () => monitor.noteProcessActivity(),+ });},onLog: async (stream, chunk) => {logs.push({ stream, chunk });@@ -84,11 +144,12 @@ describe("codex inactivity monitor (integration: real subprocess)", () => {// The errorMessage shape mirrors the AdapterExecutionResult that// execute.ts will produce for this case.expect(formatOutputInactivityMonitorErrorMessage(elapsedMs)).toMatch(- /^monitor: no codex output for \d+m \d+s$/,+ /^monitor: no codex activity \(output or process\) for \d+m \d+s$/,);// We should have observed exactly one parsed JSONL event before silence.expect(monitor.state().parsedEventCount).toBe(1);} finally {+ processActivityMonitor.current?.stop();monitor.stop();if (sigkillTimer) clearTimeout(sigkillTimer);}diff --git a/packages/adapters/codex-local/src/server/output-inactivity-monitor.test.ts b/packages/adapters/codex-local/src/server/output-inactivity-monitor.test.tsindex 5bf64ad901..c4e7570f07 100644--- a/packages/adapters/codex-local/src/server/output-inactivity-monitor.test.ts+++ b/packages/adapters/codex-local/src/server/output-inactivity-monitor.test.ts@@ -99,10 +99,16 @@ describe("resolveCodexInactivityTimeout", () => {describe("formatOutputInactivityMonitorErrorMessage", () => {it("formats minutes and seconds", () => {- expect(formatOutputInactivityMonitorErrorMessage(0)).toBe("monitor: no codex output for 0m 0s");- expect(formatOutputInactivityMonitorErrorMessage(7 * 60 * 1000)).toBe("monitor: no codex output for 7m 0s");- expect(formatOutputInactivityMonitorErrorMessage(7 * 60 * 1000 + 12_000)).toBe("monitor: no codex output for 7m 12s");- expect(formatOutputInactivityMonitorErrorMessage(45_000)).toBe("monitor: no codex output for 0m 45s");+ expect(formatOutputInactivityMonitorErrorMessage(0)).toBe("monitor: no codex activity (output or process) for 0m 0s");+ expect(formatOutputInactivityMonitorErrorMessage(7 * 60 * 1000)).toBe(+ "monitor: no codex activity (output or process) for 7m 0s",+ );+ expect(formatOutputInactivityMonitorErrorMessage(7 * 60 * 1000 + 12_000)).toBe(+ "monitor: no codex activity (output or process) for 7m 12s",+ );+ expect(formatOutputInactivityMonitorErrorMessage(45_000)).toBe(+ "monitor: no codex activity (output or process) for 0m 45s",+ );});});@@ -186,6 +192,29 @@ describe("createCodexOutputInactivityMonitor (acceptance criteria 1: fires)", ()expect(fireCount).toBe(1);monitor.stop();});++ it("resets on process activity without output", () => {+ const clock = new FakeClock();+ let fireCount = 0;+ const monitor = createCodexOutputInactivityMonitor({+ timeoutMs: 1_000,+ now: () => clock.now(),+ setTimer: (cb, ms) => clock.setTimer(cb, ms),+ clearTimer: (handle) => clock.clearTimer(handle),+ onFire: () => {+ fireCount += 1;+ },+ });++ clock.advance(900);+ monitor.noteProcessActivity();+ expect(monitor.state().processActivityCount).toBe(1);+ clock.advance(999);+ expect(fireCount).toBe(0);+ clock.advance(1);+ expect(fireCount).toBe(1);+ monitor.stop();+ });});describe("createCodexOutputInactivityMonitor (acceptance criteria 2: does not fire)", () => {diff --git a/packages/adapters/codex-local/src/server/output-inactivity-monitor.ts b/packages/adapters/codex-local/src/server/output-inactivity-monitor.tsindex e1e0ca2f2f..20c1251cef 100644--- a/packages/adapters/codex-local/src/server/output-inactivity-monitor.ts+++ b/packages/adapters/codex-local/src/server/output-inactivity-monitor.ts@@ -34,6 +34,7 @@ export interface CodexOutputInactivityMonitorState {outputChunkCount: number;outputBytes: number;parsedEventCount: number;+ processActivityCount: number;}export interface CodexOutputInactivityMonitorOptions {@@ -51,6 +52,7 @@ export interface CodexOutputInactivityMonitorOptions {export interface CodexOutputInactivityMonitorHandle {noteOutputChunk(stream: "stdout" | "stderr", chunk: string): void;+ noteProcessActivity(): void;/** Returns the current state without stopping the timer. */state(): CodexOutputInactivityMonitorState;/** Cancels any pending timer and returns the final state. */@@ -85,6 +87,7 @@ export function createCodexOutputInactivityMonitor(outputChunkCount: 0,outputBytes: 0,parsedEventCount: 0,+ processActivityCount: 0,};let timerHandle: unknown = null;let stopped = false;@@ -120,6 +123,12 @@ export function createCodexOutputInactivityMonitor(state.lastEventAt = now();arm();},+ noteProcessActivity() {+ if (stopped || state.fired) return;+ state.processActivityCount += 1;+ state.lastEventAt = now();+ arm();+ },state() {return { ...state };},@@ -136,11 +145,11 @@ export function createCodexOutputInactivityMonitor(/*** Format the inactivity monitor error message in the canonical- * `monitor: no codex output for {N}m {S}s` shape consumed by NEE-81.+ * `monitor: no codex activity (output or process) for {N}m {S}s` shape consumed by NEE-81.*/export function formatOutputInactivityMonitorErrorMessage(elapsedMs: number): string {const total = Math.max(0, Math.round(elapsedMs / 1000));const minutes = Math.floor(total / 60);const seconds = total - minutes * 60;- return `monitor: no codex output for ${minutes}m ${seconds}s`;+ return `monitor: no codex activity (output or process) for ${minutes}m ${seconds}s`;}diff --git a/packages/adapters/codex-local/src/server/process-activity-monitor.test.ts b/packages/adapters/codex-local/src/server/process-activity-monitor.test.tsnew file mode 100644index 0000000000..5e18f17fc7--- /dev/null+++ b/packages/adapters/codex-local/src/server/process-activity-monitor.test.ts@@ -0,0 +1,95 @@+import { describe, expect, it, vi } from "vitest";+import {+ createCodexProcessActivityMonitor,+ type CodexProcessActivitySnapshot,+} from "./process-activity-monitor.js";++class PollHarness {+ private callback: (() => void) | null = null;++ setTimer = (callback: () => void) => {+ this.callback = callback;+ return callback;+ };++ clearTimer = () => {+ this.callback = null;+ };++ async poll(): Promise<void> {+ await vi.waitFor(() => expect(this.callback).not.toBeNull());+ const callback = this.callback;+ this.callback = null;+ callback?.();+ }+}++function snapshot(cpuTicks: number, ioBytes: number, processIds = "100"): CodexProcessActivitySnapshot {+ return { cpuTicks, ioBytes, processIds };+}++describe("createCodexProcessActivityMonitor", () => {+ it("requires a baseline and ignores sub-threshold CPU changes", async () => {+ const samples = [snapshot(100, 1_000), snapshot(114, 1_000)];+ const harness = new PollHarness();+ const onActivity = vi.fn();+ const monitor = createCodexProcessActivityMonitor({+ pid: 100,+ processGroupId: 100,+ intervalMs: 15_000,+ sample: async () => samples.shift() ?? null,+ setTimer: harness.setTimer,+ clearTimer: harness.clearTimer,+ onActivity,+ });++ await harness.poll();+ await vi.waitFor(() => expect(onActivity).not.toHaveBeenCalled());+ monitor.stop();+ });++ it.each([+ ["CPU growth", snapshot(115, 1_000)],+ ["I/O growth", snapshot(100, 1_001)],+ ["process-group churn", snapshot(100, 1_000, "100,101")],+ ])("reports %s as process activity", async (_label, activeSnapshot) => {+ const samples = [snapshot(100, 1_000), activeSnapshot];+ const harness = new PollHarness();+ const onActivity = vi.fn();+ const monitor = createCodexProcessActivityMonitor({+ pid: 100,+ processGroupId: 100,+ intervalMs: 15_000,+ sample: async () => samples.shift() ?? null,+ setTimer: harness.setTimer,+ clearTimer: harness.clearTimer,+ onActivity,+ });++ await harness.poll();+ await vi.waitFor(() => expect(onActivity).toHaveBeenCalledTimes(1));+ monitor.stop();+ });++ it("resets its comparison baseline after an unavailable sample", async () => {+ const samples = [snapshot(100, 1_000), null, snapshot(200, 2_000), snapshot(215, 2_000)];+ const harness = new PollHarness();+ const onActivity = vi.fn();+ const monitor = createCodexProcessActivityMonitor({+ pid: 100,+ processGroupId: 100,+ intervalMs: 15_000,+ sample: async () => samples.shift() ?? null,+ setTimer: harness.setTimer,+ clearTimer: harness.clearTimer,+ onActivity,+ });++ await harness.poll();+ await harness.poll();+ await vi.waitFor(() => expect(onActivity).not.toHaveBeenCalled());+ await harness.poll();+ await vi.waitFor(() => expect(onActivity).toHaveBeenCalledTimes(1));+ monitor.stop();+ });+});diff --git a/packages/adapters/codex-local/src/server/process-activity-monitor.ts b/packages/adapters/codex-local/src/server/process-activity-monitor.tsnew file mode 100644index 0000000000..e896759fc9--- /dev/null+++ b/packages/adapters/codex-local/src/server/process-activity-monitor.ts@@ -0,0 +1,129 @@+import fs from "node:fs/promises";++export const CODEX_PROCESS_ACTIVITY_POLL_INTERVAL_MS = 15_000;++export interface CodexProcessActivitySnapshot {+ cpuTicks: number;+ ioBytes: number;+ processIds: string;+}++export interface CodexProcessActivityMonitorOptions {+ pid: number;+ processGroupId: number | null;+ onActivity: () => void;+ intervalMs?: number;+ sample?: () => Promise<CodexProcessActivitySnapshot | null>;+ setTimer?: (cb: () => void, ms: number) => unknown;+ clearTimer?: (handle: unknown) => void;+}++export interface CodexProcessActivityMonitorHandle {+ stop(): void;+}++function parseProcStat(stat: string): { processGroupId: number; cpuTicks: number } | null {+ const commandEnd = stat.lastIndexOf(")");+ if (commandEnd < 0) return null;+ const fields = stat.slice(commandEnd + 2).trim().split(/\s+/);+ const processGroupId = Number(fields[2]);+ const userTicks = Number(fields[11]);+ const systemTicks = Number(fields[12]);+ if (![processGroupId, userTicks, systemTicks].every(Number.isFinite)) return null;+ return { processGroupId, cpuTicks: userTicks + systemTicks };+}++function parseProcIo(io: string): number {+ let bytes = 0;+ for (const line of io.split("\n")) {+ const match = /^(?:read_bytes|write_bytes):\s+(\d+)$/.exec(line.trim());+ if (match) bytes += Number(match[1]);+ }+ return bytes;+}++export async function sampleCodexProcessActivity(+ pid: number,+ processGroupId: number | null,+): Promise<CodexProcessActivitySnapshot | null> {+ if (process.platform !== "linux") return null;+ const targetProcessGroupId = processGroupId && processGroupId > 0 ? processGroupId : null;+ const entries = targetProcessGroupId ? await fs.readdir("/proc") : [String(pid)];+ const processIds: number[] = [];+ let cpuTicks = 0;+ let ioBytes = 0;++ await Promise.all(+ entries.map(async (entry) => {+ if (!/^\d+$/.test(entry)) return;+ try {+ const parsed = parseProcStat(await fs.readFile(`/proc/${entry}/stat`, "utf8"));+ if (!parsed) return;+ if (targetProcessGroupId !== null && parsed.processGroupId !== targetProcessGroupId) return;+ if (targetProcessGroupId === null && Number(entry) !== pid) return;+ const io = await fs.readFile(`/proc/${entry}/io`, "utf8").catch(() => "");+ processIds.push(Number(entry));+ cpuTicks += parsed.cpuTicks;+ ioBytes += parseProcIo(io);+ } catch {+ // Processes can exit between listing /proc and reading their stat file.+ }+ }),+ );++ if (processIds.length === 0) return null;+ processIds.sort((left, right) => left - right);+ return { cpuTicks, ioBytes, processIds: processIds.join(",") };+}++export function createCodexProcessActivityMonitor(+ options: CodexProcessActivityMonitorOptions,+): CodexProcessActivityMonitorHandle {+ const intervalMs = options.intervalMs ?? CODEX_PROCESS_ACTIVITY_POLL_INTERVAL_MS;+ const sample = options.sample ?? (() => sampleCodexProcessActivity(options.pid, options.processGroupId));+ const setTimer = options.setTimer ?? ((cb, ms) => setTimeout(cb, ms));+ const clearTimer = options.clearTimer ?? ((handle) => clearTimeout(handle as ReturnType<typeof setTimeout>));+ const minimumCpuTickDelta = Math.max(1, Math.floor(intervalMs / 1_000));+ let previous: CodexProcessActivitySnapshot | null = null;+ let timer: unknown = null;+ let stopped = false;++ const schedule = () => {+ if (stopped) return;+ timer = setTimer(() => {+ void poll();+ }, intervalMs);+ if (typeof (timer as { unref?: () => void }).unref === "function") {+ (timer as { unref: () => void }).unref();+ }+ };++ const poll = async () => {+ if (stopped) return;+ const current = await sample().catch(() => null);+ if (stopped) return;+ if (+ current &&+ previous &&+ (current.cpuTicks - previous.cpuTicks >= minimumCpuTickDelta ||+ current.ioBytes > previous.ioBytes ||+ current.processIds !== previous.processIds)+ ) {+ options.onActivity();+ }+ previous = current;+ schedule();+ };++ void poll();++ return {+ stop() {+ stopped = true;+ if (timer != null) {+ clearTimer(timer);+ timer = null;+ }+ },+ };+}