Multi-Agent Collaboration for Team and Friends
TypeScript agent-automationagent-orchestrationagent-workflowai-agentsai-collaborationai-memoryai-workforceautonomous-agentsclaude-codeclaude-code-skillscodexcoding-agentmulti-agentmultiple-agentone-person-businessone-person-companyone-person-teamopencodesolopreneur
fix(daemon): retry transient runtime spawn failures
Gus0fa64525e374
diff --git a/src/daemon/src/daemon/createDaemon.test.ts b/src/daemon/src/daemon/createDaemon.test.tsindex fc85b002b..1adf86dba 100644--- a/src/daemon/src/daemon/createDaemon.test.ts+++ b/src/daemon/src/daemon/createDaemon.test.ts@@ -563,6 +563,83 @@ describe("createDaemon — logging", () => {await daemon.stop();});+ it("retries a later wake after handshake_timeout instead of globally disabling the runtime", async () => {+ global.fetch = vi.fn(async (url: string | URL) => {+ const href = String(url);+ if (href.includes("/enroll-agent")) {+ return new Response(JSON.stringify({ runnerKey: "rk_retry" }), { status: 200 });+ }+ return new Response(JSON.stringify({ bots: [] }), { status: 200 });+ }) as unknown as typeof fetch;++ const spawn = vi.fn(async () => {+ const proc = new EventEmitter() as unknown as { kill: () => void };+ proc.kill = () => {};+ return { process: proc as never };+ });+ const driver = {+ ...fullFakeDriver("cursor"),+ spawn,+ } as unknown as Driver;+ const sockets: FakeSocket[] = [];+ const logger = stubLogger();+ const daemon = await createDaemon({+ machineKey: "cmk_handshake_retry",+ serverUrl: "http://localhost:9999",+ serverWsUrl: "ws://x",+ webSocketFactory: factory(sockets) as any,+ runtimeReport: [{ id: "cursor" }],+ driverFor: () => driver,+ capabilities: [],+ handshakeTimeoutMs: 10,+ logger,+ });++ try {+ sockets[0].emit("open");+ await new Promise((resolve) => setTimeout(resolve, 20));+ sockets[0].emit(+ "message",+ JSON.stringify({ type: "bot:added", botId: "bot_retry", name: "Retry Bot", discriminator: "0001" }),+ );+ const wake = (launchId: string, latestSeq: number) =>+ sockets[0].emit(+ "message",+ JSON.stringify({+ type: "agent:wake",+ agentId: "bot_retry",+ config: { version: 1, runtime: "cursor", model: { kind: "default" }, mode: { kind: "default" } },+ launchId,+ unreadNotice: { kind: "unread_notice", channel: "/demo#1234/general", latestSeq },+ }),+ );++ wake("launch_1", 1);+ await vi.waitFor(() => expect(spawn).toHaveBeenCalledTimes(1));+ await vi.waitFor(() =>+ expect(+ logger.calls.warn.some(+ ([, message, detail]) =>+ message === "spawn failed" && (detail[0] as { reason?: string })?.reason === "handshake_timeout",+ ),+ ).toBe(true),+ );++ wake("launch_2", 2);+ await vi.waitFor(() => expect(spawn).toHaveBeenCalledTimes(2));+ expect(logger.calls.warn.some(([, message]) => message === "runtime marked unhealthy")).toBe(false);+ expect(+ logger.calls.info.some(+ ([, message, detail]) =>+ message === "agent:wake ack" &&+ (detail[0] as { status?: string; "error.code"?: string })?.["error.code"] === "bot_runtime_missing",+ ),+ ).toBe(false);+ } finally {+ await daemon.stop();+ }+ });+it("threads the shared logger into WsControlChannel/AgentRouter/AgentProcessManager, and logs bot:removed + a successful wake through the manager", async () => {global.fetch = vi.fn(async (url: string | URL) => {const href = String(url);diff --git a/src/daemon/src/daemon/createDaemon.ts b/src/daemon/src/daemon/createDaemon.tsindex 6efe8c252..a0537e9c4 100644--- a/src/daemon/src/daemon/createDaemon.ts+++ b/src/daemon/src/daemon/createDaemon.ts@@ -248,6 +248,8 @@ export interface CreateDaemonOptions {capabilities: string[];/** Working directory base for agent launch contexts. */workingDirectoryBase?: string;+ /** Override the manager's pre-handshake watchdog (primarily for tests). */+ handshakeTimeoutMs?: number;/*** Directory for the DEFAULT-ON bounded FSM transition trace* (`<fsmTraceDir>/fsm-trace.jsonl`, size-capped/rotating — batch E1). When@@ -719,7 +721,7 @@ export async function createDaemon(opts: CreateDaemonOptions): Promise<RunningDareturn opts.driverFor(agentId, runtimeConfig);},onRuntimeSpawnFailed: (runtimeId, reason) => {- router?.markRuntimeUnhealthy(runtimeId, reason);+ router?.recordRuntimeSpawnFailure(runtimeId, reason);},onRuntimeSessionEstablished: (runtimeId) => {router?.markRuntimeHealthy(runtimeId);@@ -749,6 +751,9 @@ export async function createDaemon(opts: CreateDaemonOptions): Promise<RunningDa} as LaunchContext["config"],} as Omit<LaunchContext, "prompt" | "standingPrompt"> & { config?: LaunchContext["config"] };},+ ...(opts.handshakeTimeoutMs !== undefined+ ? { handshakeTimeoutMs: opts.handshakeTimeoutMs }+ : {}),tickIntervalMs: opts.tickIntervalMs ?? 2000,onAgentSession: (info) => void channel.reportAgentSession(info),onAgentActivity: (info) => {diff --git a/src/daemon/src/manager/agentRouter.test.ts b/src/daemon/src/manager/agentRouter.test.tsindex fe365ab0f..ec87dd633 100644--- a/src/daemon/src/manager/agentRouter.test.ts+++ b/src/daemon/src/manager/agentRouter.test.ts@@ -628,6 +628,48 @@ describe("AgentRouter — runtime health map", () => {expect(scheduled).toHaveLength(2);});+ it.each(["handshake_timeout", "pre_handshake_exit", "spawn_threw", "other"])(+ "keeps the runtime retryable after the transient spawn failure %s",+ (reason) => {+ const { mgr } = fakeManager();+ const { ch } = fakeChannel();+ const scheduled: Array<() => void> = [];+ const router = new AgentRouter({+ manager: mgr,+ channel: ch,+ runtimeReport: [{ id: "cursor" }],+ scheduleReadyResend: (fn) => scheduled.push(fn),+ });++ router.recordRuntimeSpawnFailure("cursor", reason);++ expect(router.isRuntimeHealthy("cursor")).toBe(true);+ expect(router.healthyRuntimeIds()).toEqual(["cursor"]);+ expect(scheduled).toHaveLength(0);+ },+ );++ it.each(["ENOENT", "EACCES", "ENOEXEC", "EPERM"])(+ "marks the runtime globally unhealthy after the definitive executable failure %s",+ (reason) => {+ const { mgr } = fakeManager();+ const { ch } = fakeChannel();+ const router = new AgentRouter({+ manager: mgr,+ channel: ch,+ runtimeReport: [{ id: "cursor" }],+ scheduleReadyResend: (fn) => fn(),+ });++ router.recordRuntimeSpawnFailure("cursor", reason);++ expect(router.isRuntimeHealthy("cursor")).toBe(false);+ expect(router.buildReady().runtimeReport).toEqual([+ expect.objectContaining({ id: "cursor", status: "unhealthy", lastError: reason }),+ ]);+ },+ );+it("markRuntimeHealthy clears lastError/lastErrorAt when flipping back", () => {const { mgr } = fakeManager();const { ch, readyResends } = fakeChannelWithSendReady();diff --git a/src/daemon/src/manager/agentRouter.ts b/src/daemon/src/manager/agentRouter.tsindex f03ebc029..ce181bfd8 100644--- a/src/daemon/src/manager/agentRouter.ts+++ b/src/daemon/src/manager/agentRouter.ts@@ -23,6 +23,8 @@ import type { AgentProcessManager } from "./managerRuntime.js";import type { TypingScopeTracker } from "./typingScopeTracker.js";import { createLogger, type Logger } from "../logger.js";+const DEFINITIVE_EXECUTABLE_FAILURES = new Set(["ENOENT", "EACCES", "ENOEXEC", "EPERM"]);+/*** Thrown by a `driverFor` implementation when the server asked for a runtime* that isn't available on this host. Caught by `AgentRouter` and forwarded@@ -243,10 +245,23 @@ export class AgentRouter {}/**- * Mark a runtime unhealthy after a spawn failure (ENOENT, pre-handshake- * exit, etc.). Idempotent: a call whose (status, lastError) matches the- * existing entry is a no-op and does NOT schedule a resend. Silent no-op- * on unknown ids — the daemon never advertised them, so we don't want to+ * Apply a pre-handshake spawn failure to host-wide runtime health only when+ * the error proves the executable is missing or cannot be executed. A+ * handshake timeout, early process exit, or unclassified start rejection is+ * scoped to that one launch: globally disabling the runtime would make+ * `driverFor` reject every later wake before another handshake could heal it.+ */+ recordRuntimeSpawnFailure(id: string, reason: string): void {+ if (!DEFINITIVE_EXECUTABLE_FAILURES.has(reason)) return;+ this.markRuntimeUnhealthy(id, reason);+ }++ /**+ * Mark a runtime explicitly unhealthy. Spawn-failure callers should use+ * `recordRuntimeSpawnFailure`, which keeps transient launch failures scoped+ * to their session. Idempotent: a call whose (status, lastError) matches the+ * existing entry is a no-op and does NOT schedule a resend. Silent no-op on+ * unknown ids — the daemon never advertised them, so we don't want to* synthesize a phantom entry on the wire.*/markRuntimeUnhealthy(id: string, reason: string): void {diff --git a/src/daemon/src/manager/managerRuntime.ts b/src/daemon/src/manager/managerRuntime.tsindex 138c65a4e..ede9faaee 100644--- a/src/daemon/src/manager/managerRuntime.ts+++ b/src/daemon/src/manager/managerRuntime.ts@@ -346,9 +346,10 @@ export interface ManagerRuntimeOpts {stampWakePromptTime?: boolean;/*** Notified when a spawn fails BEFORE the runtime emits its handshake- * `runtime_event` (pre-establishment error). Typically wired to- * `AgentRouter.markRuntimeUnhealthy` so /community reflects the broken- * runtime; see plans/community-machine-presence-fix.md.+ * `runtime_event` (pre-establishment error). The host may use definitive+ * executable errors to update global runtime health, but transient failures+ * such as `handshake_timeout` must remain scoped to this launch so a later+ * wake can retry.** `reason` is a short code like `"ENOENT"` or `"pre_handshake_exit"`.*/