Skip to content

armgit / alook

Public Imported from GitHub
0

Multi-Agent Collaboration for Team and Friends

fix(daemon): retry transient runtime spawn failures

Gus0fa64525e374
diff --git a/src/daemon/src/daemon/createDaemon.test.ts b/src/daemon/src/daemon/createDaemon.test.ts
index fc85b002b..1adf86dba 100644
--- a/src/daemon/src/daemon/createDaemon.test.ts
+++ b/src/daemon/src/daemon/createDaemon.test.ts
@@ -563,6 +563,83 @@ describe("createDaemon — logging", () => {
await daemon.stop();
});
+ it("retries a later wake after handshake_timeout instead of globally disabling the runtime", async () => {
+ global.fetch = vi.fn(async (url: string | URL) => {
+ const href = String(url);
+ if (href.includes("/enroll-agent")) {
+ return new Response(JSON.stringify({ runnerKey: "rk_retry" }), { status: 200 });
+ }
+ return new Response(JSON.stringify({ bots: [] }), { status: 200 });
+ }) as unknown as typeof fetch;
+
+ const spawn = vi.fn(async () => {
+ const proc = new EventEmitter() as unknown as { kill: () => void };
+ proc.kill = () => {};
+ return { process: proc as never };
+ });
+ const driver = {
+ ...fullFakeDriver("cursor"),
+ spawn,
+ } as unknown as Driver;
+ const sockets: FakeSocket[] = [];
+ const logger = stubLogger();
+ const daemon = await createDaemon({
+ machineKey: "cmk_handshake_retry",
+ serverUrl: "http://localhost:9999",
+ serverWsUrl: "ws://x",
+ webSocketFactory: factory(sockets) as any,
+ runtimeReport: [{ id: "cursor" }],
+ driverFor: () => driver,
+ capabilities: [],
+ handshakeTimeoutMs: 10,
+ logger,
+ });
+
+ try {
+ sockets[0].emit("open");
+ await new Promise((resolve) => setTimeout(resolve, 20));
+ sockets[0].emit(
+ "message",
+ JSON.stringify({ type: "bot:added", botId: "bot_retry", name: "Retry Bot", discriminator: "0001" }),
+ );
+ const wake = (launchId: string, latestSeq: number) =>
+ sockets[0].emit(
+ "message",
+ JSON.stringify({
+ type: "agent:wake",
+ agentId: "bot_retry",
+ config: { version: 1, runtime: "cursor", model: { kind: "default" }, mode: { kind: "default" } },
+ launchId,
+ unreadNotice: { kind: "unread_notice", channel: "/demo#1234/general", latestSeq },
+ }),
+ );
+
+ wake("launch_1", 1);
+ await vi.waitFor(() => expect(spawn).toHaveBeenCalledTimes(1));
+ await vi.waitFor(() =>
+ expect(
+ logger.calls.warn.some(
+ ([, message, detail]) =>
+ message === "spawn failed" && (detail[0] as { reason?: string })?.reason === "handshake_timeout",
+ ),
+ ).toBe(true),
+ );
+
+ wake("launch_2", 2);
+ await vi.waitFor(() => expect(spawn).toHaveBeenCalledTimes(2));
+ expect(logger.calls.warn.some(([, message]) => message === "runtime marked unhealthy")).toBe(false);
+ expect(
+ logger.calls.info.some(
+ ([, message, detail]) =>
+ message === "agent:wake ack" &&
+ (detail[0] as { status?: string; "error.code"?: string })?.["error.code"] === "bot_runtime_missing",
+ ),
+ ).toBe(false);
+ } finally {
+ await daemon.stop();
+ }
+ });
+
it("threads the shared logger into WsControlChannel/AgentRouter/AgentProcessManager, and logs bot:removed + a successful wake through the manager", async () => {
global.fetch = vi.fn(async (url: string | URL) => {
const href = String(url);
diff --git a/src/daemon/src/daemon/createDaemon.ts b/src/daemon/src/daemon/createDaemon.ts
index 6efe8c252..a0537e9c4 100644
--- a/src/daemon/src/daemon/createDaemon.ts
+++ b/src/daemon/src/daemon/createDaemon.ts
@@ -248,6 +248,8 @@ export interface CreateDaemonOptions {
capabilities: string[];
/** Working directory base for agent launch contexts. */
workingDirectoryBase?: string;
+ /** Override the manager's pre-handshake watchdog (primarily for tests). */
+ handshakeTimeoutMs?: number;
/**
* Directory for the DEFAULT-ON bounded FSM transition trace
* (`<fsmTraceDir>/fsm-trace.jsonl`, size-capped/rotating — batch E1). When
@@ -719,7 +721,7 @@ export async function createDaemon(opts: CreateDaemonOptions): Promise<RunningDa
return opts.driverFor(agentId, runtimeConfig);
},
onRuntimeSpawnFailed: (runtimeId, reason) => {
- router?.markRuntimeUnhealthy(runtimeId, reason);
+ router?.recordRuntimeSpawnFailure(runtimeId, reason);
},
onRuntimeSessionEstablished: (runtimeId) => {
router?.markRuntimeHealthy(runtimeId);
@@ -749,6 +751,9 @@ export async function createDaemon(opts: CreateDaemonOptions): Promise<RunningDa
} as LaunchContext["config"],
} as Omit<LaunchContext, "prompt" | "standingPrompt"> & { config?: LaunchContext["config"] };
},
+ ...(opts.handshakeTimeoutMs !== undefined
+ ? { handshakeTimeoutMs: opts.handshakeTimeoutMs }
+ : {}),
tickIntervalMs: opts.tickIntervalMs ?? 2000,
onAgentSession: (info) => void channel.reportAgentSession(info),
onAgentActivity: (info) => {
diff --git a/src/daemon/src/manager/agentRouter.test.ts b/src/daemon/src/manager/agentRouter.test.ts
index fe365ab0f..ec87dd633 100644
--- a/src/daemon/src/manager/agentRouter.test.ts
+++ b/src/daemon/src/manager/agentRouter.test.ts
@@ -628,6 +628,48 @@ describe("AgentRouter — runtime health map", () => {
expect(scheduled).toHaveLength(2);
});
+ it.each(["handshake_timeout", "pre_handshake_exit", "spawn_threw", "other"])(
+ "keeps the runtime retryable after the transient spawn failure %s",
+ (reason) => {
+ const { mgr } = fakeManager();
+ const { ch } = fakeChannel();
+ const scheduled: Array<() => void> = [];
+ const router = new AgentRouter({
+ manager: mgr,
+ channel: ch,
+ runtimeReport: [{ id: "cursor" }],
+ scheduleReadyResend: (fn) => scheduled.push(fn),
+ });
+
+ router.recordRuntimeSpawnFailure("cursor", reason);
+
+ expect(router.isRuntimeHealthy("cursor")).toBe(true);
+ expect(router.healthyRuntimeIds()).toEqual(["cursor"]);
+ expect(scheduled).toHaveLength(0);
+ },
+ );
+
+ it.each(["ENOENT", "EACCES", "ENOEXEC", "EPERM"])(
+ "marks the runtime globally unhealthy after the definitive executable failure %s",
+ (reason) => {
+ const { mgr } = fakeManager();
+ const { ch } = fakeChannel();
+ const router = new AgentRouter({
+ manager: mgr,
+ channel: ch,
+ runtimeReport: [{ id: "cursor" }],
+ scheduleReadyResend: (fn) => fn(),
+ });
+
+ router.recordRuntimeSpawnFailure("cursor", reason);
+
+ expect(router.isRuntimeHealthy("cursor")).toBe(false);
+ expect(router.buildReady().runtimeReport).toEqual([
+ expect.objectContaining({ id: "cursor", status: "unhealthy", lastError: reason }),
+ ]);
+ },
+ );
+
it("markRuntimeHealthy clears lastError/lastErrorAt when flipping back", () => {
const { mgr } = fakeManager();
const { ch, readyResends } = fakeChannelWithSendReady();
diff --git a/src/daemon/src/manager/agentRouter.ts b/src/daemon/src/manager/agentRouter.ts
index f03ebc029..ce181bfd8 100644
--- a/src/daemon/src/manager/agentRouter.ts
+++ b/src/daemon/src/manager/agentRouter.ts
@@ -23,6 +23,8 @@ import type { AgentProcessManager } from "./managerRuntime.js";
import type { TypingScopeTracker } from "./typingScopeTracker.js";
import { createLogger, type Logger } from "../logger.js";
+const DEFINITIVE_EXECUTABLE_FAILURES = new Set(["ENOENT", "EACCES", "ENOEXEC", "EPERM"]);
+
/**
* Thrown by a `driverFor` implementation when the server asked for a runtime
* that isn't available on this host. Caught by `AgentRouter` and forwarded
@@ -243,10 +245,23 @@ export class AgentRouter {
}
/**
- * Mark a runtime unhealthy after a spawn failure (ENOENT, pre-handshake
- * exit, etc.). Idempotent: a call whose (status, lastError) matches the
- * existing entry is a no-op and does NOT schedule a resend. Silent no-op
- * on unknown ids — the daemon never advertised them, so we don't want to
+ * Apply a pre-handshake spawn failure to host-wide runtime health only when
+ * the error proves the executable is missing or cannot be executed. A
+ * handshake timeout, early process exit, or unclassified start rejection is
+ * scoped to that one launch: globally disabling the runtime would make
+ * `driverFor` reject every later wake before another handshake could heal it.
+ */
+ recordRuntimeSpawnFailure(id: string, reason: string): void {
+ if (!DEFINITIVE_EXECUTABLE_FAILURES.has(reason)) return;
+ this.markRuntimeUnhealthy(id, reason);
+ }
+
+ /**
+ * Mark a runtime explicitly unhealthy. Spawn-failure callers should use
+ * `recordRuntimeSpawnFailure`, which keeps transient launch failures scoped
+ * to their session. Idempotent: a call whose (status, lastError) matches the
+ * existing entry is a no-op and does NOT schedule a resend. Silent no-op on
+ * unknown ids — the daemon never advertised them, so we don't want to
* synthesize a phantom entry on the wire.
*/
markRuntimeUnhealthy(id: string, reason: string): void {
diff --git a/src/daemon/src/manager/managerRuntime.ts b/src/daemon/src/manager/managerRuntime.ts
index 138c65a4e..ede9faaee 100644
--- a/src/daemon/src/manager/managerRuntime.ts
+++ b/src/daemon/src/manager/managerRuntime.ts
@@ -346,9 +346,10 @@ export interface ManagerRuntimeOpts {
stampWakePromptTime?: boolean;
/**
* Notified when a spawn fails BEFORE the runtime emits its handshake
- * `runtime_event` (pre-establishment error). Typically wired to
- * `AgentRouter.markRuntimeUnhealthy` so /community reflects the broken
- * runtime; see plans/community-machine-presence-fix.md.
+ * `runtime_event` (pre-establishment error). The host may use definitive
+ * executable errors to update global runtime health, but transient failures
+ * such as `handshake_timeout` must remain scoped to this launch so a later
+ * wake can retry.
*
* `reason` is a short code like `"ENOENT"` or `"pre_handshake_exit"`.
*/