Skip to content

armgit / alook

Public Imported from GitHub
0

Multi-Agent Collaboration for Team and Friends

feat: add session status update for not-connection

Gusa06f5818727e
diff --git a/src/cli/daemon/daemon.test.ts b/src/cli/daemon/daemon.test.ts
index 5d70c3c9a..c55cc9b58 100644
--- a/src/cli/daemon/daemon.test.ts
+++ b/src/cli/daemon/daemon.test.ts
@@ -138,6 +138,17 @@ vi.mock("fs", () => ({
unlinkSync: (...args: any[]) => mockUnlinkSync(...args),
}));
+const mockReaddir = vi.fn(async () => [] as string[]);
+const mockReadFile = vi.fn(async () => "");
+const mockUnlink = vi.fn(async () => undefined);
+const mockFsStat = vi.fn(async () => ({ mtimeMs: Date.now() }));
+vi.mock("fs/promises", () => ({
+ readdir: (...args: any[]) => mockReaddir(...args),
+ readFile: (...args: any[]) => mockReadFile(...args),
+ unlink: (...args: any[]) => mockUnlink(...args),
+ stat: (...args: any[]) => mockFsStat(...args),
+}));
+
vi.mock("url", () => ({
fileURLToPath: vi.fn(() => "/fake/daemon.ts"),
}));
@@ -198,7 +209,7 @@ import { spawn } from "child_process";
import { loadCLIConfigForProfile, saveCLIConfigForProfile } from "../lib/config.js";
import { releaseDaemonPid } from "./pidfile.js";
import { handleCliUpdate, readUpdateMarker, clearUpdateMarker } from "./update-handler.js";
-import { startDaemon, spawnSessionRunner, pruneSessionRunnerLogs } from "./daemon.js";
+import { startDaemon, spawnSessionRunner, pruneSessionRunnerLogs, isClientError, reconcilePendingCompletions } from "./daemon.js";
const mockReleaseDaemonPid = vi.mocked(releaseDaemonPid);
@@ -1581,3 +1592,213 @@ describe("daemon kill_task handling", () => {
expect(mockClientInstance.startTask).not.toHaveBeenCalled();
});
});
+
+describe("isClientError", () => {
+ it("HTTP 400: bad request → true", () => {
+ expect(isClientError(new Error("HTTP 400: bad request"))).toBe(true);
+ });
+
+ it("HTTP 404: not found → true", () => {
+ expect(isClientError(new Error("HTTP 404: not found"))).toBe(true);
+ });
+
+ it("HTTP 408: request timeout → false (transient)", () => {
+ expect(isClientError(new Error("HTTP 408: request timeout"))).toBe(false);
+ });
+
+ it("HTTP 429: too many requests → false (transient)", () => {
+ expect(isClientError(new Error("HTTP 429: too many requests"))).toBe(false);
+ });
+
+ it("HTTP 500: internal server error → false", () => {
+ expect(isClientError(new Error("HTTP 500: internal server error"))).toBe(false);
+ });
+
+ it("HTTP 503: service unavailable → false", () => {
+ expect(isClientError(new Error("HTTP 503: service unavailable"))).toBe(false);
+ });
+
+ it("Network error (no HTTP prefix) → false", () => {
+ expect(isClientError(new Error("TypeError: fetch failed"))).toBe(false);
+ });
+
+ it("Non-Error value → false", () => {
+ expect(isClientError("some string")).toBe(false);
+ expect(isClientError(42)).toBe(false);
+ expect(isClientError(null)).toBe(false);
+ });
+});
+
+describe("reconcilePendingCompletions", () => {
+ beforeEach(() => {
+ vi.clearAllMocks();
+ mockClientInstance.completeTask.mockResolvedValue({});
+ mockClientInstance.failTask.mockResolvedValue({});
+ });
+
+ function makeMarker(overrides?: Record<string, unknown>) {
+ return JSON.stringify({
+ taskId: "t1",
+ type: "complete",
+ payload: { output: "Done!" },
+ token: "tok_123",
+ serverURL: "http://localhost:8080",
+ createdAt: new Date().toISOString(),
+ ...overrides,
+ });
+ }
+
+ it("delivers 'complete' marker and deletes file", async () => {
+ mockReaddir.mockResolvedValueOnce(["t1.json"] as any);
+ mockReadFile.mockResolvedValueOnce(makeMarker());
+
+ await reconcilePendingCompletions("/tmp/ws");
+
+ expect(mockClientInstance.completeTask).toHaveBeenCalledWith("tok_123", "t1", { output: "Done!" });
+ expect(mockUnlink).toHaveBeenCalledWith("/tmp/ws/.pending_completions/t1.json");
+ });
+
+ it("delivers 'fail' marker and deletes file", async () => {
+ mockReaddir.mockResolvedValueOnce(["t1.json"] as any);
+ mockReadFile.mockResolvedValueOnce(makeMarker({ type: "fail", payload: { error: "boom" } }));
+
+ await reconcilePendingCompletions("/tmp/ws");
+
+ expect(mockClientInstance.failTask).toHaveBeenCalledWith("tok_123", "t1", "boom");
+ expect(mockUnlink).toHaveBeenCalled();
+ });
+
+ it("deletes marker on 4xx", async () => {
+ mockReaddir.mockResolvedValueOnce(["t1.json"] as any);
+ mockReadFile.mockResolvedValueOnce(makeMarker());
+ mockClientInstance.completeTask.mockRejectedValueOnce(new Error("HTTP 400: bad request"));
+
+ await reconcilePendingCompletions("/tmp/ws");
+
+ expect(mockUnlink).toHaveBeenCalledWith("/tmp/ws/.pending_completions/t1.json");
+ });
+
+ it("retains marker on network error", async () => {
+ mockReaddir.mockResolvedValueOnce(["t1.json"] as any);
+ mockReadFile.mockResolvedValueOnce(makeMarker());
+ mockClientInstance.completeTask.mockRejectedValueOnce(new Error("TypeError: fetch failed"));
+
+ await reconcilePendingCompletions("/tmp/ws");
+
+ expect(mockUnlink).not.toHaveBeenCalled();
+ });
+
+ it("retains marker on 5xx", async () => {
+ mockReaddir.mockResolvedValueOnce(["t1.json"] as any);
+ mockReadFile.mockResolvedValueOnce(makeMarker());
+ mockClientInstance.completeTask.mockRejectedValueOnce(new Error("HTTP 500: internal server error"));
+
+ await reconcilePendingCompletions("/tmp/ws");
+
+ expect(mockUnlink).not.toHaveBeenCalled();
+ });
+
+ it("deletes stale markers (>24h)", async () => {
+ const oldDate = new Date(Date.now() - 25 * 60 * 60 * 1000).toISOString();
+ mockReaddir.mockResolvedValueOnce(["t1.json"] as any);
+ mockReadFile.mockResolvedValueOnce(makeMarker({ createdAt: oldDate }));
+
+ await reconcilePendingCompletions("/tmp/ws");
+
+ expect(mockClientInstance.completeTask).not.toHaveBeenCalled();
+ expect(mockUnlink).toHaveBeenCalledWith("/tmp/ws/.pending_completions/t1.json");
+ });
+
+ it("deletes malformed JSON marker files and continues processing", async () => {
+ mockReaddir.mockResolvedValueOnce(["bad.json", "t1.json"] as any);
+ mockReadFile.mockResolvedValueOnce("{invalid json");
+ mockReadFile.mockResolvedValueOnce(makeMarker());
+
+ await reconcilePendingCompletions("/tmp/ws");
+
+ expect(mockUnlink).toHaveBeenCalledWith("/tmp/ws/.pending_completions/bad.json");
+ expect(mockClientInstance.completeTask).toHaveBeenCalledTimes(1);
+ });
+
+ it("deletes markers with invalid createdAt dates", async () => {
+ mockReaddir.mockResolvedValueOnce(["t1.json"] as any);
+ mockReadFile.mockResolvedValueOnce(makeMarker({ createdAt: "not-a-date" }));
+
+ await reconcilePendingCompletions("/tmp/ws");
+
+ expect(mockClientInstance.completeTask).not.toHaveBeenCalled();
+ expect(mockUnlink).toHaveBeenCalledWith("/tmp/ws/.pending_completions/t1.json");
+ });
+
+ it("skips structurally invalid marker files and deletes them", async () => {
+ mockReaddir.mockResolvedValueOnce(["bad.json"] as any);
+ mockReadFile.mockResolvedValueOnce(JSON.stringify({ type: "complete", taskId: "x", payload: {} }));
+
+ await reconcilePendingCompletions("/tmp/ws");
+
+ expect(mockClientInstance.completeTask).not.toHaveBeenCalled();
+ expect(mockUnlink).toHaveBeenCalled();
+ });
+
+ it("handles empty directory", async () => {
+ mockReaddir.mockResolvedValueOnce([] as any);
+
+ await reconcilePendingCompletions("/tmp/ws");
+
+ expect(mockClientInstance.completeTask).not.toHaveBeenCalled();
+ expect(mockClientInstance.failTask).not.toHaveBeenCalled();
+ });
+
+ it("handles missing directory", async () => {
+ mockReaddir.mockRejectedValueOnce(new Error("ENOENT"));
+
+ await expect(reconcilePendingCompletions("/tmp/ws")).resolves.toBeUndefined();
+ });
+
+ it("processes markers independently — failure on one doesn't abort rest", async () => {
+ mockReaddir.mockResolvedValueOnce(["t1.json", "t2.json", "t3.json"] as any);
+ mockReadFile.mockResolvedValueOnce(makeMarker({ taskId: "t1" }));
+ mockReadFile.mockResolvedValueOnce(makeMarker({ taskId: "t2" }));
+ mockReadFile.mockResolvedValueOnce(makeMarker({ taskId: "t3" }));
+ mockClientInstance.completeTask
+ .mockResolvedValueOnce({})
+ .mockRejectedValueOnce(new Error("TypeError: fetch failed"))
+ .mockResolvedValueOnce({});
+
+ await reconcilePendingCompletions("/tmp/ws");
+
+ expect(mockClientInstance.completeTask).toHaveBeenCalledTimes(3);
+ // t1 and t3 delivered → deleted, t2 failed → retained
+ expect(mockUnlink).toHaveBeenCalledWith("/tmp/ws/.pending_completions/t1.json");
+ expect(mockUnlink).not.toHaveBeenCalledWith("/tmp/ws/.pending_completions/t2.json");
+ expect(mockUnlink).toHaveBeenCalledWith("/tmp/ws/.pending_completions/t3.json");
+ });
+
+ it("ignores .tmp files (not processed as markers)", async () => {
+ mockReaddir.mockResolvedValueOnce(["t1.tmp"] as any);
+ mockFsStat.mockResolvedValueOnce({ mtimeMs: Date.now() } as any);
+
+ await reconcilePendingCompletions("/tmp/ws");
+
+ expect(mockClientInstance.completeTask).not.toHaveBeenCalled();
+ expect(mockReadFile).not.toHaveBeenCalled();
+ });
+
+ it("cleans up stale .tmp files (>1h)", async () => {
+ mockReaddir.mockResolvedValueOnce(["t1.tmp"] as any);
+ mockFsStat.mockResolvedValueOnce({ mtimeMs: Date.now() - 2 * 60 * 60 * 1000 } as any);
+
+ await reconcilePendingCompletions("/tmp/ws");
+
+ expect(mockUnlink).toHaveBeenCalledWith("/tmp/ws/.pending_completions/t1.tmp");
+ });
+
+ it("keeps recent .tmp files", async () => {
+ mockReaddir.mockResolvedValueOnce(["t1.tmp"] as any);
+ mockFsStat.mockResolvedValueOnce({ mtimeMs: Date.now() - 5 * 60 * 1000 } as any);
+
+ await reconcilePendingCompletions("/tmp/ws");
+
+ expect(mockUnlink).not.toHaveBeenCalled();
+ });
+});
diff --git a/src/cli/daemon/daemon.ts b/src/cli/daemon/daemon.ts
index f78769736..1cf79ea54 100644
--- a/src/cli/daemon/daemon.ts
+++ b/src/cli/daemon/daemon.ts
@@ -3,6 +3,7 @@ import { type DaemonConfig, loadDaemonConfig, sessionRunnerLogDir, daemonLogFile
import { createHealthServer } from "./health.js";
import { detectVersion } from "./agent/index.js";
import { type Task, type SessionRunnerInput, fromApiTask } from "./types.js";
+import type { MarkerData } from "./session-runner.js";
import { loadCLIConfigForProfile, saveCLIConfigForProfile } from "../lib/config.js";
import { log } from "../lib/logger.js";
import { cmdPrefix } from "../lib/env.js";
@@ -18,6 +19,7 @@ import {
} from "./execenv/steering.js";
import { TASK_TYPES } from "@alook/shared";
import { existsSync, mkdirSync, openSync, closeSync, readdirSync, statSync, unlinkSync } from "fs";
+import { readdir, readFile, unlink, stat as fsStat } from "fs/promises";
import { execSync, spawn, type ChildProcess } from "child_process";
import { fileURLToPath } from "url";
import { dirname, join } from "path";
@@ -80,6 +82,113 @@ export function pruneSessionRunnerLogs(): void {
}
}
+export function isClientError(error: unknown): boolean {
+ if (!(error instanceof Error)) return false;
+ const match = error.message.match(/^HTTP (\d+):/);
+ if (!match) return false;
+ const status = Number(match[1]);
+ if (status === 408 || status === 429) return false;
+ return status >= 400 && status < 500;
+}
+
+function isValidMarker(data: unknown): data is MarkerData {
+ if (!data || typeof data !== "object") return false;
+ const d = data as Record<string, unknown>;
+ if (typeof d.taskId !== "string") return false;
+ if (typeof d.token !== "string") return false;
+ if (typeof d.serverURL !== "string") return false;
+ if (typeof d.createdAt !== "string" || isNaN(new Date(d.createdAt).getTime())) return false;
+ if (!d.payload || typeof d.payload !== "object") return false;
+ const payload = d.payload as Record<string, unknown>;
+ if (d.type === "complete") {
+ return typeof payload.output === "string";
+ }
+ if (d.type === "fail") {
+ return typeof payload.error === "string";
+ }
+ return false;
+}
+
+const MARKER_STALE_MS = 24 * 60 * 60 * 1000;
+const TMP_STALE_MS = 60 * 60 * 1000;
+
+export async function reconcilePendingCompletions(workspacesRoot: string): Promise<void> {
+ const dir = join(workspacesRoot, ".pending_completions");
+ let entries: string[];
+ try {
+ entries = await readdir(dir);
+ } catch {
+ return;
+ }
+
+ // Housekeeping: clean up stale .tmp files
+ for (const name of entries) {
+ if (!name.endsWith(".tmp")) continue;
+ try {
+ const s = await fsStat(join(dir, name));
+ if (Date.now() - s.mtimeMs > TMP_STALE_MS) {
+ await unlink(join(dir, name));
+ }
+ } catch { /* best-effort */ }
+ }
+
+ const jsonFiles = entries.filter((f) => f.endsWith(".json"));
+ for (const name of jsonFiles) {
+ const filePath = join(dir, name);
+ try {
+ let raw: string;
+ try {
+ raw = await readFile(filePath, "utf-8");
+ } catch {
+ continue;
+ }
+
+ let parsed: unknown;
+ try {
+ parsed = JSON.parse(raw);
+ } catch {
+ log.warn(`reconcile: malformed marker ${name}, deleting`);
+ try { await unlink(filePath); } catch { /* best-effort */ }
+ continue;
+ }
+
+ if (!isValidMarker(parsed)) {
+ log.warn(`reconcile: invalid marker structure ${name}, deleting`);
+ try { await unlink(filePath); } catch { /* best-effort */ }
+ continue;
+ }
+
+ const marker = parsed;
+ const age = Date.now() - new Date(marker.createdAt).getTime();
+ if (age > MARKER_STALE_MS) {
+ log.warn(`reconcile: stale marker ${name} (${Math.round(age / 3600000)}h old), deleting`);
+ try { await unlink(filePath); } catch { /* best-effort */ }
+ continue;
+ }
+
+ const client = new DaemonClient(marker.serverURL);
+ try {
+ if (marker.type === "complete") {
+ await client.completeTask(marker.token, marker.taskId, marker.payload);
+ } else {
+ await client.failTask(marker.token, marker.taskId, marker.payload.error);
+ }
+ try { await unlink(filePath); } catch (delErr) {
+ log.warn(`reconcile: delivered marker ${name} but failed to delete: ${delErr}`);
+ }
+ } catch (deliverErr) {
+ if (isClientError(deliverErr)) {
+ try { await unlink(filePath); } catch { /* best-effort */ }
+ } else {
+ log.debug(`reconcile: delivery failed for ${name}, will retry next cycle`);
+ }
+ }
+ } catch (e) {
+ log.debug(`reconcile: error processing ${name}`, e);
+ }
+ }
+}
+
export async function startDaemon(
profile?: string,
serverUrl?: string,
@@ -326,6 +435,12 @@ export async function startDaemon(
evictWorkspace(id);
}
+ try {
+ await reconcilePendingCompletions(config.workspacesRoot);
+ } catch (e) {
+ log.debug("reconciliation error", e);
+ }
+
if (workspaceStates.length === 0) {
log.info("All workspaces evicted — shutting down");
shutdown();
diff --git a/src/cli/daemon/session-runner.test.ts b/src/cli/daemon/session-runner.test.ts
index 0a0d720a1..aaef6965d 100644
--- a/src/cli/daemon/session-runner.test.ts
+++ b/src/cli/daemon/session-runner.test.ts
@@ -86,7 +86,22 @@ vi.mock("../lib/logger.js", () => ({
log: { info: vi.fn(), error: vi.fn(), debug: vi.fn(), warn: vi.fn() },
}));
-import { runSession } from "./session-runner.js";
+const mockMkdir = vi.fn(async () => undefined);
+const mockWriteFile = vi.fn(async () => undefined);
+const mockRename = vi.fn(async () => undefined);
+const mockRm = vi.fn(async () => undefined);
+vi.mock("fs/promises", () => ({
+ mkdir: (...args: any[]) => mockMkdir(...args),
+ writeFile: (...args: any[]) => mockWriteFile(...args),
+ rename: (...args: any[]) => mockRename(...args),
+ rm: (...args: any[]) => mockRm(...args),
+ readdir: vi.fn(async () => []),
+ readFile: vi.fn(async () => ""),
+ unlink: vi.fn(async () => undefined),
+ stat: vi.fn(async () => ({ mtimeMs: 0 })),
+}));
+
+import { runSession, writeMarkerFile, reportToServer, type MarkerData } from "./session-runner.js";
import { createBackend } from "./agent/index.js";
import { buildPrompt } from "./prompt.js";
import { log as mockLog } from "../lib/logger.js";
@@ -953,3 +968,208 @@ describe("session-runner runSession", () => {
});
});
});
+
+describe("writeMarkerFile", () => {
+ beforeEach(() => {
+ vi.clearAllMocks();
+ });
+
+ const marker: MarkerData = {
+ taskId: "t1",
+ type: "complete",
+ payload: { output: "Done!", session_id: "sess-1" },
+ token: "tok",
+ serverURL: "http://localhost",
+ createdAt: "2026-04-20T21:00:00Z",
+ };
+
+ it("writes marker with correct JSON structure", async () => {
+ await writeMarkerFile("/tmp/ws", marker);
+
+ expect(mockMkdir).toHaveBeenCalledWith("/tmp/ws/.pending_completions", { recursive: true, mode: 0o700 });
+ expect(mockWriteFile).toHaveBeenCalledWith(
+ "/tmp/ws/.pending_completions/t1.tmp",
+ JSON.stringify(marker),
+ { mode: 0o600 },
+ );
+ expect(mockRename).toHaveBeenCalledWith(
+ "/tmp/ws/.pending_completions/t1.tmp",
+ "/tmp/ws/.pending_completions/t1.json",
+ );
+ });
+
+ it("creates .pending_completions/ directory if missing", async () => {
+ await writeMarkerFile("/tmp/ws", marker);
+ expect(mockMkdir).toHaveBeenCalledWith("/tmp/ws/.pending_completions", { recursive: true, mode: 0o700 });
+ });
+
+ it("works when directory already exists", async () => {
+ await writeMarkerFile("/tmp/ws", marker);
+ await writeMarkerFile("/tmp/ws", { ...marker, taskId: "t2" });
+ expect(mockMkdir).toHaveBeenCalledTimes(2);
+ expect(mockWriteFile).toHaveBeenCalledTimes(2);
+ });
+
+ it("atomic write: uses .tmp then renames to .json", async () => {
+ await writeMarkerFile("/tmp/ws", marker);
+ const writeCall = mockWriteFile.mock.calls[0][0];
+ const renameArgs = mockRename.mock.calls[0];
+ expect(writeCall).toContain(".tmp");
+ expect(renameArgs[0]).toContain(".tmp");
+ expect(renameArgs[1]).toContain(".json");
+ });
+
+ it("write failure propagates", async () => {
+ mockWriteFile.mockRejectedValueOnce(new Error("ENOSPC"));
+ await expect(writeMarkerFile("/tmp/ws", marker)).rejects.toThrow("ENOSPC");
+ });
+});
+
+describe("reportToServer", () => {
+ beforeEach(() => {
+ vi.clearAllMocks();
+ });
+
+ const marker: MarkerData = {
+ taskId: "t1",
+ type: "complete",
+ payload: { output: "Done!" },
+ token: "tok",
+ serverURL: "http://localhost",
+ createdAt: "2026-04-20T21:00:00Z",
+ };
+
+ it("succeeds without writing marker", async () => {
+ await reportToServer(async () => ({}), marker, "/tmp/ws");
+ expect(mockWriteFile).not.toHaveBeenCalled();
+ });
+
+ it("writes marker on fn failure", async () => {
+ await reportToServer(async () => { throw new Error("network"); }, marker, "/tmp/ws");
+ expect(mockWriteFile).toHaveBeenCalled();
+ const written = JSON.parse(mockWriteFile.mock.calls[0][1] as string);
+ expect(written.taskId).toBe("t1");
+ expect(written.type).toBe("complete");
+ });
+
+ it("does not throw when fn fails", async () => {
+ await expect(
+ reportToServer(async () => { throw new Error("network"); }, marker, "/tmp/ws"),
+ ).resolves.toBeUndefined();
+ });
+
+ it("does not throw when both fn and marker write fail", async () => {
+ mockWriteFile.mockRejectedValueOnce(new Error("ENOSPC"));
+ await expect(
+ reportToServer(async () => { throw new Error("network"); }, marker, "/tmp/ws"),
+ ).resolves.toBeUndefined();
+ });
+});
+
+describe("session-runner marker integration", () => {
+ beforeEach(() => {
+ vi.clearAllMocks();
+ });
+
+ it("completeTask failure writes 'complete' marker", async () => {
+ mockClientInstance.completeTask.mockRejectedValueOnce(new Error("server down"));
+ setupBackend([], {
+ status: "completed",
+ output: "Done!",
+ error: "",
+ durationMs: 1000,
+ sessionId: "sess-1",
+ });
+
+ await runSession(makeInput());
+
+ expect(mockWriteFile).toHaveBeenCalled();
+ const written = JSON.parse(mockWriteFile.mock.calls[0][1] as string);
+ expect(written.type).toBe("complete");
+ expect(written.taskId).toBe("t1");
+ expect(written.payload.output).toBe("Done!");
+ });
+
+ it("failTask failure writes 'fail' marker", async () => {
+ mockClientInstance.failTask.mockRejectedValueOnce(new Error("server down"));
+ setupBackend([], {
+ status: "failed",
+ output: "",
+ error: "something broke",
+ durationMs: 100,
+ sessionId: "sess-1",
+ });
+
+ await runSession(makeInput());
+
+ expect(mockWriteFile).toHaveBeenCalled();
+ const written = JSON.parse(mockWriteFile.mock.calls[0][1] as string);
+ expect(written.type).toBe("fail");
+ expect(written.taskId).toBe("t1");
+ expect(written.payload.error).toBe("something broke");
+ });
+
+ it("onKill cancel writes marker on failure", async () => {
+ const killSpy = vi.spyOn(process, "kill").mockImplementation(() => true);
+ const exitSpy = vi.spyOn(process, "exit").mockImplementation((() => {}) as any);
+ mockClientInstance.failTask.mockRejectedValueOnce(new Error("server down"));
+ mockReadKillIntent.mockReturnValueOnce({ reason: "cancelled", targetTaskId: "t1", expectedPid: process.pid });
+
+ let resolveMessage: (() => void) | null = null;
+ mockBackendExecute.mockReturnValue({
+ pid: 99999,
+ messages: (async function* () {
+ yield { type: "text", content: "working..." };
+ await new Promise<void>((resolve) => { resolveMessage = resolve; });
+ })(),
+ sessionId: Promise.resolve("sess-cancel"),
+ result: new Promise(() => {}),
+ });
+
+ const sessionPromise = runSession(makeInput());
+ await new Promise((r) => setTimeout(r, 50));
+ process.emit("SIGTERM", "SIGTERM");
+ if (resolveMessage) resolveMessage();
+ await new Promise((r) => setTimeout(r, 50));
+
+ expect(mockWriteFile).toHaveBeenCalled();
+ const written = JSON.parse(mockWriteFile.mock.calls[0][1] as string);
+ expect(written.type).toBe("fail");
+ expect(written.payload.error).toBe("cancelled by user");
+
+ killSpy.mockRestore();
+ exitSpy.mockRestore();
+ });
+
+ it("onKill kill writes marker on failure", async () => {
+ const killSpy = vi.spyOn(process, "kill").mockImplementation(() => true);
+ const exitSpy = vi.spyOn(process, "exit").mockImplementation((() => {}) as any);
+ mockClientInstance.failTask.mockRejectedValueOnce(new Error("server down"));
+ mockReadKillIntent.mockReturnValueOnce(null);
+
+ let resolveMessage: (() => void) | null = null;
+ mockBackendExecute.mockReturnValue({
+ pid: 99999,
+ messages: (async function* () {
+ yield { type: "text", content: "working..." };
+ await new Promise<void>((resolve) => { resolveMessage = resolve; });
+ })(),
+ sessionId: Promise.resolve("sess-kill"),
+ result: new Promise(() => {}),
+ });
+
+ const sessionPromise = runSession(makeInput());
+ await new Promise((r) => setTimeout(r, 50));
+ process.emit("SIGTERM", "SIGTERM");
+ if (resolveMessage) resolveMessage();
+ await new Promise((r) => setTimeout(r, 50));
+
+ expect(mockWriteFile).toHaveBeenCalled();
+ const written = JSON.parse(mockWriteFile.mock.calls[0][1] as string);
+ expect(written.type).toBe("fail");
+ expect(written.payload.error).toBe("killed by signal");
+
+ killSpy.mockRestore();
+ exitSpy.mockRestore();
+ });
+});
diff --git a/src/cli/daemon/session-runner.ts b/src/cli/daemon/session-runner.ts
index 61ba73b55..7c97cdc21 100644
--- a/src/cli/daemon/session-runner.ts
+++ b/src/cli/daemon/session-runner.ts
@@ -6,7 +6,7 @@
* log capture, and server completion reporting.
*/
-import { mkdir, writeFile, rm } from "fs/promises";
+import { mkdir, writeFile, rm, rename } from "fs/promises";
import path from "path";
import { DaemonClient } from "./client.js";
import { createBackend } from "./agent/index.js";
@@ -24,6 +24,55 @@ import type { SessionRunnerInput, Attachment } from "./types.js";
const ATTACHMENTS_BASE = "/tmp/alook-attachments";
+// --- Marker file support for resilient server reporting ---
+
+export type MarkerData =
+ | {
+ taskId: string;
+ type: "complete";
+ payload: { output: string; session_id?: string; branch_name?: string };
+ token: string;
+ serverURL: string;
+ createdAt: string;
+ }
+ | {
+ taskId: string;
+ type: "fail";
+ payload: { error: string };
+ token: string;
+ serverURL: string;
+ createdAt: string;
+ };
+
+export async function writeMarkerFile(
+ workspacesRoot: string,
+ marker: MarkerData,
+): Promise<void> {
+ const dir = path.join(workspacesRoot, ".pending_completions");
+ await mkdir(dir, { recursive: true, mode: 0o700 });
+ const tmpPath = path.join(dir, `${marker.taskId}.tmp`);
+ const finalPath = path.join(dir, `${marker.taskId}.json`);
+ await writeFile(tmpPath, JSON.stringify(marker), { mode: 0o600 });
+ await rename(tmpPath, finalPath);
+}
+
+export async function reportToServer(
+ fn: () => Promise<unknown>,
+ markerData: MarkerData,
+ workspacesRoot: string,
+): Promise<void> {
+ try {
+ await fn();
+ } catch (e) {
+ log.warn(`server report failed for task ${markerData.taskId}, writing marker: ${e}`);
+ try {
+ await writeMarkerFile(workspacesRoot, markerData);
+ } catch (writeErr) {
+ log.error(`marker write also failed for task ${markerData.taskId}: ${writeErr}`);
+ }
+ }
+}
+
function sanitizeFilename(name: string): string {
return path.basename(name).replace(/[/\\]/g, "_").replace(/\.\./g, "_").slice(0, 255) || "file";
}
@@ -190,9 +239,11 @@ export async function runSession(input: SessionRunnerInput): Promise<void> {
entry.status = "cancelled";
entry.errmsg = "cancelled by user";
});
- try {
- await client.failTask(token, task.id, "cancelled by user");
- } catch { /* best-effort */ }
+ await reportToServer(
+ () => client.failTask(token, task.id, "cancelled by user"),
+ { taskId: task.id, type: "fail", payload: { error: "cancelled by user" }, token, serverURL, createdAt: new Date().toISOString() },
+ workspacesRoot,
+ );
} else {
// No intent file: preserve existing behavior
updateEntry(timelineDir, task.id, (entry) => {
@@ -200,9 +251,11 @@ export async function runSession(input: SessionRunnerInput): Promise<void> {
entry.status = "killed";
entry.errmsg = "killed by signal";
});
- try {
- await client.failTask(token, task.id, "killed by signal");
- } catch { /* best-effort */ }
+ await reportToServer(
+ () => client.failTask(token, task.id, "killed by signal"),
+ { taskId: task.id, type: "fail", payload: { error: "killed by signal" }, token, serverURL, createdAt: new Date().toISOString() },
+ workspacesRoot,
+ );
}
process.exit(1);
@@ -285,11 +338,20 @@ export async function runSession(input: SessionRunnerInput): Promise<void> {
};
if (result.sessionId) body.session_id = result.sessionId;
// branchName is currently always undefined — forward-compat passthrough
- await client.completeTask(token, task.id, body);
+ await reportToServer(
+ () => client.completeTask(token, task.id, body),
+ { taskId: task.id, type: "complete", payload: body, token, serverURL, createdAt: new Date().toISOString() },
+ workspacesRoot,
+ );
const dur = (result.durationMs / 1000).toFixed(1);
log.info(`completed (duration=${dur}s, messages=${seq}, tools=${toolCount})`);
} else {
- await client.failTask(token, task.id, result.error || "unknown error");
+ const errorMsg = result.error || "unknown error";
+ await reportToServer(
+ () => client.failTask(token, task.id, errorMsg),
+ { taskId: task.id, type: "fail", payload: { error: errorMsg }, token, serverURL, createdAt: new Date().toISOString() },
+ workspacesRoot,
+ );
const dur = (result.durationMs / 1000).toFixed(1);
log.info(`failed (duration=${dur}s, messages=${seq}, tools=${toolCount}) — ${result.error}`);
}
@@ -320,11 +382,12 @@ async function main(): Promise<void> {
} catch (e) {
log.error(`session-runner: unhandled error for task ${input.task.id}`, e);
await cleanupAttachments(input.task.id);
- try {
- await client.failTask(input.token, input.task.id, `session-runner crash: ${e}`);
- } catch {
- // best-effort
- }
+ const errorMsg = `session-runner crash: ${e}`;
+ await reportToServer(
+ () => client.failTask(input.token, input.task.id, errorMsg),
+ { taskId: input.task.id, type: "fail", payload: { error: errorMsg }, token: input.token, serverURL: input.serverURL, createdAt: new Date().toISOString() },
+ input.workspacesRoot,
+ );
process.exit(1);
}
}
diff --git a/src/shared/src/db/queries/task.ts b/src/shared/src/db/queries/task.ts
index ba8336664..c5a249a94 100644
--- a/src/shared/src/db/queries/task.ts
+++ b/src/shared/src/db/queries/task.ts
@@ -1,5 +1,5 @@
import { eq, and, desc, asc, inArray, notInArray, ne, count, lt, sql } from "drizzle-orm";
-import { agentTaskQueue } from "../schema";
+import { agentTaskQueue, taskMessage } from "../schema";
import type { Database } from "../index";
import { ClaimedTaskRowSchema } from "../../schemas";
import { TASK_TYPES } from "../../constants";
@@ -499,3 +499,31 @@ export async function failStaleKillTasks(db: Database, workspaceId: string) {
.returning({ id: agentTaskQueue.id });
return rows;
}
+
+const DEFAULT_STALE_RUNNING_SECONDS = Number(process.env.ALOOK_STALE_RUNNING_TIMEOUT_S) || 3600;
+
+export async function failStaleRunningTasks(db: Database, workspaceId: string, staleSeconds = DEFAULT_STALE_RUNNING_SECONDS) {
+ const threshold = new Date(Date.now() - staleSeconds * 1000).toISOString();
+ const rows = await db
+ .update(agentTaskQueue)
+ .set({
+ status: "failed",
+ completedAt: new Date().toISOString(),
+ error: `timed out in running state (no message activity for ${Math.round(staleSeconds / 60)} minutes)`,
+ })
+ .where(
+ and(
+ eq(agentTaskQueue.workspaceId, workspaceId),
+ eq(agentTaskQueue.status, "running"),
+ lt(
+ sql`coalesce(
+ (select max(${taskMessage.createdAt}) from ${taskMessage} where ${taskMessage.taskId} = ${agentTaskQueue.id}),
+ ${agentTaskQueue.startedAt}
+ )`,
+ threshold
+ )
+ )
+ )
+ .returning({ agentId: agentTaskQueue.agentId, workspaceId: agentTaskQueue.workspaceId, conversationId: agentTaskQueue.conversationId });
+ return rows;
+}
diff --git a/src/shared/test/queries/task.test.ts b/src/shared/test/queries/task.test.ts
index 6863bdff1..ca0d31ca2 100644
--- a/src/shared/test/queries/task.test.ts
+++ b/src/shared/test/queries/task.test.ts
@@ -17,6 +17,10 @@ describe("task query module exports", () => {
it("exports getActiveTaskByConversation", () => {
expect(typeof taskQueries.getActiveTaskByConversation).toBe("function");
});
+
+ it("exports failStaleRunningTasks", () => {
+ expect(typeof taskQueries.failStaleRunningTasks).toBe("function");
+ });
});
describe("task query function signatures", () => {
diff --git a/src/web/src/lib/services/sweep.test.ts b/src/web/src/lib/services/sweep.test.ts
index 276a3fbea..8d35c083c 100644
--- a/src/web/src/lib/services/sweep.test.ts
+++ b/src/web/src/lib/services/sweep.test.ts
@@ -2,6 +2,7 @@ import { describe, it, expect, vi, beforeEach } from "vitest";
const mockFailStaleDispatchedTasks = vi.fn();
const mockFailStaleKillTasks = vi.fn();
+const mockFailStaleRunningTasks = vi.fn();
const mockCountRunningTasks = vi.fn();
const mockUpdateAgentStatus = vi.fn();
const mockActivateNextBufferedMessage = vi.fn();
@@ -16,6 +17,7 @@ vi.mock("@alook/shared", () => ({
task: {
failStaleDispatchedTasks: (...args: unknown[]) => mockFailStaleDispatchedTasks(...args),
failStaleKillTasks: (...args: unknown[]) => mockFailStaleKillTasks(...args),
+ failStaleRunningTasks: (...args: unknown[]) => mockFailStaleRunningTasks(...args),
countRunningTasks: (...args: unknown[]) => mockCountRunningTasks(...args),
createTask: (...args: unknown[]) => mockCreateTask(...args),
},
@@ -56,6 +58,7 @@ describe("sweepStaleState", () => {
beforeEach(() => {
vi.clearAllMocks();
mockFailStaleKillTasks.mockResolvedValue([]);
+ mockFailStaleRunningTasks.mockResolvedValue([]);
mockActivateNextBufferedMessage.mockResolvedValue(null);
});
@@ -140,4 +143,72 @@ describe("sweepStaleState", () => {
expect(mockFailStaleKillTasks).toHaveBeenCalledWith(db, "w1");
});
+
+ it("calls failStaleRunningTasks with workspace", async () => {
+ mockFailStaleDispatchedTasks.mockResolvedValue([]);
+
+ await sweepStaleState(db, "w1");
+
+ expect(mockFailStaleRunningTasks).toHaveBeenCalledWith(db, "w1");
+ });
+
+ it("reconciles agents from both dispatched and running sweeps", async () => {
+ mockFailStaleDispatchedTasks.mockResolvedValue([
+ { agentId: "a1", workspaceId: "w1", conversationId: "c1" },
+ ]);
+ mockFailStaleRunningTasks.mockResolvedValue([
+ { agentId: "a2", workspaceId: "w1", conversationId: "c2" },
+ ]);
+ mockCountRunningTasks.mockResolvedValue(0);
+ mockUpdateAgentStatus.mockResolvedValue(undefined);
+
+ await sweepStaleState(db, "w1");
+
+ expect(mockCountRunningTasks).toHaveBeenCalledTimes(2);
+ expect(mockUpdateAgentStatus).toHaveBeenCalledTimes(2);
+ });
+
+ it("deduplicates agents across both sweeps", async () => {
+ mockFailStaleDispatchedTasks.mockResolvedValue([
+ { agentId: "a1", workspaceId: "w1", conversationId: "c1" },
+ ]);
+ mockFailStaleRunningTasks.mockResolvedValue([
+ { agentId: "a1", workspaceId: "w1", conversationId: "c2" },
+ ]);
+ mockCountRunningTasks.mockResolvedValue(0);
+ mockUpdateAgentStatus.mockResolvedValue(undefined);
+
+ await sweepStaleState(db, "w1");
+
+ expect(mockCountRunningTasks).toHaveBeenCalledTimes(1);
+ expect(mockUpdateAgentStatus).toHaveBeenCalledTimes(1);
+ });
+
+ it("dispatches buffered messages for stale running tasks", async () => {
+ mockFailStaleDispatchedTasks.mockResolvedValue([]);
+ mockFailStaleRunningTasks.mockResolvedValue([
+ { agentId: "a1", workspaceId: "w1", conversationId: "c1" },
+ ]);
+ mockCountRunningTasks.mockResolvedValue(0);
+ mockUpdateAgentStatus.mockResolvedValue(undefined);
+
+ await sweepStaleState(db, "w1");
+
+ expect(mockActivateNextBufferedMessage).toHaveBeenCalledWith(db, "c1");
+ });
+
+ it("deduplicates conversations across both sweeps", async () => {
+ mockFailStaleDispatchedTasks.mockResolvedValue([
+ { agentId: "a1", workspaceId: "w1", conversationId: "c1" },
+ ]);
+ mockFailStaleRunningTasks.mockResolvedValue([
+ { agentId: "a2", workspaceId: "w1", conversationId: "c1" },
+ ]);
+ mockCountRunningTasks.mockResolvedValue(0);
+ mockUpdateAgentStatus.mockResolvedValue(undefined);
+
+ await sweepStaleState(db, "w1");
+
+ expect(mockActivateNextBufferedMessage).toHaveBeenCalledTimes(1);
+ });
});
diff --git a/src/web/src/lib/services/sweep.ts b/src/web/src/lib/services/sweep.ts
index b6f5dc458..8ac266d34 100644
--- a/src/web/src/lib/services/sweep.ts
+++ b/src/web/src/lib/services/sweep.ts
@@ -8,25 +8,28 @@ import { TaskService } from "./task";
*/
export async function sweepStaleState(db: Database, workspaceId: string) {
// 1. Fail tasks stuck in "dispatched" for >20s (daemon crashed between claim and start)
- const stale = await queries.task.failStaleDispatchedTasks(db, workspaceId);
+ const staleDispatched = await queries.task.failStaleDispatchedTasks(db, workspaceId);
// 1b. Fail kill_tasks stuck for >30s (daemon offline or crashed after claim)
await queries.task.failStaleKillTasks(db, workspaceId);
- // 2. Reconcile agent status for any agents affected by step 1
- if (stale.length > 0) {
+ // 2. Fail tasks stuck in "running" with no message activity for >1h
+ const staleRunning = await queries.task.failStaleRunningTasks(db, workspaceId);
+
+ // 3. Reconcile agent status + dispatch buffered messages for all affected
+ const allStale = [...staleDispatched, ...staleRunning];
+ if (allStale.length > 0) {
const taskService = new TaskService(db);
- const seen = new Set<string>();
- for (const r of stale) {
+ const seenAgents = new Set<string>();
+ for (const r of allStale) {
const key = `${r.agentId}:${r.workspaceId}`;
- if (seen.has(key)) continue;
- seen.add(key);
+ if (seenAgents.has(key)) continue;
+ seenAgents.add(key);
await taskService.reconcileAgentStatus(r.agentId, r.workspaceId);
}
- // 3. Dispatch buffered messages for affected conversations
const seenConversations = new Set<string>();
- for (const r of stale) {
+ for (const r of allStale) {
if (seenConversations.has(r.conversationId)) continue;
seenConversations.add(r.conversationId);
await taskService.dispatchNextBufferedMessage(r.conversationId, r.workspaceId);