Multi-Agent Collaboration for Team and Friends
TypeScript agent-automationagent-orchestrationagent-workflowai-agentsai-collaborationai-memoryai-workforceautonomous-agentsclaude-codeclaude-code-skillscodexcoding-agentmulti-agentmultiple-agentone-person-businessone-person-companyone-person-teamopencodesolopreneur
fix: stale workspace eviction
Gus73007b8e1ccd
diff --git a/src/cli/daemon/client.test.ts b/src/cli/daemon/client.test.tsindex 67c83bb69..dd8e7b9d5 100644--- a/src/cli/daemon/client.test.ts+++ b/src/cli/daemon/client.test.ts@@ -172,7 +172,7 @@ describe("DaemonClient.poll() with mocked fetch", () => {);});- it("returns TaskApi[] on valid response", async () => {+ it("returns tasks and evicted: false on valid response", async () => {globalThis.fetch = vi.fn().mockResolvedValue({ok: true,status: 200,@@ -180,12 +180,13 @@ describe("DaemonClient.poll() with mocked fetch", () => {});const client = new DaemonClient("http://localhost:8080");- const tasks = await client.poll("tok", "d1", 1);- expect(tasks).toHaveLength(1);- expect(tasks[0].id).toBe("t1");+ const result = await client.poll("tok", "d1", 1);+ expect(result.tasks).toHaveLength(1);+ expect(result.tasks[0].id).toBe("t1");+ expect(result.evicted).toBe(false);});- it("returns empty array when no tasks", async () => {+ it("returns empty tasks and evicted: false when no tasks", async () => {globalThis.fetch = vi.fn().mockResolvedValue({ok: true,status: 200,@@ -193,8 +194,22 @@ describe("DaemonClient.poll() with mocked fetch", () => {});const client = new DaemonClient("http://localhost:8080");- const tasks = await client.poll("tok", "d1", 1);- expect(tasks).toEqual([]);+ const result = await client.poll("tok", "d1", 1);+ expect(result.tasks).toEqual([]);+ expect(result.evicted).toBe(false);+ });++ it("returns evicted: true when server signals eviction", async () => {+ globalThis.fetch = vi.fn().mockResolvedValue({+ ok: true,+ status: 200,+ json: () => Promise.resolve({ tasks: [], evicted: true }),+ });++ const client = new DaemonClient("http://localhost:8080");+ const result = await client.poll("tok", "d1", 1);+ expect(result.tasks).toEqual([]);+ expect(result.evicted).toBe(true);});it("throws ZodError when API returns wrong shape", async () => {diff --git a/src/cli/daemon/client.ts b/src/cli/daemon/client.tsindex 631bfe763..c6cf45561 100644--- a/src/cli/daemon/client.ts+++ b/src/cli/daemon/client.ts@@ -58,7 +58,7 @@ export class DaemonClient {});}- async poll(token: string, daemonId: string, maxTasks: number): Promise<TaskApi[]> {+ async poll(token: string, daemonId: string, maxTasks: number): Promise<{ tasks: TaskApi[], evicted: boolean }> {const raw = await this.request<unknown>("POST","/api/daemon/tasks/poll",@@ -66,7 +66,7 @@ export class DaemonClient {{ daemon_id: daemonId, max_tasks: maxTasks },);const resp: PollResponse = PollResponseSchema.parse(raw);- return resp.tasks;+ return { tasks: resp.tasks, evicted: resp.evicted ?? false };}startTask(token: string, taskId: string) {diff --git a/src/cli/daemon/daemon.test.ts b/src/cli/daemon/daemon.test.tsindex daf31d885..b6464818c 100644--- a/src/cli/daemon/daemon.test.ts+++ b/src/cli/daemon/daemon.test.ts@@ -7,7 +7,7 @@ const mockClientInstance = {runtimes: [{ id: "rt1" }],})),deregister: vi.fn(async () => {}),- poll: vi.fn(async () => []),+ poll: vi.fn(async () => ({ tasks: [], evicted: false })),startTask: vi.fn(async () => ({})),completeTask: vi.fn(async () => ({})),failTask: vi.fn(async () => ({})),@@ -216,9 +216,9 @@ describe("daemon session runner dispatch", () => {mockClientInstance.poll.mockImplementation(async () => {if (!claimed) {claimed = true;- return [fakeTask];+ return { tasks: [fakeTask], evicted: false };}- return [];+ return { tasks: [], evicted: false };});return fakeTask;@@ -278,9 +278,9 @@ describe("daemon session runner dispatch", () => {mockClientInstance.poll.mockImplementation(async () => {if (!claimed) {claimed = true;- return [fakeTask];+ return { tasks: [fakeTask], evicted: false };}- return [];+ return { tasks: [], evicted: false };});await startDaemon();@@ -326,7 +326,7 @@ describe("daemon session runner dispatch", () => {// After close, we verify indirectly: the next poll should request full capacity (20)// If activeTasks wasn't decremented, it would request 19- mockClientInstance.poll.mockResolvedValue([]);+ mockClientInstance.poll.mockResolvedValue({ tasks: [], evicted: false });// Manually invoke a poll by calling the function startDaemon set up// We verify the listener was attached — the decrement logic is straightforward});@@ -400,7 +400,7 @@ describe("daemon with multi-workspace config", () => {});it("polls each workspace with its own token", async () => {- mockClientInstance.poll.mockResolvedValue([]);+ mockClientInstance.poll.mockResolvedValue({ tasks: [], evicted: false });await startDaemon();@@ -447,8 +447,8 @@ describe("daemon with multi-workspace config", () => {let pollCall = 0;mockClientInstance.poll.mockImplementation(async () => {pollCall++;- if (pollCall === 1) return [fakeTask, { ...fakeTask, id: "t2" }, { ...fakeTask, id: "t3" }];- return [];+ if (pollCall === 1) return { tasks: [fakeTask, { ...fakeTask, id: "t2" }, { ...fakeTask, id: "t3" }], evicted: false };+ return { tasks: [], evicted: false };});await startDaemon();@@ -489,9 +489,9 @@ describe("daemon with multi-workspace config", () => {let pollCall = 0;mockClientInstance.poll.mockImplementation(async () => {pollCall++;- if (pollCall === 1) return [fakeTaskWs1];- if (pollCall === 2) return [fakeTaskWs2];- return [];+ if (pollCall === 1) return { tasks: [fakeTaskWs1], evicted: false };+ if (pollCall === 2) return { tasks: [fakeTaskWs2], evicted: false };+ return { tasks: [], evicted: false };});await startDaemon();@@ -517,7 +517,7 @@ describe("daemon shutdown", () => {// Restore default mock implementations after clearAllMocksmockClientInstance.register.mockResolvedValue({ runtimes: [{ id: "rt1" }] });- mockClientInstance.poll.mockResolvedValue([]);+ mockClientInstance.poll.mockResolvedValue({ tasks: [], evicted: false });mockClientInstance.startTask.mockResolvedValue({});});@@ -603,9 +603,9 @@ describe("daemon shutdown", () => {mockClientInstance.poll.mockImplementation(async () => {if (!claimed) {claimed = true;- return [fakeTask];+ return { tasks: [fakeTask], evicted: false };}- return [];+ return { tasks: [], evicted: false };});mockClientInstance.startTask.mockResolvedValue({});@@ -673,8 +673,8 @@ describe("daemon agent_ids lazy sync", () => {let claimed = false;mockClientInstance.poll.mockImplementation(async () => {- if (!claimed) { claimed = true; return [makeFakeTask()]; }- return [];+ if (!claimed) { claimed = true; return { tasks: [makeFakeTask()], evicted: false }; }+ return { tasks: [], evicted: false };});await startDaemon();@@ -694,8 +694,8 @@ describe("daemon agent_ids lazy sync", () => {let claimed = false;mockClientInstance.poll.mockImplementation(async () => {- if (!claimed) { claimed = true; return [makeFakeTask()]; }- return [];+ if (!claimed) { claimed = true; return { tasks: [makeFakeTask()], evicted: false }; }+ return { tasks: [], evicted: false };});await startDaemon();@@ -714,10 +714,9 @@ describe("daemon agent_ids lazy sync", () => {mockClientInstance.poll.mockImplementation(async () => {if (!claimed) {claimed = true;- // Two tasks from the same agent- return [makeFakeTask({ id: "t1" }), makeFakeTask({ id: "t2" })];+ return { tasks: [makeFakeTask({ id: "t1" }), makeFakeTask({ id: "t2" })], evicted: false };}- return [];+ return { tasks: [], evicted: false };});await startDaemon();@@ -746,7 +745,7 @@ describe("daemon startup failures release pidfile", () => {],});mockClientInstance.register.mockResolvedValue({ runtimes: [{ id: "rt1" }] });- mockClientInstance.poll.mockResolvedValue([]);+ mockClientInstance.poll.mockResolvedValue({ tasks: [], evicted: false });const cp = await import("child_process");vi.mocked(cp.execSync).mockImplementation(() => undefined as never);});@@ -963,3 +962,202 @@ describe("pruneSessionRunnerLogs", () => {expect(mockUnlinkSync).not.toHaveBeenCalled();});});++describe("daemon workspace eviction", () => {+ beforeEach(() => {+ signalHandlers.clear();+ intervalTimers.length = 0;+ clearedTimers.length = 0;+ spawnedChildren.length = 0;+ nextPid = 50000;+ vi.clearAllMocks();+ mockProcessExit.mockImplementation((() => {}) as any);+ });++ afterEach(() => {+ for (const t of intervalTimers) realClearInterval(t);+ });++ it("removes workspace from polling on eviction and stops polling it", async () => {+ vi.mocked(loadCLIConfigForProfile).mockReturnValue({+ server_url: "",+ watched_workspaces: [+ { id: "ws1", name: "Personal", token: "al_tok_ws1" },+ { id: "ws2", name: "Team", token: "al_tok_ws2" },+ ],+ });++ let registerCall = 0;+ mockClientInstance.register.mockImplementation(async () => {+ registerCall++;+ return { runtimes: [{ id: `rt${registerCall}` }] };+ });++ let pollCall = 0;+ mockClientInstance.poll.mockImplementation(async () => {+ pollCall++;+ // First cycle: ws1 normal, ws2 evicted+ if (pollCall === 1) return { tasks: [], evicted: false };+ if (pollCall === 2) return { tasks: [], evicted: true };+ // Subsequent cycles should only get ws1+ return { tasks: [], evicted: false };+ });++ await startDaemon();+ await new Promise((r) => setTimeout(r, 50));++ // ws2 was evicted — only ws1 should remain+ // Verify config was saved to remove ws2+ expect(saveCLIConfigForProfile).toHaveBeenCalled();+ const savedConfig = vi.mocked(saveCLIConfigForProfile).mock.calls[0][1];+ const wsIds = savedConfig.watched_workspaces!.map((w: any) => w.id);+ expect(wsIds).toContain("ws1");+ expect(wsIds).not.toContain("ws2");+ });++ it("removes evicted workspace from local config file", async () => {+ vi.mocked(loadCLIConfigForProfile).mockReturnValue({+ server_url: "",+ watched_workspaces: [{ id: "ws1", name: "Test WS", token: "al_test_token" }],+ });+ mockClientInstance.register.mockResolvedValue({ runtimes: [{ id: "rt1" }] });+ mockClientInstance.poll.mockResolvedValue({ tasks: [], evicted: true });++ await startDaemon();+ await new Promise((r) => setTimeout(r, 50));++ expect(saveCLIConfigForProfile).toHaveBeenCalled();+ const savedConfig = vi.mocked(saveCLIConfigForProfile).mock.calls[0][1];+ expect(savedConfig.watched_workspaces).toEqual([]);+ });++ it("shuts down gracefully when all workspaces are evicted", async () => {+ vi.mocked(loadCLIConfigForProfile).mockReturnValue({+ server_url: "",+ watched_workspaces: [{ id: "ws1", name: "Test WS", token: "al_test_token" }],+ });+ mockClientInstance.register.mockResolvedValue({ runtimes: [{ id: "rt1" }] });+ mockClientInstance.poll.mockResolvedValue({ tasks: [], evicted: true });++ await startDaemon();+ await new Promise((r) => setTimeout(r, 50));++ // Should trigger shutdown → process.exit(0)+ expect(mockProcessExit).toHaveBeenCalledWith(0);+ });++ it("continues polling remaining workspaces when middle one is evicted (3 workspaces)", async () => {+ vi.mocked(loadCLIConfigForProfile).mockReturnValue({+ server_url: "",+ watched_workspaces: [+ { id: "ws1", name: "First", token: "al_tok_ws1" },+ { id: "ws2", name: "Middle", token: "al_tok_ws2" },+ { id: "ws3", name: "Last", token: "al_tok_ws3" },+ ],+ });++ let registerCall = 0;+ mockClientInstance.register.mockImplementation(async () => {+ registerCall++;+ return { runtimes: [{ id: `rt${registerCall}` }] };+ });++ const pollTokens: string[] = [];+ mockClientInstance.poll.mockImplementation(async (token: string) => {+ pollTokens.push(token);+ // Evict ws2 (middle workspace)+ if (token === "al_tok_ws2") return { tasks: [], evicted: true };+ return { tasks: [], evicted: false };+ });++ await startDaemon();+ await new Promise((r) => setTimeout(r, 50));++ // First poll cycle should hit all 3+ expect(pollTokens).toContain("al_tok_ws1");+ expect(pollTokens).toContain("al_tok_ws2");+ expect(pollTokens).toContain("al_tok_ws3");++ // Verify config saved without ws2+ expect(saveCLIConfigForProfile).toHaveBeenCalled();+ const savedConfig = vi.mocked(saveCLIConfigForProfile).mock.calls[0][1];+ const wsIds = savedConfig.watched_workspaces!.map((w: any) => w.id);+ expect(wsIds).toEqual(["ws1", "ws3"]);++ // Daemon should NOT have shut down (still has ws1 and ws3)+ expect(mockProcessExit).not.toHaveBeenCalled();+ });++ it("updates health.setRuntimeCount after eviction", async () => {+ const mockHealthSetRuntimeCount = vi.fn();+ const { createHealthServer } = await import("./health.js");+ vi.mocked(createHealthServer).mockReturnValue({+ setRuntimeCount: mockHealthSetRuntimeCount,+ server: { close: vi.fn() },+ } as any);++ vi.mocked(loadCLIConfigForProfile).mockReturnValue({+ server_url: "",+ watched_workspaces: [+ { id: "ws1", name: "First", token: "al_tok_ws1" },+ { id: "ws2", name: "Second", token: "al_tok_ws2" },+ ],+ });++ let registerCall = 0;+ mockClientInstance.register.mockImplementation(async () => {+ registerCall++;+ return { runtimes: [{ id: `rt${registerCall}` }] };+ });++ mockClientInstance.poll.mockImplementation(async (token: string) => {+ if (token === "al_tok_ws2") return { tasks: [], evicted: true };+ return { tasks: [], evicted: false };+ });++ await startDaemon();+ await new Promise((r) => setTimeout(r, 50));++ // Initial count was 2 (one per workspace), after evicting ws2 it should be 1+ const calls = mockHealthSetRuntimeCount.mock.calls;+ const lastCall = calls[calls.length - 1];+ expect(lastCall[0]).toBe(1);+ });++ it("config write failure does not block in-memory eviction", async () => {+ vi.mocked(loadCLIConfigForProfile).mockReturnValue({+ server_url: "",+ watched_workspaces: [+ { id: "ws1", name: "First", token: "al_tok_ws1" },+ { id: "ws2", name: "Second", token: "al_tok_ws2" },+ ],+ });++ let registerCall = 0;+ mockClientInstance.register.mockImplementation(async () => {+ registerCall++;+ return { runtimes: [{ id: `rt${registerCall}` }] };+ });++ // Make config save throw+ vi.mocked(saveCLIConfigForProfile).mockImplementation(() => {+ throw new Error("disk full");+ });++ mockClientInstance.poll.mockImplementation(async (token: string) => {+ if (token === "al_tok_ws2") return { tasks: [], evicted: true };+ return { tasks: [], evicted: false };+ });++ await startDaemon();+ await new Promise((r) => setTimeout(r, 50));++ // saveCLIConfigForProfile threw, but daemon should NOT crash+ // and should NOT shut down (ws1 still active)+ expect(mockProcessExit).not.toHaveBeenCalled();++ // ws2 should be evicted from in-memory state (only ws1 polled in future)+ // Verify by checking the saveCLIConfigForProfile was attempted+ expect(saveCLIConfigForProfile).toHaveBeenCalled();+ });+});diff --git a/src/cli/daemon/daemon.ts b/src/cli/daemon/daemon.tsindex 55dae2c6c..0a4bbbcfb 100644--- a/src/cli/daemon/daemon.ts+++ b/src/cli/daemon/daemon.ts@@ -220,6 +220,33 @@ export async function startDaemon(}}+ function evictWorkspace(workspaceId: string): void {+ const idx = workspaceStates.findIndex((ws) => ws.workspaceId === workspaceId);+ if (idx === -1) return;++ const ws = workspaceStates[idx];+ for (const rid of ws.runtimeIds) {+ runtimeIndex.delete(rid);+ }+ workspaceStates.splice(idx, 1);++ health.setRuntimeCount(+ workspaceStates.reduce((sum, w) => sum + w.runtimeIds.length, 0),+ );++ try {+ const cfg = loadCLIConfigForProfile(profile);+ cfg.watched_workspaces = (cfg.watched_workspaces || []).filter(+ (w) => w.id !== workspaceId,+ );+ saveCLIConfigForProfile(profile, cfg);+ } catch {+ // Best-effort — config write failure must not block eviction+ }++ log.info(`Workspace ${workspaceId} evicted — runtimes removed server-side`);+ }+// Staggered per-workspace pollingconst pollCycle = async () => {let remaining = config.maxConcurrentTasks - activeTasks.size;@@ -227,6 +254,7 @@ export async function startDaemon(const N = workspaceStates.length;const staggerMs = N > 1 ? Math.floor(config.pollInterval / N) : 0;+ const evictedIds: string[] = [];for (let i = 0; i < N; i++) {if (remaining <= 0) break;@@ -237,8 +265,14 @@ export async function startDaemon(}try {- const tasks = await client.poll(ws.token, config.daemonId, remaining);- for (const apiTask of tasks) {+ const { tasks: apiTasks, evicted } = await client.poll(ws.token, config.daemonId, remaining);++ if (evicted) {+ evictedIds.push(ws.workspaceId);+ continue;+ }++ for (const apiTask of apiTasks) {const task = fromApiTask(apiTask);syncAgentId(task.agentId, ws.workspaceId);activeTasks.add(task.id);@@ -253,6 +287,15 @@ export async function startDaemon(log.debug("Poll error", e);}}++ for (const id of evictedIds) {+ evictWorkspace(id);+ }++ if (workspaceStates.length === 0) {+ log.info("All workspaces evicted — shutting down");+ shutdown();+ }};const pollTimer = setInterval(pollCycle, config.pollInterval);diff --git a/src/shared/schemas.test.ts b/src/shared/schemas.test.tsindex 10b2ffa2f..185f35189 100644--- a/src/shared/schemas.test.ts+++ b/src/shared/schemas.test.ts@@ -198,6 +198,16 @@ describe("PollResponseSchema", () => {});expect(parsed.tasks[0].id).toBe("t1");});++ it("accepts evicted: true", () => {+ const parsed = PollResponseSchema.parse({ tasks: [], evicted: true });+ expect(parsed.evicted).toBe(true);+ });++ it("accepts missing evicted field", () => {+ const parsed = PollResponseSchema.parse({ tasks: [] });+ expect(parsed.evicted).toBeUndefined();+ });});// ---------------------------------------------------------------------------diff --git a/src/shared/schemas.ts b/src/shared/schemas.tsindex d5bd907cd..85f0b0e23 100644--- a/src/shared/schemas.ts+++ b/src/shared/schemas.ts@@ -94,6 +94,7 @@ export type PollRequest = z.infer<typeof PollRequestSchema>;export const PollResponseSchema = z.object({tasks: z.array(TaskApiSchema),+ evicted: z.boolean().optional(),});export type PollResponse = z.infer<typeof PollResponseSchema>;diff --git a/src/shared/src/schemas.ts b/src/shared/src/schemas.tsindex dd814c035..5e730469d 100644--- a/src/shared/src/schemas.ts+++ b/src/shared/src/schemas.ts@@ -96,6 +96,7 @@ export type PollRequest = z.infer<typeof PollRequestSchema>;export const PollResponseSchema = z.object({tasks: z.array(TaskApiSchema),+ evicted: z.boolean().optional(),});export type PollResponse = z.infer<typeof PollResponseSchema>;diff --git a/src/web/src/app/api/daemon/tasks/poll/route.test.ts b/src/web/src/app/api/daemon/tasks/poll/route.test.tsindex c5a8f1c4d..876e5797c 100644--- a/src/web/src/app/api/daemon/tasks/poll/route.test.ts+++ b/src/web/src/app/api/daemon/tasks/poll/route.test.ts@@ -89,21 +89,32 @@ function postReq(body: unknown) {describe("POST /api/daemon/tasks/poll", () => {beforeEach(() => vi.clearAllMocks());- it("returns empty tasks array when daemon has no runtimes but still updates liveness", async () => {- mockUpsertMachine.mockResolvedValue({});+ it("returns evicted: true and skips heartbeat when daemon has no runtimes", async () => {mockGetRuntimeIdsByDaemon.mockResolvedValue([]);- mockBroadcastToUser.mockResolvedValue(undefined);const res = await POST(postReq({ daemon_id: "d1" }));const body = await res.json();expect(res.status).toBe(200);expect(body.tasks).toEqual([]);- expect(mockUpsertMachine).toHaveBeenCalledWith({}, {- daemonId: "d1",- workspaceId: "w1",- deviceInfo: "d1",- });+ expect(body.evicted).toBe(true);+ expect(mockUpsertMachine).not.toHaveBeenCalled();+ expect(mockBroadcastToUser).not.toHaveBeenCalled();+ });++ it("omits evicted field for normal polls with active runtimes", async () => {+ mockUpsertMachine.mockResolvedValue({});+ mockGetRuntimeIdsByDaemon.mockResolvedValue(["r1"]);+ mockSweepStaleState.mockResolvedValue(undefined);+ mockBroadcastToUser.mockResolvedValue(undefined);+ mockClaimTasksForRuntimes.mockResolvedValue([]);++ const res = await POST(postReq({ daemon_id: "d1" }));+ const body = await res.json();++ expect(res.status).toBe(200);+ expect(body.evicted).toBeUndefined();+ expect(mockUpsertMachine).toHaveBeenCalled();expect(mockBroadcastToUser).toHaveBeenCalled();});diff --git a/src/web/src/app/api/daemon/tasks/poll/route.ts b/src/web/src/app/api/daemon/tasks/poll/route.tsindex b641adcca..1432a989e 100644--- a/src/web/src/app/api/daemon/tasks/poll/route.ts+++ b/src/web/src/app/api/daemon/tasks/poll/route.ts@@ -21,7 +21,18 @@ export const POST = withAuth(async (req: NextRequest, ctx) => {return writeError("Forbidden: machine token required", 403);}- // 1. Liveness: upsert machine row (creates if missing, updates lastSeenAt if exists)+ // 1. Resolve runtime IDs from daemon_id + workspaceId+ const runtimeIds = await queries.runtime.getRuntimeIdsByDaemon(+ db,+ body.daemon_id,+ ctx.workspaceId,+ );++ if (runtimeIds.length === 0) {+ return writeJSON({ tasks: [], evicted: true });+ }++ // 2. Liveness: upsert machine row only when runtimes existawait queries.machine.upsertMachine(db, {daemonId: body.daemon_id,workspaceId: ctx.workspaceId,@@ -35,17 +46,6 @@ export const POST = withAuth(async (req: NextRequest, ctx) => {status: "online",}).catch(() => {});- // 2. Resolve runtime IDs from daemon_id + workspaceId- const runtimeIds = await queries.runtime.getRuntimeIdsByDaemon(- db,- body.daemon_id,- ctx.workspaceId,- );-- if (runtimeIds.length === 0) {- return writeJSON({ tasks: [] });- }-// 3. Housekeeping: sweep stale stateawait sweepStaleState(db, ctx.workspaceId);