The open-source app everyone uses to manage agents at work
TypeScript acpxagent-orchestrationagent-skillsagentic-aiai-agentsai-engineeringai-governanceai-toolsautonomous-agentsclaudeclaude-codeclaude-skillscodexhermesllmmcpmulti-agentopenclawproject-managementworkflow-automation
Fix active heartbeat run reaping (#8776)
Devin Foleyf019f54bb33e
diff --git a/server/src/__tests__/heartbeat-process-recovery.test.ts b/server/src/__tests__/heartbeat-process-recovery.test.tsindex df7ffae7d9..af0c89a8bd 100644--- a/server/src/__tests__/heartbeat-process-recovery.test.ts+++ b/server/src/__tests__/heartbeat-process-recovery.test.ts@@ -575,6 +575,72 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => {return { environmentId, leaseId };}+ it("does not reap active adapter executions started by another heartbeat service instance", async () => {+ let releaseAdapter: (() => void) | null = null;+ const adapterStarted = new Promise<void>((resolve) => {+ mockAdapterExecute.mockImplementationOnce(async () => {+ resolve();+ await new Promise<void>((release) => {+ releaseAdapter = release;+ });+ return {+ exitCode: 0,+ signal: null,+ timedOut: false,+ errorMessage: null,+ summary: "Remote run completed.",+ provider: "test",+ model: "test-model",+ };+ });+ });++ const { runId, wakeupRequestId } = await seedRunFixture({+ adapterType: "openclaw_gateway",+ agentStatus: "idle",+ runStatus: "queued",+ processPid: null,+ processGroupId: null,+ includeIssue: false,+ });+ const executorHeartbeat = heartbeatService(db);+ const reaperHeartbeat = heartbeatService(db);++ await executorHeartbeat.resumeQueuedRuns();+ await Promise.race([+ adapterStarted,+ new Promise<never>((_, reject) => {+ setTimeout(() => reject(new Error("Timed out waiting for adapter execution to start")), 3_000);+ }),+ ]);++ await db+ .update(heartbeatRuns)+ .set({+ updatedAt: new Date("2026-03-19T00:00:00.000Z"),+ })+ .where(eq(heartbeatRuns.id, runId));++ const result = await reaperHeartbeat.reapOrphanedRuns({ staleThresholdMs: 1 });+ expect(result).toEqual({ reaped: 0, runIds: [] });++ const activeRun = await reaperHeartbeat.getRun(runId);+ expect(activeRun?.status).toBe("running");+ expect(activeRun?.errorCode).toBeNull();++ const wakeup = await db+ .select()+ .from(agentWakeupRequests)+ .where(eq(agentWakeupRequests.id, wakeupRequestId))+ .then((rows) => rows[0] ?? null);+ expect(wakeup?.status).toBe("claimed");++ if (!releaseAdapter) throw new Error("Adapter release handle was not captured");+ releaseAdapter();+ const settledRun = await waitForRunToSettle(executorHeartbeat, runId, 5_000);+ expect(settledRun?.status).toBe("succeeded");+ });+async function seedStrandedIssueFixture(input: {status: "todo" | "in_progress";runStatus: "failed" | "timed_out" | "cancelled" | "succeeded";diff --git a/server/src/services/heartbeat.ts b/server/src/services/heartbeat.tsindex c530f09513..739b4dc47e 100644--- a/server/src/services/heartbeat.ts+++ b/server/src/services/heartbeat.ts@@ -409,6 +409,9 @@ const SESSIONED_LOCAL_ADAPTERS = new Set(["opencode_local","pi_local",]);+// Routes and the scheduler construct separate heartbeatService instances, but+// they must agree on in-process adapter executions when reaping stale runs.+const activeRunExecutions = new Set<string>();const INLINE_BASE64_IMAGE_DATA_RE = /("type":"image","source":\{"type":"base64","data":")([A-Za-z0-9+/=]{1024,})(")/g;type RuntimeConfigSecretResolver = Pick<@@ -3556,7 +3559,6 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {})environmentRuntime,});const workspaceOperationsSvc = workspaceOperationService(db);- const activeRunExecutions = new Set<string>();const liveRunExecutions = {has(id: string) {return runningProcesses.has(id) || activeRunExecutions.has(id);