The open-source app everyone uses to manage agents at work
TypeScript acpxagent-orchestrationagent-skillsagentic-aiai-agentsai-engineeringai-governanceai-toolsautonomous-agentsclaudeclaude-codeclaude-skillscodexhermesllmmcpmulti-agentopenclawproject-managementworkflow-automation
fix(heartbeat): avoid large run reads during deferred-message recovery (#17460)
Devin Foley14e1f59fbf4d
diff --git a/doc/DEVELOPING.md b/doc/DEVELOPING.mdindex fd25491f0d..a8c500a438 100644--- a/doc/DEVELOPING.md+++ b/doc/DEVELOPING.md@@ -2871,6 +2871,13 @@ package avatar PNGs using the API renderer; static hosting needs no API proxy.### Investigating polling load+Deferred-message recovery reads run admission fields before calling the release+policy. The release coordinator reads authentication evidence and then refreshes+completion identity after post-commit effects. These checks do not load stored+prompts, Runner profiles, or output excerpts. Full execution and detail reads+retain those fields. Queue cadence, delivery admission, and cleanup ownership+remain unchanged.+Runtime progress updates recheck the persisted run status before publication.This check selects only the run ID, company ID, agent ID, and status through`createHeartbeatRunState().getRunStatus`. It does not fetch prompts, runnerdiff --git a/server/src/services/heartbeat/queue.test.ts b/server/src/services/heartbeat/queue.test.tsindex 9af4a5ac95..5a31b5ee13 100644--- a/server/src/services/heartbeat/queue.test.ts+++ b/server/src/services/heartbeat/queue.test.ts@@ -161,6 +161,9 @@ describe.skipIf(!support.supported)("heartbeat queue database wiring", () => {it.each(["legacy", "native"])("resumes an accepted non-assignee queue through its %s owner", async runtimeMode => {const { run, pending } = await strandedMention(runtimeMode);+ await db.update(heartbeatRuns).set({ contextSnapshot: { ...run.contextSnapshot, prompt: "large context".repeat(20_000) },+ runnerProfileJson: { instructions: "large profile".repeat(20_000) }, stdoutExcerpt: "large output".repeat(20_000),+ }).where(eq(heartbeatRuns.id, run.id));const deps = callbacks(db);await createHeartbeatQueue(db, deps).resumeQueuedRuns();expect(deps.releaseIssueExecutionAndPromote).toHaveBeenCalledExactlyOnceWith(@@ -170,6 +173,11 @@ describe.skipIf(!support.supported)("heartbeat queue database wiring", () => {status: "deferred_issue_execution", runId: null, payload: pending.payload,});expect(deps.executeRun).not.toHaveBeenCalled();+ const selected = deps.releaseIssueExecutionAndPromote.mock.calls[0][0];+ expect(Buffer.byteLength(JSON.stringify(selected))).toBeLessThan(512);+ expect(selected).not.toHaveProperty("contextSnapshot");+ expect(selected).not.toHaveProperty("runnerProfileJson");+ expect(selected).not.toHaveProperty("stdoutExcerpt");});it("preserves a former owner's ordinary messages for the assignment handoff", async () => {diff --git a/server/src/services/heartbeat/queue.ts b/server/src/services/heartbeat/queue.tsindex c6ac63b6a5..515bd683fb 100644--- a/server/src/services/heartbeat/queue.ts+++ b/server/src/services/heartbeat/queue.ts@@ -1837,7 +1837,15 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDependefor (const { wake } of strandedQueues) {if (!queuedCommentIdsFromWakePayload(wake.payload).length &&!await readQueuedInteractionResponse(db, wake.companyId, String(wake.payload?.issueId), wake.payload)) continue;- const [latest] = await db.select().from(heartbeatRuns).where(and(+ const [latest] = await db.select({+ id: heartbeatRuns.id,+ companyId: heartbeatRuns.companyId,+ status: heartbeatRuns.status,+ finishedAt: heartbeatRuns.finishedAt,+ startedAt: heartbeatRuns.startedAt,+ runtimeMode: heartbeatRuns.runtimeMode,+ errorCode: heartbeatRuns.errorCode,+ }).from(heartbeatRuns).where(and(eq(heartbeatRuns.companyId, wake.companyId),sql`${heartbeatRuns.contextSnapshot}->>'issueId' = ${String(wake.payload?.issueId)}`,or(isNull(heartbeatRuns.nativeIssueId), sql`${heartbeatRuns.nativeIssueId}::text = ${String(wake.payload?.issueId)}`),diff --git a/server/src/services/heartbeat/run-control-reads.test.ts b/server/src/services/heartbeat/run-control-reads.test.tsnew file mode 100644index 0000000000..eb0248cc46--- /dev/null+++ b/server/src/services/heartbeat/run-control-reads.test.ts@@ -0,0 +1,87 @@+import { randomBytes } from "node:crypto";+import type { Db } from "@paperclipai/db";+import { heartbeatRuns, issues } from "@paperclipai/db";+import { eq } from "drizzle-orm";+import { describe, expect, it, vi } from "vitest";+import { isAiAuthenticationBlocked } from "../ai-auth-failure.js";+import { settleSlackConversation } from "../slack-conversation-lifecycle.js";+import { createHeartbeatRunControl } from "./run-control.js";+import { callbacks, runControlPostgresSupport, setupRunControlDatabase } from "./__tests__/run-control-fixtures.js";++vi.mock("../local-service-supervisor.js", async importOriginal => ({+ ...await importOriginal<typeof import("../local-service-supervisor.js")>(),+ terminateLocalService: vi.fn(async () => undefined),+}));+vi.mock("../slack-conversation-lifecycle.js", () => ({ settleSlackConversation: vi.fn(async () => undefined) }));++describe.skipIf(!runControlPostgresSupport.supported)("execution release reads", () => {+ let db: Db;+ const { fixture } = setupRunControlDatabase("heartbeat-release-reads-", value => { db = value; });++ function recordRunReads() {+ const reads: unknown[][] = [];+ const database = new Proxy(db, { get(target, key) {+ if (key !== "select") return Reflect.get(target, key);+ return (...fields: Parameters<Db["select"]>) => {+ const builder = target.select(...fields);+ const from = builder.from.bind(builder);+ builder.from = ((...args: Parameters<typeof from>) => {+ const query = from(...args);+ if (args[0] === heartbeatRuns) {+ const execute = query.execute.bind(query);+ query.execute = async (...executeArgs: Parameters<typeof execute>) => {+ const rows = await execute(...executeArgs);+ reads.push(rows);+ return rows;+ };+ }+ return query;+ }) as typeof builder.from;+ return builder;+ };+ }});+ return { database, reads };+ }++ it.each([+ { errorCode: "codex_auth_required", livenessState: "blocked", resultJson: null },+ { errorCode: "codex_auth_required", livenessState: "advanced", resultJson: null },+ { errorCode: "configuration_incomplete", livenessState: "blocked", resultJson: { configurationIncomplete: { reason: "ai_connection_unavailable" } } },+ { errorCode: "configuration_incomplete", livenessState: "blocked", resultJson: { configurationIncomplete: { reason: "secret_binding_missing", agentId: "owner", missingBindings: [{ bindingType: "user_secret_ref", consumerType: "agent", consumerId: "owner", errorCode: "user_secret_missing", envKey: "OPENAI_API_KEY", configPath: "env.OPENAI_API_KEY" }] } } },+ { errorCode: "configuration_incomplete", livenessState: "blocked", resultJson: { configurationIncomplete: { reason: "workspace_unavailable" } } },+ ])("preserves authentication admission for %j without loading execution payloads", async failure => {+ const { company, agent } = await fixture();+ const payload = randomBytes(128 * 1024).toString("base64");+ const [run] = await db.insert(heartbeatRuns).values({ companyId: company.id, agentId: agent.id,+ status: "failed", ...failure, contextSnapshot: { prompt: payload }, runnerProfileJson: { prompt: payload }, stdoutExcerpt: payload,+ }).returning();+ const { database, reads } = recordRunReads();+ const deps = callbacks(database);+ await createHeartbeatRunControl(database, deps).releaseIssueExecutionAndPromote(run);+ expect(deps.wakeQueue.releaseIssueExecution).toHaveBeenCalledExactlyOnceWith(expect.objectContaining({+ companyId: company.id, runId: run.id, suppressImmediateRecovery: isAiAuthenticationBlocked(run),+ }));+ expect(deps.getRun).not.toHaveBeenCalled();+ expect(reads).toHaveLength(2);+ expect(Buffer.byteLength(JSON.stringify(reads))).toBeLessThan(2048);+ const [stored] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, run.id));+ expect(stored).toEqual(run);+ });++ it("reads the completed identity after post-commit effects and preserves the full record", async () => {+ const { company, agent } = await fixture();+ const [issue] = await db.insert(issues).values({ companyId: company.id, title: "Release destination" }).returning();+ const [run] = await db.insert(heartbeatRuns).values({ companyId: company.id, agentId: agent.id,+ status: "running", contextSnapshot: { issueId: 42, taskId: "", prompt: "retained" },+ }).returning();+ const deps = callbacks(db);+ vi.spyOn(deps, "applyWakeQueuePostCommitEffects").mockImplementationOnce(async () => {+ await db.update(heartbeatRuns).set({ status: "succeeded", contextSnapshot: { issueId: 42, taskId: issue.id, prompt: "retained" } }).where(eq(heartbeatRuns.id, run.id));+ });+ vi.mocked(settleSlackConversation).mockClear();+ await createHeartbeatRunControl(db, deps).releaseIssueExecutionAndPromote(run, { resumeStrandedQueue: true });+ expect(settleSlackConversation).toHaveBeenCalledExactlyOnceWith(db, company.id, issue.id);+ expect(deps.wakeQueue.releaseIssueExecution).toHaveBeenCalledWith(expect.objectContaining({ resumeStrandedQueue: true }));+ expect(deps.getRun).not.toHaveBeenCalled();+ });+});diff --git a/server/src/services/heartbeat/run-control.ts b/server/src/services/heartbeat/run-control.tsindex e9b21a98aa..7f80f99321 100644--- a/server/src/services/heartbeat/run-control.ts+++ b/server/src/services/heartbeat/run-control.ts@@ -815,7 +815,15 @@ export function createHeartbeatRunControl(db: Db, dependencies: HeartbeatRunContoptions: { suppressImmediateRecovery?: boolean; resumeStrandedQueue?: boolean; deferredPostCommitEffects?: WakeQueuePostCommitEffect[] } = {},) {try {- const source = await getRun(run.id);+ const [source] = await db.select({+ errorCode: heartbeatRuns.errorCode,+ livenessState: heartbeatRuns.livenessState,+ resultJson: sql<Record<string, unknown>>`jsonb_build_object(+ 'configurationIncomplete', ${heartbeatRuns.resultJson}->'configurationIncomplete'+ )`,+ }).from(heartbeatRuns).where(and(+ eq(heartbeatRuns.id, run.id), eq(heartbeatRuns.companyId, run.companyId),+ ));const sourceHandoff = await pendingCaseHandoffForRun(db, run.id, run.companyId);const { postCommitEffects } = await wakeQueue.releaseIssueExecution({companyId: run.companyId,@@ -831,9 +839,18 @@ export function createHeartbeatRunControl(db: Db, dependencies: HeartbeatRunCont} else {await applyWakeQueuePostCommitEffects(postCommitEffects);}- const completed = await getRun(run.id);- const issueId = readNonEmptyString(completed?.contextSnapshot?.issueId)- ?? readNonEmptyString(completed?.contextSnapshot?.taskId) ?? completed?.nativeIssueId;+ const [completed] = await db.select({+ id: heartbeatRuns.id,+ status: heartbeatRuns.status,+ startedAt: heartbeatRuns.startedAt,+ nativeIssueId: heartbeatRuns.nativeIssueId,+ contextIssueId: sql<unknown>`${heartbeatRuns.contextSnapshot}->'issueId'`,+ contextTaskId: sql<unknown>`${heartbeatRuns.contextSnapshot}->'taskId'`,+ }).from(heartbeatRuns).where(and(+ eq(heartbeatRuns.id, run.id), eq(heartbeatRuns.companyId, run.companyId),+ ));+ const issueId = readNonEmptyString(completed?.contextIssueId)+ ?? readNonEmptyString(completed?.contextTaskId) ?? completed?.nativeIssueId;if (sourceHandoff && completed?.status === "cancelled" && !completed.startedAt && issueId) {// A cancelled queue entry has no executeRun.finally. The canonical// bootstrap proof still needs to release and wake its durable handoff.