The open-source app everyone uses to manage agents at work
TypeScript acpxagent-orchestrationagent-skillsagentic-aiai-agentsai-engineeringai-governanceai-toolsautonomous-agentsclaudeclaude-codeclaude-skillscodexhermesllmmcpmulti-agentopenclawproject-managementworkflow-automation
Retry pooled execution projection reads after disconnects (#15300)
Devin Foleyd21fe312432d
diff --git a/doc/DATABASE.md b/doc/DATABASE.mdindex a0f30babbb..2b4480f9d6 100644--- a/doc/DATABASE.md+++ b/doc/DATABASE.md@@ -186,6 +186,12 @@ workflow, or issue label and watchdog enrichment. Missing resources,authentication errors, and other database errors retain their usual behavior.This does not enable general SQL replay or retry a full request.+The task run list also opts its execution-status projection into these bounded+retries. Each of its four read-only lookups rebuilds only the failed query;+completed lookups and the separate liveness backfill are not replayed. Projection+callers that use a transaction retain single-attempt reads. The opt-in is only+for a pooled database handle outside a transaction.+## Execution identity row locksIdentity initialization, credential acquisition, and steering reconciliation lockdiff --git a/server/src/__tests__/activity-routes.test.ts b/server/src/__tests__/activity-routes.test.tsindex e6d9d61113..45871e3236 100644--- a/server/src/__tests__/activity-routes.test.ts+++ b/server/src/__tests__/activity-routes.test.ts@@ -277,6 +277,26 @@ describe.sequential("activity routes", () => {expect(mockActivityService.create).not.toHaveBeenCalled();});+ it.each([null, { id: "issue-uuid-1", companyId: "company-2" }])(+ "does not start run reads or retries for a missing or cross-company task", async (issue) => {+ mockIssueService.getByIdentifier.mockResolvedValue(issue);+ const app = await createApp();+ const res = await request(app).get("/api/issues/TEST-1/runs");+ expect(res.status).toBe(404);+ expect(res.body).toEqual({ error: "Issue not found" });+ expect(mockActivityService.runsForIssue).not.toHaveBeenCalled();+ },+ );++ it("does not start run reads or retries when task read authorization is denied", async () => {+ mockIssueService.getByIdentifier.mockResolvedValue({ id: "issue-uuid-1", companyId: "company-1" });+ mockAccessService.decide.mockResolvedValue({ allowed: false, reason: "denied", explanation: "Denied by test policy." });+ const app = await createApp();+ const res = await request(app).get("/api/issues/TEST-1/runs");+ expect(res.status).toBe(403);+ expect(mockActivityService.runsForIssue).not.toHaveBeenCalled();+ });+it("returns 200 [] (not 404) when listing issues for another company's run, preserving API contract and the cross-tenant oracle", async () => {mockHeartbeatService.getRun.mockResolvedValue({id: "run-2",diff --git a/server/src/services/activity.ts b/server/src/services/activity.tsindex 5d2b3ebdba..8a2310cbad 100644--- a/server/src/services/activity.ts+++ b/server/src/services/activity.ts@@ -497,7 +497,7 @@ export function activityService(db: Db) {const [exhaustionRows, leaseRows, executionByRunId, [savedPlan]] = await Promise.all([exhaustionRowsQuery,leaseRowsQuery,- executionProjectionsForRuns(db, companyId, runIds),+ executionProjectionsForRuns(db, companyId, runIds, new Date(), { retryDatabaseReads: true }),savedPlanQuery,]);const retryExhaustedReasonByRunId = new Map<string, string>();diff --git a/server/src/services/execution-projection-read-retry.test.ts b/server/src/services/execution-projection-read-retry.test.tsnew file mode 100644index 0000000000..383835d802--- /dev/null+++ b/server/src/services/execution-projection-read-retry.test.ts@@ -0,0 +1,150 @@+import { getTableName, type SQL } from "drizzle-orm";+import { PgDialect } from "drizzle-orm/pg-core";+import type { Db } from "@paperclipai/db";+import { afterEach, describe, expect, it, vi } from "vitest";+import { activityService } from "./activity.js";+import { executionProjectionsForRuns } from "./execution-projection.js";+import { logger } from "../middleware/logger.js";++const companyId = "00000000-0000-4000-8000-000000000001";+const issueId = "00000000-0000-4000-8000-000000000002";+const runId = "00000000-0000-4000-8000-000000000003";+const now = new Date("2026-01-01T00:00:00Z");+const tables = ["heartbeat_runs", "native_run_finalizations", "issue_thread_interactions", "issue_recovery_actions"];+const dialect = new PgDialect();+function disconnected(code = "CONNECTION_CLOSED") {+ return new Error("Failed query", { cause: Object.assign(new Error("Connection failed"), { code }) });+}++function fixture(options: {+ failures?: Record<string, Error[]>;+ missing?: boolean;+ noIssue?: boolean;+ backfill?: boolean;+ writeFailure?: Error;+} = {}) {+ const attempts: Array<{ key: string; params: unknown[]; order: string[] }> = [];+ const row = {+ id: runId, companyId, runId, status: "succeeded", runtimeMode: "native", errorCode: null,+ nativeIssueId: options.noIssue ? null : issueId, contextSnapshot: {},+ finishedAt: now, startedAt: now, createdAt: now, resultJson: null, usageJson: null,+ scheduledRetryAt: null, scheduledRetryAttempt: 0, retryOfRunId: null, nextAction: null,+ };+ const select = vi.fn((fields: Record<string, unknown> = {}) => ({+ from(table: Parameters<typeof getTableName>[0]) {+ const name = getTableName(table);+ const key = name === "heartbeat_runs" && "runId" in fields ? "run_list"+ : name === "heartbeat_runs" && "stdoutExcerpt" in fields ? "liveness_backfill"+ : name;+ let condition: SQL;+ let order: SQL[] = [];+ let result: Promise<unknown[]> | undefined;+ const query = {+ where(value: SQL) { condition = value; return query; },+ innerJoin() { return query; },+ orderBy(...values: SQL[]) { order = values; return query; },+ limit() { return query; },+ then(resolve: (value: unknown[]) => unknown, reject: (error: unknown) => unknown) {+ // Re-awaiting the same failed builder cannot recover the connection.+ result ??= Promise.resolve().then(() => {+ const attempt = attempts.filter((read) => read.key === key).length;+ attempts.push({ key, params: dialect.sqlToQuery(condition).params,+ order: order.map((value) => dialect.sqlToQuery(value).sql) });+ const error = options.failures?.[key]?.[attempt];+ if (error) throw error;+ if (options.missing) return [];+ if (key === "heartbeat_runs" || key === "run_list") return [row];+ if (key === "liveness_backfill") return options.backfill ? [row] : [];+ if (key === "issues") return [{ status: "in_progress", title: "Fixture", workMode: "standard" }];+ if (key === "issue_thread_interactions") return [{ issueId, kind: "ask_user_questions" }];+ return [];+ });+ return result.then(resolve, reject);+ },+ };+ return query;+ },+ }));+ const updateWhere = vi.fn(async () => { if (options.writeFailure) throw options.writeFailure; });+ const update = vi.fn(() => ({ set: () => ({ where: updateWhere }) }));+ const transaction = vi.fn(() => { throw new Error("Must not start or replay a transaction"); });+ const db = { select, update, transaction } as unknown as Db;+ return { db, attempts, select, update, updateWhere, transaction,+ count: (key: string) => attempts.filter((read) => read.key === key).length };+}++afterEach(() => vi.restoreAllMocks());++describe("execution projection read retries", () => {+ it.each(tables)("rebuilds only the failed %s SELECT within the existing retry budget", async (table) => {+ const test = fixture({ failures: { [table]: [disconnected(), disconnected("CONNECTION_ENDED")] } });+ const result = await executionProjectionsForRuns(test.db, companyId, [runId], now, { retryDatabaseReads: true });+ expect(result.get(runId)).toMatchObject({ phase: "waiting_for_answer", label: "Waiting for answer" });+ for (const name of tables) expect(test.count(name)).toBe(name === table ? 3 : 1);+ expect(test.select).toHaveBeenCalledTimes(6);+ const reads = test.attempts.filter((read) => read.key === table);+ expect(reads[1]).toEqual(reads[0]);+ expect(reads[2]).toEqual(reads[0]);+ expect(reads[0].params).toContain(companyId);+ expect(reads[0].params).toContain(table === "heartbeat_runs" || table === "native_run_finalizations" ? runId : issueId);+ if (table === "issue_thread_interactions") expect(reads[0].params).toContain("pending");+ if (table === "issue_recovery_actions") expect(reads[0].order).toEqual(['"issue_recovery_actions"."updated_at" desc']);+ expect(test.update).not.toHaveBeenCalled();+ expect(test.transaction).not.toHaveBeenCalled();+ });++ it.each(tables)("preserves the last %s error after three attempts", async (table) => {+ const errors = [disconnected(), disconnected(), disconnected()];+ const test = fixture({ failures: { [table]: errors } });+ await expect(executionProjectionsForRuns(test.db, companyId, [runId], now, { retryDatabaseReads: true })).rejects.toBe(errors[2]);+ for (const [index, name] of tables.entries()) {+ expect(test.count(name)).toBe(index < tables.indexOf(table) ? 1 : name === table ? 3 : 0);+ }+ });++ it.each(tables)("keeps the default transaction-compatible %s path single-attempt", async (table) => {+ const error = disconnected();+ const test = fixture({ failures: { [table]: [error] } });+ await expect(executionProjectionsForRuns(test.db, companyId, [runId], now)).rejects.toBe(error);+ expect(test.count(table)).toBe(1);+ expect(test.transaction).not.toHaveBeenCalled();+ });++ it.each([new Error("CONNECTION_CLOSED"), disconnected("28P01"), disconnected("42501"), disconnected("22P02"), disconnected("25P02")])(+ "does not retry an unclassified or non-transient error", async (error) => {+ const test = fixture({ failures: { issue_thread_interactions: [error] } });+ await expect(executionProjectionsForRuns(test.db, companyId, [runId], now, { retryDatabaseReads: true })).rejects.toBe(error);+ expect(test.count("issue_thread_interactions")).toBe(1);+ expect(test.count("issue_recovery_actions")).toBe(0);+ },+ );++ it("preserves empty and taskless projection behavior without unnecessary reads", async () => {+ const empty = fixture();+ expect(await executionProjectionsForRuns(empty.db, companyId, [], now, { retryDatabaseReads: true })).toEqual(new Map());+ expect(empty.select).not.toHaveBeenCalled();+ for (const options of [{ missing: true }, { noIssue: true }]) {+ const test = fixture(options);+ const result = await executionProjectionsForRuns(test.db, companyId, [runId], now, { retryDatabaseReads: true });+ expect(test.select).toHaveBeenCalledTimes(2);+ expect(options.missing ? result.size : result.get(runId)?.phase).toBe(options.missing ? 0 : "completed");+ }+ });++ it.each([false, true])("the run list retries projection reads without replaying other reads or backfill writes (write fails: %s)", async (writeFails) => {+ vi.spyOn(logger, "warn").mockImplementation(() => undefined);+ const test = fixture({ backfill: true, writeFailure: writeFails ? disconnected() : undefined,+ failures: { issue_thread_interactions: [disconnected(), disconnected()] } });+ const result = await activityService(test.db).runsForIssue(companyId, issueId);+ expect(result).toMatchObject([{ runId, execution: { phase: "waiting_for_answer" } }]);+ expect(test.count("issue_thread_interactions")).toBe(3);+ for (const key of ["run_list", "liveness_backfill", "heartbeat_runs", "native_run_finalizations", "environment_leases", "issue_recovery_actions"]) {+ expect(test.count(key)).toBe(1);+ }+ // One retry-exhaustion lookup plus the independent backfill's event count.+ expect(test.count("heartbeat_run_events")).toBe(2);+ expect(test.update).toHaveBeenCalledTimes(1);+ expect(test.updateWhere).toHaveBeenCalledTimes(1);+ expect(test.transaction).not.toHaveBeenCalled();+ });+});diff --git a/server/src/services/execution-projection.ts b/server/src/services/execution-projection.tsindex cab39d73e3..ae88b4b739 100644--- a/server/src/services/execution-projection.ts+++ b/server/src/services/execution-projection.ts@@ -9,6 +9,7 @@ import {import type { ExecutionProjection } from "@paperclipai/shared";import { EXECUTION_CONTROL_DEADLINE_MS } from "./execution-control-deadline.js";import { executionFailureRetryCount } from "./execution-recovery-attempt.js";+import { retryIdempotentDatabaseOperation } from "../database-retry.js";const text = (v: unknown) => (typeof v === "string" ? v : null);const executionRunColumns = {id: heartbeatRuns.id,@@ -48,10 +49,16 @@ export async function executionProjectionsForRuns(companyId: string,runIds: string[],now = new Date(),+ options: { retryDatabaseReads?: boolean } = {},) {const projections = new Map<string, ExecutionProjection>();if (!runIds.length) return projections;- const runs = await db+ // Opt in only with a pooled Db outside a transaction. Issue enrichment also+ // calls this helper with a transaction, whose failed reads must not replay.+ const read = <T>(query: () => Promise<T>) => options.retryDatabaseReads+ ? retryIdempotentDatabaseOperation(query)+ : query();+ const runs = await read(async () => db.select(executionRunColumns).from(heartbeatRuns).where(@@ -59,8 +66,8 @@ export async function executionProjectionsForRuns(eq(heartbeatRuns.companyId, companyId),inArray(heartbeatRuns.id, runIds),),- );- const coordinators = await db+ ));+ const coordinators = await read(async () => db.select().from(nativeRunFinalizations).where(@@ -68,7 +75,7 @@ export async function executionProjectionsForRuns(eq(nativeRunFinalizations.companyId, companyId),inArray(nativeRunFinalizations.runId, runIds),),- );+ ));const issueIds = [...new Set(runs@@ -77,7 +84,7 @@ export async function executionProjectionsForRuns(),];const pending = issueIds.length- ? await db+ ? await read(async () => db.select({issueId: issueThreadInteractions.issueId,kind: issueThreadInteractions.kind,@@ -89,10 +96,10 @@ export async function executionProjectionsForRuns(inArray(issueThreadInteractions.issueId, issueIds),eq(issueThreadInteractions.status, "pending"),),- )+ )): [];const recovery = issueIds.length- ? await db+ ? await read(async () => db.select({issueId: issueRecoveryActions.sourceIssueId,cause: issueRecoveryActions.cause,@@ -112,7 +119,7 @@ export async function executionProjectionsForRuns(]),),)- .orderBy(desc(issueRecoveryActions.updatedAt))+ .orderBy(desc(issueRecoveryActions.updatedAt))): [];const coordinatorByRun = new Map(coordinators.map((row) => [row.runId, row]));for (const run of runs) {