Skip to content

The open-source app everyone uses to manage agents at work

Retry pooled execution projection reads after disconnects (#15300)

Devin Foleyd21fe312432d
diff --git a/doc/DATABASE.md b/doc/DATABASE.md
index a0f30babbb..2b4480f9d6 100644
--- a/doc/DATABASE.md
+++ b/doc/DATABASE.md
@@ -186,6 +186,12 @@ workflow, or issue label and watchdog enrichment. Missing resources,
authentication errors, and other database errors retain their usual behavior.
This does not enable general SQL replay or retry a full request.
+The task run list also opts its execution-status projection into these bounded
+retries. Each of its four read-only lookups rebuilds only the failed query;
+completed lookups and the separate liveness backfill are not replayed. Projection
+callers that use a transaction retain single-attempt reads. The opt-in is only
+for a pooled database handle outside a transaction.
+
## Execution identity row locks
Identity initialization, credential acquisition, and steering reconciliation lock
diff --git a/server/src/__tests__/activity-routes.test.ts b/server/src/__tests__/activity-routes.test.ts
index e6d9d61113..45871e3236 100644
--- a/server/src/__tests__/activity-routes.test.ts
+++ b/server/src/__tests__/activity-routes.test.ts
@@ -277,6 +277,26 @@ describe.sequential("activity routes", () => {
expect(mockActivityService.create).not.toHaveBeenCalled();
});
+ it.each([null, { id: "issue-uuid-1", companyId: "company-2" }])(
+ "does not start run reads or retries for a missing or cross-company task", async (issue) => {
+ mockIssueService.getByIdentifier.mockResolvedValue(issue);
+ const app = await createApp();
+ const res = await request(app).get("/api/issues/TEST-1/runs");
+ expect(res.status).toBe(404);
+ expect(res.body).toEqual({ error: "Issue not found" });
+ expect(mockActivityService.runsForIssue).not.toHaveBeenCalled();
+ },
+ );
+
+ it("does not start run reads or retries when task read authorization is denied", async () => {
+ mockIssueService.getByIdentifier.mockResolvedValue({ id: "issue-uuid-1", companyId: "company-1" });
+ mockAccessService.decide.mockResolvedValue({ allowed: false, reason: "denied", explanation: "Denied by test policy." });
+ const app = await createApp();
+ const res = await request(app).get("/api/issues/TEST-1/runs");
+ expect(res.status).toBe(403);
+ expect(mockActivityService.runsForIssue).not.toHaveBeenCalled();
+ });
+
it("returns 200 [] (not 404) when listing issues for another company's run, preserving API contract and the cross-tenant oracle", async () => {
mockHeartbeatService.getRun.mockResolvedValue({
id: "run-2",
diff --git a/server/src/services/activity.ts b/server/src/services/activity.ts
index 5d2b3ebdba..8a2310cbad 100644
--- a/server/src/services/activity.ts
+++ b/server/src/services/activity.ts
@@ -497,7 +497,7 @@ export function activityService(db: Db) {
const [exhaustionRows, leaseRows, executionByRunId, [savedPlan]] = await Promise.all([
exhaustionRowsQuery,
leaseRowsQuery,
- executionProjectionsForRuns(db, companyId, runIds),
+ executionProjectionsForRuns(db, companyId, runIds, new Date(), { retryDatabaseReads: true }),
savedPlanQuery,
]);
const retryExhaustedReasonByRunId = new Map<string, string>();
diff --git a/server/src/services/execution-projection-read-retry.test.ts b/server/src/services/execution-projection-read-retry.test.ts
new file mode 100644
index 0000000000..383835d802
--- /dev/null
+++ b/server/src/services/execution-projection-read-retry.test.ts
@@ -0,0 +1,150 @@
+import { getTableName, type SQL } from "drizzle-orm";
+import { PgDialect } from "drizzle-orm/pg-core";
+import type { Db } from "@paperclipai/db";
+import { afterEach, describe, expect, it, vi } from "vitest";
+import { activityService } from "./activity.js";
+import { executionProjectionsForRuns } from "./execution-projection.js";
+import { logger } from "../middleware/logger.js";
+
+const companyId = "00000000-0000-4000-8000-000000000001";
+const issueId = "00000000-0000-4000-8000-000000000002";
+const runId = "00000000-0000-4000-8000-000000000003";
+const now = new Date("2026-01-01T00:00:00Z");
+const tables = ["heartbeat_runs", "native_run_finalizations", "issue_thread_interactions", "issue_recovery_actions"];
+const dialect = new PgDialect();
+function disconnected(code = "CONNECTION_CLOSED") {
+ return new Error("Failed query", { cause: Object.assign(new Error("Connection failed"), { code }) });
+}
+
+function fixture(options: {
+ failures?: Record<string, Error[]>;
+ missing?: boolean;
+ noIssue?: boolean;
+ backfill?: boolean;
+ writeFailure?: Error;
+} = {}) {
+ const attempts: Array<{ key: string; params: unknown[]; order: string[] }> = [];
+ const row = {
+ id: runId, companyId, runId, status: "succeeded", runtimeMode: "native", errorCode: null,
+ nativeIssueId: options.noIssue ? null : issueId, contextSnapshot: {},
+ finishedAt: now, startedAt: now, createdAt: now, resultJson: null, usageJson: null,
+ scheduledRetryAt: null, scheduledRetryAttempt: 0, retryOfRunId: null, nextAction: null,
+ };
+ const select = vi.fn((fields: Record<string, unknown> = {}) => ({
+ from(table: Parameters<typeof getTableName>[0]) {
+ const name = getTableName(table);
+ const key = name === "heartbeat_runs" && "runId" in fields ? "run_list"
+ : name === "heartbeat_runs" && "stdoutExcerpt" in fields ? "liveness_backfill"
+ : name;
+ let condition: SQL;
+ let order: SQL[] = [];
+ let result: Promise<unknown[]> | undefined;
+ const query = {
+ where(value: SQL) { condition = value; return query; },
+ innerJoin() { return query; },
+ orderBy(...values: SQL[]) { order = values; return query; },
+ limit() { return query; },
+ then(resolve: (value: unknown[]) => unknown, reject: (error: unknown) => unknown) {
+ // Re-awaiting the same failed builder cannot recover the connection.
+ result ??= Promise.resolve().then(() => {
+ const attempt = attempts.filter((read) => read.key === key).length;
+ attempts.push({ key, params: dialect.sqlToQuery(condition).params,
+ order: order.map((value) => dialect.sqlToQuery(value).sql) });
+ const error = options.failures?.[key]?.[attempt];
+ if (error) throw error;
+ if (options.missing) return [];
+ if (key === "heartbeat_runs" || key === "run_list") return [row];
+ if (key === "liveness_backfill") return options.backfill ? [row] : [];
+ if (key === "issues") return [{ status: "in_progress", title: "Fixture", workMode: "standard" }];
+ if (key === "issue_thread_interactions") return [{ issueId, kind: "ask_user_questions" }];
+ return [];
+ });
+ return result.then(resolve, reject);
+ },
+ };
+ return query;
+ },
+ }));
+ const updateWhere = vi.fn(async () => { if (options.writeFailure) throw options.writeFailure; });
+ const update = vi.fn(() => ({ set: () => ({ where: updateWhere }) }));
+ const transaction = vi.fn(() => { throw new Error("Must not start or replay a transaction"); });
+ const db = { select, update, transaction } as unknown as Db;
+ return { db, attempts, select, update, updateWhere, transaction,
+ count: (key: string) => attempts.filter((read) => read.key === key).length };
+}
+
+afterEach(() => vi.restoreAllMocks());
+
+describe("execution projection read retries", () => {
+ it.each(tables)("rebuilds only the failed %s SELECT within the existing retry budget", async (table) => {
+ const test = fixture({ failures: { [table]: [disconnected(), disconnected("CONNECTION_ENDED")] } });
+ const result = await executionProjectionsForRuns(test.db, companyId, [runId], now, { retryDatabaseReads: true });
+ expect(result.get(runId)).toMatchObject({ phase: "waiting_for_answer", label: "Waiting for answer" });
+ for (const name of tables) expect(test.count(name)).toBe(name === table ? 3 : 1);
+ expect(test.select).toHaveBeenCalledTimes(6);
+ const reads = test.attempts.filter((read) => read.key === table);
+ expect(reads[1]).toEqual(reads[0]);
+ expect(reads[2]).toEqual(reads[0]);
+ expect(reads[0].params).toContain(companyId);
+ expect(reads[0].params).toContain(table === "heartbeat_runs" || table === "native_run_finalizations" ? runId : issueId);
+ if (table === "issue_thread_interactions") expect(reads[0].params).toContain("pending");
+ if (table === "issue_recovery_actions") expect(reads[0].order).toEqual(['"issue_recovery_actions"."updated_at" desc']);
+ expect(test.update).not.toHaveBeenCalled();
+ expect(test.transaction).not.toHaveBeenCalled();
+ });
+
+ it.each(tables)("preserves the last %s error after three attempts", async (table) => {
+ const errors = [disconnected(), disconnected(), disconnected()];
+ const test = fixture({ failures: { [table]: errors } });
+ await expect(executionProjectionsForRuns(test.db, companyId, [runId], now, { retryDatabaseReads: true })).rejects.toBe(errors[2]);
+ for (const [index, name] of tables.entries()) {
+ expect(test.count(name)).toBe(index < tables.indexOf(table) ? 1 : name === table ? 3 : 0);
+ }
+ });
+
+ it.each(tables)("keeps the default transaction-compatible %s path single-attempt", async (table) => {
+ const error = disconnected();
+ const test = fixture({ failures: { [table]: [error] } });
+ await expect(executionProjectionsForRuns(test.db, companyId, [runId], now)).rejects.toBe(error);
+ expect(test.count(table)).toBe(1);
+ expect(test.transaction).not.toHaveBeenCalled();
+ });
+
+ it.each([new Error("CONNECTION_CLOSED"), disconnected("28P01"), disconnected("42501"), disconnected("22P02"), disconnected("25P02")])(
+ "does not retry an unclassified or non-transient error", async (error) => {
+ const test = fixture({ failures: { issue_thread_interactions: [error] } });
+ await expect(executionProjectionsForRuns(test.db, companyId, [runId], now, { retryDatabaseReads: true })).rejects.toBe(error);
+ expect(test.count("issue_thread_interactions")).toBe(1);
+ expect(test.count("issue_recovery_actions")).toBe(0);
+ },
+ );
+
+ it("preserves empty and taskless projection behavior without unnecessary reads", async () => {
+ const empty = fixture();
+ expect(await executionProjectionsForRuns(empty.db, companyId, [], now, { retryDatabaseReads: true })).toEqual(new Map());
+ expect(empty.select).not.toHaveBeenCalled();
+ for (const options of [{ missing: true }, { noIssue: true }]) {
+ const test = fixture(options);
+ const result = await executionProjectionsForRuns(test.db, companyId, [runId], now, { retryDatabaseReads: true });
+ expect(test.select).toHaveBeenCalledTimes(2);
+ expect(options.missing ? result.size : result.get(runId)?.phase).toBe(options.missing ? 0 : "completed");
+ }
+ });
+
+ it.each([false, true])("the run list retries projection reads without replaying other reads or backfill writes (write fails: %s)", async (writeFails) => {
+ vi.spyOn(logger, "warn").mockImplementation(() => undefined);
+ const test = fixture({ backfill: true, writeFailure: writeFails ? disconnected() : undefined,
+ failures: { issue_thread_interactions: [disconnected(), disconnected()] } });
+ const result = await activityService(test.db).runsForIssue(companyId, issueId);
+ expect(result).toMatchObject([{ runId, execution: { phase: "waiting_for_answer" } }]);
+ expect(test.count("issue_thread_interactions")).toBe(3);
+ for (const key of ["run_list", "liveness_backfill", "heartbeat_runs", "native_run_finalizations", "environment_leases", "issue_recovery_actions"]) {
+ expect(test.count(key)).toBe(1);
+ }
+ // One retry-exhaustion lookup plus the independent backfill's event count.
+ expect(test.count("heartbeat_run_events")).toBe(2);
+ expect(test.update).toHaveBeenCalledTimes(1);
+ expect(test.updateWhere).toHaveBeenCalledTimes(1);
+ expect(test.transaction).not.toHaveBeenCalled();
+ });
+});
diff --git a/server/src/services/execution-projection.ts b/server/src/services/execution-projection.ts
index cab39d73e3..ae88b4b739 100644
--- a/server/src/services/execution-projection.ts
+++ b/server/src/services/execution-projection.ts
@@ -9,6 +9,7 @@ import {
import type { ExecutionProjection } from "@paperclipai/shared";
import { EXECUTION_CONTROL_DEADLINE_MS } from "./execution-control-deadline.js";
import { executionFailureRetryCount } from "./execution-recovery-attempt.js";
+import { retryIdempotentDatabaseOperation } from "../database-retry.js";
const text = (v: unknown) => (typeof v === "string" ? v : null);
const executionRunColumns = {
id: heartbeatRuns.id,
@@ -48,10 +49,16 @@ export async function executionProjectionsForRuns(
companyId: string,
runIds: string[],
now = new Date(),
+ options: { retryDatabaseReads?: boolean } = {},
) {
const projections = new Map<string, ExecutionProjection>();
if (!runIds.length) return projections;
- const runs = await db
+ // Opt in only with a pooled Db outside a transaction. Issue enrichment also
+ // calls this helper with a transaction, whose failed reads must not replay.
+ const read = <T>(query: () => Promise<T>) => options.retryDatabaseReads
+ ? retryIdempotentDatabaseOperation(query)
+ : query();
+ const runs = await read(async () => db
.select(executionRunColumns)
.from(heartbeatRuns)
.where(
@@ -59,8 +66,8 @@ export async function executionProjectionsForRuns(
eq(heartbeatRuns.companyId, companyId),
inArray(heartbeatRuns.id, runIds),
),
- );
- const coordinators = await db
+ ));
+ const coordinators = await read(async () => db
.select()
.from(nativeRunFinalizations)
.where(
@@ -68,7 +75,7 @@ export async function executionProjectionsForRuns(
eq(nativeRunFinalizations.companyId, companyId),
inArray(nativeRunFinalizations.runId, runIds),
),
- );
+ ));
const issueIds = [
...new Set(
runs
@@ -77,7 +84,7 @@ export async function executionProjectionsForRuns(
),
];
const pending = issueIds.length
- ? await db
+ ? await read(async () => db
.select({
issueId: issueThreadInteractions.issueId,
kind: issueThreadInteractions.kind,
@@ -89,10 +96,10 @@ export async function executionProjectionsForRuns(
inArray(issueThreadInteractions.issueId, issueIds),
eq(issueThreadInteractions.status, "pending"),
),
- )
+ ))
: [];
const recovery = issueIds.length
- ? await db
+ ? await read(async () => db
.select({
issueId: issueRecoveryActions.sourceIssueId,
cause: issueRecoveryActions.cause,
@@ -112,7 +119,7 @@ export async function executionProjectionsForRuns(
]),
),
)
- .orderBy(desc(issueRecoveryActions.updatedAt))
+ .orderBy(desc(issueRecoveryActions.updatedAt)))
: [];
const coordinatorByRun = new Map(coordinators.map((row) => [row.runId, row]));
for (const run of runs) {