The open-source app everyone uses to manage agents at work
TypeScript acpxagent-orchestrationagent-skillsagentic-aiai-agentsai-engineeringai-governanceai-toolsautonomous-agentsclaudeclaude-codeclaude-skillscodexhermesllmmcpmulti-agentopenclawproject-managementworkflow-automation
fix(server): remove the fixture-cleanup deadlock in the stale queued-run test (#14968)
Nicky Leach41c1431d66fc
diff --git a/server/src/__tests__/heartbeat-active-run-output-watchdog.test.ts b/server/src/__tests__/heartbeat-active-run-output-watchdog.test.tsindex eb04d057a4..197dc260ed 100644--- a/server/src/__tests__/heartbeat-active-run-output-watchdog.test.ts+++ b/server/src/__tests__/heartbeat-active-run-output-watchdog.test.ts@@ -1,5 +1,5 @@import { randomUUID } from "node:crypto";-import { and, eq, sql } from "drizzle-orm";+import { and, eq } from "drizzle-orm";import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest";import {activityLog,@@ -19,6 +19,7 @@ import {getEmbeddedPostgresTestSupport,startEmbeddedPostgresTestDatabase,} from "./helpers/embedded-postgres.js";+import { truncateTablesWithDeadlockRetry } from "./helpers/truncate-with-deadlock-retry.js";import { appendHeartbeatRunEvent } from "../services/heartbeat-run-events.js";import {ACTIVE_RUN_OUTPUT_CONTINUE_REARM_MS,@@ -43,29 +44,6 @@ if (!embeddedPostgresSupport.supported) {);}-function errorHasPostgresCode(error: unknown, code: string): boolean {- let current: unknown = error;- for (let depth = 0; depth < 4; depth += 1) {- if (!current || typeof current !== "object") return false;- const record = current as { code?: unknown; cause?: unknown };- if (record.code === code) return true;- current = record.cause;- }- return false;-}--async function truncateCompaniesWithDeadlockRetry(db: ReturnType<typeof createDb>) {- for (let attempt = 0; attempt < 5; attempt += 1) {- try {- await db.execute(sql.raw(`TRUNCATE TABLE "companies" CASCADE`));- return;- } catch (error) {- if (!errorHasPostgresCode(error, "40P01") || attempt === 4) throw error;- await new Promise((resolve) => setTimeout(resolve, 50 * (attempt + 1)));- }- }-}-describeEmbeddedPostgres("active-run output watchdog", () => {let tempDb: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>> | null = null;let db: ReturnType<typeof createDb>;@@ -77,7 +55,10 @@ describeEmbeddedPostgres("active-run output watchdog", () => {afterEach(async () => {mockedAppendHeartbeatRunEvent.mockClear();- await truncateCompaniesWithDeadlockRetry(db);+ await truncateTablesWithDeadlockRetry(db, `TRUNCATE TABLE "companies" CASCADE`, {+ attempts: 5,+ delayMs: (attempt) => 50 * (attempt + 1),+ });});afterAll(async () => {diff --git a/server/src/__tests__/heartbeat-stale-queue-invalidation.test.ts b/server/src/__tests__/heartbeat-stale-queue-invalidation.test.tsindex 882a81e5d2..99efb01d32 100644--- a/server/src/__tests__/heartbeat-stale-queue-invalidation.test.ts+++ b/server/src/__tests__/heartbeat-stale-queue-invalidation.test.ts@@ -22,6 +22,8 @@ import {getEmbeddedPostgresTestSupport,startEmbeddedPostgresTestDatabase,} from "./helpers/embedded-postgres.js";+import { drainHeartbeatRunsToQuiescence } from "./helpers/drain-heartbeat-runs.js";+import { truncateTablesWithDeadlockRetry } from "./helpers/truncate-with-deadlock-retry.js";import {MAX_TURN_CONTINUATION_RETRY_REASON,MAX_TURN_CONTINUATION_WAKE_REASON,@@ -89,42 +91,30 @@ async function waitForCondition(fn: () => Promise<boolean>, timeoutMs = 3_000) {}async function cleanupHeartbeatInvalidationFixture(db: ReturnType<typeof createDb>) {- for (let attempt = 0; attempt < 10; attempt += 1) {- try {- await db.execute(sql.raw(`- TRUNCATE TABLE- "company_skills",- "issue_comments",- "issue_documents",- "document_revisions",- "documents",- "issue_relations",- "issue_tree_holds",- "issues",- "heartbeat_run_events",- "cost_events",- "activity_log",- "heartbeat_runs",- "agent_wakeup_requests",- "agent_runtime_state",- "agents",- "companies"- RESTART IDENTITY CASCADE- `));- return;- } catch (error) {- const isLateCommentRace =- error instanceof Error &&- error.message.includes("issue_comments_issue_id_issues_id_fk");- if (!isLateCommentRace || attempt === 9) {- throw error;- }-- // Heartbeat completion can write issue-thread comments shortly after the- // run leaves queued/running. Retry the dependent deletes once those land.- await new Promise((resolve) => setTimeout(resolve, 100));- }- }+ await truncateTablesWithDeadlockRetry(+ db,+ `+ TRUNCATE TABLE+ "company_skills",+ "issue_comments",+ "issue_documents",+ "document_revisions",+ "documents",+ "issue_relations",+ "issue_tree_holds",+ "issues",+ "heartbeat_run_events",+ "cost_events",+ "activity_log",+ "heartbeat_runs",+ "agent_wakeup_requests",+ "agent_runtime_state",+ "agents",+ "companies"+ RESTART IDENTITY CASCADE+ `,+ { attempts: 10, delayMs: () => 100 },+ );}type SeedOptions = {@@ -182,21 +172,12 @@ describeEmbeddedPostgres("heartbeat stale queued-run invalidation", () => {model: "test-model",}));runningProcesses.clear();- let idlePolls = 0;- for (let attempt = 0; attempt < 100; attempt += 1) {- const runs = await db- .select({ status: heartbeatRuns.status })- .from(heartbeatRuns);- const hasActiveRun = runs.some((run) => run.status === "queued" || run.status === "running");- if (!hasActiveRun) {- idlePolls += 1;- if (idlePolls >= 3) break;- } else {- idlePolls = 0;- }- await new Promise((resolve) => setTimeout(resolve, 50));- }- await new Promise((resolve) => setTimeout(resolve, 50));+ // A table poll for "queued" or "running" rows cannot see a wakeup promise+ // that has not yet written its run row. The shared drain helper awaits+ // both in-flight wakeup promises and in-flight run executions first, then+ // re-checks the run table, so a late writer cannot still be in flight when+ // the TRUNCATE below runs.+ await drainHeartbeatRunsToQuiescence(db, heartbeat);await cleanupHeartbeatInvalidationFixture(db);});diff --git a/server/src/__tests__/helpers/truncate-with-deadlock-retry.test.ts b/server/src/__tests__/helpers/truncate-with-deadlock-retry.test.tsnew file mode 100644index 0000000000..98d9d657ee--- /dev/null+++ b/server/src/__tests__/helpers/truncate-with-deadlock-retry.test.ts@@ -0,0 +1,96 @@+import { describe, expect, it, vi } from "vitest";+import { errorHasPostgresCode, truncateTablesWithDeadlockRetry } from "./truncate-with-deadlock-retry.js";++function deadlockError(): Error & { code: string } {+ return Object.assign(new Error("deadlock detected"), { code: "40P01" });+}++function lateCommentForeignKeyError(): Error {+ return new Error(+ 'insert or update on table "issue_comments" violates foreign key constraint "issue_comments_issue_id_issues_id_fk"',+ );+}++function unrelatedError(): Error & { code: string } {+ return Object.assign(new Error("relation does not exist"), { code: "42P01" });+}++function fakeDb(execute: ReturnType<typeof vi.fn>) {+ return { execute } as unknown as Parameters<typeof truncateTablesWithDeadlockRetry>[0];+}++describe("truncateTablesWithDeadlockRetry", () => {+ it("retries a deadlock (40P01) and succeeds once the lock clears", async () => {+ const execute = vi.fn()+ .mockRejectedValueOnce(deadlockError())+ .mockRejectedValueOnce(deadlockError())+ .mockResolvedValueOnce(undefined);++ await truncateTablesWithDeadlockRetry(fakeDb(execute), "TRUNCATE TABLE \"companies\" CASCADE", {+ attempts: 5,+ delayMs: () => 0,+ });++ expect(execute).toHaveBeenCalledTimes(3);+ });++ it("retries the late-comment foreign key violation and succeeds once the dependent row lands", async () => {+ const execute = vi.fn()+ .mockRejectedValueOnce(lateCommentForeignKeyError())+ .mockResolvedValueOnce(undefined);++ await truncateTablesWithDeadlockRetry(fakeDb(execute), "TRUNCATE TABLE \"issue_comments\" CASCADE", {+ attempts: 5,+ delayMs: () => 0,+ });++ expect(execute).toHaveBeenCalledTimes(2);+ });++ it("rethrows once attempts are exhausted on a persistent deadlock", async () => {+ const error = deadlockError();+ const execute = vi.fn().mockRejectedValue(error);++ await expect(+ truncateTablesWithDeadlockRetry(fakeDb(execute), "TRUNCATE TABLE \"companies\" CASCADE", {+ attempts: 3,+ delayMs: () => 0,+ }),+ ).rejects.toBe(error);++ expect(execute).toHaveBeenCalledTimes(3);+ });++ it("rethrows immediately on an error that is neither a deadlock nor the late-comment race", async () => {+ const error = unrelatedError();+ const execute = vi.fn().mockRejectedValue(error);++ await expect(+ truncateTablesWithDeadlockRetry(fakeDb(execute), "TRUNCATE TABLE \"companies\" CASCADE", {+ attempts: 5,+ delayMs: () => 0,+ }),+ ).rejects.toBe(error);++ expect(execute).toHaveBeenCalledTimes(1);+ });+});++describe("errorHasPostgresCode", () => {+ it("finds the code on the error itself", () => {+ expect(errorHasPostgresCode(deadlockError(), "40P01")).toBe(true);+ });++ it("finds the code on a wrapped cause", () => {+ const wrapped = new Error("wrapper", { cause: deadlockError() });+ expect(errorHasPostgresCode(wrapped, "40P01")).toBe(true);+ });++ it("returns false when the code never matches", () => {+ expect(errorHasPostgresCode(unrelatedError(), "40P01")).toBe(false);+ });++ it("returns false for a non-object error", () => {+ expect(errorHasPostgresCode("plain string failure", "40P01")).toBe(false);+ });+});diff --git a/server/src/__tests__/helpers/truncate-with-deadlock-retry.ts b/server/src/__tests__/helpers/truncate-with-deadlock-retry.tsnew file mode 100644index 0000000000..98f64b7894--- /dev/null+++ b/server/src/__tests__/helpers/truncate-with-deadlock-retry.ts@@ -0,0 +1,49 @@+import { sql } from "drizzle-orm";+import type { createDb } from "@paperclipai/db";++type Db = ReturnType<typeof createDb>;++const LATE_COMMENT_FOREIGN_KEY = "issue_comments_issue_id_issues_id_fk";+const DEADLOCK_DETECTED_CODE = "40P01";++export function errorHasPostgresCode(error: unknown, code: string): boolean {+ let current: unknown = error;+ for (let depth = 0; depth < 4; depth += 1) {+ if (!current || typeof current !== "object") return false;+ const record = current as { code?: unknown; cause?: unknown };+ if (record.code === code) return true;+ current = record.cause;+ }+ return false;+}++function isLateWriterRace(error: unknown): boolean {+ if (errorHasPostgresCode(error, DEADLOCK_DETECTED_CODE)) return true;+ return error instanceof Error && error.message.includes(LATE_COMMENT_FOREIGN_KEY);+}++// A fixture cleanup can race a background write that the test's drain step+// cannot see yet. Two forms of that race show up in this suite: a PostgreSQL+// deadlock (code 40P01) when a late writer holds a row lock that the+// TRUNCATE's lock ordering also needs, and a foreign-key violation when a+// dependent row lands between two deletes. Retry a fixed number of times with+// a backoff delay before the caller gives up and rethrows.+export async function truncateTablesWithDeadlockRetry(+ db: Db,+ truncateStatement: string,+ opts: { attempts?: number; delayMs?: (attempt: number) => number } = {},+): Promise<void> {+ const attempts = opts.attempts ?? 10;+ const delayMs = opts.delayMs ?? ((attempt: number) => 50 * (attempt + 1));+ for (let attempt = 0; attempt < attempts; attempt += 1) {+ try {+ await db.execute(sql.raw(truncateStatement));+ return;+ } catch (error) {+ if (!isLateWriterRace(error) || attempt === attempts - 1) {+ throw error;+ }+ await new Promise((resolve) => setTimeout(resolve, delayMs(attempt)));+ }+ }+}