Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

fix(shared): retry Prisma reads on transient Postgres connection errors (#19480)

Niklas Semmler8ddebc0713bf
diff --git a/packages/shared/src/db.ts b/packages/shared/src/db.ts
index 7ef667dec2..f7f18a1568 100644
--- a/packages/shared/src/db.ts
+++ b/packages/shared/src/db.ts
@@ -6,6 +6,7 @@ import { Prisma, PrismaClient } from "@prisma/client";
import { env } from "process";
import { logger } from "./server";
import { withPostgresApplicationName } from "./server/postgresApplicationName";
+import { postgresConnectionRetryExtension } from "./dbConnectionRetry";
export class PrismaClientSingleton {
private static instance: PrismaClient;
@@ -58,7 +59,12 @@ const createPrismaInstance = () => {
client.$on("error", (event) => {
logger.error(`prisma:error ${event.message}`);
});
- return client;
+
+ // A query extension leaves model and result types untouched, so the
+ // extended client is still a PrismaClient for every caller.
+ return client.$extends(
+ postgresConnectionRetryExtension,
+ ) as unknown as PrismaClient;
};
declare const globalThis: {
diff --git a/packages/shared/src/dbConnectionRetry.test.ts b/packages/shared/src/dbConnectionRetry.test.ts
new file mode 100644
index 0000000000..2576c1ca8a
--- /dev/null
+++ b/packages/shared/src/dbConnectionRetry.test.ts
@@ -0,0 +1,110 @@
+import { afterAll, beforeEach, describe, expect, it, vi } from "vitest";
+import { Prisma, PrismaClient } from "@prisma/client";
+import {
+ postgresConnectionRetryExtension,
+ withPostgresConnectionRetry,
+} from "./dbConnectionRetry";
+import { logger } from "./server/logger";
+
+const unreachable = () =>
+ new Prisma.PrismaClientKnownRequestError(
+ "Can't reach database server at `db:5432`",
+ { code: "P1001", clientVersion: "test" },
+ );
+const startingUp = () =>
+ new Prisma.PrismaClientUnknownRequestError(
+ "Error in connector: Error querying the database: FATAL: the database system is starting up",
+ { clientVersion: "test" },
+ );
+const connectionClosedOnInit = () =>
+ new Prisma.PrismaClientInitializationError(
+ "Server has closed the connection.",
+ "test",
+ "P1017",
+ );
+
+describe("withPostgresConnectionRetry", () => {
+ it.each([
+ ["unreachable server", unreachable],
+ ["server starting up", startingUp],
+ ["connection closed", connectionClosedOnInit],
+ ])("retries reads on %s", async (_, makeError) => {
+ const run = vi
+ .fn()
+ .mockRejectedValueOnce(makeError())
+ .mockResolvedValueOnce("ok");
+
+ await expect(
+ withPostgresConnectionRetry({ operation: "findFirst" }, run, 0),
+ ).resolves.toBe("ok");
+ expect(run).toHaveBeenCalledTimes(2);
+ });
+
+ it("does not retry writes on connection errors", async () => {
+ const error = unreachable();
+ const run = vi.fn().mockRejectedValue(error);
+
+ await expect(
+ withPostgresConnectionRetry({ operation: "create" }, run, 0),
+ ).rejects.toBe(error);
+ expect(run).toHaveBeenCalledTimes(1);
+ });
+
+ it("does not retry query errors", async () => {
+ const error = new Prisma.PrismaClientKnownRequestError(
+ "Unique constraint failed",
+ { code: "P2002", clientVersion: "test" },
+ );
+ const run = vi.fn().mockRejectedValue(error);
+
+ await expect(
+ withPostgresConnectionRetry({ operation: "findFirst" }, run, 0),
+ ).rejects.toBe(error);
+ expect(run).toHaveBeenCalledTimes(1);
+ });
+
+ it("throws the last error once retries are exhausted", async () => {
+ const last = unreachable();
+ const run = vi
+ .fn()
+ .mockRejectedValueOnce(startingUp())
+ .mockRejectedValueOnce(startingUp())
+ .mockRejectedValueOnce(startingUp())
+ .mockRejectedValueOnce(last);
+
+ await expect(
+ withPostgresConnectionRetry({ operation: "findMany" }, run, 0),
+ ).rejects.toBe(last);
+ expect(run).toHaveBeenCalledTimes(4);
+ });
+});
+
+describe("postgresConnectionRetryExtension", () => {
+ // Nothing listens on port 1, so every query fails to connect.
+ const base = new PrismaClient({
+ datasourceUrl: "postgresql://postgres:[email protected]:1/postgres",
+ });
+ const client = base.$extends(postgresConnectionRetryExtension);
+ const warn = vi.spyOn(logger, "warn");
+ const retryWarnings = () =>
+ warn.mock.calls.filter(([message]) =>
+ String(message).startsWith("Retrying Postgres"),
+ );
+
+ beforeEach(() => warn.mockClear());
+ afterAll(() => base.$disconnect());
+
+ it("retries reads issued through the extended client", async () => {
+ await expect(client.project.findFirst()).rejects.toThrow(
+ "Can't reach database server",
+ );
+ expect(retryWarnings()).toHaveLength(3);
+ });
+
+ it("does not retry writes issued through the extended client", async () => {
+ await expect(
+ client.organization.create({ data: { name: "unreachable" } }),
+ ).rejects.toThrow("Can't reach database server");
+ expect(retryWarnings()).toHaveLength(0);
+ });
+});
diff --git a/packages/shared/src/dbConnectionRetry.ts b/packages/shared/src/dbConnectionRetry.ts
new file mode 100644
index 0000000000..54c290dec0
--- /dev/null
+++ b/packages/shared/src/dbConnectionRetry.ts
@@ -0,0 +1,88 @@
+import { Prisma } from "@prisma/client";
+import { backOff } from "exponential-backoff";
+import { logger } from "./server/logger";
+
+const READ_OPERATIONS = new Set([
+ "findUnique",
+ "findUniqueOrThrow",
+ "findFirst",
+ "findFirstOrThrow",
+ "findMany",
+ "count",
+ "aggregate",
+ "groupBy",
+]);
+
+const errorCode = (error: unknown): string | undefined => {
+ if (error instanceof Prisma.PrismaClientKnownRequestError) return error.code;
+ if (error instanceof Prisma.PrismaClientInitializationError)
+ return error.errorCode;
+ return undefined;
+};
+
+const errorMessage = (error: unknown): string =>
+ error instanceof Error ? error.message : "";
+
+// Matched on message too: a client without an open connection reports an
+// unreachable server as PrismaClientInitializationError without an error code.
+// Administrator termination is not matched: a deliberately killed query must not rerun.
+const CONNECTION_ERROR_MESSAGES = [
+ "Can't reach database server",
+ "the database system is starting up",
+ "the database system is shutting down",
+ "the database system is in recovery mode",
+ "the database system is not yet accepting connections",
+ "terminating connection because of crash of another server process",
+ "Error opening a TLS connection",
+];
+
+// P1001: server unreachable, P1017: server closed the connection.
+const isConnectionError = (error: unknown): boolean => {
+ const code = errorCode(error);
+ if (code === "P1001" || code === "P1017") return true;
+ const message = errorMessage(error);
+ return CONNECTION_ERROR_MESSAGES.some((m) => message.includes(m));
+};
+
+// Retries cover a Postgres restart (e.g. an Aurora minor-version upgrade or
+// failover), which refuses connections for one to two seconds.
+// Writes and raw queries are never retried: a dropped connection may already
+// have committed them, and a retried operation from a batch $transaction would
+// run outside that transaction.
+const isRetryable = (error: unknown, operation: string): boolean =>
+ READ_OPERATIONS.has(operation) && isConnectionError(error);
+
+const MAX_ATTEMPTS = 4;
+
+export const withPostgresConnectionRetry = <T>(
+ context: { model?: string; operation: string },
+ run: () => Promise<T>,
+ startingDelayMs = 250,
+): Promise<T> =>
+ backOff(run, {
+ numOfAttempts: MAX_ATTEMPTS,
+ startingDelay: startingDelayMs,
+ timeMultiple: 3,
+ maxDelay: 1500,
+ // Also called after the final attempt, which is never retried.
+ retry: (error, attemptNumber) => {
+ if (attemptNumber >= MAX_ATTEMPTS) return false;
+ if (!isRetryable(error, context.operation)) return false;
+ logger.warn(
+ `Retrying Postgres ${context.model ?? "raw"}.${context.operation} after connection error (attempt ${attemptNumber})`,
+ { error: errorMessage(error) },
+ );
+ return true;
+ },
+ });
+
+export const postgresConnectionRetryExtension = Prisma.defineExtension({
+ name: "postgresConnectionRetry",
+ query: {
+ $allOperations({ model, operation, args, query }) {
+ return withPostgresConnectionRetry({ model, operation }, () =>
+ query(args),
+ );
+ },
+ },
+});