🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
fix(shared): retry Prisma reads on transient Postgres connection errors (#19480)
Niklas Semmler8ddebc0713bf
diff --git a/packages/shared/src/db.ts b/packages/shared/src/db.tsindex 7ef667dec2..f7f18a1568 100644--- a/packages/shared/src/db.ts+++ b/packages/shared/src/db.ts@@ -6,6 +6,7 @@ import { Prisma, PrismaClient } from "@prisma/client";import { env } from "process";import { logger } from "./server";import { withPostgresApplicationName } from "./server/postgresApplicationName";+import { postgresConnectionRetryExtension } from "./dbConnectionRetry";export class PrismaClientSingleton {private static instance: PrismaClient;@@ -58,7 +59,12 @@ const createPrismaInstance = () => {client.$on("error", (event) => {logger.error(`prisma:error ${event.message}`);});- return client;++ // A query extension leaves model and result types untouched, so the+ // extended client is still a PrismaClient for every caller.+ return client.$extends(+ postgresConnectionRetryExtension,+ ) as unknown as PrismaClient;};declare const globalThis: {diff --git a/packages/shared/src/dbConnectionRetry.test.ts b/packages/shared/src/dbConnectionRetry.test.tsnew file mode 100644index 0000000000..2576c1ca8a--- /dev/null+++ b/packages/shared/src/dbConnectionRetry.test.ts@@ -0,0 +1,110 @@+import { afterAll, beforeEach, describe, expect, it, vi } from "vitest";+import { Prisma, PrismaClient } from "@prisma/client";+import {+ postgresConnectionRetryExtension,+ withPostgresConnectionRetry,+} from "./dbConnectionRetry";+import { logger } from "./server/logger";++const unreachable = () =>+ new Prisma.PrismaClientKnownRequestError(+ "Can't reach database server at `db:5432`",+ { code: "P1001", clientVersion: "test" },+ );+const startingUp = () =>+ new Prisma.PrismaClientUnknownRequestError(+ "Error in connector: Error querying the database: FATAL: the database system is starting up",+ { clientVersion: "test" },+ );+const connectionClosedOnInit = () =>+ new Prisma.PrismaClientInitializationError(+ "Server has closed the connection.",+ "test",+ "P1017",+ );++describe("withPostgresConnectionRetry", () => {+ it.each([+ ["unreachable server", unreachable],+ ["server starting up", startingUp],+ ["connection closed", connectionClosedOnInit],+ ])("retries reads on %s", async (_, makeError) => {+ const run = vi+ .fn()+ .mockRejectedValueOnce(makeError())+ .mockResolvedValueOnce("ok");++ await expect(+ withPostgresConnectionRetry({ operation: "findFirst" }, run, 0),+ ).resolves.toBe("ok");+ expect(run).toHaveBeenCalledTimes(2);+ });++ it("does not retry writes on connection errors", async () => {+ const error = unreachable();+ const run = vi.fn().mockRejectedValue(error);++ await expect(+ withPostgresConnectionRetry({ operation: "create" }, run, 0),+ ).rejects.toBe(error);+ expect(run).toHaveBeenCalledTimes(1);+ });++ it("does not retry query errors", async () => {+ const error = new Prisma.PrismaClientKnownRequestError(+ "Unique constraint failed",+ { code: "P2002", clientVersion: "test" },+ );+ const run = vi.fn().mockRejectedValue(error);++ await expect(+ withPostgresConnectionRetry({ operation: "findFirst" }, run, 0),+ ).rejects.toBe(error);+ expect(run).toHaveBeenCalledTimes(1);+ });++ it("throws the last error once retries are exhausted", async () => {+ const last = unreachable();+ const run = vi+ .fn()+ .mockRejectedValueOnce(startingUp())+ .mockRejectedValueOnce(startingUp())+ .mockRejectedValueOnce(startingUp())+ .mockRejectedValueOnce(last);++ await expect(+ withPostgresConnectionRetry({ operation: "findMany" }, run, 0),+ ).rejects.toBe(last);+ expect(run).toHaveBeenCalledTimes(4);+ });+});++describe("postgresConnectionRetryExtension", () => {+ // Nothing listens on port 1, so every query fails to connect.+ const base = new PrismaClient({+ datasourceUrl: "postgresql://postgres:[email protected]:1/postgres",+ });+ const client = base.$extends(postgresConnectionRetryExtension);+ const warn = vi.spyOn(logger, "warn");+ const retryWarnings = () =>+ warn.mock.calls.filter(([message]) =>+ String(message).startsWith("Retrying Postgres"),+ );++ beforeEach(() => warn.mockClear());+ afterAll(() => base.$disconnect());++ it("retries reads issued through the extended client", async () => {+ await expect(client.project.findFirst()).rejects.toThrow(+ "Can't reach database server",+ );+ expect(retryWarnings()).toHaveLength(3);+ });++ it("does not retry writes issued through the extended client", async () => {+ await expect(+ client.organization.create({ data: { name: "unreachable" } }),+ ).rejects.toThrow("Can't reach database server");+ expect(retryWarnings()).toHaveLength(0);+ });+});diff --git a/packages/shared/src/dbConnectionRetry.ts b/packages/shared/src/dbConnectionRetry.tsnew file mode 100644index 0000000000..54c290dec0--- /dev/null+++ b/packages/shared/src/dbConnectionRetry.ts@@ -0,0 +1,88 @@+import { Prisma } from "@prisma/client";+import { backOff } from "exponential-backoff";+import { logger } from "./server/logger";++const READ_OPERATIONS = new Set([+ "findUnique",+ "findUniqueOrThrow",+ "findFirst",+ "findFirstOrThrow",+ "findMany",+ "count",+ "aggregate",+ "groupBy",+]);++const errorCode = (error: unknown): string | undefined => {+ if (error instanceof Prisma.PrismaClientKnownRequestError) return error.code;+ if (error instanceof Prisma.PrismaClientInitializationError)+ return error.errorCode;+ return undefined;+};++const errorMessage = (error: unknown): string =>+ error instanceof Error ? error.message : "";++// Matched on message too: a client without an open connection reports an+// unreachable server as PrismaClientInitializationError without an error code.+// Administrator termination is not matched: a deliberately killed query must not rerun.+const CONNECTION_ERROR_MESSAGES = [+ "Can't reach database server",+ "the database system is starting up",+ "the database system is shutting down",+ "the database system is in recovery mode",+ "the database system is not yet accepting connections",+ "terminating connection because of crash of another server process",+ "Error opening a TLS connection",+];++// P1001: server unreachable, P1017: server closed the connection.+const isConnectionError = (error: unknown): boolean => {+ const code = errorCode(error);+ if (code === "P1001" || code === "P1017") return true;+ const message = errorMessage(error);+ return CONNECTION_ERROR_MESSAGES.some((m) => message.includes(m));+};++// Retries cover a Postgres restart (e.g. an Aurora minor-version upgrade or+// failover), which refuses connections for one to two seconds.+// Writes and raw queries are never retried: a dropped connection may already+// have committed them, and a retried operation from a batch $transaction would+// run outside that transaction.+const isRetryable = (error: unknown, operation: string): boolean =>+ READ_OPERATIONS.has(operation) && isConnectionError(error);++const MAX_ATTEMPTS = 4;++export const withPostgresConnectionRetry = <T>(+ context: { model?: string; operation: string },+ run: () => Promise<T>,+ startingDelayMs = 250,+): Promise<T> =>+ backOff(run, {+ numOfAttempts: MAX_ATTEMPTS,+ startingDelay: startingDelayMs,+ timeMultiple: 3,+ maxDelay: 1500,+ // Also called after the final attempt, which is never retried.+ retry: (error, attemptNumber) => {+ if (attemptNumber >= MAX_ATTEMPTS) return false;+ if (!isRetryable(error, context.operation)) return false;+ logger.warn(+ `Retrying Postgres ${context.model ?? "raw"}.${context.operation} after connection error (attempt ${attemptNumber})`,+ { error: errorMessage(error) },+ );+ return true;+ },+ });++export const postgresConnectionRetryExtension = Prisma.defineExtension({+ name: "postgresConnectionRetry",+ query: {+ $allOperations({ model, operation, args, query }) {+ return withPostgresConnectionRetry({ model, operation }, () =>+ query(args),+ );+ },+ },+});