Skip to content

The open-source app everyone uses to manage agents at work

Improve embedded Postgres bootstrap errors

dotta76a692c260ea
diff --git a/cli/src/commands/worktree.ts b/cli/src/commands/worktree.ts
index a528bf5b26..65e748497f 100644
--- a/cli/src/commands/worktree.ts
+++ b/cli/src/commands/worktree.ts
@@ -41,6 +41,8 @@ import {
projects,
runDatabaseBackup,
runDatabaseRestore,
+ createEmbeddedPostgresLogBuffer,
+ formatEmbeddedPostgresError,
} from "@paperclipai/db";
import type { Command } from "commander";
import { ensureAgentJwtSecret, loadPaperclipEnvFile, mergePaperclipEnvEntries, readPaperclipEnvEntries, resolvePaperclipEnvFile } from "../config/env.js";
@@ -806,6 +808,7 @@ async function ensureEmbeddedPostgres(dataDir: string, preferredPort: number): P
}
const port = await findAvailablePort(preferredPort);
+ const logBuffer = createEmbeddedPostgresLogBuffer();
const instance = new EmbeddedPostgres({
databaseDir: dataDir,
user: "paperclip",
@@ -813,17 +816,31 @@ async function ensureEmbeddedPostgres(dataDir: string, preferredPort: number): P
port,
persistent: true,
initdbFlags: ["--encoding=UTF8", "--locale=C", "--lc-messages=C"],
- onLog: () => {},
- onError: () => {},
+ onLog: logBuffer.append,
+ onError: logBuffer.append,
});
if (!existsSync(path.resolve(dataDir, "PG_VERSION"))) {
- await instance.initialise();
+ try {
+ await instance.initialise();
+ } catch (error) {
+ throw formatEmbeddedPostgresError(error, {
+ fallbackMessage: `Failed to initialize embedded PostgreSQL cluster in ${dataDir} on port ${port}`,
+ recentLogs: logBuffer.getRecentLogs(),
+ });
+ }
}
if (existsSync(postmasterPidFile)) {
rmSync(postmasterPidFile, { force: true });
}
- await instance.start();
+ try {
+ await instance.start();
+ } catch (error) {
+ throw formatEmbeddedPostgresError(error, {
+ fallbackMessage: `Failed to start embedded PostgreSQL on port ${port}`,
+ recentLogs: logBuffer.getRecentLogs(),
+ });
+ }
return {
port,
diff --git a/packages/db/src/embedded-postgres-error.test.ts b/packages/db/src/embedded-postgres-error.test.ts
new file mode 100644
index 0000000000..dba1ad4615
--- /dev/null
+++ b/packages/db/src/embedded-postgres-error.test.ts
@@ -0,0 +1,28 @@
+import { describe, expect, it } from "vitest";
+import { createEmbeddedPostgresLogBuffer, formatEmbeddedPostgresError } from "./embedded-postgres-error.js";
+
+describe("formatEmbeddedPostgresError", () => {
+ it("adds a shared-memory hint when initdb logs expose the real cause", () => {
+ const error = formatEmbeddedPostgresError("Postgres init script exited with code 1.", {
+ fallbackMessage: "Failed to initialize embedded PostgreSQL cluster",
+ recentLogs: [
+ "running bootstrap script ...",
+ "FATAL: could not create shared memory segment: Cannot allocate memory",
+ "DETAIL: Failed system call was shmget(key=123, size=56, 03600).",
+ ],
+ });
+
+ expect(error.message).toContain("could not allocate shared memory");
+ expect(error.message).toContain("kern.sysv.shm");
+ expect(error.message).toContain("could not create shared memory segment");
+ });
+
+ it("keeps only recent non-empty log lines in the collector", () => {
+ const buffer = createEmbeddedPostgresLogBuffer(2);
+ buffer.append("line one\n\n");
+ buffer.append("line two");
+ buffer.append("line three");
+
+ expect(buffer.getRecentLogs()).toEqual(["line two", "line three"]);
+ });
+});
diff --git a/packages/db/src/embedded-postgres-error.ts b/packages/db/src/embedded-postgres-error.ts
new file mode 100644
index 0000000000..9862a0f3be
--- /dev/null
+++ b/packages/db/src/embedded-postgres-error.ts
@@ -0,0 +1,89 @@
+const DEFAULT_RECENT_LOG_LIMIT = 40;
+const RECENT_LOG_SUMMARY_LINES = 8;
+
+function toError(error: unknown, fallbackMessage: string): Error {
+ if (error instanceof Error) return error;
+ if (error === undefined) return new Error(fallbackMessage);
+ if (typeof error === "string") return new Error(`${fallbackMessage}: ${error}`);
+
+ try {
+ return new Error(`${fallbackMessage}: ${JSON.stringify(error)}`);
+ } catch {
+ return new Error(`${fallbackMessage}: ${String(error)}`);
+ }
+}
+
+function summarizeRecentLogs(recentLogs: string[]): string | null {
+ if (recentLogs.length === 0) return null;
+ return recentLogs
+ .slice(-RECENT_LOG_SUMMARY_LINES)
+ .map((line) => line.trim())
+ .filter((line) => line.length > 0)
+ .join(" | ");
+}
+
+function detectEmbeddedPostgresHint(recentLogs: string[]): string | null {
+ const haystack = recentLogs.join("\n").toLowerCase();
+ if (!haystack.includes("could not create shared memory segment")) {
+ return null;
+ }
+
+ return (
+ "Embedded PostgreSQL bootstrap could not allocate shared memory. " +
+ "On macOS, this usually means the host's kern.sysv.shm* limits are too low for another local PostgreSQL cluster. " +
+ "Stop other local PostgreSQL servers or raise the shared-memory sysctls, then retry."
+ );
+}
+
+export function createEmbeddedPostgresLogBuffer(limit = DEFAULT_RECENT_LOG_LIMIT): {
+ append(message: unknown): void;
+ getRecentLogs(): string[];
+} {
+ const recentLogs: string[] = [];
+
+ return {
+ append(message: unknown) {
+ const text =
+ typeof message === "string"
+ ? message
+ : message instanceof Error
+ ? message.message
+ : String(message ?? "");
+
+ for (const rawLine of text.split(/\r?\n/)) {
+ const line = rawLine.trim();
+ if (!line) continue;
+ recentLogs.push(line);
+ if (recentLogs.length > limit) {
+ recentLogs.splice(0, recentLogs.length - limit);
+ }
+ }
+ },
+ getRecentLogs() {
+ return [...recentLogs];
+ },
+ };
+}
+
+export function formatEmbeddedPostgresError(
+ error: unknown,
+ input: {
+ fallbackMessage: string;
+ recentLogs?: string[];
+ },
+): Error {
+ const baseError = toError(error, input.fallbackMessage);
+ const recentLogs = input.recentLogs ?? [];
+ const parts = [baseError.message];
+ const hint = detectEmbeddedPostgresHint(recentLogs);
+ const recentSummary = summarizeRecentLogs(recentLogs);
+
+ if (hint) {
+ parts.push(hint);
+ }
+ if (recentSummary) {
+ parts.push(`Recent embedded Postgres logs: ${recentSummary}`);
+ }
+
+ return new Error(parts.join(" "));
+}
diff --git a/packages/db/src/index.ts b/packages/db/src/index.ts
index b5ccb5d43d..6c45acbc43 100644
--- a/packages/db/src/index.ts
+++ b/packages/db/src/index.ts
@@ -25,4 +25,8 @@ export {
type RunDatabaseBackupResult,
type RunDatabaseRestoreOptions,
} from "./backup-lib.js";
+export {
+ createEmbeddedPostgresLogBuffer,
+ formatEmbeddedPostgresError,
+} from "./embedded-postgres-error.js";
export * from "./schema/index.js";
diff --git a/packages/db/src/migration-runtime.ts b/packages/db/src/migration-runtime.ts
index 921de612a3..5aa2b6a27d 100644
--- a/packages/db/src/migration-runtime.ts
+++ b/packages/db/src/migration-runtime.ts
@@ -2,6 +2,7 @@ import { existsSync, readFileSync, rmSync } from "node:fs";
import { createServer } from "node:net";
import path from "node:path";
import { ensurePostgresDatabase, getPostgresDataDirectory } from "./client.js";
+import { createEmbeddedPostgresLogBuffer, formatEmbeddedPostgresError } from "./embedded-postgres-error.js";
import { resolveDatabaseTarget } from "./runtime-config.js";
type EmbeddedPostgresInstance = {
@@ -27,18 +28,6 @@ export type MigrationConnection = {
stop: () => Promise<void>;
};
-function toError(error: unknown, fallbackMessage: string): Error {
- if (error instanceof Error) return error;
- if (error === undefined) return new Error(fallbackMessage);
- if (typeof error === "string") return new Error(`${fallbackMessage}: ${error}`);
-
- try {
- return new Error(`${fallbackMessage}: ${JSON.stringify(error)}`);
- } catch {
- return new Error(`${fallbackMessage}: ${String(error)}`);
- }
-}
-
function readRunningPostmasterPid(postmasterPidFile: string): number | null {
if (!existsSync(postmasterPidFile)) return null;
try {
@@ -109,6 +98,7 @@ async function ensureEmbeddedPostgresConnection(
const runningPid = readRunningPostmasterPid(postmasterPidFile);
const runningPort = readPidFilePort(postmasterPidFile);
const preferredAdminConnectionString = `postgres://paperclip:[email protected]:${preferredPort}/postgres`;
+ const logBuffer = createEmbeddedPostgresLogBuffer();
if (!runningPid && existsSync(pgVersionFile)) {
try {
@@ -151,18 +141,19 @@ async function ensureEmbeddedPostgresConnection(
port: selectedPort,
persistent: true,
initdbFlags: ["--encoding=UTF8", "--locale=C", "--lc-messages=C"],
- onLog: () => {},
- onError: () => {},
+ onLog: logBuffer.append,
+ onError: logBuffer.append,
});
if (!existsSync(path.resolve(dataDir, "PG_VERSION"))) {
try {
await instance.initialise();
} catch (error) {
- throw toError(
- error,
- `Failed to initialize embedded PostgreSQL cluster in ${dataDir} on port ${selectedPort}`,
- );
+ throw formatEmbeddedPostgresError(error, {
+ fallbackMessage:
+ `Failed to initialize embedded PostgreSQL cluster in ${dataDir} on port ${selectedPort}`,
+ recentLogs: logBuffer.getRecentLogs(),
+ });
}
}
if (existsSync(postmasterPidFile)) {
@@ -171,7 +162,10 @@ async function ensureEmbeddedPostgresConnection(
try {
await instance.start();
} catch (error) {
- throw toError(error, `Failed to start embedded PostgreSQL on port ${selectedPort}`);
+ throw formatEmbeddedPostgresError(error, {
+ fallbackMessage: `Failed to start embedded PostgreSQL on port ${selectedPort}`,
+ recentLogs: logBuffer.getRecentLogs(),
+ });
}
const adminConnectionString = `postgres://paperclip:[email protected]:${selectedPort}/postgres`;
diff --git a/server/src/index.ts b/server/src/index.ts
index c37157c053..7ebfa7d1c5 100644
--- a/server/src/index.ts
+++ b/server/src/index.ts
@@ -10,9 +10,11 @@ import { and, eq } from "drizzle-orm";
import {
createDb,
ensurePostgresDatabase,
+ formatEmbeddedPostgresError,
getPostgresDataDirectory,
inspectMigrations,
applyPendingMigrations,
+ createEmbeddedPostgresLogBuffer,
reconcilePendingMigrationHistory,
formatDatabaseBackupResult,
runDatabaseBackup,
@@ -272,29 +274,31 @@ export async function startServer(): Promise<StartedServer> {
const dataDir = resolve(config.embeddedPostgresDataDir);
const configuredPort = config.embeddedPostgresPort;
let port = configuredPort;
- const embeddedPostgresLogBuffer: string[] = [];
- const EMBEDDED_POSTGRES_LOG_BUFFER_LIMIT = 120;
+ const logBuffer = createEmbeddedPostgresLogBuffer(120);
const verboseEmbeddedPostgresLogs = process.env.PAPERCLIP_EMBEDDED_POSTGRES_VERBOSE === "true";
const appendEmbeddedPostgresLog = (message: unknown) => {
- const text = typeof message === "string" ? message : message instanceof Error ? message.message : String(message ?? "");
- for (const lineRaw of text.split(/\r?\n/)) {
+ logBuffer.append(message);
+ if (!verboseEmbeddedPostgresLogs) {
+ return;
+ }
+ const lines = typeof message === "string"
+ ? message.split(/\r?\n/)
+ : message instanceof Error
+ ? [message.message]
+ : [String(message ?? "")];
+ for (const lineRaw of lines) {
const line = lineRaw.trim();
if (!line) continue;
- embeddedPostgresLogBuffer.push(line);
- if (embeddedPostgresLogBuffer.length > EMBEDDED_POSTGRES_LOG_BUFFER_LIMIT) {
- embeddedPostgresLogBuffer.splice(0, embeddedPostgresLogBuffer.length - EMBEDDED_POSTGRES_LOG_BUFFER_LIMIT);
- }
- if (verboseEmbeddedPostgresLogs) {
- logger.info({ embeddedPostgresLog: line }, "embedded-postgres");
- }
+ logger.info({ embeddedPostgresLog: line }, "embedded-postgres");
}
};
const logEmbeddedPostgresFailure = (phase: "initialise" | "start", err: unknown) => {
- if (embeddedPostgresLogBuffer.length > 0) {
+ const recentLogs = logBuffer.getRecentLogs();
+ if (recentLogs.length > 0) {
logger.error(
{
phase,
- recentLogs: embeddedPostgresLogBuffer,
+ recentLogs,
err,
},
"Embedded PostgreSQL failed; showing buffered startup logs",
@@ -371,7 +375,10 @@ export async function startServer(): Promise<StartedServer> {
await embeddedPostgres.initialise();
} catch (err) {
logEmbeddedPostgresFailure("initialise", err);
- throw err;
+ throw formatEmbeddedPostgresError(err, {
+ fallbackMessage: `Failed to initialize embedded PostgreSQL cluster in ${dataDir} on port ${port}`,
+ recentLogs: logBuffer.getRecentLogs(),
+ });
}
} else {
logger.info(`Embedded PostgreSQL cluster already exists (${clusterVersionFile}); skipping init`);
@@ -385,7 +392,10 @@ export async function startServer(): Promise<StartedServer> {
await embeddedPostgres.start();
} catch (err) {
logEmbeddedPostgresFailure("start", err);
- throw err;
+ throw formatEmbeddedPostgresError(err, {
+ fallbackMessage: `Failed to start embedded PostgreSQL on port ${port}`,
+ recentLogs: logBuffer.getRecentLogs(),
+ });
}
embeddedPostgresStartedByThisProcess = true;
}