Skip to content

The open-source app everyone uses to manage agents at work

perf(test): reuse a migrated embedded Postgres template across suites (#15861)

Devin Foley85f4205de391
diff --git a/doc/DEVELOPING.md b/doc/DEVELOPING.md
index 3425955fed..6fc8fa4f26 100644
--- a/doc/DEVELOPING.md
+++ b/doc/DEVELOPING.md
@@ -531,6 +531,35 @@ verifying Vite/HMR behavior.
For normal issue work, start with the smallest targeted check that proves the change. Reserve repo-wide typecheck/build/test runs for PR-ready handoff or changes broad enough that narrow checks do not cover the risk.
+#### Embedded Postgres template cache
+
+Suites that call `startEmbeddedPostgresTestDatabase` (from `@paperclipai/db`)
+get their own embedded Postgres cluster. The first such suite on a host builds
+one fully migrated cluster and publishes its data directory under
+`$TMPDIR/paperclip-embedded-postgres-templates-<uid>/<key>`. Every later suite
+copies that template and starts Postgres on the copy instead of running
+`initdb` and all migrations again, and the `getEmbeddedPostgresTestSupport`
+probe reuses it instead of starting a throwaway cluster. The key hashes the
+embedded-postgres version, the initdb flags, and every migration file, so a
+changed migration builds a new template. Each suite still runs
+`applyPendingMigrations` on its copy, which is a no-op when the template is
+current. Templates for other keys are removed after seven days without use.
+
+A template is a Postgres data directory, and starting it executes its
+configuration, so the cache is private to the current OS user: the root is
+created with mode `0700` and is only used when it is a real directory (not a
+symlink) owned by the current user with no group or world write bit, and each
+template is checked the same way before it is copied. A root that fails those
+checks (for example one pre-created by another user in a shared `/tmp`) is left
+alone and every suite in that process uses a fresh cluster.
+
+Set `PAPERCLIP_TEST_POSTGRES_TEMPLATE=0` to bypass the cache and give every
+suite a fresh `initdb`, for example when debugging cluster startup itself. A
+failure on the template path (root checks, build, copy, start, or publish)
+falls back to the fresh path automatically, so the cache can make a suite
+slower but never fail it or skip it: a migration that cannot apply still fails
+the suite on the fresh path.
+
### Task search evaluation
The task search relevance rubric and regression corpus are documented in
diff --git a/packages/db/src/test-embedded-postgres.test.ts b/packages/db/src/test-embedded-postgres.test.ts
index 1d4f78c797..aeec953d73 100644
--- a/packages/db/src/test-embedded-postgres.test.ts
+++ b/packages/db/src/test-embedded-postgres.test.ts
@@ -1,9 +1,21 @@
import fs from "node:fs";
+import os from "node:os";
+import path from "node:path";
+import postgres from "postgres";
import { afterEach, describe, expect, it } from "vitest";
+import { inspectMigrations } from "./client.js";
import {
__embeddedPostgresStartMaxAttemptsForTests as MAX_ATTEMPTS,
+ __embeddedPostgresTemplateRootDirForTests as templateRootDir,
+ __isUsableEmbeddedPostgresTemplateForTests as isUsableTemplate,
+ __lastEmbeddedPostgresStartModeForTests as lastStartMode,
+ __resetEmbeddedPostgresTemplateStateForTests as resetTemplateState,
__setEmbeddedPostgresCtorProviderForTests,
__startEmbeddedPostgresWithRetryForTests as startWithRetry,
+ computeEmbeddedPostgresTemplateKey,
+ EMBEDDED_POSTGRES_TEST_TIMEOUT_MS,
+ getEmbeddedPostgresTestSupport,
+ startEmbeddedPostgresTestDatabase,
} from "./test-embedded-postgres.js";
// A fake embedded-postgres constructor. It records every constructed instance so
@@ -111,3 +123,371 @@ describe("startEmbeddedPostgresWithRetry", () => {
expect((error as Error).message).toContain("Address already in use");
});
});
+
+describe("computeEmbeddedPostgresTemplateKey", () => {
+ function writeMigrations(dir: string, files: Record<string, string>) {
+ fs.mkdirSync(path.join(dir, "meta"), { recursive: true });
+ fs.writeFileSync(path.join(dir, "meta", "_journal.json"), JSON.stringify({ entries: Object.keys(files) }));
+ for (const [name, content] of Object.entries(files)) {
+ fs.writeFileSync(path.join(dir, name), content);
+ }
+ }
+
+ it("is stable for identical inputs and changes with any migration, flag, or version", () => {
+ const root = fs.mkdtempSync(path.join(os.tmpdir(), "paperclip-template-key-"));
+ try {
+ const a = path.join(root, "a");
+ const b = path.join(root, "b");
+ writeMigrations(a, { "0000_one.sql": "create table one();", "0001_two.sql": "create table two();" });
+ writeMigrations(b, { "0000_one.sql": "create table one();", "0001_two.sql": "create table two();" });
+ const fixed = { packageVersion: "18.1.0", initdbFlags: ["--locale=C"], user: "paperclip" };
+
+ const keyA = computeEmbeddedPostgresTemplateKey(a, fixed);
+ expect(keyA).toMatch(/^[0-9a-f]{24}$/);
+ expect(computeEmbeddedPostgresTemplateKey(a, fixed)).toBe(keyA);
+ expect(computeEmbeddedPostgresTemplateKey(b, fixed)).toBe(keyA);
+
+ // Content change, new file, flags, version, and user each change the key.
+ fs.writeFileSync(path.join(b, "0001_two.sql"), "create table two(id int);");
+ const contentChanged = computeEmbeddedPostgresTemplateKey(b, fixed);
+ expect(contentChanged).not.toBe(keyA);
+ fs.writeFileSync(path.join(b, "0002_three.sql"), "create table three();");
+ expect(computeEmbeddedPostgresTemplateKey(b, fixed)).not.toBe(contentChanged);
+ expect(computeEmbeddedPostgresTemplateKey(a, { ...fixed, initdbFlags: ["--locale=C", "--encoding=UTF8"] })).not.toBe(keyA);
+ expect(computeEmbeddedPostgresTemplateKey(a, { ...fixed, packageVersion: "18.2.0" })).not.toBe(keyA);
+ expect(computeEmbeddedPostgresTemplateKey(a, { ...fixed, user: "other" })).not.toBe(keyA);
+ } finally {
+ fs.rmSync(root, { recursive: true, force: true });
+ }
+ });
+
+ it("hashes the real migrations directory of this package", () => {
+ const key = computeEmbeddedPostgresTemplateKey();
+ expect(key).toMatch(/^[0-9a-f]{24}$/);
+ expect(computeEmbeddedPostgresTemplateKey()).toBe(key);
+ });
+});
+
+const describeOnPosix = process.platform === "win32" ? describe.skip : describe;
+
+describe("isUsableTemplate", () => {
+ let root: string;
+
+ function writeCluster(dir: string): void {
+ fs.mkdirSync(dir, { recursive: true, mode: 0o700 });
+ fs.writeFileSync(path.join(dir, "PG_VERSION"), "18\n");
+ }
+
+ afterEach(() => {
+ fs.rmSync(root, { recursive: true, force: true });
+ });
+
+ it("accepts a private directory that holds a cleanly stopped cluster", () => {
+ root = fs.mkdtempSync(path.join(os.tmpdir(), "paperclip-template-usable-"));
+ const template = path.join(root, "template");
+ writeCluster(template);
+ expect(isUsableTemplate(template)).toBe(true);
+
+ // A running or crashed cluster leaves `postmaster.pid` behind.
+ fs.writeFileSync(path.join(template, "postmaster.pid"), "1\n");
+ expect(isUsableTemplate(template)).toBe(false);
+ fs.rmSync(path.join(template, "postmaster.pid"));
+ expect(isUsableTemplate(template)).toBe(true);
+
+ // A directory without a cluster, or no directory at all.
+ fs.rmSync(path.join(template, "PG_VERSION"));
+ expect(isUsableTemplate(template)).toBe(false);
+ expect(isUsableTemplate(path.join(root, "missing"))).toBe(false);
+ });
+
+ it("rejects a symbolic link even when it points at a usable cluster", () => {
+ root = fs.mkdtempSync(path.join(os.tmpdir(), "paperclip-template-usable-"));
+ const template = path.join(root, "template");
+ writeCluster(template);
+ const link = path.join(root, "link");
+ fs.symlinkSync(template, link, "dir");
+ expect(isUsableTemplate(template)).toBe(true);
+ expect(isUsableTemplate(link)).toBe(false);
+ });
+
+ describeOnPosix("on POSIX", () => {
+ it("rejects a cluster directory that group or others can write", () => {
+ root = fs.mkdtempSync(path.join(os.tmpdir(), "paperclip-template-usable-"));
+ const template = path.join(root, "template");
+ writeCluster(template);
+ for (const mode of [0o720, 0o702, 0o777]) {
+ fs.chmodSync(template, mode);
+ expect(isUsableTemplate(template), `mode ${mode.toString(8)}`).toBe(false);
+ }
+ // Read bits for others are not a write path; Postgres narrows the copy
+ // to 0700 before it starts anyway.
+ fs.chmodSync(template, 0o755);
+ expect(isUsableTemplate(template)).toBe(true);
+ fs.chmodSync(template, 0o700);
+ expect(isUsableTemplate(template)).toBe(true);
+ });
+ });
+});
+
+// Real-cluster coverage of the template path. Skipped on hosts that cannot run
+// embedded Postgres, like every other suite that uses the fixture.
+const support = await getEmbeddedPostgresTestSupport();
+const describeWithPostgres = support.supported ? describe : describe.skip;
+
+// Points `os.tmpdir()` at a private scratch directory so a test can drive the
+// cache root from an empty state. `os.tmpdir()` reads the environment on every
+// call: TMPDIR on POSIX, TEMP and then TMP on Windows, so all three are set.
+// Returns the scratch directory; `restore` puts the environment and the
+// fixture's cached state back.
+const TMPDIR_VARIABLES = ["TMPDIR", "TEMP", "TMP"] as const;
+
+function useScratchTmpdir(prefix: string): { scratch: string; restore(): void } {
+ const scratch = fs.mkdtempSync(path.join(os.tmpdir(), prefix));
+ const previous = new Map(TMPDIR_VARIABLES.map((name) => [name, process.env[name]] as const));
+ for (const name of TMPDIR_VARIABLES) process.env[name] = scratch;
+ return {
+ scratch,
+ restore() {
+ for (const [name, value] of previous) {
+ if (value === undefined) delete process.env[name];
+ else process.env[name] = value;
+ }
+ resetTemplateState();
+ fs.rmSync(scratch, { recursive: true, force: true });
+ },
+ };
+}
+
+describeWithPostgres("startEmbeddedPostgresTestDatabase template path", () => {
+ const originalTemplateSetting = process.env.PAPERCLIP_TEST_POSTGRES_TEMPLATE;
+
+ afterEach(() => {
+ if (originalTemplateSetting === undefined) delete process.env.PAPERCLIP_TEST_POSTGRES_TEMPLATE;
+ else process.env.PAPERCLIP_TEST_POSTGRES_TEMPLATE = originalTemplateSetting;
+ });
+
+ it("publishes a complete template and starts suites from a copy of it", { timeout: EMBEDDED_POSTGRES_TEST_TIMEOUT_MS }, async () => {
+ // This test covers the template path, so it enables the cache even when
+ // the suite runs under PAPERCLIP_TEST_POSTGRES_TEMPLATE=0. The first
+ // start then builds the template if the support probe did not.
+ delete process.env.PAPERCLIP_TEST_POSTGRES_TEMPLATE;
+
+ const database = await startEmbeddedPostgresTestDatabase("paperclip-template-copy-");
+ try {
+ expect(lastStartMode()).toBe("template");
+ // The copy is fully migrated and independent of the template.
+ const state = await inspectMigrations(database.connectionString);
+ expect(state.status).toBe("upToDate");
+ const sql = postgres(database.connectionString, { max: 1 });
+ try {
+ await sql`create table paperclip_template_copy_probe (id int)`;
+ } finally {
+ await sql.end();
+ }
+ } finally {
+ await database.cleanup();
+ }
+
+ const rootDir = templateRootDir();
+ const templateDir = path.join(rootDir, computeEmbeddedPostgresTemplateKey());
+ expect(fs.existsSync(path.join(templateDir, "PG_VERSION"))).toBe(true);
+ expect(fs.existsSync(path.join(templateDir, "postmaster.pid"))).toBe(false);
+ expect(isUsableTemplate(templateDir)).toBe(true);
+ if (process.platform !== "win32") {
+ // The root is private to this user, and the template is a data
+ // directory Postgres accepts: both are 0700.
+ expect(path.basename(rootDir)).toBe(`paperclip-embedded-postgres-templates-${process.getuid?.()}`);
+ expect(fs.statSync(rootDir).mode & 0o777).toBe(0o700);
+ expect(fs.statSync(templateDir).mode & 0o777).toBe(0o700);
+ }
+ // Writing to the copy never touches the template, so a second copy does
+ // not see the table.
+ const second = await startEmbeddedPostgresTestDatabase("paperclip-template-copy-");
+ try {
+ expect(lastStartMode()).toBe("template");
+ const sql = postgres(second.connectionString, { max: 1 });
+ try {
+ const rows = await sql`select to_regclass('paperclip_template_copy_probe') as name`;
+ expect(rows[0]?.name).toBeNull();
+ } finally {
+ await sql.end();
+ }
+ } finally {
+ await second.cleanup();
+ }
+ });
+
+ it("refreshes the template's timestamp on every reuse so idle time drives pruning", { timeout: EMBEDDED_POSTGRES_TEST_TIMEOUT_MS }, async () => {
+ delete process.env.PAPERCLIP_TEST_POSTGRES_TEMPLATE;
+ // Make sure a template exists before ageing it: the support probe does
+ // not publish one under PAPERCLIP_TEST_POSTGRES_TEMPLATE=0, and a name
+ // filter can run this test on its own.
+ resetTemplateState();
+ const first = await startEmbeddedPostgresTestDatabase("paperclip-template-touch-");
+ await first.cleanup();
+ expect(lastStartMode()).toBe("template");
+ const templateDir = path.join(templateRootDir(), computeEmbeddedPostgresTemplateKey());
+ expect(isUsableTemplate(templateDir)).toBe(true);
+ // Age the template as if it had not been used for ten days.
+ const old = new Date(Date.now() - 10 * 24 * 60 * 60 * 1000);
+ fs.utimesSync(templateDir, old, old);
+ expect(fs.statSync(templateDir).mtimeMs).toBeLessThan(Date.now() - 9 * 24 * 60 * 60 * 1000);
+
+ resetTemplateState();
+ const database = await startEmbeddedPostgresTestDatabase("paperclip-template-touch-");
+ try {
+ expect(lastStartMode()).toBe("template");
+ } finally {
+ await database.cleanup();
+ }
+ expect(fs.statSync(templateDir).mtimeMs).toBeGreaterThan(Date.now() - 60 * 1000);
+ });
+
+ it("rebuilds the template when it was pruned after this process resolved it", { timeout: EMBEDDED_POSTGRES_TEST_TIMEOUT_MS }, async () => {
+ delete process.env.PAPERCLIP_TEST_POSTGRES_TEMPLATE;
+ const scratch = useScratchTmpdir("paperclip-template-pruned-");
+ try {
+ resetTemplateState();
+ const first = await startEmbeddedPostgresTestDatabase("paperclip-template-pruned-copy-");
+ await first.cleanup();
+ expect(lastStartMode()).toBe("template");
+ const templateDir = path.join(templateRootDir(), computeEmbeddedPostgresTemplateKey());
+ expect(isUsableTemplate(templateDir)).toBe(true);
+
+ // Another process prunes the template while this one still holds its path.
+ fs.rmSync(templateDir, { recursive: true, force: true });
+ const second = await startEmbeddedPostgresTestDatabase("paperclip-template-pruned-copy-");
+ await second.cleanup();
+ // This start fell back without retrying the missing path; the fallback
+ // forgot the stale path, so the next start rebuilt the template.
+ expect(lastStartMode()).toBe("fresh");
+ const third = await startEmbeddedPostgresTestDatabase("paperclip-template-pruned-copy-");
+ await third.cleanup();
+ expect(lastStartMode()).toBe("template");
+ expect(isUsableTemplate(templateDir)).toBe(true);
+ } finally {
+ scratch.restore();
+ }
+ });
+
+ it("runs the fresh initdb path when PAPERCLIP_TEST_POSTGRES_TEMPLATE=0", { timeout: EMBEDDED_POSTGRES_TEST_TIMEOUT_MS }, async () => {
+ process.env.PAPERCLIP_TEST_POSTGRES_TEMPLATE = "0";
+ const database = await startEmbeddedPostgresTestDatabase("paperclip-template-bypass-");
+ try {
+ expect(lastStartMode()).toBe("fresh");
+ const state = await inspectMigrations(database.connectionString);
+ expect(state.status).toBe("upToDate");
+ } finally {
+ await database.cleanup();
+ }
+ });
+
+ describeOnPosix("cache root safety", () => {
+ it("uses fresh clusters when the cache root is not a private directory of this user", { timeout: EMBEDDED_POSTGRES_TEST_TIMEOUT_MS }, async () => {
+ delete process.env.PAPERCLIP_TEST_POSTGRES_TEMPLATE;
+ const scratch = useScratchTmpdir("paperclip-template-root-");
+ const warnings: string[] = [];
+ const warn = console.warn;
+ console.warn = (...args: unknown[]) => {
+ warnings.push(args.map(String).join(" "));
+ };
+ try {
+ // Someone pre-created the predictable root path, world-writable, the
+ // way a foreign user on a shared host would.
+ const rootDir = templateRootDir();
+ expect(rootDir.startsWith(scratch.scratch)).toBe(true);
+ fs.mkdirSync(rootDir);
+ fs.chmodSync(rootDir, 0o777);
+
+ resetTemplateState();
+ const database = await startEmbeddedPostgresTestDatabase("paperclip-template-root-copy-");
+ try {
+ expect(lastStartMode()).toBe("fresh");
+ } finally {
+ await database.cleanup();
+ }
+ // Nothing was published into the rejected root, and the reason was logged once.
+ expect(fs.readdirSync(rootDir)).toEqual([]);
+ expect(warnings.filter((line) => line.includes("template cache root"))).toHaveLength(1);
+ expect(warnings[0]).toContain("writable by group or others");
+
+ // The probe still reports support, from the throwaway cluster.
+ resetTemplateState();
+ expect(await getEmbeddedPostgresTestSupport()).toEqual({ supported: true });
+ } finally {
+ console.warn = warn;
+ scratch.restore();
+ }
+ });
+
+ it("uses fresh clusters when the cache root path is a symbolic link", { timeout: EMBEDDED_POSTGRES_TEST_TIMEOUT_MS }, async () => {
+ delete process.env.PAPERCLIP_TEST_POSTGRES_TEMPLATE;
+ const scratch = useScratchTmpdir("paperclip-template-root-");
+ const warn = console.warn;
+ console.warn = () => {};
+ try {
+ const target = path.join(scratch.scratch, "elsewhere");
+ fs.mkdirSync(target, { mode: 0o700 });
+ fs.symlinkSync(target, templateRootDir(), "dir");
+
+ resetTemplateState();
+ const database = await startEmbeddedPostgresTestDatabase("paperclip-template-root-copy-");
+ try {
+ expect(lastStartMode()).toBe("fresh");
+ } finally {
+ await database.cleanup();
+ }
+ expect(fs.readdirSync(target)).toEqual([]);
+ } finally {
+ console.warn = warn;
+ scratch.restore();
+ }
+ });
+ });
+});
+
+describe("support probe when the template cannot be built", () => {
+ afterEach(() => {
+ __setEmbeddedPostgresCtorProviderForTests(null);
+ resetTemplateState();
+ });
+
+ it("falls through to the fresh probe instead of reporting the host as unsupported", { timeout: EMBEDDED_POSTGRES_TEST_TIMEOUT_MS }, async () => {
+ const originalTemplateSetting = process.env.PAPERCLIP_TEST_POSTGRES_TEMPLATE;
+ delete process.env.PAPERCLIP_TEST_POSTGRES_TEMPLATE;
+ const scratch = useScratchTmpdir("paperclip-template-build-fail-");
+ const warnings: string[] = [];
+ const warn = console.warn;
+ console.warn = (...args: unknown[]) => {
+ warnings.push(args.map(String).join(" "));
+ };
+ try {
+ // This test runs on every platform, so check the redirect took effect
+ // before asserting on paths under it.
+ expect(os.tmpdir()).toBe(scratch.scratch);
+ // The fake cluster "starts" without listening, so the template build
+ // fails after the start, at the migration step, the way a broken
+ // migration would. A real broken migration then fails the suite on
+ // the fresh path instead of being hidden by `describe.skip`.
+ const { ctor, constructed } = makeFakeCtor(0);
+ __setEmbeddedPostgresCtorProviderForTests(async () => ctor);
+ resetTemplateState();
+
+ expect(await getEmbeddedPostgresTestSupport()).toEqual({ supported: true });
+ // One build attempt, then the throwaway probe, both under the scratch root.
+ expect(constructed).toHaveLength(2);
+ for (const options of constructed) expect(options.databaseDir.startsWith(scratch.scratch)).toBe(true);
+ expect(constructed[0]?.databaseDir).toContain("paperclip-embedded-postgres-template-build-");
+ expect(constructed[1]?.databaseDir).toContain("paperclip-embedded-postgres-probe-");
+ expect(warnings.filter((line) => line.includes("template build failed"))).toHaveLength(1);
+ // Nothing was published.
+ const rootDir = templateRootDir();
+ expect(fs.existsSync(rootDir) ? fs.readdirSync(rootDir) : []).toEqual([]);
+ } finally {
+ console.warn = warn;
+ scratch.restore();
+ if (originalTemplateSetting === undefined) delete process.env.PAPERCLIP_TEST_POSTGRES_TEMPLATE;
+ else process.env.PAPERCLIP_TEST_POSTGRES_TEMPLATE = originalTemplateSetting;
+ }
+ });
+});
diff --git a/packages/db/src/test-embedded-postgres.ts b/packages/db/src/test-embedded-postgres.ts
index 67a52cd1b0..862829c823 100644
--- a/packages/db/src/test-embedded-postgres.ts
+++ b/packages/db/src/test-embedded-postgres.ts
@@ -1,7 +1,10 @@
+import { createHash } from "node:crypto";
import fs from "node:fs";
+import { createRequire } from "node:module";
import net from "node:net";
import os from "node:os";
import path from "node:path";
+import { fileURLToPath } from "node:url";
import { applyPendingMigrations, closeRegisteredClients, ensurePostgresDatabase } from "./client.js";
import {
createEmbeddedPostgresLogBuffer,
@@ -113,8 +116,19 @@ async function getAvailablePort(): Promise<number> {
);
}
+const EMBEDDED_POSTGRES_USER = "paperclip";
+const EMBEDDED_POSTGRES_PASSWORD = "paperclip";
+const EMBEDDED_POSTGRES_INITDB_FLAGS = ["--encoding=UTF8", "--locale=C", "--lc-messages=C"];
+
async function createEmbeddedPostgresTestInstance(tempDirPrefix: string) {
const dataDir = fs.mkdtempSync(path.join(os.tmpdir(), tempDirPrefix));
+ return await createEmbeddedPostgresTestInstanceAt(dataDir);
+}
+
+// Wraps an embedded-postgres instance around `dataDir`. The caller decides
+// whether the directory still needs `initialise()` (a fresh cluster) or already
+// holds a copied cluster that only needs `start()`.
+async function createEmbeddedPostgresTestInstanceAt(dataDir: string) {
const port = await getAvailablePort();
const EmbeddedPostgres = await getEmbeddedPostgresCtor();
// Postgres writes the true reason for a failed start to its output, for
@@ -124,11 +138,11 @@ async function createEmbeddedPostgresTestInstance(tempDirPrefix: string) {
const logBuffer = createEmbeddedPostgresLogBuffer();
const instance = new EmbeddedPostgres({
databaseDir: dataDir,
- user: "paperclip",
- password: "paperclip",
+ user: EMBEDDED_POSTGRES_USER,
+ password: EMBEDDED_POSTGRES_PASSWORD,
port,
persistent: true,
- initdbFlags: ["--encoding=UTF8", "--locale=C", "--lc-messages=C"],
+ initdbFlags: EMBEDDED_POSTGRES_INITDB_FLAGS,
onLog: (message) => logBuffer.append(message),
onError: (message) => logBuffer.append(message),
});
@@ -136,6 +150,14 @@ async function createEmbeddedPostgresTestInstance(tempDirPrefix: string) {
return { dataDir, port, instance, getRecentLogs: () => logBuffer.getRecentLogs() };
}
+function adminConnectionStringFor(port: number): string {
+ return `postgres://${EMBEDDED_POSTGRES_USER}:${EMBEDDED_POSTGRES_PASSWORD}@127.0.0.1:${port}/postgres`;
+}
+
+function databaseConnectionStringFor(port: number): string {
+ return `postgres://${EMBEDDED_POSTGRES_USER}:${EMBEDDED_POSTGRES_PASSWORD}@127.0.0.1:${port}/paperclip`;
+}
+
function cleanupEmbeddedPostgresTestDirs(dataDir: string) {
fs.rmSync(dataDir, { recursive: true, force: true });
}
@@ -237,13 +259,391 @@ async function startEmbeddedPostgresWithRetry(tempDirPrefix: string): Promise<{
);
}
+// ---------------------------------------------------------------------------
+// Template cluster cache
+//
+// Every suite that calls `startEmbeddedPostgresTestDatabase` used to pay for
+// `initdb`, a cluster start, and all pending migrations (331 files at the time
+// of writing) before its first test ran, and the `getEmbeddedPostgresTestSupport`
+// probe started and stopped a second throwaway cluster per test file. On the
+// serial CI shards that was 5-6s per suite plus 2.5-3s per probe; measured
+// locally it is 3.2s per suite plus 0.75s per probe.
+//
+// A cluster data directory that was shut down cleanly can be copied and started
+// again as an independent cluster. So the first caller on a host builds one
+// fully migrated cluster, stops it, and publishes the data directory as a
+// template under the OS temp directory. Every later caller copies the template
+// (about 90 MB, ~0.2s) and starts Postgres on the copy (~0.05s), then still runs
+// `ensurePostgresDatabase` and `applyPendingMigrations` so the returned database
+// is exactly what the fresh path produces; both are no-ops on a template copy.
+//
+// The template key hashes everything that determines the cluster's contents:
+// the embedded-postgres package version, the initdb flags, the user, and the
+// name and content of every migration file plus the journal. A changed
+// migration therefore builds a new template instead of reusing a stale one.
+// Publishing is a rename, which is atomic on the same filesystem, so a
+// concurrent builder either wins the rename or discards its copy and uses the
+// winner's; a template that exists is always complete.
+//
+// A template is a Postgres data directory, and starting one executes its
+// configuration (for example `shared_preload_libraries`). The cache root is
+// therefore private to the current OS user: its name carries the user id, it
+// is created with mode 0700, and it is only used when it is a real directory
+// (not a symlink) owned by this user with no group or world write bit. Each
+// template is checked the same way before it is copied. On a host with a
+// shared temp directory another user can pre-create the predictable path, and
+// those checks reject it. Windows has no uid; its temp directory is already
+// per user, so the ownership and mode checks are skipped there.
+//
+// Set PAPERCLIP_TEST_POSTGRES_TEMPLATE=0 to bypass the cache and run the fresh
+// path for every suite. Any failure on the template path (root checks, build,
+// copy, start, or publish) falls back to the fresh path, so a broken cache can
+// make a suite slower but never fail it or skip it: the fresh path still
+// reports a migration that cannot apply or a cluster that cannot start.
+// ---------------------------------------------------------------------------
+
+const TEMPLATE_ROOT_DIR_NAME = "paperclip-embedded-postgres-templates";
+// Templates whose key no longer matches are pruned once they have not been
+// used for this long. Every reuse refreshes the template's timestamp.
+const TEMPLATE_PRUNE_AGE_MS = 7 * 24 * 60 * 60 * 1000;
+// Publishing a template requires the builder cluster to have exited for real,
+// so its data files are quiescent before the rename. Give that stop more room
+// than the per-suite bounded stop; a template build happens once per host.
+const TEMPLATE_BUILD_STOP_TIMEOUT_MS = 30_000;
+
+export type EmbeddedPostgresStartMode = "template" | "fresh";
+
+function isTemplateCacheDisabled(): boolean {
+ return process.env.PAPERCLIP_TEST_POSTGRES_TEMPLATE === "0";
+}
+
+function migrationsDirectory(): string {
+ return fileURLToPath(new URL("./migrations/", import.meta.url));
+}
+
+function embeddedPostgresPackageVersion(): string {
+ try {
+ const require = createRequire(import.meta.url);
+ const manifest = require("embedded-postgres/package.json") as { version?: unknown };
+ return typeof manifest.version === "string" ? manifest.version : "unknown";
+ } catch {
+ return "unknown";
+ }
+}
+
+// Hash of everything that determines the contents of a migrated cluster. Reads
+// every migration file, which costs a few milliseconds per process.
+export function computeEmbeddedPostgresTemplateKey(
+ migrationsDir: string = migrationsDirectory(),
+ input: { packageVersion?: string; initdbFlags?: string[]; user?: string } = {},
+): string {
+ const hash = createHash("sha256");
+ hash.update(`embedded-postgres=${input.packageVersion ?? embeddedPostgresPackageVersion()}\n`);
+ hash.update(`initdb=${(input.initdbFlags ?? EMBEDDED_POSTGRES_INITDB_FLAGS).join(" ")}\n`);
+ hash.update(`user=${input.user ?? EMBEDDED_POSTGRES_USER}\n`);
+ const journalPath = path.join(migrationsDir, "meta", "_journal.json");
+ if (fs.existsSync(journalPath)) {
+ hash.update(`journal\n`);
+ hash.update(fs.readFileSync(journalPath));
+ hash.update(`\n`);
+ }
+ const migrationFiles = fs
+ .readdirSync(migrationsDir)
+ .filter((name) => name.endsWith(".sql"))
+ .sort((a, b) => a.localeCompare(b));
+ for (const name of migrationFiles) {
+ hash.update(`${name}\n`);
+ hash.update(fs.readFileSync(path.join(migrationsDir, name)));
+ hash.update(`\n`);
+ }
+ return hash.digest("hex").slice(0, 24);
+}
+
+// The uid on POSIX, else the user name. Part of the cache root's name so two
+// users on one host never share a root.
+function templateOwnerTag(): string {
+ const uid = process.getuid?.();
+ if (uid !== undefined) return String(uid);
+ let username = "user";
+ try {
+ username = os.userInfo().username;
+ } catch {
+ // Fall through to the generic tag.
+ }
+ return username.replace(/[^A-Za-z0-9_.-]/g, "_");
+}
+
+function templateRootDir(): string {
+ return path.join(os.tmpdir(), `${TEMPLATE_ROOT_DIR_NAME}-${templateOwnerTag()}`);
+}
+
+// Why `dir` cannot be trusted as a private directory of the current user, or
+// null when it can. Uses `lstat` so a symlink planted at the path is rejected
+// rather than followed. On Windows (no uid) only the directory check applies.
+function describePrivateDirectoryProblem(dir: string): string | null {
+ let stat: fs.Stats;
+ try {
+ stat = fs.lstatSync(dir);
+ } catch (error) {
+ return `cannot be inspected (${(error as NodeJS.ErrnoException).code ?? "unknown error"})`;
+ }
+ if (stat.isSymbolicLink()) return "is a symbolic link";
+ if (!stat.isDirectory()) return "is not a directory";
+ const uid = process.getuid?.();
+ if (uid === undefined) return null;
+ if (stat.uid !== uid) return `is owned by uid ${stat.uid}, not the current user (uid ${uid})`;
+ if ((stat.mode & 0o022) !== 0) {
+ return `is writable by group or others (mode ${(stat.mode & 0o777).toString(8)})`;
+ }
+ return null;
+}
+
+// Creates the cache root with mode 0700 and verifies it is private to the
+// current user. Returns null, after one warning, when it is not: the cache is
+// then unused for this process and every suite takes the fresh path.
+function prepareTemplateRootDir(): string | null {
+ const rootDir = templateRootDir();
+ try {
+ fs.mkdirSync(rootDir, { recursive: true, mode: 0o700 });
+ const problem = describePrivateDirectoryProblem(rootDir);
+ if (problem !== null) {
+ warnTemplateOnce("root", `template cache root ${rootDir} ${problem}; using fresh clusters`);
+ return null;
+ }
+ // A root created by an earlier version may be wider than 0700. It is
+ // ours (checked above), so narrow it.
+ if (process.getuid !== undefined && (fs.lstatSync(rootDir).mode & 0o777) !== 0o700) {
+ fs.chmodSync(rootDir, 0o700);
+ }
+ return rootDir;
+ } catch (error) {
+ warnTemplateOnce(
+ "root",
+ `template cache root ${rootDir} could not be prepared: ${(error as Error).message}; using fresh clusters`,
+ );
+ return null;
+ }
+}
+
+// A template is usable when it is a private directory of the current user
+// that holds a cleanly stopped cluster: `PG_VERSION` present, no
+// `postmaster.pid`. The ownership check repeats the root's check so a
+// template is never copied from a directory this user does not own.
+function isUsableTemplate(dir: string): boolean {
+ if (describePrivateDirectoryProblem(dir) !== null) return false;
+ return fs.existsSync(path.join(dir, "PG_VERSION")) && !fs.existsSync(path.join(dir, "postmaster.pid"));
+}
+
+// Records a use of the template so `pruneStaleTemplates` measures idle time,
+// not age. Best effort.
+function touchTemplate(dir: string): void {
+ try {
+ const now = new Date();
+ fs.utimesSync(dir, now, now);
+ } catch {
+ // The template still works without the refreshed timestamp.
+ }
+}
+
+const warnedTemplateTopics = new Set<string>();
+
+function warnTemplateOnce(topic: string, message: string): void {
+ if (warnedTemplateTopics.has(topic)) return;
+ warnedTemplateTopics.add(topic);
+ console.warn(`[embedded-postgres] ${message}`);
+}
+
+// Best-effort removal of templates for other keys that have not been used for
+// a week, plus abandoned build directories. Every reuse refreshes a template's
+// timestamp, so a template that any process resolved in the last week stays.
+// A suite never depends on the template after its copy is made, so pruning
+// can only affect a process that resolved the path and has not copied yet;
+// that process then rebuilds the template instead of failing. Never throws.
+function pruneStaleTemplates(rootDir: string, keepDir: string): void {
+ let entries: fs.Dirent[];
+ try {
+ entries = fs.readdirSync(rootDir, { withFileTypes: true });
+ } catch {
+ return;
+ }
+ const now = Date.now();
+ for (const entry of entries) {
+ const entryPath = path.join(rootDir, entry.name);
+ if (entryPath === keepDir || !entry.isDirectory()) continue;
+ try {
+ const age = now - fs.statSync(entryPath).mtimeMs;
+ if (age > TEMPLATE_PRUNE_AGE_MS) fs.rmSync(entryPath, { recursive: true, force: true });
+ } catch {
+ // Another process may be using or removing it; leave it alone.
+ }
+ }
+}
+
+// Waits for the raw `stop()` so the builder cluster has really exited before
+// its data directory is published. Returns false if the stop overran.
+async function stopForTemplatePublish(instance: EmbeddedPostgresInstance): Promise<boolean> {
+ let timer: ReturnType<typeof setTimeout> | undefined;
+ try {
+ return await Promise.race([
+ instance.stop().then(() => true, () => false),
+ new Promise<boolean>((resolve) => {
+ timer = setTimeout(() => resolve(false), TEMPLATE_BUILD_STOP_TIMEOUT_MS);
+ timer.unref?.();
+ }),
+ ]);
+ } finally {
+ if (timer) clearTimeout(timer);
+ }
+}
+
+// Builds a migrated cluster and publishes its data directory at `templateDir`.
+// Throws when the cluster cannot start or a migration cannot apply. Returns
+// false when the cluster started but the template could not be published;
+// callers then use the fresh path for this process.
+async function buildEmbeddedPostgresTemplate(templateDir: string): Promise<boolean> {
+ const rootDir = path.dirname(templateDir);
+ const { port, dataDir, instance } = await startEmbeddedPostgresWithRetry(
+ "paperclip-embedded-postgres-template-build-",
+ );
+ let published = false;
+ try {
+ await ensurePostgresDatabase(adminConnectionStringFor(port), "paperclip");
+ await applyPendingMigrations(databaseConnectionStringFor(port));
+ const stopped = await stopForTemplatePublish(instance);
+ if (!stopped || !isUsableTemplate(dataDir)) return false;
+ try {
+ fs.renameSync(dataDir, templateDir);
+ published = true;
+ } catch (error) {
+ // Another process published the same key first. Use theirs.
+ const code = (error as NodeJS.ErrnoException).code;
+ if (code === "EEXIST" || code === "ENOTEMPTY" || code === "EPERM") {
+ return isUsableTemplate(templateDir);
+ }
+ throw error;
+ }
+ pruneStaleTemplates(rootDir, templateDir);
+ return true;
+ } finally {
+ if (!published) {
+ await stopEmbeddedPostgresBounded(instance, () => cleanupEmbeddedPostgresTestDirs(dataDir));
+ }
+ }
+}
+
+let templateDirPromise: Promise<string | null> | null = null;
+
+// Resolves the template data directory for this host, building it on first
+// use. Resolves null when the cache is disabled, the cache root is not private
+// to this user, or the template could not be built or published; every caller
+// then takes the fresh path, which reports the real error itself (a cluster
+// that cannot start, a migration that cannot apply). Never rejects, so a
+// broken cache can never turn into `supported: false` and skip a suite.
+async function getEmbeddedPostgresTemplateDir(): Promise<string | null> {
+ if (isTemplateCacheDisabled()) return null;
+ if (!templateDirPromise) {
+ templateDirPromise = (async () => {
+ const rootDir = prepareTemplateRootDir();
+ if (rootDir === null) return null;
+ const templateDir = path.join(rootDir, computeEmbeddedPostgresTemplateKey());
+ if (isUsableTemplate(templateDir)) {
+ touchTemplate(templateDir);
+ return templateDir;
+ }
+ try {
+ const built = await buildEmbeddedPostgresTemplate(templateDir);
+ return built && isUsableTemplate(templateDir) ? templateDir : null;
+ } catch (error) {
+ warnTemplateOnce(
+ "build",
+ `template build failed, using fresh clusters: ${(error as Error).message}`,
+ );
+ return null;
+ }
+ })();
+ }
+ return await templateDirPromise;
+}
+
+// Forgets the resolved template so the next caller in this process checks the
+// cache again. Used after a template start failed, for example because the
+// template was pruned by another process after this one resolved its path.
+function resetTemplateDirCache(): void {
+ templateDirPromise = null;
+}
+
+// Copies the template and starts a cluster on the copy. Each attempt uses a
+// fresh copy and a fresh port, mirroring `startEmbeddedPostgresWithRetry`. A
+// template that is no longer usable (pruned, or no longer private) is not
+// retried: the caller falls back and the next caller rebuilds it.
+async function startEmbeddedPostgresFromTemplateWithRetry(
+ templateDir: string,
+ tempDirPrefix: string,
+): Promise<{ port: number; dataDir: string; instance: EmbeddedPostgresInstance }> {
+ let lastError = new Error("embedded Postgres startup from template failed");
+
+ for (let attempt = 1; attempt <= EMBEDDED_POSTGRES_START_MAX_ATTEMPTS; attempt += 1) {
+ if (!isUsableTemplate(templateDir)) {
+ throw new Error(`embedded Postgres template ${templateDir} is no longer usable`);
+ }
+ const rootDir = fs.mkdtempSync(path.join(os.tmpdir(), tempDirPrefix));
+ const dataDir = path.join(rootDir, "data");
+ let created: Awaited<ReturnType<typeof createEmbeddedPostgresTestInstanceAt>> | null = null;
+ try {
+ fs.cpSync(templateDir, dataDir, { recursive: true });
+ // Postgres refuses a data directory that is group or world accessible.
+ fs.chmodSync(dataDir, 0o700);
+ created = await createEmbeddedPostgresTestInstanceAt(dataDir);
+ await created.instance.start();
+ return { port: created.port, dataDir: rootDir, instance: created.instance };
+ } catch (error) {
+ lastError = formatEmbeddedPostgresError(error, {
+ fallbackMessage: "embedded Postgres startup from template failed",
+ recentLogs: created?.getRecentLogs(),
+ });
+ await stopEmbeddedPostgresBounded(created?.instance ?? null, () =>
+ cleanupEmbeddedPostgresTestDirs(rootDir),
+ );
+ }
+ }
+
+ throw new Error(
+ `Failed to start embedded PostgreSQL from template after ${EMBEDDED_POSTGRES_START_MAX_ATTEMPTS} attempts: ${lastError.message}`,
+ );
+}
+
+let templateFallbackWarned = false;
+let lastStartMode: EmbeddedPostgresStartMode | null = null;
+
// Test-only accessors. Production callers use `startEmbeddedPostgresTestDatabase`
// or `getEmbeddedPostgresTestSupport`. A test drives the bounded retry directly
// so it does not need a real Postgres connection.
export const __startEmbeddedPostgresWithRetryForTests = startEmbeddedPostgresWithRetry;
export const __embeddedPostgresStartMaxAttemptsForTests = EMBEDDED_POSTGRES_START_MAX_ATTEMPTS;
+export const __embeddedPostgresTemplateRootDirForTests = templateRootDir;
+export const __isUsableEmbeddedPostgresTemplateForTests = isUsableTemplate;
+export function __lastEmbeddedPostgresStartModeForTests(): EmbeddedPostgresStartMode | null {
+ return lastStartMode;
+}
+// Forgets the support probe result, the resolved template, and the one-time
+// warnings so a test can drive the probe and the cache again in one process.
+export function __resetEmbeddedPostgresTemplateStateForTests(): void {
+ embeddedPostgresSupportPromise = null;
+ templateDirPromise = null;
+ templateFallbackWarned = false;
+ warnedTemplateTopics.clear();
+ lastStartMode = null;
+}
async function probeEmbeddedPostgresSupport(): Promise<EmbeddedPostgresTestSupport> {
+ // A published template proves this host can run the cluster: it was built
+ // here. Reusing it makes the probe free for every file after the first.
+ // Building it costs one cluster start, the same as the throwaway probe did.
+ // When the template cannot be built or used, the throwaway probe below
+ // decides, so only a cluster that cannot start reports `supported: false`;
+ // a migration that cannot apply fails the suite loudly on the fresh path.
+ if ((await getEmbeddedPostgresTemplateDir()) !== null) return { supported: true };
+
let started: { dataDir: string; instance: EmbeddedPostgresInstance } | null = null;
try {
@@ -271,17 +671,48 @@ export async function getEmbeddedPostgresTestSupport(): Promise<EmbeddedPostgres
return await embeddedPostgresSupportPromise;
}
-export async function startEmbeddedPostgresTestDatabase(
+// Starts the cluster for one suite: from the host template when one is
+// available, otherwise a fresh `initdb`. Template failures fall back to the
+// fresh path so the cache can only make a suite faster, never fail it.
+async function startEmbeddedPostgresClusterForSuite(
tempDirPrefix: string,
-): Promise<EmbeddedPostgresTestDatabase> {
+): Promise<{ port: number; dataDir: string; instance: EmbeddedPostgresInstance; mode: EmbeddedPostgresStartMode }> {
+ const templateDir = await getEmbeddedPostgresTemplateDir();
+
+ if (templateDir !== null) {
+ try {
+ const started = await startEmbeddedPostgresFromTemplateWithRetry(templateDir, tempDirPrefix);
+ return { ...started, mode: "template" };
+ } catch (error) {
+ // The next caller in this process re-resolves the template: if it was
+ // pruned it is rebuilt, instead of retrying a path that is gone.
+ resetTemplateDirCache();
+ if (!templateFallbackWarned) {
+ templateFallbackWarned = true;
+ console.warn(
+ `[embedded-postgres] template start failed, using a fresh cluster: ${(error as Error).message}`,
+ );
+ }
+ }
+ }
+
// The bounded retry hardens the cluster start against the port race. It throws
// with the real Postgres output if every attempt fails.
- const { port, dataDir, instance } = await startEmbeddedPostgresWithRetry(tempDirPrefix);
+ const started = await startEmbeddedPostgresWithRetry(tempDirPrefix);
+ return { ...started, mode: "fresh" };
+}
+
+export async function startEmbeddedPostgresTestDatabase(
+ tempDirPrefix: string,
+): Promise<EmbeddedPostgresTestDatabase> {
+ const { port, dataDir, instance, mode } = await startEmbeddedPostgresClusterForSuite(tempDirPrefix);
+ lastStartMode = mode;
try {
- const adminConnectionString = `postgres://paperclip:[email protected]:${port}/postgres`;
- await ensurePostgresDatabase(adminConnectionString, "paperclip");
- const connectionString = `postgres://paperclip:[email protected]:${port}/paperclip`;
+ // Both calls are no-ops on a template copy and keep the returned database
+ // identical to the fresh path even if the template were ever stale.
+ await ensurePostgresDatabase(adminConnectionStringFor(port), "paperclip");
+ const connectionString = databaseConnectionStringFor(port);
await applyPendingMigrations(connectionString);
return {