TypeScript
fix(server): keep writes moving with a larger WAL limit (#31804)
Yash Singh9ef12568cb1f
diff --git a/apps/server/src/persistence/Sqlite.test.ts b/apps/server/src/persistence/Sqlite.test.tsindex 940d6265143..a2d29f12f18 100644--- a/apps/server/src/persistence/Sqlite.test.ts+++ b/apps/server/src/persistence/Sqlite.test.ts@@ -2,6 +2,8 @@import * as NodeFS from "node:fs";import * as NodeOS from "node:os";import * as NodePath from "node:path";+import * as NodeSqlite from "node:sqlite";+import type * as NodeWorkerThreads from "node:worker_threads";import * as NodeServices from "@effect/platform-node/NodeServices";import { assert, it } from "@effect/vitest";@@ -9,9 +11,94 @@ import * as HostProcess from "@t3tools/shared/HostProcess";import * as Effect from "effect/Effect";import * as Layer from "effect/Layer";import * as SqlClient from "effect/sql/SqlClient";+import { vi } from "vite-plus/test";import * as SqlitePersistence from "./Sqlite.ts";+const workers = vi.hoisted((): Array<NodeWorkerThreads.Worker> => []);++vi.mock("node:worker_threads", async (importOriginal) => {+ const actual = await importOriginal<typeof NodeWorkerThreads>();+ class RecordedWorker extends actual.Worker {+ constructor(...args: ConstructorParameters<typeof actual.Worker>) {+ super(...args);+ workers.push(this);+ }+ }+ return { ...actual, Worker: RecordedWorker };+});++it.effect("keeps writes moving past a 48 MiB WAL while a reader retains its snapshot", () => {+ const tempDir = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "t3-sqlite-burst-"));+ const dbPath = NodePath.join(tempDir, "state.sqlite");+ const persistence = SqlitePersistence.layerFromPath(dbPath).pipe(+ Layer.provide(NodeServices.layer),+ );++ return Effect.gen(function* () {+ const sql = yield* SqlClient.SqlClient;+ const checkpointWorker = workers.at(-1)!;+ yield* sql`CREATE TABLE burst_probe(id INTEGER PRIMARY KEY, payload BLOB)`;+ yield* sql`INSERT INTO burst_probe VALUES (1, NULL)`;+ yield* sql`PRAGMA wal_checkpoint(TRUNCATE)`;+ const reader = yield* Effect.acquireRelease(+ Effect.sync(() => new NodeSqlite.DatabaseSync(dbPath, { readOnly: true })),+ (database) => Effect.sync(() => database.close()),+ );+ reader.exec("BEGIN");+ assert.equal(reader.prepare("SELECT COUNT(*) AS count FROM burst_probe").get()?.count, 1);++ yield* sql.withTransaction(+ sql`INSERT INTO burst_probe VALUES (2, zeroblob(${48 * 1024 * 1024}))`,+ );+ const state = yield* Effect.callback<{+ readonly log: number;+ readonly checkpointed: number;+ readonly blocked: boolean;+ }>((resume) => {+ const onMessage = (message: {+ readonly type?: string;+ readonly log?: number;+ readonly checkpointed?: number;+ readonly blocked?: boolean;+ }) => {+ if (+ message.type === "checkpoint" &&+ typeof message.log === "number" &&+ message.log > 10_000 &&+ typeof message.checkpointed === "number" &&+ typeof message.blocked === "boolean"+ ) {+ checkpointWorker.off("message", onMessage);+ resume(+ Effect.succeed({+ log: message.log,+ checkpointed: message.checkpointed,+ blocked: message.blocked,+ }),+ );+ }+ };+ checkpointWorker.on("message", onMessage);+ checkpointWorker.ref();+ // oxlint-disable-next-line unicorn/require-post-message-target-origin -- Node workers do not accept a target origin.+ checkpointWorker.postMessage("barrier");+ return Effect.sync(() => checkpointWorker.off("message", onMessage));+ });+ assert.isBelow(state.checkpointed, state.log);+ assert.isFalse(state.blocked);++ yield* sql.withTransaction(sql`INSERT INTO burst_probe VALUES (3, NULL)`);+ assert.equal(reader.prepare("SELECT COUNT(*) AS count FROM burst_probe").get()?.count, 1);+ reader.exec("COMMIT");+ assert.equal(reader.prepare("SELECT COUNT(*) AS count FROM burst_probe").get()?.count, 3);+ }).pipe(+ Effect.scoped,+ Effect.provide(persistence),+ Effect.ensuring(Effect.sync(() => NodeFS.rmSync(tempDir, { recursive: true, force: true }))),+ );+});+// Linux names an open, already-deleted file "<path> (deleted)" in /proc/self/fd.// SQLite names its temp files "etilqs_*" and, without TMPDIR, puts them in// /var/tmp while os.tmpdir() is /tmp, so match the name, not the directory.diff --git a/apps/server/src/persistence/Sqlite.ts b/apps/server/src/persistence/Sqlite.tsindex 9b95422c4f5..38913376cd1 100644--- a/apps/server/src/persistence/Sqlite.ts+++ b/apps/server/src/persistence/Sqlite.ts@@ -9,7 +9,7 @@ import { runMigrations } from "./Migrations.ts";import { initializeV2Database } from "./initializeV2Database.ts";import * as ServerConfig from "../config.ts";-const WAL_SIZE_LIMIT_BYTES = 32 * 1024 * 1024;+const WAL_SIZE_LIMIT_BYTES = 128 * 1024 * 1024;const layerSetup = Layer.effectDiscard(Effect.gen(function* () {diff --git a/packages/shared/src/nodeSqliteClient.ts b/packages/shared/src/nodeSqliteClient.tsindex 481b7af0377..4c2c2fe9677 100644--- a/packages/shared/src/nodeSqliteClient.ts+++ b/packages/shared/src/nodeSqliteClient.ts@@ -486,7 +486,7 @@ const makeReader = Effect.fnUntraced(function* (const CHECKPOINT_FRAMES = 1000;const CHECKPOINT_IDLE_MS = 100;-const CHECKPOINT_BACKPRESSURE_FRAMES = 10 * CHECKPOINT_FRAMES;+const CHECKPOINT_BACKPRESSURE_FRAMES = 32_768;const readSafeSqliteInteger = (value: unknown) => {const number = typeof value === "bigint" ? Number(value) : value;