Where agent teams gather. Cross-platform team chat where AI agents are first-class teammates — with cloud or bring-your-own (Claude Code / Codex) brains.
TypeScript
fix(documents): compact only persisted update sets atomically (#262)
LILACd0c46ba56041
diff --git a/server/src/__integration__/document-compaction.test.ts b/server/src/__integration__/document-compaction.test.tsnew file mode 100644index 0000000..ef6d994--- /dev/null+++ b/server/src/__integration__/document-compaction.test.ts@@ -0,0 +1,192 @@+import assert from 'node:assert/strict'+import { randomUUID } from 'node:crypto'+import { after, before, beforeEach, test } from 'node:test'+import { Pool, type PoolClient } from 'pg'+import * as Y from 'yjs'+import { compactDocument, readDocumentState } from '../documents/persistence.js'++// Use an isolated schema: these tests need real PostgreSQL transaction and+// sequence semantics but do not need the app's Redis or provider services.+const schema = `compaction_test_${randomUUID().replaceAll('-', '')}`+const connectionString = process.env.INTEGRATION_DATABASE_URL ?? process.env.DATABASE_URL+const admin = new Pool({ connectionString })+const pool = new Pool({ connectionString, options: `-c search_path=${schema}` })++before(async () => {+ await admin.query(`CREATE SCHEMA ${schema}`)+ await pool.query(`+ CREATE TABLE document_updates (+ id BIGSERIAL PRIMARY KEY, document_id TEXT NOT NULL, update_bytes BYTEA NOT NULL+ );+ CREATE TABLE document_snapshots (+ document_id TEXT PRIMARY KEY, state_bytes BYTEA NOT NULL,+ snapshot_at_update_id BIGINT NOT NULL DEFAULT 0, updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()+ )`)+})+beforeEach(async () => {+ await pool.query('TRUNCATE document_updates, document_snapshots RESTART IDENTITY')+})+after(async () => {+ await pool.end()+ await admin.query(`DROP SCHEMA IF EXISTS ${schema} CASCADE`)+ await admin.end()+})++function edits() {+ const doc = new Y.Doc()+ const updates: Uint8Array[] = []+ doc.on('update', (update: Uint8Array) => updates.push(update))+ doc.getText('text').insert(0, 'A')+ doc.getText('text').insert(1, 'B')+ doc.getText('text').insert(2, 'C')+ doc.destroy()+ return updates+}++async function insert(bytes: Uint8Array, client: Pick<PoolClient, 'query'> = pool, documentId = 'doc') {+ const { rows } = await client.query<{ id: string }>(+ 'INSERT INTO document_updates (document_id, update_bytes) VALUES ($1, $2) RETURNING id::text',+ [documentId, Buffer.from(bytes)],+ )+ return rows[0].id+}++async function coldText(client: Pick<PoolClient, 'query'> = pool) {+ const doc = new Y.Doc()+ try {+ for (const row of await readDocumentState(client, 'doc')) Y.applyUpdate(doc, row.bytes)+ return doc.getText('text').toString()+ } finally {+ doc.destroy()+ }+}++// Intercept a real query's result, keeping the transaction open so another+// connection can commit at a deterministic point in the compaction.+function interceptedPool(hook: (sql: string) => Promise<void>): Pick<Pool, 'connect'> {+ return {+ async connect() {+ const client = await pool.connect()+ return {+ query: async (sql: string, params?: unknown[]) => {+ const result = await client.query(sql, params)+ await hook(sql)+ return result+ },+ release: () => client.release(),+ } as unknown as PoolClient+ },+ } as Pick<Pool, 'connect'>+}++test('compaction includes persisted edits missing from a stale room', async () => {+ const [a, b] = edits()+ const staleRoom = new Y.Doc()+ Y.applyUpdate(staleRoom, a)+ await insert(a)+ await insert(b) // Redis has not delivered B to the live room.+ assert.equal(staleRoom.getText('text').toString(), 'A')+ await compactDocument(pool, 'doc')+ assert.equal(await coldText(), 'AB')+ assert.equal((await pool.query('SELECT * FROM document_updates')).rowCount, 0)+ staleRoom.destroy()+})++test('a commit after the read remains in the log and survives another compaction', async () => {+ const [a, b, c] = edits()+ await insert(a)+ await compactDocument(pool, 'doc')+ await insert(b)+ const otherId = await insert(a, pool, 'other-doc')+ let tailId = ''+ await compactDocument(interceptedPool(async (sql) => {+ if (sql.includes('UNION ALL')) tailId = await insert(c)+ }), 'doc')+ const { rows } = await pool.query('SELECT id::text FROM document_updates ORDER BY id')+ assert.deepEqual(rows.map((row) => row.id), [otherId, tailId])+ assert.equal(await coldText(), 'ABC')+ await compactDocument(pool, 'doc')+ assert.equal(await coldText(), 'ABC')+})++test('late lower-ID commits and pending Yjs dependencies survive compaction', async () => {+ const [a, b] = edits()+ const writer = await pool.connect()+ try {+ await writer.query('BEGIN')+ const lowerId = await insert(a, writer)+ const higherId = await insert(b)+ await compactDocument(pool, 'doc') // B is pending until A arrives.+ await writer.query('COMMIT')+ assert.ok(BigInt(lowerId) < BigInt(higherId))+ assert.equal(await coldText(), 'AB')+ await compactDocument(pool, 'doc')+ assert.equal(await coldText(), 'AB')+ assert.equal((await pool.query('SELECT * FROM document_updates')).rowCount, 0)+ } finally {+ await writer.query('ROLLBACK')+ writer.release()+ }+})++test('a lower-ID transaction committed between read and deletion is not trimmed', async () => {+ const [a, b] = edits()+ const writer = await pool.connect()+ try {+ await writer.query('BEGIN')+ const lowerId = await insert(a, writer)+ await insert(b)+ await compactDocument(interceptedPool(async (sql) => {+ if (sql.includes('UNION ALL')) await writer.query('COMMIT')+ }), 'doc')+ assert.deepEqual((await pool.query('SELECT id::text FROM document_updates')).rows, [{ id: lowerId }])+ assert.equal(await coldText(), 'AB')+ } finally {+ await writer.query('ROLLBACK')+ writer.release()+ }+})++test('concurrent compactors cannot overwrite a newer snapshot', async () => {+ const [a, b] = edits()+ await insert(a)+ await compactDocument(interceptedPool(async (sql) => {+ if (!sql.includes('UNION ALL')) return+ await insert(b)+ assert.equal(await compactDocument(pool, 'doc'), false)+ }), 'doc')+ assert.equal(await coldText(), 'AB')+ assert.equal(await compactDocument(pool, 'doc'), true)+ assert.equal(await coldText(), 'AB')+})++test('failure after deletion rolls back both snapshot replacement and log deletion', async () => {+ const [a, b] = edits()+ await insert(a)+ await compactDocument(pool, 'doc')+ const beforeSnapshot = (await pool.query('SELECT * FROM document_snapshots')).rows+ const bId = await insert(b)+ await assert.rejects(compactDocument(interceptedPool(async (sql) => {+ if (sql.startsWith('DELETE')) throw new Error('injected failure')+ }), 'doc'), /injected failure/)+ assert.deepEqual((await pool.query('SELECT * FROM document_snapshots')).rows, beforeSnapshot)+ assert.deepEqual((await pool.query('SELECT id::text FROM document_updates')).rows, [{ id: bId }])+ assert.equal(await coldText(), 'AB')+ assert.equal(await compactDocument(pool, 'doc'), true)+})++test('cold load keeps a consistent snapshot and log when compaction commits during the read', async () => {+ const [a, b] = edits()+ await insert(a)+ await compactDocument(pool, 'doc')+ await insert(b)+ const client = await interceptedPool(async (sql) => {+ if (sql.includes('UNION ALL')) await compactDocument(pool, 'doc')+ }).connect()+ try {+ assert.equal(await coldText(client), 'AB')+ assert.equal((await pool.query('SELECT * FROM document_updates')).rowCount, 0)+ } finally {+ client.release()+ }+})diff --git a/server/src/documents/persistence.ts b/server/src/documents/persistence.tsnew file mode 100644index 0000000..a1b5105--- /dev/null+++ b/server/src/documents/persistence.ts@@ -0,0 +1,68 @@+import * as Y from 'yjs'+import type { Pool, PoolClient } from 'pg'++/** One statement keeps snapshot and log on the same PostgreSQL MVCC view,+ * even when a compactor commits while a cold load is in progress. Read ALL+ * remaining rows: sequence IDs are allocated before transactions commit, so+ * a late commit can have an ID below snapshot_at_update_id. */+export async function readDocumentState(client: Pick<PoolClient, 'query'>, documentId: string) {+ const { rows } = await client.query<{+ is_snapshot: boolean+ id: string+ bytes: Buffer+ }>(+ `SELECT TRUE AS is_snapshot, snapshot_at_update_id::text AS id, state_bytes AS bytes+ FROM document_snapshots WHERE document_id = $1+ UNION ALL+ SELECT FALSE AS is_snapshot, id::text AS id, update_bytes AS bytes+ FROM document_updates WHERE document_id = $1`,+ [documentId],+ )+ return rows+}++/** Compact persisted bytes, never a possibly stale live room. The lock+ * serializes compactors across instances, including the first snapshot.+ * Snapshot replacement and deletion commit together; only the exact rows+ * merged below may be removed. mergeUpdates preserves pending Yjs structs+ * and deletes whose dependencies have not been persisted yet. */+export async function compactDocument(pool: Pick<Pool, 'connect'>, documentId: string): Promise<boolean> {+ const client = await pool.connect()+ try {+ await client.query('BEGIN')+ const { rows: locks } = await client.query<{ locked: boolean }>(+ `SELECT pg_try_advisory_xact_lock(hashtextextended($1, 0)) AS locked`,+ [`document-compaction:${documentId}`],+ )+ if (!locks[0].locked) {+ await client.query('ROLLBACK')+ return false+ }+ const rows = await readDocumentState(client, documentId)+ const updateIds = rows.filter((row) => !row.is_snapshot).map((row) => row.id)+ if (updateIds.length > 0) {+ const state = Y.mergeUpdates(rows.map((row) => new Uint8Array(row.bytes)))+ const maxId = rows.reduce((max, row) => BigInt(row.id) > max ? BigInt(row.id) : max, 0n)+ await client.query(+ `INSERT INTO document_snapshots (document_id, state_bytes, snapshot_at_update_id, updated_at)+ VALUES ($1, $2, $3, NOW())+ ON CONFLICT (document_id)+ DO UPDATE SET state_bytes = EXCLUDED.state_bytes,+ snapshot_at_update_id = EXCLUDED.snapshot_at_update_id,+ updated_at = NOW()`,+ [documentId, Buffer.from(state), maxId.toString()],+ )+ await client.query(+ `DELETE FROM document_updates WHERE document_id = $1 AND id = ANY($2::bigint[])`,+ [documentId, updateIds],+ )+ }+ await client.query('COMMIT')+ return true+ } catch (error) {+ await client.query('ROLLBACK').catch(() => {})+ throw error+ } finally {+ client.release()+ }+}diff --git a/server/src/documents/rooms.ts b/server/src/documents/rooms.tsindex fefce9e..2a6883c 100644--- a/server/src/documents/rooms.ts+++ b/server/src/documents/rooms.ts@@ -17,6 +17,7 @@import * as Y from 'yjs'import type { PoolClient } from 'pg'import { pool } from '../db/pool.js'+import { compactDocument, readDocumentState } from './persistence.js'import {redis, sub, publish,CH_DOC_UPDATE, CH_DOC_AWARENESS,@@ -49,6 +50,7 @@ interface Room {subs: Set<DocSubscriber>/** Updates since last snapshot — drives the compaction threshold. */updatesSinceSnapshot: number+ compacting: boolean/** Set during cold-load to coalesce concurrent waiters. */loaded: Promise<void>/** Marked true after the doc is hydrated from DB; flips OFF doc.on('update')@@ -68,36 +70,6 @@ const evictions = new Map<string, NodeJS.Timeout>()* echo-suppressed when they originated here. */const INSTANCE_ORIGIN = `instance:${env.INSTANCE_ID}`-async function loadSnapshot(- documentId: string,- dbClient?: PoolClient,-): Promise<{ state: Uint8Array | null; lastIncluded: bigint }> {- const { rows } = await (dbClient ?? pool).query<{ state_bytes: Buffer; snapshot_at_update_id: string }>(- `SELECT state_bytes, snapshot_at_update_id- FROM document_snapshots- WHERE document_id = $1`,- [documentId],- )- const row = rows[0]- if (!row) return { state: null, lastIncluded: 0n }- return { state: new Uint8Array(row.state_bytes), lastIncluded: BigInt(row.snapshot_at_update_id) }-}--async function loadUpdatesAfter(- documentId: string,- afterId: bigint,- dbClient?: PoolClient,-): Promise<Array<{ id: bigint; bytes: Uint8Array }>> {- const { rows } = await (dbClient ?? pool).query<{ id: string; update_bytes: Buffer }>(- `SELECT id, update_bytes- FROM document_updates- WHERE document_id = $1 AND id > $2- ORDER BY id ASC`,- [documentId, afterId.toString()],- )- return rows.map((r) => ({ id: BigInt(r.id), bytes: new Uint8Array(r.update_bytes) }))-}-async function persistUpdate(documentId: string, authorId: string, bytes: Uint8Array): Promise<void> {await pool.query(`INSERT INTO document_updates (document_id, author_id, update_bytes)@@ -113,51 +85,24 @@ async function persistUpdate(documentId: string, authorId: string, bytes: Uint8A}async function maybeCompact(room: Room): Promise<void> {- if (room.updatesSinceSnapshot < COMPACT_AFTER_UPDATES) return- // Snapshot the current state and find the latest update id covered.- const state = Y.encodeStateAsUpdate(room.doc)- const { rows } = await pool.query<{ max_id: string | null }>(- `SELECT MAX(id)::text AS max_id FROM document_updates WHERE document_id = $1`,- [room.documentId],- )- const maxId = rows[0]?.max_id ? BigInt(rows[0].max_id) : 0n- await pool.query(- `INSERT INTO document_snapshots (document_id, state_bytes, snapshot_at_update_id, updated_at)- VALUES ($1, $2, $3, NOW())- ON CONFLICT (document_id)- DO UPDATE SET state_bytes = EXCLUDED.state_bytes,- snapshot_at_update_id = EXCLUDED.snapshot_at_update_id,- updated_at = NOW()`,- [room.documentId, Buffer.from(state), maxId.toString()],- )- // Trim updates that are now safely captured in the snapshot.- await pool.query(- `DELETE FROM document_updates WHERE document_id = $1 AND id <= $2`,- [room.documentId, maxId.toString()],- )- room.updatesSinceSnapshot = 0-}--async function hydrateDoc(documentId: string, doc: Y.Doc, dbClient?: PoolClient): Promise<void> {- // A cold load is a two-query logical read. When the caller does not already- // own a transaction connection, reserve one for the whole hydration rather- // than returning it between snapshot and tail. Otherwise a pool-full wave- // of locked CLI callers can occupy every released slot while waiting on the- // shared `room.loaded`, leaving each hydration's tail query queued behind- // the callers that depend on it.- const client = dbClient ?? await pool.connect()+ if (room.compacting || room.updatesSinceSnapshot < COMPACT_AFTER_UPDATES) return+ room.compacting = true+ const updatesAtStart = room.updatesSinceSnapshottry {- const snap = await loadSnapshot(documentId, client)- if (snap.state) Y.applyUpdate(doc, snap.state, 'hydrate')- const tail = await loadUpdatesAfter(documentId, snap.lastIncluded, client)- for (const u of tail) {- Y.applyUpdate(doc, u.bytes, 'hydrate')+ if (await compactDocument(pool, room.documentId)) {+ // Keep edits persisted while compaction was running in the next count.+ room.updatesSinceSnapshot -= updatesAtStart}} finally {- if (!dbClient) client.release()+ room.compacting = false}}+async function hydrateDoc(documentId: string, doc: Y.Doc, dbClient?: PoolClient): Promise<void> {+ const rows = await readDocumentState(dbClient ?? pool, documentId)+ for (const row of rows) Y.applyUpdate(doc, new Uint8Array(row.bytes), 'hydrate')+}+function roomKey(documentId: string): string {return documentId}@@ -184,6 +129,7 @@ async function getOrCreateRoom(doc,subs: new Set(),updatesSinceSnapshot: 0,+ compacting: false,hydrated: false,loaded: Promise.resolve(),}@@ -218,8 +164,10 @@ async function getOrCreateRoom(// Persist + fan-out unless this update arrived FROM another instance// (it's already persisted there + already on the bus).if (!isRemote) {- room.updatesSinceSnapshot += 1- void persistUpdate(documentId, authorId, update).catch((e) => {+ void persistUpdate(documentId, authorId, update).then(() => {+ room.updatesSinceSnapshot += 1+ void maybeCompact(room).catch((e) => console.warn('[docs] compact failed', e))+ }).catch((e) => {console.warn('[docs] persistUpdate failed', e)})void publish(CH_DOC_UPDATE, {@@ -230,7 +178,6 @@ async function getOrCreateRoom(originId,authorId,}).catch(() => { /* swallow */ })- void maybeCompact(room).catch((e) => console.warn('[docs] compact failed', e))}})normalizeMarkdownImageParagraphs(doc, pmFragment(doc), { originId: 'system:doc-image-normalize', authorId: 'system' })