Beautiful, AI-native markdown IDE and LLM wiki
TypeScript 2nd-brainagent-skillsclaudecodexcompany-braindocsknowledge-knowledge-baseknowledge-managementllmllm-wikillm-wiki-karpathymarkdownmarkdown-editormdmdxnotespkmsecond-brainskills
fix(ok): serialize store publishes per document (PRD-8574) (#4549)
Nick Gomez30e09381e508
diff --git a/.changeset/late-store-cannot-overwrite-newer-save.md b/.changeset/late-store-cannot-overwrite-newer-save.mdnew file mode 100644index 000000000..3e4646131--- /dev/null+++ b/.changeset/late-store-cannot-overwrite-newer-save.md@@ -0,0 +1,5 @@+---+'@inkeep/open-knowledge': patch+---++A markdown document save that finishes after a newer save of the same document no longer overwrites the newer saved content on disk; the late save is skipped and logged.diff --git a/packages/server/src/document-durability-state.test.ts b/packages/server/src/document-durability-state.test.tsindex 219f432b1..2a64db2ce 100644--- a/packages/server/src/document-durability-state.test.ts+++ b/packages/server/src/document-durability-state.test.ts@@ -24,6 +24,7 @@ import {DISPLACED_VERSION_TTL_MS,DocumentDurabilityState,DocumentDurabilityStateError,+ type StorePublishOutcome,} from './document-durability-state.ts';import * as tracedFs from './fs-traced.ts';import { getLogger } from './logger.ts';@@ -834,3 +835,432 @@ describe('displaced-version history', () => {}});});++describe('store publish generation primitives', () => {+ test('beginStoreAttempt issues monotonically increasing tokens per document', () => {+ const state = new DocumentDurabilityState();+ expect(state.beginStoreAttempt('doc')).toEqual({ docName: 'doc', generation: 1 });+ expect(state.beginStoreAttempt('doc')).toEqual({ docName: 'doc', generation: 2 });+ expect(state.beginStoreAttempt('doc')).toEqual({ docName: 'doc', generation: 3 });+ expect(state.beginStoreAttempt('other')).toEqual({ docName: 'other', generation: 1 });+ });++ test('tryPublishStore refuses a stale token without publishing and without moving the published watermark', () => {+ const state = new DocumentDurabilityState();+ const first = state.beginStoreAttempt('doc');+ const second = state.beginStoreAttempt('doc');+ let publishCalls = 0;+ expect(+ state.tryPublishStore(second, () => {+ publishCalls += 1;+ }),+ ).toBe('published');+ expect(publishCalls).toBe(1);+ expect(+ state.tryPublishStore(first, () => {+ publishCalls += 1;+ }),+ ).toBe('stale');+ expect(publishCalls).toBe(1);+ expect(+ state.tryPublishStore(first, () => {+ publishCalls += 1;+ }),+ ).toBe('stale');+ expect(publishCalls).toBe(1);+ });++ test('tryPublishStore refuses an already-published token and advances the watermark only on publish', () => {+ const state = new DocumentDurabilityState();+ const first = state.beginStoreAttempt('doc');+ const second = state.beginStoreAttempt('doc');+ const third = state.beginStoreAttempt('doc');+ let publishCalls = 0;+ expect(+ state.tryPublishStore(second, () => {+ publishCalls += 1;+ }),+ ).toBe('published');+ expect(+ state.tryPublishStore(second, () => {+ publishCalls += 1;+ }),+ ).toBe('stale');+ expect(+ state.tryPublishStore(first, () => {+ publishCalls += 1;+ }),+ ).toBe('stale');+ expect(+ state.tryPublishStore(third, () => {+ publishCalls += 1;+ }),+ ).toBe('published');+ expect(+ state.tryPublishStore(second, () => {+ publishCalls += 1;+ }),+ ).toBe('stale');+ expect(+ state.tryPublishStore(third, () => {+ publishCalls += 1;+ }),+ ).toBe('stale');+ expect(publishCalls).toBe(2);+ });++ test('tryPublishStore invokes the publish callback synchronously before returning', () => {+ const state = new DocumentDurabilityState();+ const token = state.beginStoreAttempt('doc');+ let committed = false;+ const accepted = state.tryPublishStore(token, () => {+ committed = true;+ });+ expect(accepted).toBe('published');+ expect(committed).toBe(true);+ });++ test('tryPublishStore rejects a publish callback that returns a thenable instead of publishing synchronously', () => {+ const state = new DocumentDurabilityState();+ const token = state.beginStoreAttempt('doc');+ let renamed = false;+ expect(() =>+ state.tryPublishStore(token, async () => {+ await Promise.resolve();+ renamed = true;+ }),+ ).toThrow(TypeError);+ expect(renamed).toBe(false);+ expect(+ state.tryPublishStore(token, () => {+ renamed = true;+ }),+ ).toBe('published');+ expect(renamed).toBe(true);+ });++ test('a rejecting async publish callback throws without leaking an unhandled rejection', async () => {+ const state = new DocumentDurabilityState();+ const token = state.beginStoreAttempt('doc');+ const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {});+ try {+ expect(() =>+ state.tryPublishStore(token, async () => {+ await Promise.reject(new Error('in-flight rename failed'));+ }),+ ).toThrow(TypeError);+ await new Promise((resolve) => setImmediate(resolve));+ const events = warnSpy.mock.calls+ .map((call) => call.map(String).join(' '))+ .filter((line) => line.includes('"event":"persistence-store-publish-async-rejected"'));+ expect(events).toHaveLength(1);+ const payload = JSON.parse(events[0]) as Record<string, unknown>;+ expect(payload['doc.name']).toBe('doc');+ expect(payload.generation).toBe(1);+ expect(payload.error).toBe('in-flight rename failed');+ } finally {+ warnSpy.mockRestore();+ }+ });++ test('a non-Error rejection reason still lands the async-rejected event without crashing the handler', async () => {+ const state = new DocumentDurabilityState();+ const token = state.beginStoreAttempt('doc');+ const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {});+ try {+ expect(() =>+ state.tryPublishStore(token, async () => {+ await Promise.reject({ toString: 1 });+ }),+ ).toThrow(TypeError);+ await new Promise((resolve) => setImmediate(resolve));+ const events = warnSpy.mock.calls+ .map((call) => call.map(String).join(' '))+ .filter((line) => line.includes('"event":"persistence-store-publish-async-rejected"'));+ expect(events).toHaveLength(1);+ const payload = JSON.parse(events[0]) as Record<string, unknown>;+ expect(payload['doc.name']).toBe('doc');+ expect(payload.generation).toBe(1);+ expect(typeof payload.error).toBe('string');+ expect(payload.error).not.toBe('');+ } finally {+ warnSpy.mockRestore();+ }+ });++ test('an Error cause carrying a non-string message still lands the event with a string reason', async () => {+ const state = new DocumentDurabilityState();+ const token = state.beginStoreAttempt('doc');+ const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {});+ try {+ const hostile = new Error('placeholder');+ const circular: Record<string, unknown> = {};+ circular.self = circular;+ hostile.message = circular;+ expect(() =>+ state.tryPublishStore(token, async () => {+ await Promise.reject(hostile);+ }),+ ).toThrow(TypeError);+ await new Promise((resolve) => setImmediate(resolve));+ const events = warnSpy.mock.calls+ .map((call) => call.map(String).join(' '))+ .filter((line) => line.includes('"event":"persistence-store-publish-async-rejected"'));+ expect(events).toHaveLength(1);+ const payload = JSON.parse(events[0]) as Record<string, unknown>;+ expect(payload['doc.name']).toBe('doc');+ expect(payload.generation).toBe(1);+ expect(typeof payload.error).toBe('string');+ } finally {+ warnSpy.mockRestore();+ }+ });++ test('a sync-throwing then on a refused thenable still surfaces the TypeError contract', () => {+ const state = new DocumentDurabilityState();+ const token = state.beginStoreAttempt('doc');+ const syncThrowing = () => {};+ Reflect.set(syncThrowing, 'then', () => {+ throw new Error('then threw synchronously');+ });+ expect(() => state.tryPublishStore(token, () => syncThrowing)).toThrow(TypeError);+ expect(state.tryPublishStore(token, () => {})).toBe('published');+ });++ test('a self-resolving thenable return is refused without assimilating it into the promise machinery', () => {+ const state = new DocumentDurabilityState();+ const token = state.beginStoreAttempt('doc');+ const selfResolving = () => {};+ Reflect.set(selfResolving, 'then', (onFulfilled: (value: unknown) => void) => {+ onFulfilled(selfResolving);+ });+ expect(() => state.tryPublishStore(token, () => selfResolving)).toThrow(TypeError);+ expect(state.tryPublishStore(token, () => {})).toBe('published');+ });++ test('tryPublishStore rejects a thenable that is a function carrying a then method', () => {+ const state = new DocumentDurabilityState();+ const token = state.beginStoreAttempt('doc');+ const callableThenable = () => {};+ Reflect.set(callableThenable, 'then', () => {});+ expect(() => state.tryPublishStore(token, () => callableThenable)).toThrow(TypeError);+ expect(state.tryPublishStore(token, () => {})).toBe('published');+ });++ test('a nested publish attempt inside the gate is refused before its callback runs and the outer attempt completes', () => {+ const state = new DocumentDurabilityState();+ const stale = state.beginStoreAttempt('doc');+ const outer = state.beginStoreAttempt('doc');+ const afterOuter = state.beginStoreAttempt('doc');+ let publishCalls = 0;+ let nestedOutcome: StorePublishOutcome | undefined;+ expect(+ state.tryPublishStore(outer, () => {+ publishCalls += 1;+ nestedOutcome = state.tryPublishStore(stale, () => {+ publishCalls += 1;+ });+ }),+ ).toBe('published');+ expect(nestedOutcome).toBe('reentrant');+ expect(publishCalls).toBe(1);+ expect(+ state.tryPublishStore(outer, () => {+ publishCalls += 1;+ }),+ ).toBe('stale');+ expect(+ state.tryPublishStore(afterOuter, () => {+ publishCalls += 1;+ }),+ ).toBe('published');+ expect(publishCalls).toBe(2);+ });++ test('a reentrant attempt with a higher generation is still refused while the outer publish is in flight', () => {+ const state = new DocumentDurabilityState();+ const outer = state.beginStoreAttempt('doc');+ const nested = state.beginStoreAttempt('doc');+ let publishCalls = 0;+ let nestedOutcome: StorePublishOutcome | undefined;+ expect(+ state.tryPublishStore(outer, () => {+ publishCalls += 1;+ nestedOutcome = state.tryPublishStore(nested, () => {+ publishCalls += 1;+ });+ }),+ ).toBe('published');+ expect(nestedOutcome).toBe('reentrant');+ expect(publishCalls).toBe(1);+ expect(+ state.tryPublishStore(outer, () => {+ publishCalls += 1;+ }),+ ).toBe('stale');+ expect(+ state.tryPublishStore(nested, () => {+ publishCalls += 1;+ }),+ ).toBe('published');+ expect(publishCalls).toBe(2);+ });++ test('an in-flight publish for one document does not refuse a nested publish for another document', () => {+ const state = new DocumentDurabilityState();+ const docToken = state.beginStoreAttempt('doc');+ const otherToken = state.beginStoreAttempt('other');+ let docPublishCalls = 0;+ let otherPublishCalls = 0;+ let nestedOtherOutcome: StorePublishOutcome | undefined;+ let docReentryOutcome: StorePublishOutcome | undefined;+ expect(+ state.tryPublishStore(docToken, () => {+ docPublishCalls += 1;+ nestedOtherOutcome = state.tryPublishStore(otherToken, () => {+ otherPublishCalls += 1;+ });+ docReentryOutcome = state.tryPublishStore(docToken, () => {+ docPublishCalls += 1;+ });+ }),+ ).toBe('published');+ expect(nestedOtherOutcome).toBe('published');+ expect(otherPublishCalls).toBe(1);+ expect(docReentryOutcome).toBe('reentrant');+ expect(docPublishCalls).toBe(1);+ const otherFollowUp = state.beginStoreAttempt('other');+ expect(+ state.tryPublishStore(otherFollowUp, () => {+ otherPublishCalls += 1;+ }),+ ).toBe('published');+ expect(otherPublishCalls).toBe(2);+ });++ test('a reentrant attempt that is also stale is still refused as reentrant while the outer publish is in flight', () => {+ const state = new DocumentDurabilityState();+ const first = state.beginStoreAttempt('doc');+ expect(state.tryPublishStore(first, () => {})).toBe('published');+ const outer = state.beginStoreAttempt('doc');+ let publishCalls = 0;+ let nestedOutcome: StorePublishOutcome | undefined;+ expect(+ state.tryPublishStore(outer, () => {+ publishCalls += 1;+ nestedOutcome = state.tryPublishStore(first, () => {+ publishCalls += 1;+ });+ }),+ ).toBe('published');+ expect(nestedOutcome).toBe('reentrant');+ expect(publishCalls).toBe(1);+ });++ test('a throwing publish callback propagates without advancing the published watermark', () => {+ const state = new DocumentDurabilityState();+ const first = state.beginStoreAttempt('doc');+ expect(state.tryPublishStore(first, () => {})).toBe('published');+ const second = state.beginStoreAttempt('doc');+ expect(() =>+ state.tryPublishStore(second, () => {+ throw new Error('rename failed');+ }),+ ).toThrow('rename failed');+ expect(state.tryPublishStore(second, () => {})).toBe('published');+ const third = state.beginStoreAttempt('doc');+ expect(state.tryPublishStore(third, () => {})).toBe('published');+ });++ test('a pre-delete store token stays outranked across the delete boundary', () => {+ const state = new DocumentDurabilityState();+ const stale = state.beginStoreAttempt('doc');+ const current = state.beginStoreAttempt('doc');+ const otherFirst = state.beginStoreAttempt('other');+ const otherSecond = state.beginStoreAttempt('other');+ let publishCalls = 0;+ expect(+ state.tryPublishStore(current, () => {+ publishCalls += 1;+ }),+ ).toBe('published');+ expect(+ state.tryPublishStore(stale, () => {+ publishCalls += 1;+ }),+ ).toBe('stale');+ expect(+ state.tryPublishStore(otherSecond, () => {+ publishCalls += 1;+ }),+ ).toBe('published');++ state.deleteReconciledBase('doc');++ expect(state.beginStoreAttempt('doc')).toEqual({ docName: 'doc', generation: 3 });+ expect(+ state.tryPublishStore(stale, () => {+ publishCalls += 1;+ }),+ ).toBe('stale');+ const postDelete = state.beginStoreAttempt('doc');+ expect(postDelete).toEqual({ docName: 'doc', generation: 4 });+ expect(+ state.tryPublishStore(postDelete, () => {+ publishCalls += 1;+ }),+ ).toBe('published');+ expect(+ state.tryPublishStore(stale, () => {+ publishCalls += 1;+ }),+ ).toBe('stale');+ expect(publishCalls).toBe(3);+ expect(state.beginStoreAttempt('other')).toEqual({ docName: 'other', generation: 3 });+ expect(+ state.tryPublishStore(otherFirst, () => {+ publishCalls += 1;+ }),+ ).toBe('stale');+ expect(publishCalls).toBe(3);+ });++ test('a pre-delete store token stays outranked across a persisted delete', () => {+ const state = new DocumentDurabilityState();+ state.recordDisplacedVersion('doc', 'displaced bytes');+ const stale = state.beginStoreAttempt('doc');+ const current = state.beginStoreAttempt('doc');+ let publishCalls = 0;+ expect(+ state.tryPublishStore(current, () => {+ publishCalls += 1;+ }),+ ).toBe('published');+ expect(+ state.tryPublishStore(stale, () => {+ publishCalls += 1;+ }),+ ).toBe('stale');++ state.deleteReconciledBase('doc');++ expect(state.beginStoreAttempt('doc')).toEqual({ docName: 'doc', generation: 3 });+ expect(+ state.tryPublishStore(stale, () => {+ publishCalls += 1;+ }),+ ).toBe('stale');+ const postDelete = state.beginStoreAttempt('doc');+ expect(+ state.tryPublishStore(postDelete, () => {+ publishCalls += 1;+ }),+ ).toBe('published');+ expect(+ state.tryPublishStore(stale, () => {+ publishCalls += 1;+ }),+ ).toBe('stale');+ expect(publishCalls).toBe(2);+ });+});diff --git a/packages/server/src/document-durability-state.ts b/packages/server/src/document-durability-state.tsindex 49671ed5f..e83570b78 100644--- a/packages/server/src/document-durability-state.ts+++ b/packages/server/src/document-durability-state.ts@@ -16,6 +16,17 @@ export interface StoreFailure {message: string;}+export interface StoreAttemptToken {+ readonly docName: string;+ readonly generation: number;+}++export type StorePublishOutcome = 'published' | 'stale' | 'reentrant';++export function assertNeverStorePublishOutcome(outcome: never): never {+ throw new Error(`[StorePublishOutcome] unhandled variant: ${JSON.stringify(outcome)}`);+}+export const OK_DOC_REMOVED = 'OK_DOC_REMOVED';export const OK_PATH_UNRESOLVABLE = 'OK_PATH_UNRESOLVABLE';@@ -85,6 +96,33 @@ function snapshotParts(branches: ReadonlyMap<string, string[]>): string[] {return parts;}+function isThenable(+ value: unknown,+): value is { then: (onFulfilled: () => void, onRejected: (cause: unknown) => void) => unknown } {+ return (+ (typeof value === 'object' || typeof value === 'function') &&+ value !== null &&+ typeof (value as { then?: unknown }).then === 'function'+ );+}++function logRefusedPublishRejection(token: StoreAttemptToken, cause: unknown): void {+ let reason: string;+ try {+ reason = cause instanceof Error ? String(cause.message) : String(cause);+ } catch {+ reason = 'unserializable rejection reason';+ }+ console.warn(+ JSON.stringify({+ event: 'persistence-store-publish-async-rejected',+ 'doc.name': token.docName,+ generation: token.generation,+ error: reason,+ }),+ );+}+function sameStaleConflict(previous: StaleExternalWriteConflict | undefined,current: StaleExternalWriteConflict | undefined,@@ -209,6 +247,9 @@ export class DocumentDurabilityState {Map<string, StaleExternalWriteConflict>>();private readonly inFlightFlushByDoc = new Map<string, PendingFlush[]>();+ private readonly storeAttemptGenerations = new Map<string, number>();+ private readonly publishedStoreGenerations = new Map<string, number>();+ private readonly documentsWithPublishInFlight = new Set<string>();private readonly agentWriteStores = new Set<string>();private readonly storeFailures = new Map<string, StoreFailure>();private readonly storeDivergences = new Set<string>();@@ -691,6 +732,50 @@ export class DocumentDurabilityState {}}+ beginStoreAttempt(docName: string): StoreAttemptToken {+ const generation = (this.storeAttemptGenerations.get(docName) ?? 0) + 1;+ this.storeAttemptGenerations.set(docName, generation);+ return { docName, generation };+ }++ /**+ * STOP: `publish` must complete the disk write synchronously before returning — a+ * `'published'` return promises the caller the bytes are already on disk, and the call+ * site's post-publish bookkeeping (writeTracker registration, recordSuccessfulStore) runs+ * on that promise. The call site must pass a synchronous rename (`tracedRenameSync`), never+ * the async `tracedRename`: a callback that returns before the rename lands advances the+ * published watermark over content that is not durably persisted. A thenable return throws,+ * with a rejection handler attached first so its in-flight work cannot crash the process. A+ * nested `tryPublishStore` for the same document while a publish is in flight returns+ * `'reentrant'` before its callback runs — distinct from `'stale'` — and the in-flight+ * attempt owns the document's publish gate and completes normally: a `'reentrant'` result+ * is not a superseded drop, the in-flight capture is the one that lands.+ */+ tryPublishStore(token: StoreAttemptToken, publish: () => void): StorePublishOutcome {+ if (this.documentsWithPublishInFlight.has(token.docName)) return 'reentrant';+ const publishedAtEntry = this.publishedStoreGenerations.get(token.docName) ?? 0;+ if (token.generation <= publishedAtEntry) return 'stale';+ this.documentsWithPublishInFlight.add(token.docName);+ try {+ const outcome: unknown = publish();+ if (isThenable(outcome)) {+ try {+ outcome.then(+ () => {},+ (cause: unknown) => logRefusedPublishRejection(token, cause),+ );+ } catch {}+ throw new TypeError(+ `tryPublishStore: the publish callback must complete synchronously, but it returned a thenable (${token.docName} generation ${token.generation})`,+ );+ }+ this.publishedStoreGenerations.set(token.docName, token.generation);+ return 'published';+ } finally {+ this.documentsWithPublishInFlight.delete(token.docName);+ }+ }+setBatchInProgress(value: boolean): void {this.batchInProgress = value;}diff --git a/packages/server/src/index.ts b/packages/server/src/index.tsindex efc3111f8..e042b8038 100644--- a/packages/server/src/index.ts+++ b/packages/server/src/index.ts@@ -159,7 +159,9 @@ export { FILE_WATCHER_ORIGIN } from './disk-content-intake.ts';export {DocumentDurabilityState,DocumentDurabilityStateError,+ type StoreAttemptToken,type StoreFailure,+ type StorePublishOutcome,} from './document-durability-state.ts';export {canonicalProjectKey,diff --git a/packages/server/src/metrics.ts b/packages/server/src/metrics.tsindex 6f565f816..3133c794a 100644--- a/packages/server/src/metrics.ts+++ b/packages/server/src/metrics.ts@@ -20,6 +20,7 @@ export interface ReconciliationMetrics {batchCount: number;upstreamImportCount: number;persistenceStoreRemovedDocCount: number;+ persistenceStoreSupersededCount: number;rescueBufferCount: number;branchSwitchCount: number;parkCount: number;@@ -125,6 +126,7 @@ const counters: ReconciliationMetrics = {batchCount: 0,upstreamImportCount: 0,persistenceStoreRemovedDocCount: 0,+ persistenceStoreSupersededCount: 0,rescueBufferCount: 0,branchSwitchCount: 0,parkCount: 0,@@ -237,6 +239,10 @@ export function incrementPersistenceStoreRemovedDoc(): void {counters.persistenceStoreRemovedDocCount++;}+export function incrementPersistenceStoreSuperseded(): void {+ counters.persistenceStoreSupersededCount++;+}+export function incrementRescueBuffer(): void {counters.rescueBufferCount++;}@@ -602,6 +608,7 @@ export function resetMetrics(): void {counters.batchCount = 0;counters.upstreamImportCount = 0;counters.persistenceStoreRemovedDocCount = 0;+ counters.persistenceStoreSupersededCount = 0;counters.rescueBufferCount = 0;counters.branchSwitchCount = 0;counters.parkCount = 0;diff --git a/packages/server/src/persistence-deferred-store.test.ts b/packages/server/src/persistence-deferred-store.test.tsindex d29babeab..1a5a32cc0 100644--- a/packages/server/src/persistence-deferred-store.test.ts+++ b/packages/server/src/persistence-deferred-store.test.ts@@ -625,7 +625,7 @@ describe('FR-9 — deferred-store-failed event + counter', () => {await storeDocument(persistence, document, docName);setBatchInProgress(false);- const renameSpy = vi.spyOn(fsTraced, 'tracedRename').mockImplementation(async () => {+ const renameSpy = vi.spyOn(fsTraced, 'tracedRenameSync').mockImplementation(() => {const malformed = Object.create(Error.prototype) as Error & { name: string };Object.defineProperty(malformed, 'message', { value: 'malformed-error', enumerable: true });Object.defineProperty(malformed, 'name', { value: 'MalformedError', enumerable: true });@@ -683,7 +683,7 @@ describe('FR-9 — deferred-store-failed event + counter', () => {await storeDocument(persistence, document, docName);setBatchInProgress(false);- const renameSpy = vi.spyOn(fsTraced, 'tracedRename').mockImplementation(async () => {+ const renameSpy = vi.spyOn(fsTraced, 'tracedRenameSync').mockImplementation(() => {const malformed = Object.create(Error.prototype) as Error & { name: string };Object.defineProperty(malformed, 'name', { value: 'MalformedError', enumerable: true });Object.defineProperty(malformed, 'message', {@@ -731,7 +731,7 @@ describe('FR-9 — deferred-store-failed event + counter', () => {await storeDocument(persistence, document, docName);setBatchInProgress(false);- const renameSpy = vi.spyOn(fsTraced, 'tracedRename').mockImplementation(async () => {+ const renameSpy = vi.spyOn(fsTraced, 'tracedRenameSync').mockImplementation(() => {const malformed = Object.create(Error.prototype) as Error & { name: string };Object.defineProperty(malformed, 'message', {value: 'malformed-error-verbose',diff --git a/packages/server/src/persistence-store-overlap.test-helper.ts b/packages/server/src/persistence-store-overlap.test-helper.tsnew file mode 100644index 000000000..1dfb65c01--- /dev/null+++ b/packages/server/src/persistence-store-overlap.test-helper.ts@@ -0,0 +1,57 @@+import * as Y from 'yjs';+import type { DocumentDurabilityState } from './document-durability-state.ts';+import type { createPersistenceExtension } from './persistence.ts';++export const BROWSER_ORIGIN = {+ source: 'connection',+ connection: { context: { principalId: 'principal-test' } },+};++export function replaceDocParagraphs(document: Y.Doc, texts: string[]): void {+ const body = `${texts.join('\n\n')}\n`;+ const fragment = document.getXmlFragment('default');+ const ytext = document.getText('source');+ if (fragment.length > 0) {+ fragment.delete(0, fragment.length);+ }+ fragment.insert(+ 0,+ texts.map((text) => {+ const paragraph = new Y.XmlElement('paragraph');+ paragraph.insert(0, [new Y.XmlText(text)]);+ return paragraph;+ }),+ );+ if (ytext.length > 0) {+ ytext.delete(0, ytext.length);+ }+ ytext.insert(0, body);+}++function hookStorePayload(document: Y.Doc, documentName: string) {+ return {+ document,+ documentName,+ lastTransactionOrigin: BROWSER_ORIGIN,+ lastContext: {},+ };+}++export async function runHookStore(+ persistence: ReturnType<typeof createPersistenceExtension>,+ document: Y.Doc,+ documentName: string,+): Promise<void> {+ await persistence.extension.onStoreDocument?.(hookStorePayload(document, documentName) as never);+}++export async function queueDeferredStore(+ persistence: ReturnType<typeof createPersistenceExtension>,+ durabilityState: DocumentDurabilityState,+ document: Y.Doc,+ documentName: string,+): Promise<void> {+ durabilityState.setBatchInProgress(true);+ await runHookStore(persistence, document, documentName);+ durabilityState.setBatchInProgress(false);+}diff --git a/packages/server/src/persistence-store-publish-order.test.ts b/packages/server/src/persistence-store-publish-order.test.tsnew file mode 100644index 000000000..01d86a3cc--- /dev/null+++ b/packages/server/src/persistence-store-publish-order.test.ts@@ -0,0 +1,188 @@+import { mkdtempSync, readFileSync, realpathSync, rmSync, writeFileSync } from 'node:fs';+import { tmpdir } from 'node:os';+import { join } from 'node:path';+import { afterEach, beforeEach, describe, expect, test, vi } from 'vitest';+import * as Y from 'yjs';+import { DocumentDurabilityState } from './document-durability-state.ts';+import * as fsTraced from './fs-traced.ts';+import { getMetrics } from './metrics.ts';+import { createPersistenceExtension } from './persistence.ts';+import {+ BROWSER_ORIGIN,+ queueDeferredStore,+ replaceDocParagraphs,+ runHookStore,+} from './persistence-store-overlap.test-helper.ts';++const STALE_PARAGRAPHS = ['alpha'];+const STALE_CONTENT = 'alpha\n';+const NEWER_PARAGRAPHS = ['beta'];+const NEWER_CONTENT = 'beta\n';+const INITIAL_CONTENT = 'initial\n';++type NewerStoreVia = 'hook' | 'forceStore';+type StaleStoreVia = 'drain' | 'hook' | 'forceStore';++const OVERLAPPING_STORE_PAIRS: ReadonlyArray<{+ staleVia: StaleStoreVia;+ newerVia: NewerStoreVia;+}> = [+ { staleVia: 'drain', newerVia: 'hook' },+ { staleVia: 'drain', newerVia: 'forceStore' },+ { staleVia: 'hook', newerVia: 'forceStore' },+ { staleVia: 'forceStore', newerVia: 'hook' },+];++interface DocStateObservation {+ disk: string;+ reconciledBase: string | undefined;+ docText: string;+}++describe('persistence publish ordering — a store must not publish bytes older than a completed store', () => {+ let tmpDir: string;+ let docName: string;+ let docPath: string;+ let document: Y.Doc;+ let durabilityState: DocumentDurabilityState;++ function createPersistence(): ReturnType<typeof createPersistenceExtension> {+ return createPersistenceExtension({+ contentDir: tmpDir,+ projectDir: tmpDir,+ gitEnabled: false,+ durabilityState,+ });+ }++ beforeEach(() => {+ tmpDir = realpathSync(mkdtempSync(join(tmpdir(), 'ok-store-publish-order-')));+ docName = 'publish-order';+ docPath = join(tmpDir, `${docName}.md`);+ document = new Y.Doc();+ durabilityState = new DocumentDurabilityState();+ });++ afterEach(() => {+ vi.restoreAllMocks();+ document.destroy();+ rmSync(tmpDir, { recursive: true, force: true });+ });++ async function loadFromDisk(+ persistence: ReturnType<typeof createPersistenceExtension>,+ ): Promise<void> {+ writeFileSync(docPath, INITIAL_CONTENT, 'utf-8');+ await persistence.extension.onLoadDocument?.({+ document,+ documentName: docName,+ context: {},+ } as never);+ expect(durabilityState.getReconciledBase(docName)).toBe(INITIAL_CONTENT);+ }++ function observeDocState(docNameForBase: string): DocStateObservation {+ return {+ disk: readFileSync(docPath, 'utf-8'),+ reconciledBase: durabilityState.getReconciledBase(docNameForBase),+ docText: document.getText('source').toString(),+ };+ }++ function interleaveNewerStoreInsideStaleStoreMkdir(+ persistence: ReturnType<typeof createPersistenceExtension>,+ newerVia: NewerStoreVia,+ observations: { afterNewerStore?: DocStateObservation },+ ): void {+ const realMkdir = fsTraced.tracedMkdir;+ let newerStoreRan = false;+ vi.spyOn(fsTraced, 'tracedMkdir').mockImplementation(async (path, options) => {+ if (!newerStoreRan) {+ newerStoreRan = true;+ document.transact(() => replaceDocParagraphs(document, NEWER_PARAGRAPHS), BROWSER_ORIGIN);+ if (newerVia === 'hook') {+ await runHookStore(persistence, document, docName);+ } else {+ await persistence.forceStore(document, docName);+ }+ observations.afterNewerStore = observeDocState(docName);+ }+ return realMkdir(path, options);+ });+ }++ test.each(OVERLAPPING_STORE_PAIRS)(+ 'a late $staleVia store whose capture predates a completed $newerVia publish leaves the newer bytes on disk and in the reconciled base',+ async ({ staleVia, newerVia }) => {+ const persistence = createPersistence();+ await loadFromDisk(persistence);++ document.transact(() => replaceDocParagraphs(document, STALE_PARAGRAPHS), BROWSER_ORIGIN);+ if (staleVia === 'drain') {+ await queueDeferredStore(persistence, durabilityState, document, docName);+ expect(readFileSync(docPath, 'utf-8')).toBe(INITIAL_CONTENT);+ }++ const observations: { afterNewerStore?: DocStateObservation } = {};+ interleaveNewerStoreInsideStaleStoreMkdir(persistence, newerVia, observations);+ const supersededBefore = getMetrics().persistenceStoreSupersededCount;++ try {+ if (staleVia === 'drain') {+ await persistence.flushDeferredStores('within-branch');+ } else if (staleVia === 'forceStore') {+ await persistence.forceStore(document, docName);+ } else {+ await runHookStore(persistence, document, docName);+ }+ } finally {+ vi.restoreAllMocks();+ }++ expect(observations.afterNewerStore).toEqual({+ disk: NEWER_CONTENT,+ reconciledBase: NEWER_CONTENT,+ docText: NEWER_CONTENT,+ });+ expect(readFileSync(docPath, 'utf-8')).toBe(NEWER_CONTENT);+ expect(durabilityState.getReconciledBase(docName)).toBe(NEWER_CONTENT);+ expect(document.getText('source').toString()).toBe(NEWER_CONTENT);+ expect(durabilityState.inFlightFlushCount(docName)).toBe(0);+ expect(getMetrics().persistenceStoreSupersededCount).toBe(supersededBefore + 1);+ },+ );++ test('a store whose capture is still the newest published when it lands is not dropped', async () => {+ const persistence = createPersistence();+ await loadFromDisk(persistence);++ document.transact(() => replaceDocParagraphs(document, STALE_PARAGRAPHS), BROWSER_ORIGIN);+ await queueDeferredStore(persistence, durabilityState, document, docName);+ expect(readFileSync(docPath, 'utf-8')).toBe(INITIAL_CONTENT);++ const realMkdir = fsTraced.tracedMkdir;+ let docEditedInFlight = false;+ vi.spyOn(fsTraced, 'tracedMkdir').mockImplementation(async (path, options) => {+ if (!docEditedInFlight) {+ docEditedInFlight = true;+ document.transact(() => replaceDocParagraphs(document, NEWER_PARAGRAPHS), BROWSER_ORIGIN);+ }+ return realMkdir(path, options);+ });++ try {+ await persistence.flushDeferredStores('within-branch');+ } finally {+ vi.restoreAllMocks();+ }++ expect(docEditedInFlight).toBe(true);+ expect(readFileSync(docPath, 'utf-8')).toBe(STALE_CONTENT);+ expect(durabilityState.getReconciledBase(docName)).toBe(STALE_CONTENT);+ expect(document.getText('source').toString()).toBe(NEWER_CONTENT);++ await runHookStore(persistence, document, docName);+ expect(readFileSync(docPath, 'utf-8')).toBe(NEWER_CONTENT);+ expect(durabilityState.getReconciledBase(docName)).toBe(NEWER_CONTENT);+ });+});diff --git a/packages/server/src/persistence-store-superseded-drop.test.ts b/packages/server/src/persistence-store-superseded-drop.test.tsnew file mode 100644index 000000000..b0d38600d--- /dev/null+++ b/packages/server/src/persistence-store-superseded-drop.test.ts@@ -0,0 +1,241 @@+import {+ mkdtempSync,+ readdirSync,+ readFileSync,+ realpathSync,+ rmSync,+ writeFileSync,+} from 'node:fs';+import { tmpdir } from 'node:os';+import { join } from 'node:path';+import { ATOMIC_TEMP_PATH_RE } from '@inkeep/open-knowledge-core/server';+import { afterEach, beforeEach, describe, expect, test, vi } from 'vitest';+import * as Y from 'yjs';+import { __setQuiescentOverrideForTests } from './bridge-quiescence.ts';+import { DocumentDurabilityState } from './document-durability-state.ts';+import { contentHash, writeTracker } from './file-watcher.ts';+import * as fsTraced from './fs-traced.ts';+import { getMetrics, resetMetrics } from './metrics.ts';+import { createPersistenceExtension } from './persistence.ts';+import {+ BROWSER_ORIGIN,+ queueDeferredStore,+ replaceDocParagraphs,+ runHookStore,+} from './persistence-store-overlap.test-helper.ts';++const STALE_PARAGRAPHS = ['alpha'];+const STALE_CONTENT = 'alpha\n';+const NEWER_PARAGRAPHS = ['beta'];+const NEWER_CONTENT = 'beta\n';+const INITIAL_CONTENT = 'initial\n';++function supersededCounter(): number {+ return getMetrics().persistenceStoreSupersededCount;+}++interface DocStateObservation {+ disk: string;+ reconciledBase: string | undefined;+ docText: string;+}++describe('persistence superseded-store drop — a store that loses the publish race exits without publishing side effects', () => {+ let tmpDir: string;+ let docName: string;+ let docPath: string;+ let document: Y.Doc;+ let durabilityState: DocumentDurabilityState;+ let warnSpy: ReturnType<typeof vi.spyOn>;+ let warnings: string[];+ let diskFlushMarkdowns: string[];++ function findEventLines(eventName: string): Array<Record<string, unknown>> {+ const matches: Array<Record<string, unknown>> = [];+ for (const line of warnings) {+ if (!line.includes(`"event":"${eventName}"`)) continue;+ try {+ matches.push(JSON.parse(line) as Record<string, unknown>);+ } catch {}+ }+ return matches;+ }++ function createPersistence(): ReturnType<typeof createPersistenceExtension> {+ return createPersistenceExtension({+ contentDir: tmpDir,+ projectDir: tmpDir,+ gitEnabled: false,+ durabilityState,+ onDiskFlush: (_docName, _sv, persistedMarkdown) => {+ diskFlushMarkdowns.push(persistedMarkdown);+ },+ });+ }++ beforeEach(() => {+ tmpDir = realpathSync(mkdtempSync(join(tmpdir(), 'ok-store-superseded-drop-')));+ docName = 'superseded-drop';+ docPath = join(tmpDir, `${docName}.md`);+ document = new Y.Doc();+ durabilityState = new DocumentDurabilityState();+ resetMetrics();+ warnings = [];+ diskFlushMarkdowns = [];+ warnSpy = vi.spyOn(console, 'warn').mockImplementation((...args: unknown[]) => {+ warnings.push(args.map(String).join(' '));+ });+ });++ afterEach(() => {+ __setQuiescentOverrideForTests(document, undefined);+ warnSpy.mockRestore();+ vi.restoreAllMocks();+ document.destroy();+ rmSync(tmpDir, { recursive: true, force: true });+ });++ async function loadFromDisk(+ persistence: ReturnType<typeof createPersistenceExtension>,+ ): Promise<void> {+ writeFileSync(docPath, INITIAL_CONTENT, 'utf-8');+ await persistence.extension.onLoadDocument?.({+ document,+ documentName: docName,+ context: {},+ } as never);+ expect(durabilityState.getReconciledBase(docName)).toBe(INITIAL_CONTENT);+ }++ function observeDocState(): DocStateObservation {+ return {+ disk: readFileSync(docPath, 'utf-8'),+ reconciledBase: durabilityState.getReconciledBase(docName),+ docText: document.getText('source').toString(),+ };+ }++ test('a superseded store leaves the newer bytes published and drops without a disk ack, a write-tracker entry, temp residue, or a kept defer count', async () => {+ const persistence = createPersistence();+ await loadFromDisk(persistence);++ document.transact(() => replaceDocParagraphs(document, STALE_PARAGRAPHS), BROWSER_ORIGIN);+ await queueDeferredStore(persistence, durabilityState, document, docName);+ expect(readFileSync(docPath, 'utf-8')).toBe(INITIAL_CONTENT);+ expect(writeTracker.get(docPath)).toBeUndefined();+ expect(diskFlushMarkdowns).toEqual([]);++ const realMkdir = fsTraced.tracedMkdir;+ let newerStoreRan = false;+ const observations: {+ afterNewerStore?: DocStateObservation;+ deferCountAfterNewerStore?: number;+ } = {};+ vi.spyOn(fsTraced, 'tracedMkdir').mockImplementation(async (path, options) => {+ if (!newerStoreRan) {+ newerStoreRan = true;+ document.transact(() => replaceDocParagraphs(document, NEWER_PARAGRAPHS), BROWSER_ORIGIN);+ await runHookStore(persistence, document, docName);+ __setQuiescentOverrideForTests(document, false);+ await runHookStore(persistence, document, docName);+ __setQuiescentOverrideForTests(document, undefined);+ observations.afterNewerStore = observeDocState();+ observations.deferCountAfterNewerStore = persistence.getQueueDepths().quiescenceDeferred;+ durabilityState.recordStoreFailure(docName, {+ code: 'EIO',+ message: 'recorded store failure that the superseded drop must not clear',+ });+ }+ return realMkdir(path, options);+ });++ const supersededBefore = supersededCounter();++ try {+ await persistence.flushDeferredStores('within-branch');+ } finally {+ vi.restoreAllMocks();+ }++ expect(observations.afterNewerStore).toEqual({+ disk: NEWER_CONTENT,+ reconciledBase: NEWER_CONTENT,+ docText: NEWER_CONTENT,+ });+ expect(observations.deferCountAfterNewerStore).toBe(1);+ expect(readFileSync(docPath, 'utf-8')).toBe(NEWER_CONTENT);+ expect(durabilityState.getReconciledBase(docName)).toBe(NEWER_CONTENT);+ expect(document.getText('source').toString()).toBe(NEWER_CONTENT);++ expect(diskFlushMarkdowns).toEqual([NEWER_CONTENT]);+ expect((writeTracker.get(docPath) ?? []).map((entry) => entry.hash)).toEqual([+ contentHash(NEWER_CONTENT),+ ]);++ const events = findEventLines('persistence-store-superseded');+ expect(events).toHaveLength(1);+ const ev = events[0] as Record<string, unknown>;+ expect(ev.event).toBe('persistence-store-superseded');+ expect(ev['doc.name']).toBe(docName);+ expect(ev.baseBytes).toBe(INITIAL_CONTENT.length);+ expect(ev.candidateBytes).toBe(STALE_CONTENT.length);+ expect(ev.generation).toBe(1);+ expect(Object.keys(ev).sort()).toEqual([+ 'baseBytes',+ 'candidateBytes',+ 'doc.name',+ 'event',+ 'generation',+ ]);++ expect(supersededCounter() - supersededBefore).toBe(1);+ expect(persistence.getQueueDepths().quiescenceDeferred).toBe(0);+ expect(readdirSync(tmpDir).filter((name) => ATOMIC_TEMP_PATH_RE.test(name))).toEqual([]);+ expect(durabilityState.takeStoreFailure(docName)).toEqual({+ code: 'EIO',+ message: 'recorded store failure that the superseded drop must not clear',+ });+ });++ test('a reentrant store refusal emits its own refusal event, drops without advancing the superseded counter, and still removes the temp file and the defer count', async () => {+ const persistence = createPersistence();+ await loadFromDisk(persistence);++ document.transact(() => replaceDocParagraphs(document, STALE_PARAGRAPHS), BROWSER_ORIGIN);+ __setQuiescentOverrideForTests(document, false);+ await runHookStore(persistence, document, docName);+ __setQuiescentOverrideForTests(document, undefined);+ expect(persistence.getQueueDepths().quiescenceDeferred).toBe(1);++ let deferCountAtDispatch: number | undefined;+ vi.spyOn(durabilityState, 'tryPublishStore').mockImplementation(() => {+ deferCountAtDispatch = persistence.getQueueDepths().quiescenceDeferred;+ return 'reentrant';+ });++ const supersededBefore = supersededCounter();+ await runHookStore(persistence, document, docName);++ const events = findEventLines('persistence-store-reentrant-refused');+ expect(events).toHaveLength(1);+ const ev = events[0] as Record<string, unknown>;+ expect(ev.event).toBe('persistence-store-reentrant-refused');+ expect(ev['doc.name']).toBe(docName);+ expect(ev.generation).toBe(1);+ expect(ev.baseBytes).toBe(INITIAL_CONTENT.length);+ expect(ev.candidateBytes).toBe(STALE_CONTENT.length);+ expect(Object.keys(ev).sort()).toEqual([+ 'baseBytes',+ 'candidateBytes',+ 'doc.name',+ 'event',+ 'generation',+ ]);++ expect(findEventLines('persistence-store-superseded')).toEqual([]);+ expect(supersededCounter() - supersededBefore).toBe(0);+ expect(deferCountAtDispatch).toBe(1);+ expect(persistence.getQueueDepths().quiescenceDeferred).toBe(0);+ expect(readdirSync(tmpDir).filter((name) => ATOMIC_TEMP_PATH_RE.test(name))).toEqual([]);+ });+});diff --git a/packages/server/src/persistence.ts b/packages/server/src/persistence.tsindex bfd6b71dd..05068c21d 100644--- a/packages/server/src/persistence.ts+++ b/packages/server/src/persistence.ts@@ -57,6 +57,7 @@ import {import type { DerivedDocumentIndexPersistencePort } from './derived-document-index.ts';import { applyDiskContentToDoc, FILE_WATCHER_ORIGIN } from './disk-content-intake.ts';import {+ assertNeverStorePublishOutcome,DocumentDurabilityState,OK_DOC_REMOVED,OK_PATH_UNRESOLVABLE,@@ -64,7 +65,7 @@ import {} from './document-durability-state.ts';import { refuseStaleExternalWrite } from './external-change.ts';import { contentHash, registerWrite } from './file-watcher.ts';-import { tracedMkdir, tracedRename, tracedUnlinkSync, tracedWriteFile } from './fs-traced.ts';+import { tracedMkdir, tracedRenameSync, tracedUnlinkSync, tracedWriteFile } from './fs-traced.ts';import { errnoCode } from './http/handler-utils.ts';import { getLogger } from './logger.ts';import {@@ -110,6 +111,7 @@ import {incrementPersistenceSanityCheckSerializeFailures,incrementPersistenceSkipNonQuiescent,incrementPersistenceStoreRemovedDoc,+ incrementPersistenceStoreSuperseded,} from './metrics.ts';import { toPosix } from './path-utils.ts';import { classifyDuplication } from './persistence-tripwire.ts';@@ -1291,6 +1293,7 @@ export function createPersistenceExtension(options?: PersistenceOptions): Persisconst { sv: stateVectorAtRead, json } = captureDocSnapshotForPersistence(document);const ytextSnapshot = document.getText('source').toString();+ const storeAttempt = durabilityState.beginStoreAttempt(documentName);const { frontmatter, body } = stripFrontmatter(ytextSnapshot);const markdown = prependFrontmatter(frontmatter, body);@@ -1723,7 +1726,48 @@ export function createPersistenceExtension(options?: PersistenceOptions): Persis}return;}- await tracedRename(tmpPath, canonicalPath);+ const outcome = durabilityState.tryPublishStore(storeAttempt, () => {+ tracedRenameSync(tmpPath, canonicalPath);+ });+ if (outcome !== 'published') {+ try {+ tracedUnlinkSync(tmpPath);+ } catch (cleanupErr) {+ if (errnoCode(cleanupErr) !== 'ENOENT') {+ log.warn(+ { err: cleanupErr, docName: documentName, path: tmpPath },+ '[persistence] could not remove the temp file after dropping the refused publish',+ );+ }+ }+ persistenceDeferCounts.delete(documentName);+ if (outcome === 'reentrant') {+ console.warn(+ JSON.stringify({+ event: 'persistence-store-reentrant-refused',+ 'doc.name': documentName,+ generation: storeAttempt.generation,+ baseBytes: currentBase?.length ?? null,+ candidateBytes: markdown.length,+ }),+ );+ return;+ }+ if (outcome === 'stale') {+ incrementPersistenceStoreSuperseded();+ console.warn(+ JSON.stringify({+ event: 'persistence-store-superseded',+ 'doc.name': documentName,+ generation: storeAttempt.generation,+ baseBytes: currentBase?.length ?? null,+ candidateBytes: markdown.length,+ }),+ );+ return;+ }+ return assertNeverStorePublishOutcome(outcome);+ }registerWrite(canonicalPath, contentHash(markdown));durabilityState.clearStoreFailure(documentName);incrementPersistenceDiskWrite();@@ -2016,12 +2060,11 @@ export function createPersistenceExtension(options?: PersistenceOptions): Persis* `captureDocSnapshotForPersistence` at the top of the body — its* co-capture of `{sv, json}` is what guarantees the disk-ack* watermark reflects the exact doc state that lands on disk. A- * second SV captured later (e.g., after `await tracedRename`) would- * include updates from the async write window, falsely advancing the- * watermark past content that's NOT durably persisted, and- * clients would drop those bytes from the recycle buffer →- * unsynced-edit loss on server-restart. See the helper's docstring- * for the full timing contract.+ * second SV captured later (e.g., after the `await tracedWriteFile`+ * that stages the temp file) would include updates from the async+ * write window, falsely advancing the watermark past content that's+ * NOT durably persisted, and clients would drop those bytes from+ * the recycle buffer → unsynced-edit loss on server-restart.*/async onStoreDocument({document,