Beautiful, AI-native markdown IDE and LLM wiki
TypeScript 2nd-brainagent-skillsclaudecodexcompany-braindocsknowledge-knowledge-baseknowledge-managementllmllm-wikillm-wiki-karpathymarkdownmarkdown-editormdmdxnotespkmsecond-brainskills
Preserve recreated pages across delayed watcher deletions (#6239)
Nick Gomez859042b1eaa2
diff --git a/.changeset/preserve-recreated-pages.md b/.changeset/preserve-recreated-pages.mdnew file mode 100644index 000000000..ce9464e2b--- /dev/null+++ b/.changeset/preserve-recreated-pages.md@@ -0,0 +1,5 @@+---+"@inkeep/open-knowledge": patch+---++Keep recreated pages open and preserve their comments when earlier file removals finish late.diff --git a/packages/app/tests/integration/stale-connected-write.test.ts b/packages/app/tests/integration/stale-connected-write.test.tsnew file mode 100644index 000000000..5d6c80058--- /dev/null+++ b/packages/app/tests/integration/stale-connected-write.test.ts@@ -0,0 +1,246 @@+import { randomUUID } from 'node:crypto';+import {+ existsSync,+ mkdirSync,+ mkdtempSync,+ readFileSync,+ realpathSync,+ rmSync,+ unlinkSync,+ writeFileSync,+} from 'node:fs';+import { tmpdir } from 'node:os';+import { join } from 'node:path';+import { HocuspocusProvider } from '@hocuspocus/provider';+import {+ MetricsReconciliationSuccessSchema,+ PagesSuccessSchema,+} from '@inkeep/open-knowledge-core';+import { afterEach, expect, test, vi } from 'vitest';+import * as Y from 'yjs';+import * as fileWatcher from '../../../server/src/file-watcher.ts';+import { resetMetrics } from '../../../server/src/metrics.ts';+import * as parcelSubscription from '../../../server/src/parcel-subscription.ts';+import { parseCC1DiskAck, SYSTEM_DOC_NAME } from '../../src/lib/cc1.ts';+import { runTeardownPhases } from '../stress/_helpers/teardown-fs.ts';+import {+ createTestClient,+ createTestServer,+ type TestClient,+ type TestServer,+ waitForSync,+} from './test-harness.ts';++const ORIGINAL = '# Original WS page\n\nOriginal body.\n';+const APPEND = '\nConnected edit after recreation.\n';+const EDITED = ORIGINAL + APPEND;+const cleanups: Array<() => Promise<void>> = [];++afterEach(async () => {+ await runTeardownPhases(...cleanups.splice(0).reverse());+});++test('an already-connected WebSocket edit acknowledged on disk survives a delayed native deletion', async () => {+ resetMetrics();+ const contentDir = realpathSync(mkdtempSync(join(tmpdir(), 'ok-ws-delete-adoption-')));+ const docName = `connected-${randomUUID()}`;+ const filePath = join(contentDir, `${docName}.md`);+ const entered = Promise.withResolvers<void>();+ const deliver = Promise.withResolvers<void>();+ const followers = Promise.withResolvers<void>();+ const batchFinished = Promise.withResolvers<void>();+ const diskAck = Promise.withResolvers<{+ payload: string;+ stateVector: Array<[number, number]>;+ disk: string;+ }>();+ let server: TestServer | undefined;+ let originalClient: TestClient | undefined;+ let systemProvider: HocuspocusProvider | undefined;+ let systemDoc: Y.Doc | undefined;+ let rawTail = Promise.resolve();+ let selectedDeleteSeen = false;+ let handlerHeld = false;+ let clientClosed = false;+ let closing = false;+ let desiredVector: Map<number, number> | undefined;+ let editIssued = false;++ const subscribe = parcelSubscription.subscribeParcel;+ const subscriptionSpy = vi+ .spyOn(parcelSubscription, 'subscribeParcel')+ .mockImplementation((parcel, dir, onEvents, options, platform) =>+ subscribe(+ parcel,+ dir,+ (error, events) => {+ if (dir !== contentDir || closing) return onEvents(error, events);+ const selected =+ !selectedDeleteSeen &&+ events.some((event) => event.type === 'delete' && event.path === filePath);+ const follower = selectedDeleteSeen;+ if (selected) {+ selectedDeleteSeen = true;+ }+ const work = rawTail.then(async () => {+ if (follower) await followers.promise;+ await onEvents(error, events);+ if (selected) batchFinished.resolve();+ });+ rawTail = work;+ return work;+ },+ options,+ platform,+ ),+ );+ const startWatcher = fileWatcher.startWatcher;+ const watcherSpy = vi+ .spyOn(fileWatcher, 'startWatcher')+ .mockImplementation((root, onDiskEvent, filter, options) =>+ startWatcher(+ root,+ async (event) => {+ if (!handlerHeld && event.kind === 'delete' && event.docName === docName) {+ handlerHeld = true;+ entered.resolve();+ await deliver.promise;+ }+ await onDiskEvent(event);+ },+ filter,+ { ...options, forceBackend: 'parcel' },+ ),+ );++ cleanups.push(async () => {+ closing = true;+ deliver.resolve();+ followers.resolve();+ await runTeardownPhases(+ () => rawTail,+ async () => {+ await originalClient?.cleanup();+ },+ () => {+ systemProvider?.destroy();+ systemDoc?.destroy();+ },+ async () => {+ await server?.cleanup();+ },+ () => {+ watcherSpy.mockRestore();+ subscriptionSpy.mockRestore();+ },+ () => rmSync(contentDir, { recursive: true, force: true }),+ );+ });++ writeFileSync(filePath, ORIGINAL, { flag: 'wx' });+ mkdirSync(join(contentDir, '.ok', 'templates'), { recursive: true });+ writeFileSync(join(contentDir, '.ok', 'templates', 'same-original.md'), ORIGINAL, { flag: 'wx' });+ server = await createTestServer({ contentDir });+ const running = server;+ originalClient = await createTestClient(running.port, docName, { resetOnCleanup: false });+ const client = originalClient;+ client.provider.on('close', () => {+ clientClosed = true;+ });+ expect(client.ytext.toString()).toBe(ORIGINAL);+ expect(client.provider.isAuthenticated).toBe(true);++ systemDoc = new Y.Doc();+ systemProvider = new HocuspocusProvider({+ url: `${running.wsUrl}/collab`,+ name: SYSTEM_DOC_NAME,+ document: systemDoc,+ onStateless: ({ payload }: { payload: string }) => {+ if (closing) return;+ const ack = parseCC1DiskAck(payload);+ if (!ack || ack.docName !== docName) return;+ const vector = Y.decodeStateVector(ack.sv);+ const coversEdit =+ editIssued &&+ desiredVector !== undefined &&+ [...desiredVector].every(([id, clock]) => (vector.get(id) ?? 0) >= clock);+ const disk = existsSync(filePath) ? readFileSync(filePath, 'utf8') : null;+ if (coversEdit && disk === EDITED) {+ diskAck.resolve({ payload, stateVector: [...vector], disk });+ }+ },+ });+ await waitForSync(systemProvider);+ unlinkSync(filePath);+ await entered.promise;+ expect(selectedDeleteSeen).toBe(true);+ expect(existsSync(filePath)).toBe(false);+ expect(clientClosed).toBe(false);+ expect(client.provider.isAuthenticated).toBe(true);+ const created = await fetch(`${running.baseUrl}/api/create-page`, {+ method: 'POST',+ headers: { 'Content-Type': 'application/json' },+ body: JSON.stringify({ path: `${docName}.md`, template: 'same-original' }),+ });+ const createdBody = await created.json();+ expect(created.status).toBe(200);+ expect(createdBody).toEqual({ docName });+ expect(readFileSync(filePath, 'utf8')).toBe(ORIGINAL);+ expect(client.ytext.toString()).toBe(ORIGINAL);+ expect(clientClosed).toBe(false);+ expect(client.provider.isAuthenticated).toBe(true);++ const beforeEditVector = Y.decodeStateVector(Y.encodeStateVector(client.doc));+ client.doc.transact(() => client.ytext.insert(client.ytext.length, APPEND));+ desiredVector = Y.decodeStateVector(Y.encodeStateVector(client.doc));+ editIssued = true;+ expect(desiredVector.get(client.doc.clientID) ?? 0).toBeGreaterThan(+ beforeEditVector.get(client.doc.clientID) ?? 0,+ );+ const acknowledged = await diskAck.promise;+ expect(acknowledged.disk).toBe(EDITED);+ expect(readFileSync(filePath, 'utf8')).toBe(EDITED);+ expect(client.ytext.toString()).toBe(EDITED);+ const savedResident = running.instance.hocuspocus.documents.get(docName);+ expect(savedResident).toBeDefined();+ expect(savedResident?.getText('source').toString()).toBe(EDITED);+ expect(savedResident?.getMap('lifecycle').get('status')).toBeUndefined();+ expect(clientClosed).toBe(false);++ const snapshot = async () => {+ const resident = running.instance.hocuspocus.documents.get(docName);+ const metricsResponse = await fetch(`${running.baseUrl}/api/metrics/reconciliation`);+ const pagesResponse = await fetch(`${running.baseUrl}/api/pages`);+ expect([metricsResponse.status, pagesResponse.status]).toEqual([200, 200]);+ const metrics = MetricsReconciliationSuccessSchema.parse(await metricsResponse.json());+ const pages = PagesSuccessSchema.parse(await pagesResponse.json());+ return {+ disk: existsSync(filePath) ? readFileSync(filePath, 'utf8') : null,+ residentPreserved: resident === savedResident,+ residentContent: resident?.getText('source').toString() ?? null,+ lifecycleStatus: resident?.getMap('lifecycle').get('status') ?? null,+ clientContent: client.ytext.toString(),+ clientLifecycleStatus: client.doc.getMap('lifecycle').get('status') ?? null,+ clientClosed,+ removedCount: metrics.recentlyRemovedDocsSize,+ listed: pages.pages.some((page) => page.docName === docName),+ };+ };+ const expected = {+ disk: EDITED,+ residentPreserved: true,+ residentContent: EDITED,+ lifecycleStatus: null,+ clientContent: EDITED,+ clientLifecycleStatus: null,+ clientClosed: false,+ removedCount: 0,+ listed: true,+ };+ const before = await snapshot();+ expect(before).toEqual(expected);+ deliver.resolve();+ await batchFinished.promise;+ const after = await snapshot();+ expect(after).toEqual(expected);+});diff --git a/packages/app/tests/integration/stale-delete-recreation.test-helper.ts b/packages/app/tests/integration/stale-delete-recreation.test-helper.tsnew file mode 100644index 000000000..c6de3d4b2--- /dev/null+++ b/packages/app/tests/integration/stale-delete-recreation.test-helper.ts@@ -0,0 +1,265 @@+import { randomUUID } from 'node:crypto';+import {+ existsSync,+ mkdtempSync,+ readFileSync,+ realpathSync,+ rmSync,+ unlinkSync,+ writeFileSync,+} from 'node:fs';+import { tmpdir } from 'node:os';+import { join } from 'node:path';+import type { Document } from '@hocuspocus/server';+import {+ CommentThreadMetaSchema,+ DocumentReadSuccessSchema,+ ForwardLinksSuccessSchema,+ MetricsReconciliationSuccessSchema,+ PagesSuccessSchema,+ ThreadListSuccessSchema,+} from '@inkeep/open-knowledge-core';+import { expect, vi } from 'vitest';+import { CommentService } from '../../../server/src/comments/comment-service.ts';+import { DerivedDocumentIndex } from '../../../server/src/derived-document-index.ts';+import * as fileWatcher from '../../../server/src/file-watcher.ts';+import { resetMetrics } from '../../../server/src/metrics.ts';+import * as parcelSubscription from '../../../server/src/parcel-subscription.ts';+import { runTeardownPhases } from '../stress/_helpers/teardown-fs.ts';+import {+ createTestClient,+ createTestServer,+ type TestClient,+ type TestServer,+} from './test-harness.ts';++export const ORIGINAL = '# Original page\n\nOld body.\n';+export const RECREATED = '# Recreated page\n\nNew body with [target](target.md).\n';+export type DeletionHold = 'raw' | 'classified' | 'index' | 'none';++export async function startDeletionScenario(+ hold: DeletionHold,+ registerCleanup: (cleanup: () => Promise<void>) => void,+) {+ resetMetrics();+ const docName = `deletion-${randomUUID()}`;+ const contentDir = realpathSync(mkdtempSync(join(tmpdir(), 'ok-stale-delete-')));+ const filePath = join(contentDir, `${docName}.md`);+ const reached = Promise.withResolvers<void>();+ const release = Promise.withResolvers<void>();+ const followers = Promise.withResolvers<void>();+ const completed = Promise.withResolvers<void>();+ const clients = new Set<TestClient>();+ const admittedDocuments = new WeakMap<TestClient, Document | undefined>();+ const closedClients = new WeakSet<TestClient>();+ const commentDeletes = new Set<Promise<number>>();+ let server: TestServer | undefined;+ let rawTail = Promise.resolve();+ let sawDelete = false;+ let held = false;+ const pause = async () => {+ if (held) return;+ held = true;+ reached.resolve();+ await release.promise;+ };++ const subscribe = parcelSubscription.subscribeParcel;+ const subscriptionSpy = vi+ .spyOn(parcelSubscription, 'subscribeParcel')+ .mockImplementation((parcel, dir, onEvents, options, platform) =>+ subscribe(+ parcel,+ dir,+ (error, events) => {+ if (dir !== contentDir) return onEvents(error, events);+ const selected =+ !sawDelete &&+ events.some((event) => event.type === 'delete' && event.path === filePath);+ const follower = sawDelete;+ if (selected) sawDelete = true;+ const work = rawTail.then(async () => {+ if (selected && hold === 'raw') await pause();+ if (follower) await followers.promise;+ await onEvents(error, events);+ if (selected) completed.resolve();+ });+ rawTail = work;+ return work;+ },+ options,+ platform,+ ),+ );++ const startWatcher = fileWatcher.startWatcher;+ const watcherSpy = vi+ .spyOn(fileWatcher, 'startWatcher')+ .mockImplementation((root, onDiskEvent, contentFilter, options) =>+ startWatcher(+ root,+ async (event) => {+ if (hold === 'classified' && event.kind === 'delete' && event.docName === docName)+ await pause();+ await onDiskEvent(event);+ },+ contentFilter,+ { ...options, forceBackend: 'parcel' },+ ),+ );++ const deleteIndex = DerivedDocumentIndex.prototype.recordDiskDelete;+ const indexSpy = vi+ .spyOn(DerivedDocumentIndex.prototype, 'recordDiskDelete')+ .mockImplementation(async function (this: DerivedDocumentIndex, name) {+ await deleteIndex.call(this, name);+ if (hold === 'index' && name === docName) await pause();+ });++ const deleteComments = CommentService.prototype.deleteDoc;+ const commentsSpy = vi.spyOn(CommentService.prototype, 'deleteDoc').mockImplementation(function (+ this: CommentService,+ name,+ ) {+ const work = deleteComments.call(this, name);+ commentDeletes.add(work);+ void work.then(+ () => commentDeletes.delete(work),+ () => commentDeletes.delete(work),+ );+ return work;+ });++ registerCleanup(async () => {+ release.resolve();+ followers.resolve();+ await runTeardownPhases(+ () => rawTail,+ ...Array.from(clients, (client) => () => client.cleanup()),+ async () => {+ await server?.cleanup();+ },+ () => {+ commentsSpy.mockRestore();+ indexSpy.mockRestore();+ watcherSpy.mockRestore();+ subscriptionSpy.mockRestore();+ },+ () => rmSync(contentDir, { recursive: true, force: true }),+ );+ });++ writeFileSync(filePath, ORIGINAL, { flag: 'wx' });+ writeFileSync(join(contentDir, 'target.md'), '# Target\n', { flag: 'wx' });+ server = await createTestServer({ contentDir });+ const running = server;++ const post = (route: string, body: unknown) =>+ fetch(`${running.baseUrl}${route}`, {+ method: 'POST',+ headers: { 'Content-Type': 'application/json' },+ body: JSON.stringify(body),+ });++ const open = async () => {+ const client = await createTestClient(running.port, docName, { resetOnCleanup: false });+ clients.add(client);+ admittedDocuments.set(client, running.instance.hocuspocus.documents.get(docName));+ client.provider.on('close', () => closedClients.add(client));+ return client;+ };++ const close = async (client: TestClient) => {+ await client.cleanup();+ clients.delete(client);+ };++ const comment = async (quote: string) => {+ const response = await post('/api/comments', { docName, quote, body: quote });+ expect(response.status).toBe(201);+ return CommentThreadMetaSchema.parse(await response.json());+ };++ const create = async () => {+ expect((await post('/api/create-page', { path: `${docName}.md` })).status).toBe(200);+ const client = await open();+ const response = await post('/api/agent-write-md', {+ docName,+ markdown: RECREATED,+ position: 'replace',+ agentId: 'deletion-test',+ });+ expect(response.status).toBe(200);+ await expect.poll(() => client.ytext.toString()).toBe(RECREATED);+ return client;+ };++ const finishDeletion = async () => {+ release.resolve();+ await completed.promise;+ await Promise.all(commentDeletes);+ };++ const snapshot = async (client?: TestClient) => {+ const resident = running.instance.hocuspocus.documents.get(docName);+ const pagesResponse = await fetch(`${running.baseUrl}/api/pages`);+ const metricsResponse = await fetch(`${running.baseUrl}/api/metrics/reconciliation`);+ const commentsResponse = await fetch(+ `${running.baseUrl}/api/comments?doc=${encodeURIComponent(docName)}`,+ );+ const linksResponse = await fetch(+ `${running.baseUrl}/api/forward-links?docName=${encodeURIComponent(docName)}`,+ );+ expect([+ pagesResponse.status,+ metricsResponse.status,+ commentsResponse.status,+ linksResponse.status,+ ]).toEqual([200, 200, 200, 200]);+ const pages = PagesSuccessSchema.parse(await pagesResponse.json());+ const metrics = MetricsReconciliationSuccessSchema.parse(await metricsResponse.json());+ const comments = ThreadListSuccessSchema.parse(await commentsResponse.json());+ const links = ForwardLinksSuccessSchema.parse(await linksResponse.json());+ return {+ disk: existsSync(filePath) ? readFileSync(filePath, 'utf-8') : null,+ residentPresent:+ resident !== undefined &&+ (client === undefined || resident === admittedDocuments.get(client)),+ residentContent: resident?.getText('source').toString() ?? null,+ lifecycleStatus: resident?.getMap('lifecycle').get('status') ?? null,+ clientContent: client?.ytext.toString() ?? null,+ clientLifecycleStatus: client?.doc.getMap('lifecycle').get('status') ?? null,+ clientClosed: client !== undefined && closedClients.has(client),+ removedCount: metrics.recentlyRemovedDocsSize,+ listed: pages.pages.some((page) => page.docName === docName),+ threads: comments.threads.map((thread) => thread.threadId),+ forwardLinks: links.forwardLinks.flatMap((link) =>+ link.kind === 'doc' ? [link.docName] : [],+ ),+ };+ };++ return {+ server: running,+ docName,+ open,+ close,+ comment,+ create,+ snapshot,+ waitUntilHeld: () => reached.promise,+ finishDeletion,+ releaseFollowers: followers.resolve,+ async deleteThroughApi() {+ expect((await post('/api/delete-path', { kind: 'file', path: docName })).status).toBe(200);+ },+ deleteExternally: () => unlinkSync(filePath),+ async readDocument() {+ const response = await fetch(+ `${running.baseUrl}/api/document?docName=${encodeURIComponent(docName)}`,+ );+ expect(response.status).toBe(200);+ return DocumentReadSuccessSchema.parse(await response.json());+ },+ };+}diff --git a/packages/app/tests/integration/stale-delete-recreation.test.ts b/packages/app/tests/integration/stale-delete-recreation.test.tsnew file mode 100644index 000000000..bb62d8efa--- /dev/null+++ b/packages/app/tests/integration/stale-delete-recreation.test.ts@@ -0,0 +1,235 @@+import { randomUUID } from 'node:crypto';+import {+ type EncodingOption,+ existsSync,+ mkdtempSync,+ type PathLike,+ realpathSync,+ renameSync,+ rmSync,+ statSync,+ symlinkSync,+ unlinkSync,+ writeFileSync,+} from 'node:fs';+import { tmpdir } from 'node:os';+import { basename, dirname, join } from 'node:path';+import { PagesSuccessSchema } from '@inkeep/open-knowledge-core';+import { afterEach, expect, test, vi } from 'vitest';+import { hasAdmittedMarkdownFile } from '../../../server/src/file-watcher.ts';+import { runTeardownPhases } from '../stress/_helpers/teardown-fs.ts';+import {+ ORIGINAL,+ RECREATED,+ startDeletionScenario,+} from './stale-delete-recreation.test-helper.ts';+import { createTestServer } from './test-harness.ts';++vi.mock('node:fs', async (importOriginal) => {+ const actual = await importOriginal<typeof import('node:fs')>();+ return {+ ...actual,+ statSync: vi.fn(actual.statSync),+ realpathSync: Object.assign(vi.fn(actual.realpathSync), {+ native: actual.realpathSync.native,+ }),+ };+});++const actualFs = await vi.importActual<typeof import('node:fs')>('node:fs');++const cleanups: Array<() => Promise<void>> = [];+const registerCleanup = (cleanup: () => Promise<void>) => {+ cleanups.push(cleanup);+};++afterEach(async () => {+ await runTeardownPhases(...cleanups.splice(0).reverse());+});++function foldNameCaseLikeApfs(dir: string) {+ const storedVariant = (path: unknown) => {+ if (typeof path !== 'string' || dirname(path) !== dir) return undefined;+ const name = basename(path);+ const entries = actualFs.readdirSync(dir);+ if (entries.includes(name)) return undefined;+ return entries.find((entry) => entry.toLowerCase() === name.toLowerCase());+ };+ vi.mocked(statSync).mockImplementation(((path, options) => {+ const stored = storedVariant(path);+ return actualFs.statSync(stored === undefined ? path : join(dir, stored), options);+ }) as typeof statSync);+ vi.mocked(realpathSync).mockImplementation(((path: PathLike, options?: EncodingOption) =>+ storedVariant(path) === undefined+ ? actualFs.realpathSync(path, options)+ : join(actualFs.realpathSync(dir), basename(String(path)))) as typeof realpathSync);+ registerCleanup(async () => {+ vi.mocked(statSync).mockImplementation(actualFs.statSync);+ vi.mocked(realpathSync).mockImplementation(actualFs.realpathSync);+ });+}++test.each([+ ['before raw classification', 'raw', true, 'api'],+ ['before handler entry', 'classified', true, 'api'],+ ['while indexing an open page', 'index', true, 'external'],+ ['while indexing an unloaded page', 'index', false, 'external'],+] as const)(+ 'preserves a recreated page after a delete held %s',+ async (_timing, hold, initiallyOpen, deletion) => {+ const scenario = await startDeletionScenario(hold, registerCleanup);+ const oldClient = initiallyOpen ? await scenario.open() : undefined;+ if (oldClient) expect(oldClient.ytext.toString()).toBe(ORIGINAL);+ expect(scenario.server.instance.hocuspocus.documents.has(scenario.docName)).toBe(initiallyOpen);+ const oldThread = await scenario.comment('Original page');+ if (deletion === 'api') await scenario.deleteThroughApi();+ else scenario.deleteExternally();+ await scenario.waitUntilHeld();+ if (oldClient && deletion === 'api') await scenario.close(oldClient);++ const fresh = await scenario.create();+ const newThread = await scenario.comment('Recreated page');+ expect(newThread.threadId).not.toBe(oldThread.threadId);+ const before = await scenario.snapshot(fresh);+ expect(before).toMatchObject({+ disk: RECREATED,+ residentPresent: true,+ residentContent: RECREATED,+ lifecycleStatus: null,+ clientContent: RECREATED,+ clientLifecycleStatus: null,+ clientClosed: false,+ listed: true,+ removedCount: 0,+ forwardLinks: ['target'],+ });+ expect(before.threads).toContain(newThread.threadId);++ await scenario.finishDeletion();+ expect(await scenario.snapshot(fresh)).toEqual({+ disk: RECREATED,+ residentPresent: true,+ residentContent: RECREATED,+ lifecycleStatus: null,+ clientContent: RECREATED,+ clientLifecycleStatus: null,+ clientClosed: false,+ listed: true,+ removedCount: 0,+ threads: [newThread.threadId],+ forwardLinks: ['target'],+ });+ },+);++test('a delivered deletion removes the existing page and its comments', async () => {+ const scenario = await startDeletionScenario('none', registerCleanup);+ const original = await scenario.open();+ expect(original.ytext.toString()).toBe(ORIGINAL);+ const thread = await scenario.comment('Original page');+ expect((await scenario.snapshot(original)).threads).toEqual([thread.threadId]);++ await scenario.deleteThroughApi();+ await scenario.finishDeletion();+ const deleted = await scenario.snapshot();+ expect(deleted).toMatchObject({+ disk: null,+ residentPresent: false,+ residentContent: null,+ listed: false,+ removedCount: 1,+ threads: [],+ forwardLinks: [],+ });+});++test('a recreation after deletion completes admits and saves new content', async () => {+ const scenario = await startDeletionScenario('none', registerCleanup);+ const original = await scenario.open();+ const oldThread = await scenario.comment('Original page');+ expect((await scenario.snapshot(original)).threads).toEqual([oldThread.threadId]);+ await scenario.deleteThroughApi();+ await scenario.finishDeletion();+ await scenario.close(original);+ expect((await scenario.snapshot()).threads).toEqual([]);+ scenario.releaseFollowers();++ const recreated = await scenario.create();+ const thread = await scenario.comment('Recreated page');+ expect(await scenario.snapshot(recreated)).toEqual({+ disk: RECREATED,+ residentPresent: true,+ residentContent: RECREATED,+ lifecycleStatus: null,+ clientContent: RECREATED,+ clientLifecycleStatus: null,+ clientClosed: false,+ listed: true,+ removedCount: 0,+ threads: [thread.threadId],+ forwardLinks: ['target'],+ });+ expect((await scenario.readDocument()).content).toBe(RECREATED);+});++test('an external case-only rename moves the page to its new name', async ({ skip }) => {+ const contentDir = realpathSync(mkdtempSync(join(tmpdir(), 'ok-stale-case-rename-')));+ registerCleanup(async () => rmSync(contentDir, { recursive: true, force: true }));+ const probe = join(contentDir, 'Probe.md');+ writeFileSync(probe, '');+ const caseInsensitive = existsSync(join(contentDir, 'probe.md'));+ unlinkSync(probe);+ if (!caseInsensitive) skip();++ const id = randomUUID();+ const oldName = `Case-${id}`;+ const newName = `case-${id}`;+ writeFileSync(join(contentDir, `${oldName}.md`), ORIGINAL);+ const server = await createTestServer({ contentDir });+ registerCleanup(() => server.cleanup());+ const listed = async () => {+ const response = await fetch(`${server.baseUrl}/api/pages`);+ const pages = PagesSuccessSchema.parse(await response.json());+ const names = pages.pages.map((page) => page.docName);+ return { old: names.includes(oldName), renamed: names.includes(newName) };+ };+ await expect.poll(listed).toEqual({ old: true, renamed: false });++ renameSync(join(contentDir, `${oldName}.md`), join(contentDir, `${newName}.md`));++ await expect.poll(listed).toEqual({ old: false, renamed: true });+});++test('removal currentness treats only an admitted file at the exact name as bound', () => {+ const contentDir = realpathSync(mkdtempSync(join(tmpdir(), 'ok-stale-admitted-')));+ registerCleanup(async () => rmSync(contentDir, { recursive: true, force: true }));+ const outside = realpathSync(mkdtempSync(join(tmpdir(), 'ok-stale-admitted-outside-')));+ registerCleanup(async () => rmSync(outside, { recursive: true, force: true }));+ writeFileSync(join(contentDir, 'page.md'), ORIGINAL);+ writeFileSync(join(contentDir, 'excluded.md'), ORIGINAL);+ writeFileSync(join(outside, 'elsewhere.md'), ORIGINAL);+ symlinkSync(join(outside, 'elsewhere.md'), join(contentDir, 'escaping.md'));+ symlinkSync(join(contentDir, 'looping.md'), join(contentDir, 'looping.md'));+ const filter = { isExcluded: (path: string) => path === 'excluded.md' };+ const admitted = (name: string) =>+ hasAdmittedMarkdownFile(name, join(contentDir, `${name}.md`), contentDir, filter);+ foldNameCaseLikeApfs(contentDir);++ expect({+ differentCaseIsFile: statSync(join(contentDir, 'PAGE.md')).isFile(),+ page: admitted('page'),+ absent: admitted('absent'),+ differentCase: admitted('PAGE'),+ excluded: admitted('excluded'),+ escaping: admitted('escaping'),+ looping: admitted('looping'),+ }).toEqual({+ differentCaseIsFile: true,+ page: true,+ absent: false,+ differentCase: false,+ excluded: false,+ escaping: false,+ looping: false,+ });+});diff --git a/packages/app/tests/integration/stale-managed-rename-write.test.ts b/packages/app/tests/integration/stale-managed-rename-write.test.tsnew file mode 100644index 000000000..a2365cd8f--- /dev/null+++ b/packages/app/tests/integration/stale-managed-rename-write.test.ts@@ -0,0 +1,240 @@+import {+ existsSync,+ mkdirSync,+ mkdtempSync,+ readFileSync,+ realpathSync,+ rmSync,+ unlinkSync,+ writeFileSync,+} from 'node:fs';+import { tmpdir } from 'node:os';+import { join } from 'node:path';+import { ForwardLinksSuccessSchema } from '@inkeep/open-knowledge-core';+import { afterEach, expect, test, vi } from 'vitest';+import * as fileWatcher from '../../../server/src/file-watcher.ts';+import * as fsTraced from '../../../server/src/fs-traced.ts';+import * as parcelSubscription from '../../../server/src/parcel-subscription.ts';+import * as persistenceModule from '../../../server/src/persistence.ts';+import { runTeardownPhases } from '../stress/_helpers/teardown-fs.ts';+import {+ createTestClient,+ createTestServer,+ type TestClient,+ type TestServer,+} from './test-harness.ts';++const ORIGINAL = '# Reference\n\n[[source]]\n';+const REWRITTEN = '# Reference\n\n[[destination|source]]\n';+const SAVED = `${REWRITTEN}\nStill editable.\n`;+const cleanups: Array<() => Promise<void>> = [];++afterEach(async () => {+ await runTeardownPhases(...cleanups.splice(0).reverse());+});++test('a direct managed rename rewrite adopts a recreated reference before its older delete arrives', async () => {+ const contentDir = realpathSync(mkdtempSync(join(tmpdir(), 'ok-stale-managed-rewrite-')));+ const docName = 'reference';+ const filePath = join(contentDir, `${docName}.md`);+ const entered = Promise.withResolvers<void>();+ const releaseDelete = Promise.withResolvers<void>();+ const completed = Promise.withResolvers<void>();+ const followers = Promise.withResolvers<void>();+ const restores: Array<() => void> = [];+ const writes: Array<{ before: string; after: string; residentContent: string | null }> = [];+ const atomicPublications: string[] = [];+ let server: TestServer | undefined;+ let client: TestClient | undefined;+ let persistence: persistenceModule.PersistenceHandle | undefined;+ let rawTail = Promise.resolve();+ let selectedDeleteSeen = false;+ let handlerHeld = false;+ let clientClosed = false;+ let observingRewrite = false;+ let closing = false;++ const subscribe = parcelSubscription.subscribeParcel;+ const subscriptionSpy = vi+ .spyOn(parcelSubscription, 'subscribeParcel')+ .mockImplementation((parcel, dir, onEvents, options, platform) =>+ subscribe(+ parcel,+ dir,+ (error, events) => {+ if (dir !== contentDir || closing) return onEvents(error, events);+ const selected =+ !selectedDeleteSeen &&+ events.some((event) => event.type === 'delete' && event.path === filePath);+ const follower = selectedDeleteSeen;+ if (selected) selectedDeleteSeen = true;+ const work = rawTail.then(async () => {+ if (follower) await followers.promise;+ await onEvents(error, events);+ if (selected) completed.resolve();+ });+ rawTail = work;+ return work;+ },+ options,+ platform,+ ),+ );+ restores.push(() => subscriptionSpy.mockRestore());+ const startWatcher = fileWatcher.startWatcher;+ const watcherSpy = vi+ .spyOn(fileWatcher, 'startWatcher')+ .mockImplementation((root, onDiskEvent, filter, options) =>+ startWatcher(+ root,+ async (event) => {+ if (!handlerHeld && event.kind === 'delete' && event.docName === docName) {+ handlerHeld = true;+ entered.resolve();+ await releaseDelete.promise;+ }+ await onDiskEvent(event);+ },+ filter,+ { ...options, forceBackend: 'parcel' },+ ),+ );+ restores.push(() => watcherSpy.mockRestore());+ const createPersistence = persistenceModule.createPersistenceExtension;+ const persistenceSpy = vi+ .spyOn(persistenceModule, 'createPersistenceExtension')+ .mockImplementation((options) => {+ const handle = createPersistence(options);+ if (options?.contentDir === contentDir) persistence = handle;+ return handle;+ });+ restores.push(() => persistenceSpy.mockRestore());+ const write = fsTraced.tracedWriteFileSync;+ const writeSpy = vi.spyOn(fsTraced, 'tracedWriteFileSync').mockImplementation((...args) => {+ if (!observingRewrite || args[0] !== filePath) return write(...args);+ const before = readFileSync(filePath, 'utf8');+ write(...args);+ writes.push({+ before,+ after: readFileSync(filePath, 'utf8'),+ residentContent:+ server?.instance.hocuspocus.documents.get(docName)?.getText('source').toString() ?? null,+ });+ });+ restores.push(() => writeSpy.mockRestore());+ const rename = fsTraced.tracedRenameSync;+ const renameSpy = vi.spyOn(fsTraced, 'tracedRenameSync').mockImplementation((from, to) => {+ rename(from, to);+ if (observingRewrite && to === filePath) {+ atomicPublications.push(readFileSync(filePath, 'utf8'));+ }+ });+ restores.push(() => renameSpy.mockRestore());++ cleanups.push(async () => {+ closing = true;+ releaseDelete.resolve();+ followers.resolve();+ await runTeardownPhases(+ () => rawTail,+ async () => {+ await client?.cleanup();+ },+ async () => {+ await server?.cleanup();+ },+ () => {+ for (const restore of restores.reverse()) restore();+ },+ () => rmSync(contentDir, { recursive: true, force: true }),+ );+ });++ writeFileSync(filePath, ORIGINAL, { flag: 'wx' });+ writeFileSync(join(contentDir, 'source.md'), '# Source\n', { flag: 'wx' });+ mkdirSync(join(contentDir, '.ok', 'templates'), { recursive: true });+ writeFileSync(join(contentDir, '.ok', 'templates', 'same-reference.md'), ORIGINAL, {+ flag: 'wx',+ });+ server = await createTestServer({ contentDir });+ const running = server;+ const post = (path: string, body: unknown) =>+ fetch(`${running.baseUrl}${path}`, {+ method: 'POST',+ headers: { 'Content-Type': 'application/json' },+ body: JSON.stringify(body),+ });+ client = await createTestClient(running.port, docName, { resetOnCleanup: false });+ const connected = client;+ connected.provider.on('close', () => {+ clientClosed = true;+ });+ const resident = running.instance.hocuspocus.documents.get(docName);+ if (!resident || !persistence) throw new Error('Real reference and persistence were not loaded');+ const store = persistence;+ expect(connected.ytext.toString()).toBe(ORIGINAL);+ const linksResponse = await fetch(`${running.baseUrl}/api/forward-links?docName=${docName}`);+ expect(linksResponse.status).toBe(200);+ const links = ForwardLinksSuccessSchema.parse(await linksResponse.json());+ expect(links.forwardLinks).toContainEqual(+ expect.objectContaining({ kind: 'doc', docName: 'source' }),+ );++ unlinkSync(filePath);+ await entered.promise;+ expect(selectedDeleteSeen).toBe(true);+ expect(existsSync(filePath)).toBe(false);+ const created = await post('/api/create-page', {+ path: 'reference.md',+ template: 'same-reference',+ });+ expect(created.status).toBe(200);+ expect(await created.json()).toEqual({ docName });+ expect(readFileSync(filePath, 'utf8')).toBe(ORIGINAL);+ expect(running.instance.hocuspocus.documents.get(docName)).toBe(resident);+ expect(resident.getText('source').toString()).toBe(ORIGINAL);+ expect(clientClosed).toBe(false);+ observingRewrite = true;++ const renamed = await post('/api/rename-path', {+ kind: 'file',+ fromPath: 'source',+ toPath: 'destination',+ });+ const renameBody: unknown = await renamed.json();+ expect(renamed.status, JSON.stringify(renameBody)).toBe(200);+ expect(renameBody).toMatchObject({+ renamed: [{ fromDocName: 'source', toDocName: 'destination' }],+ rewrittenDocs: expect.arrayContaining([expect.objectContaining({ docName, rewrites: 1 })]),+ });+ expect(writes).toContainEqual({ before: ORIGINAL, after: REWRITTEN, residentContent: REWRITTEN });+ expect(running.instance.hocuspocus.documents.get(docName)).toBe(resident);+ expect(resident.getText('source').toString()).toBe(REWRITTEN);+ expect(readFileSync(filePath, 'utf8')).toBe(REWRITTEN);+ expect(running.instance.durabilityState.getReconciledBase(docName)).toBe(REWRITTEN);+ await store.forceStore(resident, docName);+ expect(atomicPublications).toEqual([]);+ expect(readFileSync(filePath, 'utf8')).toBe(REWRITTEN);+ await expect.poll(() => connected.ytext.toString()).toBe(REWRITTEN);++ releaseDelete.resolve();+ await completed.promise;+ expect(running.instance.hocuspocus.documents.get(docName)).toBe(resident);+ expect(resident.getMap('lifecycle').get('status')).toBeUndefined();+ expect(resident.getText('source').toString()).toBe(REWRITTEN);+ expect(readFileSync(filePath, 'utf8')).toBe(REWRITTEN);+ expect(clientClosed).toBe(false);+ expect(atomicPublications).toEqual([]);+ observingRewrite = false;++ const saved = await post('/api/agent-write-md', {+ docName,+ markdown: '\nStill editable.\n',+ position: 'append',+ agentId: 'managed-rename-continuation',+ });+ expect(saved.status).toBe(200);+ expect(running.instance.hocuspocus.documents.get(docName)).toBe(resident);+ expect(readFileSync(filePath, 'utf8')).toBe(SAVED);+ await expect.poll(() => connected.ytext.toString()).toBe(SAVED);+});diff --git a/packages/app/tests/integration/stale-publication-burst.test.ts b/packages/app/tests/integration/stale-publication-burst.test.tsnew file mode 100644index 000000000..c2df04e7d--- /dev/null+++ b/packages/app/tests/integration/stale-publication-burst.test.ts@@ -0,0 +1,113 @@+import { mkdirSync, readFileSync, writeFileSync } from 'node:fs';+import { join } from 'node:path';+import { afterEach, expect, test, vi } from 'vitest';+import * as fsTraced from '../../../server/src/fs-traced.ts';+import { runTeardownPhases } from '../stress/_helpers/teardown-fs.ts';+import { ORIGINAL, startDeletionScenario } from './stale-delete-recreation.test-helper.ts';++const FIRST_APPEND = '\nFirst published edit after recreation.\n';+const SECOND_APPEND = '\nLater edit awaiting publication.\n';+const FIRST_CONTENT = ORIGINAL + FIRST_APPEND;+const SECOND_CONTENT = FIRST_CONTENT + SECOND_APPEND;+const cleanups: Array<() => Promise<void>> = [];+const registerCleanup = (cleanup: () => Promise<void>) => cleanups.push(cleanup);++afterEach(async () => {+ await runTeardownPhases(...cleanups.splice(0).reverse());+});++test('a published edit preserves its recreated lifetime while a later edit is still awaiting publication', async () => {+ const scenario = await startDeletionScenario('classified', registerCleanup);+ const client = await scenario.open();+ expect(client.ytext.toString()).toBe(ORIGINAL);+ const resident = scenario.server.instance.hocuspocus.documents.get(scenario.docName);+ if (!resident) throw new Error('Expected the original connected resident');+ const filePath = join(scenario.server.contentDir, `${scenario.docName}.md`);+ const templates = join(scenario.server.contentDir, '.ok', 'templates');++ scenario.deleteExternally();+ await scenario.waitUntilHeld();+ mkdirSync(templates, { recursive: true });+ writeFileSync(join(templates, 'same-before-burst.md'), ORIGINAL, { flag: 'wx' });+ const created = await fetch(`${scenario.server.baseUrl}/api/create-page`, {+ method: 'POST',+ headers: { 'Content-Type': 'application/json' },+ body: JSON.stringify({ path: `${scenario.docName}.md`, template: 'same-before-burst' }),+ });+ expect(created.status).toBe(200);+ expect(await created.json()).toEqual({ docName: scenario.docName });+ expect(readFileSync(filePath, 'utf8')).toBe(ORIGINAL);+ expect(scenario.server.instance.hocuspocus.documents.get(scenario.docName)).toBe(resident);++ const firstEntered = Promise.withResolvers<void>();+ const releaseFirst = Promise.withResolvers<void>();+ const secondEntered = Promise.withResolvers<void>();+ const releaseSecond = Promise.withResolvers<void>();+ const pendingWrites = new Set<Promise<void>>();+ const write = fsTraced.tracedWriteFile;+ let firstHeld = false;+ let secondHeld = false;+ const writeSpy = vi.spyOn(fsTraced, 'tracedWriteFile').mockImplementation((...args) => {+ const work = (async () => {+ await write(...args);+ if (!String(args[0]).startsWith(`${filePath}.tmp.`)) return;+ if (args[1] === FIRST_CONTENT && !firstHeld) {+ firstHeld = true;+ firstEntered.resolve();+ await releaseFirst.promise;+ } else if (args[1] === SECOND_CONTENT && !secondHeld) {+ secondHeld = true;+ secondEntered.resolve();+ await releaseSecond.promise;+ }+ })();+ pendingWrites.add(work);+ void work.then(+ () => pendingWrites.delete(work),+ () => pendingWrites.delete(work),+ );+ return work;+ });+ registerCleanup(async () => {+ releaseFirst.resolve();+ releaseSecond.resolve();+ await Promise.allSettled(pendingWrites);+ writeSpy.mockRestore();+ });++ client.doc.transact(() => client.ytext.insert(client.ytext.length, FIRST_APPEND));+ await firstEntered.promise;+ expect(resident.getText('source').toString()).toBe(FIRST_CONTENT);+ expect(readFileSync(filePath, 'utf8')).toBe(ORIGINAL);++ client.doc.transact(() => client.ytext.insert(client.ytext.length, SECOND_APPEND));+ await expect.poll(() => resident.getText('source').toString()).toBe(SECOND_CONTENT);+ expect(scenario.server.instance.hocuspocus.documents.get(scenario.docName)).toBe(resident);+ releaseFirst.resolve();+ await secondEntered.promise;+ expect(readFileSync(filePath, 'utf8')).toBe(FIRST_CONTENT);+ expect(resident.getText('source').toString()).toBe(SECOND_CONTENT);+ expect(resident.isDestroyed).toBe(false);++ await scenario.finishDeletion();+ expect({+ residentPreserved:+ scenario.server.instance.hocuspocus.documents.get(scenario.docName) === resident,+ destroyed: resident.isDestroyed,+ lifecycleStatus: resident.getMap('lifecycle').get('status') ?? null,+ residentContent: resident.getText('source').toString(),+ clientContent: client.ytext.toString(),+ disk: readFileSync(filePath, 'utf8'),+ }).toEqual({+ residentPreserved: true,+ destroyed: false,+ lifecycleStatus: null,+ residentContent: SECOND_CONTENT,+ clientContent: SECOND_CONTENT,+ disk: FIRST_CONTENT,+ });++ releaseSecond.resolve();+ await expect.poll(() => readFileSync(filePath, 'utf8')).toBe(SECOND_CONTENT);+ expect(scenario.server.instance.hocuspocus.documents.get(scenario.docName)).toBe(resident);+});diff --git a/packages/app/tests/integration/stale-publication-completion.test-helper.ts b/packages/app/tests/integration/stale-publication-completion.test-helper.tsnew file mode 100644index 000000000..9f4ce97c9--- /dev/null+++ b/packages/app/tests/integration/stale-publication-completion.test-helper.ts@@ -0,0 +1,349 @@+import { randomUUID } from 'node:crypto';+import {+ existsSync,+ mkdirSync,+ mkdtempSync,+ readFileSync,+ realpathSync,+ rmSync,+ unlinkSync,+ writeFileSync,+} from 'node:fs';+import { tmpdir } from 'node:os';+import { join } from 'node:path';+import type { Document } from '@hocuspocus/server';+import { expect, vi } from 'vitest';+import * as fileWatcher from '../../../server/src/file-watcher.ts';+import * as fsTraced from '../../../server/src/fs-traced.ts';+import * as parcelSubscription from '../../../server/src/parcel-subscription.ts';+import { runTeardownPhases } from '../stress/_helpers/teardown-fs.ts';+import {+ createTestClient,+ createTestServer,+ type TestClient,+ type TestServer,+} from './test-harness.ts';++export const ORIGINAL_PUBLICATION = '# Original page\n\nInitial body.\n';+export const PENDING_PUBLICATION = '# Pending publication\n\nFrom the original connection.\n';++type DeletionPlan = {+ hold: 'none' | 'classified';+ nativeSeen: boolean;+ held: boolean;+ reached: PromiseWithResolvers<void>;+ release: PromiseWithResolvers<void>;+ followers: PromiseWithResolvers<void>;+ completed: PromiseWithResolvers<void>;+};++export async function startStalePublicationScenario(+ registerCleanup: (cleanup: () => Promise<void>) => void,+) {+ const contentDir = realpathSync(mkdtempSync(join(tmpdir(), 'ok-stale-publication-')));+ const docName = `publication-${randomUUID()}`;+ const filePath = join(contentDir, `${docName}.md`);+ const clients = new Set<TestClient>();+ const closedClients = new WeakSet<TestClient>();+ const plans: DeletionPlan[] = [];+ const trace: Array<Record<string, unknown>> = [];+ const staged = Promise.withResolvers<void>();+ const releasePublication = Promise.withResolvers<void>();+ const storeCompleted = Promise.withResolvers<void>();+ const requests = new Set<Promise<Response>>();+ const nativeAppearances = new Set<PromiseWithResolvers<void>>();+ let server: TestServer | undefined;+ let originalDocument: Document | undefined;+ let upcomingPlan: DeletionPlan | undefined;+ let activePlan: DeletionPlan | undefined;+ let processingPlan: DeletionPlan | undefined;+ let rawTail = Promise.resolve();+ let heldTempPath: string | undefined;+ let published = false;+ let closing = false;+ const restore: Array<() => void> = [];++ registerCleanup(async () => {+ closing = true;+ releasePublication.resolve();+ for (const plan of plans) {+ plan.release.resolve();+ plan.followers.resolve();+ }+ await runTeardownPhases(+ () => rawTail,+ async () => {+ await Promise.allSettled(requests);+ },+ ...Array.from(clients, (client) => () => client.cleanup()),+ async () => {+ await server?.cleanup();+ },+ () => {+ for (const reset of restore.reverse()) reset();+ },+ () => rmSync(contentDir, { recursive: true, force: true }),+ () => {+ console.info(+ JSON.stringify({+ event: 'stale-publication-history',+ docName,+ trace,+ fixtureAbsent: !existsSync(contentDir),+ }),+ );+ },+ );+ });++ const subscribe = parcelSubscription.subscribeParcel;+ const subscriptionSpy = vi+ .spyOn(parcelSubscription, 'subscribeParcel')+ .mockImplementation((parcel, dir, onEvents, options, platform) =>+ subscribe(+ parcel,+ dir,+ (error, events) => {+ if (dir !== contentDir || closing) return onEvents(error, events);+ const canonicalEvents = events.filter((event) => event.path === filePath);+ if (canonicalEvents.length > 0) {+ trace.push({+ phase: 'native-canonical-events',+ types: canonicalEvents.map((event) => event.type),+ filePresent: existsSync(filePath),+ });+ }+ const appearances = canonicalEvents.some((event) => event.type !== 'delete')+ ? [...nativeAppearances]+ : [];+ const selected =+ upcomingPlan &&+ events.some((event) => event.type === 'delete' && event.path === filePath)+ ? upcomingPlan+ : undefined;+ if (selected) {+ selected.nativeSeen = true;+ upcomingPlan = undefined;+ activePlan = selected;+ trace.push({+ phase: 'native-delete',+ ordinal: plans.indexOf(selected),+ fileAbsent: !existsSync(filePath),+ });+ }+ const follower = selected ? undefined : activePlan;+ const work = rawTail.then(async () => {+ await follower?.followers.promise;+ processingPlan = selected;+ try {+ await onEvents(error, events);+ for (const appearance of appearances) {+ nativeAppearances.delete(appearance);+ trace.push({+ phase: 'native-appearance-consumed',+ filePresent: existsSync(filePath),+ });+ appearance.resolve();+ }+ selected?.completed.resolve();+ } finally {+ processingPlan = undefined;+ }+ });+ rawTail = work;+ return work;+ },+ options,+ platform,+ ),+ );+ restore.push(() => subscriptionSpy.mockRestore());++ const startWatcher = fileWatcher.startWatcher;+ const watcherSpy = vi+ .spyOn(fileWatcher, 'startWatcher')+ .mockImplementation((root, onDiskEvent, filter, options) =>+ startWatcher(+ root,+ async (event) => {+ if (+ event.kind === 'delete' &&+ event.docName === docName &&+ processingPlan?.hold === 'classified'+ ) {+ processingPlan.held = true;+ trace.push({ phase: 'classified-delete-held', ordinal: plans.indexOf(processingPlan) });+ processingPlan.reached.resolve();+ await processingPlan.release.promise;+ }+ await onDiskEvent(event);+ },+ filter,+ { ...options, forceBackend: 'parcel' },+ ),+ );+ restore.push(() => watcherSpy.mockRestore());++ const write = fsTraced.tracedWriteFile;+ const writeSpy = vi.spyOn(fsTraced, 'tracedWriteFile').mockImplementation(async (...args) => {+ await write(...args);+ if (+ heldTempPath ||+ !String(args[0]).startsWith(`${filePath}.tmp.`) ||+ args[1] !== PENDING_PUBLICATION+ )+ return;+ heldTempPath = String(args[0]);+ trace.push({+ phase: 'real-store-staged',+ residentText: originalDocument?.getText('source').toString(),+ saveMutexLocked: originalDocument?.saveMutex.isLocked(),+ });+ staged.resolve();+ await releasePublication.promise;+ });+ restore.push(() => writeSpy.mockRestore());++ const rename = fsTraced.tracedRenameSync;+ const renameSpy = vi.spyOn(fsTraced, 'tracedRenameSync').mockImplementation((...args) => {+ rename(...args);+ if (String(args[0]) !== heldTempPath || String(args[1]) !== filePath) return;+ published = true;+ trace.push({+ phase: 'old-store-published',+ originalDestroyed: originalDocument?.isDestroyed,+ mappedIsOriginal: server?.instance.hocuspocus.documents.get(docName) === originalDocument,+ disk: readFileSync(filePath, 'utf8'),+ });+ });+ restore.push(() => renameSpy.mockRestore());++ writeFileSync(filePath, ORIGINAL_PUBLICATION, { flag: 'wx' });+ mkdirSync(join(contentDir, '.ok', 'templates'), { recursive: true });+ writeFileSync(+ join(contentDir, '.ok', 'templates', 'pending-publication.md'),+ PENDING_PUBLICATION,+ { flag: 'wx' },+ );+ server = await createTestServer({ contentDir });+ const running = server;+ running.instance.hocuspocus.configuration.extensions.push({+ async afterStoreDocument({ document, documentName }) {+ if (documentName === docName && document === originalDocument && published) {+ trace.push({ phase: 'old-store-completed', originalDestroyed: document.isDestroyed });+ storeCompleted.resolve();+ }+ },+ });++ const post = (route: string, body: unknown) => {+ const request = fetch(`${running.baseUrl}${route}`, {+ method: 'POST',+ headers: { 'Content-Type': 'application/json' },+ body: JSON.stringify(body),+ });+ requests.add(request);+ void request.then(+ () => requests.delete(request),+ () => requests.delete(request),+ );+ return request;+ };++ return {+ server: running,+ docName,+ filePath,+ trace,+ disk: () => readFileSync(filePath, 'utf8'),+ resident: () => running.instance.hocuspocus.documents.get(docName),+ clientClosed: (client: TestClient) => closedClients.has(client),+ async open() {+ const client = await createTestClient(running.port, docName, { resetOnCleanup: false });+ clients.add(client);+ client.provider.on('close', () => {+ closedClients.add(client);+ client.provider.disconnect();+ });+ trace.push({+ phase: 'websocket-connected',+ originalDestroyed: originalDocument?.isDestroyed,+ mappedIsOriginal: running.instance.hocuspocus.documents.get(docName) === originalDocument,+ });+ return client;+ },+ observeNextNativeAppearance() {+ const appearance = Promise.withResolvers<void>();+ nativeAppearances.add(appearance);+ return appearance.promise;+ },+ async startPendingWrite(client: TestClient) {+ originalDocument = running.instance.hocuspocus.documents.get(docName);+ if (!originalDocument) throw new Error('Expected the original resident');+ client.doc.transact(() => {+ client.ytext.delete(0, client.ytext.length);+ client.ytext.insert(0, PENDING_PUBLICATION);+ });+ await staged.promise;+ expect(originalDocument.getText('source').toString()).toBe(PENDING_PUBLICATION);+ expect(readFileSync(filePath, 'utf8')).toBe(ORIGINAL_PUBLICATION);+ return originalDocument;+ },+ async finishPendingWrite() {+ releasePublication.resolve();+ await storeCompleted.promise;+ expect(published).toBe(true);+ expect(readFileSync(filePath, 'utf8')).toBe(PENDING_PUBLICATION);+ },+ armDeletion(hold: DeletionPlan['hold']) {+ if (upcomingPlan) throw new Error('An earlier native deletion has not arrived');+ const plan: DeletionPlan = {+ hold,+ nativeSeen: false,+ held: false,+ reached: Promise.withResolvers<void>(),+ release: Promise.withResolvers<void>(),+ followers: Promise.withResolvers<void>(),+ completed: Promise.withResolvers<void>(),+ };+ plans.push(plan);+ upcomingPlan = plan;+ return {+ waitUntilHeld: () => plan.reached.promise,+ async finish(releaseFollowers = false) {+ plan.release.resolve();+ await plan.completed.promise;+ expect(plan.nativeSeen).toBe(true);+ if (releaseFollowers) plan.followers.resolve();+ },+ };+ },+ deleteExternally: () => {+ unlinkSync(filePath);+ trace.push({ phase: 'external-unlink-completed', ordinal: plans.length - 1 });+ },+ async deleteThroughApi() {+ const response = await post('/api/delete-path', { kind: 'file', path: docName });+ expect(response.status).toBe(200);+ trace.push({+ phase: 'api-delete-completed',+ originalDestroyed: originalDocument?.isDestroyed,+ residentPresent: running.instance.hocuspocus.documents.has(docName),+ });+ },+ async recreate() {+ const response = await post('/api/create-page', {+ path: `${docName}.md`,+ template: 'pending-publication',+ });+ expect(response.status).toBe(200);+ expect(await response.json()).toEqual({ docName });+ expect(readFileSync(filePath, 'utf8')).toBe(PENDING_PUBLICATION);+ trace.push({+ phase: 'exclusive-recreation-completed',+ originalDestroyed: originalDocument?.isDestroyed,+ mappedIsOriginal: running.instance.hocuspocus.documents.get(docName) === originalDocument,+ });+ },+ };+}diff --git a/packages/app/tests/integration/stale-publication-completion.test.ts b/packages/app/tests/integration/stale-publication-completion.test.tsnew file mode 100644index 000000000..767cb1ae8--- /dev/null+++ b/packages/app/tests/integration/stale-publication-completion.test.ts@@ -0,0 +1,74 @@+import { afterEach, expect, test } from 'vitest';+import { runTeardownPhases } from '../stress/_helpers/teardown-fs.ts';+import {+ ORIGINAL_PUBLICATION,+ PENDING_PUBLICATION,+ startStalePublicationScenario,+} from './stale-publication-completion.test-helper.ts';++const cleanups: Array<() => Promise<void>> = [];+const registerCleanup = (cleanup: () => Promise<void>) => cleanups.push(cleanup);++afterEach(async () => {+ await runTeardownPhases(...cleanups.splice(0).reverse());+});++test('a publication captured before same-object recreation cannot adopt its unmodified lifetime', async () => {+ const scenario = await startStalePublicationScenario(registerCleanup);+ const originalClient = await scenario.open();+ expect(originalClient.ytext.toString()).toBe(ORIGINAL_PUBLICATION);+ const originalDocument = await scenario.startPendingWrite(originalClient);+ const deletion = scenario.armDeletion('classified');+ scenario.deleteExternally();+ await deletion.waitUntilHeld();+ await scenario.recreate();+ expect(scenario.resident()).toBe(originalDocument);+ expect(originalDocument.isDestroyed).toBe(false);+ expect(scenario.clientClosed(originalClient)).toBe(false);++ await scenario.finishPendingWrite();+ expect(scenario.resident()).toBe(originalDocument);+ expect(originalDocument.getText('source').toString()).toBe(PENDING_PUBLICATION);+ await deletion.finish();+ expect(originalDocument.isDestroyed, JSON.stringify(scenario.trace)).toBe(true);+ expect(scenario.resident()).not.toBe(originalDocument);+ expect(scenario.disk()).toBe(PENDING_PUBLICATION);+});++test('a publication from a replaced Document cannot adopt the replacement unmodified lifetime', async () => {+ const scenario = await startStalePublicationScenario(registerCleanup);+ const originalClient = await scenario.open();+ expect(originalClient.ytext.toString()).toBe(ORIGINAL_PUBLICATION);+ const originalDocument = await scenario.startPendingWrite(originalClient);+ const originalDeletion = scenario.armDeletion('none');+ await scenario.deleteThroughApi();+ expect(originalDocument.isDestroyed).toBe(true);+ await originalDeletion.finish(true);++ const replacementAppearance = scenario.observeNextNativeAppearance();+ await scenario.recreate();+ await replacementAppearance;+ const replacementClient = await scenario.open();+ const replacementDocument = scenario.resident();+ if (!replacementDocument) throw new Error('Expected the replacement resident');+ expect(replacementDocument).not.toBe(originalDocument);+ expect(replacementDocument.isDestroyed).toBe(false);+ expect(replacementClient.ytext.toString()).toBe(PENDING_PUBLICATION);+ const replacementDeletion = scenario.armDeletion('classified');+ scenario.deleteExternally();+ await replacementDeletion.waitUntilHeld();+ expect(scenario.resident()).toBe(replacementDocument);+ expect(replacementDocument.getMap('lifecycle').get('status')).toBeUndefined();+ await scenario.recreate();+ expect(scenario.resident()).toBe(replacementDocument);+ expect(replacementDocument.getMap('lifecycle').get('status')).toBeUndefined();+ expect(replacementDocument.getText('source').toString()).toBe(PENDING_PUBLICATION);++ await scenario.finishPendingWrite();+ expect(originalDocument.isDestroyed).toBe(true);+ expect(scenario.resident()).toBe(replacementDocument);+ await replacementDeletion.finish();+ expect(replacementDocument.isDestroyed, JSON.stringify(scenario.trace)).toBe(true);+ expect(scenario.resident()).not.toBe(replacementDocument);+ expect(scenario.disk()).toBe(PENDING_PUBLICATION);+});diff --git a/packages/app/tests/integration/stale-recovery-adoption.test.ts b/packages/app/tests/integration/stale-recovery-adoption.test.tsnew file mode 100644index 000000000..2de95f0e5--- /dev/null+++ b/packages/app/tests/integration/stale-recovery-adoption.test.ts@@ -0,0 +1,233 @@+import { randomUUID } from 'node:crypto';+import {+ existsSync,+ mkdtempSync,+ readFileSync,+ realpathSync,+ rmSync,+ unlinkSync,+ writeFileSync,+} from 'node:fs';+import { tmpdir } from 'node:os';+import { join } from 'node:path';+import { MetricsReconciliationSuccessSchema } from '@inkeep/open-knowledge-core';+import { afterEach, expect, test, vi } from 'vitest';+import * as fileWatcher from '../../../server/src/file-watcher.ts';+import * as headWatcher from '../../../server/src/head-watcher.ts';+import { resetMetrics } from '../../../server/src/metrics.ts';+import * as parcelSubscription from '../../../server/src/parcel-subscription.ts';+import { runTeardownPhases } from '../stress/_helpers/teardown-fs.ts';+import {+ createTestClient,+ createTestServer,+ type TestClient,+ type TestServer,+} from './test-harness.ts';++const cleanups: Array<() => Promise<void>> = [];+const LOSS_NOTICE = 'Events were dropped by the FSEvents client. File system must be re-scanned.';++afterEach(async () => {+ await runTeardownPhases(...cleanups.splice(0).reverse());+});++async function scenario(mode: 'connection' | 'writer' | 'buffer') {+ resetMetrics();+ const contentDir = realpathSync(mkdtempSync(join(tmpdir(), 'ok-recovery-adoption-')));+ const docName = `adoption-${randomUUID()}`;+ const filePath = join(contentDir, `${docName}.md`);+ const nativeCaptured = Promise.withResolvers<void>();+ const nativeRelease = Promise.withResolvers<void>();+ const handlerCaptured = Promise.withResolvers<void>();+ const handlerRelease = Promise.withResolvers<void>();+ const batchCaptured = Promise.withResolvers<void>();+ const batchRelease = Promise.withResolvers<void>();+ const batchFinished = Promise.withResolvers<void>();+ const work: Promise<void>[] = [];+ const clients: TestClient[] = [];+ const closed = new WeakSet<TestClient>();+ let armed = false;+ let recovery: Promise<void> | undefined;+ let server: TestServer | undefined;+ let recover: (() => Promise<void>) | undefined;+ let beginBatch: (() => Promise<void>) | undefined;+ let delivered = 0;+ const subscribe = parcelSubscription.subscribeParcel;+ const subscriptionSpy = vi+ .spyOn(parcelSubscription, 'subscribeParcel')+ .mockImplementation((parcel, dir, onEvents, options, platform) => {+ if (dir === contentDir)+ recover = async () => {+ await onEvents(new Error(LOSS_NOTICE), []);+ };+ if (dir === join(contentDir, '.git'))+ beginBatch = async () => {+ await onEvents(new Error(LOSS_NOTICE), []);+ };+ return subscribe(+ parcel,+ dir,+ (error, events) => {+ if (dir !== contentDir || !armed) return onEvents(error, events);+ if (events.some((event) => event.type === 'delete' && event.path === filePath))+ nativeCaptured.resolve();+ const pending = nativeRelease.promise.then(async () => {+ await onEvents(error, events);+ });+ work.push(pending);+ return pending;+ },+ options,+ platform,+ );+ });+ const startWatcher = fileWatcher.startWatcher;+ const watcherSpy = vi+ .spyOn(fileWatcher, 'startWatcher')+ .mockImplementation((root, onDiskEvent, filter, options) =>+ startWatcher(+ root,+ async (event) => {+ if (armed && event.kind === 'delete' && event.docName === docName) {+ handlerCaptured.resolve();+ if (mode !== 'buffer') await handlerRelease.promise;+ }+ await onDiskEvent(event);+ if (armed && event.kind === 'delete' && event.docName === docName) delivered += 1;+ },+ filter,+ { ...options, forceBackend: 'parcel' },+ ),+ );+ const startHeadWatcher = headWatcher.startHeadWatcher;+ const headSpy = vi+ .spyOn(headWatcher, 'startHeadWatcher')+ .mockImplementation((root, begin, end, options) =>+ startHeadWatcher(+ root,+ async (info) => {+ await begin(info);+ if (mode === 'buffer' && armed) {+ batchCaptured.resolve();+ await batchRelease.promise;+ }+ },+ async (info) => {+ await end(info);+ if (mode === 'buffer' && armed) batchFinished.resolve();+ },+ options,+ ),+ );+ cleanups.push(async () => {+ armed = false;+ batchRelease.resolve();+ handlerRelease.resolve();+ nativeRelease.resolve();+ await runTeardownPhases(+ async () => {+ await recovery;+ await Promise.all(work);+ },+ ...clients.map((client) => () => client.cleanup()),+ async () => {+ await server?.cleanup();+ },+ () => {+ headSpy.mockRestore();+ watcherSpy.mockRestore();+ subscriptionSpy.mockRestore();+ },+ () => rmSync(contentDir, { recursive: true, force: true }),+ );+ });+ writeFileSync(filePath, '# Original adoption page\n', { flag: 'wx' });+ server = await createTestServer({ contentDir });+ const running = server;+ const open = async () => {+ const client = await createTestClient(running.port, docName, { resetOnCleanup: false });+ clients.push(client);+ client.provider.on('close', () => closed.add(client));+ return client;+ };+ const initial = await open();+ expect(initial.ytext.toString()).toBe('# Original adoption page\n');+ armed = true;+ if (mode === 'buffer') {+ if (!beginBatch) throw new Error('Real head subscription not installed');+ await beginBatch();+ await batchCaptured.promise;+ expect(running.instance.durabilityState.isBatchInProgress()).toBe(true);+ }+ unlinkSync(filePath);+ await nativeCaptured.promise;+ if (!recover) throw new Error('Real content subscription not installed');+ recovery = recover();+ await handlerCaptured.promise;+ if (mode === 'buffer') {+ await recovery;+ expect(delivered).toBe(1);+ expect(running.instance.durabilityState.isBatchInProgress()).toBe(true);+ }+ const post = (route: string, body: unknown) =>+ fetch(`${running.baseUrl}${route}`, {+ method: 'POST',+ headers: { 'Content-Type': 'application/json' },+ body: JSON.stringify(body),+ });+ expect((await post('/api/create-page', { path: `${docName}.md` })).status).toBe(200);+ const admitted = mode === 'writer' ? initial : await open();+ if (mode === 'writer') {+ expect(+ (+ await post('/api/agent-write-md', {+ docName,+ markdown: '# Written after recreation\n',+ position: 'replace',+ agentId: 'adoption-writer',+ })+ ).status,+ ).toBe(200);+ await expect.poll(() => admitted.ytext.toString()).toBe('# Written after recreation\n');+ }+ const resident = running.instance.hocuspocus.documents.get(docName);+ expect(resident).toBeDefined();+ const snapshot = async () => {+ const response = await fetch(`${running.baseUrl}/api/metrics/reconciliation`);+ expect(response.status).toBe(200);+ const metrics = MetricsReconciliationSuccessSchema.parse(await response.json());+ return {+ residentPreserved: running.instance.hocuspocus.documents.get(docName) === resident,+ clientClosed: closed.has(admitted),+ clientStatus: admitted.doc.getMap('lifecycle').get('status') ?? null,+ removedCount: metrics.recentlyRemovedDocsSize,+ disk: existsSync(filePath) ? readFileSync(filePath, 'utf8') : null,+ };+ };+ const before = await snapshot();+ expect(before).toMatchObject({+ residentPreserved: true,+ clientClosed: false,+ clientStatus: null,+ removedCount: 0,+ });+ if (mode === 'buffer') {+ expect(running.instance.durabilityState.isBatchInProgress()).toBe(true);+ batchRelease.resolve();+ await batchFinished.promise;+ expect(running.instance.durabilityState.isBatchInProgress()).toBe(false);+ } else {+ handlerRelease.resolve();+ await recovery;+ }+ expect(await snapshot()).toEqual(before);+}++test('a fresh connection without a write adopts recreation before recovery deletion', () =>+ scenario('connection'));++test('a direct writer without a new client adopts recreation before recovery deletion', () =>+ scenario('writer'));++test('recreation admitted after recovery completes survives the later head-batch drain', () =>+ scenario('buffer'));diff --git a/packages/app/tests/integration/stale-recovery-removal.test.ts b/packages/app/tests/integration/stale-recovery-removal.test.tsnew file mode 100644index 000000000..fab8d2ab2--- /dev/null+++ b/packages/app/tests/integration/stale-recovery-removal.test.ts@@ -0,0 +1,210 @@+import { randomUUID } from 'node:crypto';+import {+ existsSync,+ mkdtempSync,+ readFileSync,+ realpathSync,+ rmSync,+ unlinkSync,+ writeFileSync,+} from 'node:fs';+import { tmpdir } from 'node:os';+import { join } from 'node:path';+import {+ MetricsReconciliationSuccessSchema,+ PagesSuccessSchema,+} from '@inkeep/open-knowledge-core';+import { afterEach, expect, test, vi } from 'vitest';+import * as fileWatcher from '../../../server/src/file-watcher.ts';+import { resetMetrics } from '../../../server/src/metrics.ts';+import * as parcelSubscription from '../../../server/src/parcel-subscription.ts';+import { runTeardownPhases } from '../stress/_helpers/teardown-fs.ts';+import {+ createTestClient,+ createTestServer,+ type TestClient,+ type TestServer,+} from './test-harness.ts';++const cleanups: Array<() => Promise<void>> = [];+const ORIGINAL = '# Original recovery page\n';+const RECREATED = '# Recreated recovery page\n';+const LOSS_NOTICE = 'Events were dropped by the FSEvents client. File system must be re-scanned.';++afterEach(async () => {+ await runTeardownPhases(...cleanups.splice(0).reverse());+});++async function recoveryScenario(recreate: boolean) {+ resetMetrics();+ const contentDir = realpathSync(mkdtempSync(join(tmpdir(), 'ok-recovery-removal-')));+ const docName = `recovery-${randomUUID()}`;+ const filePath = join(contentDir, `${docName}.md`);+ const nativeCaptured = Promise.withResolvers<void>();+ const nativeRelease = Promise.withResolvers<void>();+ const handlerCaptured = Promise.withResolvers<void>();+ const handlerRelease = Promise.withResolvers<void>();+ const clients: TestClient[] = [];+ const closedClients = new WeakSet<TestClient>();+ const nativeWork: Promise<void>[] = [];+ let server: TestServer | undefined;+ let armed = false;+ let recovery: Promise<void> | undefined;+ let recover: (() => Promise<void>) | undefined;+ const subscribe = parcelSubscription.subscribeParcel;+ const subscriptionSpy = vi+ .spyOn(parcelSubscription, 'subscribeParcel')+ .mockImplementation((parcel, dir, onEvents, options, platform) => {+ if (dir === contentDir) {+ recover = async () => {+ await onEvents(new Error(LOSS_NOTICE), []);+ };+ }+ return subscribe(+ parcel,+ dir,+ (error, events) => {+ if (dir !== contentDir || !armed) return onEvents(error, events);+ if (events.some((event) => event.type === 'delete' && event.path === filePath))+ nativeCaptured.resolve();+ const work = nativeRelease.promise.then(async () => {+ await onEvents(error, events);+ });+ nativeWork.push(work);+ return work;+ },+ options,+ platform,+ );+ });+ const startWatcher = fileWatcher.startWatcher;+ const watcherSpy = vi+ .spyOn(fileWatcher, 'startWatcher')+ .mockImplementation((root, onDiskEvent, filter, options) =>+ startWatcher(+ root,+ async (event) => {+ if (armed && event.kind === 'delete' && event.docName === docName) {+ handlerCaptured.resolve();+ await handlerRelease.promise;+ }+ await onDiskEvent(event);+ },+ filter,+ { ...options, forceBackend: 'parcel' },+ ),+ );+ cleanups.push(async () => {+ armed = false;+ handlerRelease.resolve();+ nativeRelease.resolve();+ await runTeardownPhases(+ async () => {+ await recovery;+ await Promise.all(nativeWork);+ },+ ...clients.map((client) => () => client.cleanup()),+ async () => {+ await server?.cleanup();+ },+ () => {+ watcherSpy.mockRestore();+ subscriptionSpy.mockRestore();+ },+ () => rmSync(contentDir, { recursive: true, force: true }),+ );+ });++ writeFileSync(filePath, ORIGINAL, { flag: 'wx' });+ server = await createTestServer({ contentDir });+ const running = server;+ const open = async () => {+ const client = await createTestClient(running.port, docName, { resetOnCleanup: false });+ clients.push(client);+ client.provider.on('close', () => closedClients.add(client));+ return client;+ };+ const post = (route: string, body: unknown) =>+ fetch(`${running.baseUrl}${route}`, {+ method: 'POST',+ headers: { 'Content-Type': 'application/json' },+ body: JSON.stringify(body),+ });+ const snapshot = async (client: TestClient) => {+ const resident = running.instance.hocuspocus.documents.get(docName);+ const metricsResponse = await fetch(`${running.baseUrl}/api/metrics/reconciliation`);+ const pagesResponse = await fetch(`${running.baseUrl}/api/pages`);+ expect([metricsResponse.status, pagesResponse.status]).toEqual([200, 200]);+ const metrics = MetricsReconciliationSuccessSchema.parse(await metricsResponse.json());+ const pages = PagesSuccessSchema.parse(await pagesResponse.json());+ return {+ disk: existsSync(filePath) ? readFileSync(filePath, 'utf-8') : null,+ resident,+ residentContent: resident?.getText('source').toString() ?? null,+ lifecycleStatus: resident?.getMap('lifecycle').get('status') ?? null,+ clientContent: client.ytext.toString(),+ clientLifecycleStatus: client.doc.getMap('lifecycle').get('status') ?? null,+ clientClosed: closedClients.has(client),+ removedCount: metrics.recentlyRemovedDocsSize,+ listed: pages.pages.some((page) => page.docName === docName),+ };+ };+ const initial = await open();+ expect(initial.ytext.toString()).toBe(ORIGINAL);+ armed = true;+ unlinkSync(filePath);+ await nativeCaptured.promise;+ if (!recover) throw new Error('Owned Parcel subscription was not established');+ recovery = recover();+ await handlerCaptured.promise;++ if (recreate) {+ const created = await post('/api/create-page', { path: `${docName}.md` });+ expect(created.status).toBe(200);+ const fresh = await open();+ const written = await post('/api/agent-write-md', {+ docName,+ markdown: RECREATED,+ position: 'replace',+ agentId: 'recovery-recreation',+ });+ expect(written.status).toBe(200);+ await expect.poll(() => fresh.ytext.toString()).toBe(RECREATED);+ const { resident: admitted, ...before } = await snapshot(fresh);+ expect(admitted).toBeDefined();+ expect(before).toEqual({+ disk: RECREATED,+ residentContent: RECREATED,+ lifecycleStatus: null,+ clientContent: RECREATED,+ clientLifecycleStatus: null,+ clientClosed: false,+ removedCount: 0,+ listed: true,+ });++ handlerRelease.resolve();+ await recovery;+ const { resident, ...after } = await snapshot(fresh);+ expect({ residentPreserved: resident === admitted, ...after }).toEqual({+ residentPreserved: true,+ ...before,+ });+ } else {+ handlerRelease.resolve();+ await recovery;+ const { resident, ...after } = await snapshot(initial);+ expect({ residentPresent: resident !== undefined, ...after }).toMatchObject({+ disk: null,+ residentPresent: false,+ residentContent: null,+ removedCount: 1,+ listed: false,+ });+ }+}++test('loss-notice recovery preserves a recreation accepted before deletion delivery', () =>+ recoveryScenario(true));++test('loss-notice recovery removes a genuinely deleted page', () => recoveryScenario(false));diff --git a/packages/app/tests/integration/stale-removal-adoption.test.ts b/packages/app/tests/integration/stale-removal-adoption.test.tsnew file mode 100644index 000000000..e1f031370--- /dev/null+++ b/packages/app/tests/integration/stale-removal-adoption.test.ts@@ -0,0 +1,179 @@+import { mkdirSync, readFileSync, writeFileSync } from 'node:fs';+import { join } from 'node:path';+import { afterEach, expect, test, vi } from 'vitest';+import { runTeardownPhases } from '../stress/_helpers/teardown-fs.ts';+import {+ ORIGINAL,+ RECREATED,+ startDeletionScenario,+} from './stale-delete-recreation.test-helper.ts';++const nativeWriteObservation = vi.hoisted(+ (): {+ current?: {+ directory: string;+ path: string;+ consumed: PromiseWithResolvers<void>;+ };+ } => ({}),+);++vi.mock('../../../server/src/parcel-subscription.ts', async (importOriginal) => {+ const actual =+ await importOriginal<typeof import('../../../server/src/parcel-subscription.ts')>();+ const subscribe = actual.subscribeParcel;+ return {+ ...actual,+ subscribeParcel: (...args: Parameters<typeof subscribe>) => {+ const [parcel, directory, callback, options, platform] = args;+ return subscribe(+ parcel,+ directory,+ (error, events) => {+ const observation = nativeWriteObservation.current;+ const selected =+ observation !== undefined &&+ observation.directory === directory &&+ events.some(+ (event) =>+ event.path === observation.path &&+ (event.type === 'create' || event.type === 'update'),+ );+ const result = callback(error, events);+ if (selected) {+ void Promise.resolve(result).then(+ () => observation.consumed.resolve(),+ (cause: unknown) => observation.consumed.reject(cause),+ );+ }+ return result;+ },+ options,+ platform,+ );+ },+ };+});++const cleanups: Array<() => Promise<void>> = [];+const registerCleanup = (cleanup: () => Promise<void>) => cleanups.push(cleanup);++afterEach(async () => {+ nativeWriteObservation.current = undefined;+ await runTeardownPhases(...cleanups.splice(0).reverse());+});++test('external deletion followed by creation keeps only the recreated page comments', async () => {+ const scenario = await startDeletionScenario('classified', registerCleanup);+ const original = await scenario.open();+ expect(original.ytext.toString()).toBe(ORIGINAL);+ const oldThread = await scenario.comment('Original page');+ expect((await scenario.snapshot(original)).threads).toEqual([oldThread.threadId]);+ scenario.deleteExternally();+ await scenario.waitUntilHeld();+ const fresh = await scenario.create();+ const newThread = await scenario.comment('Recreated page');+ expect((await scenario.snapshot(fresh)).threads).toContain(newThread.threadId);+ await scenario.finishDeletion();+ expect(await scenario.snapshot(fresh)).toMatchObject({+ disk: RECREATED,+ residentPresent: true,+ clientClosed: false,+ removedCount: 0,+ threads: [newThread.threadId],+ });+});++test('a successful write and index registration retain the current page comments', async () => {+ const scenario = await startDeletionScenario('none', registerCleanup);+ const client = await scenario.open();+ const thread = await scenario.comment('Original page');+ expect((await scenario.snapshot(client)).threads).toEqual([thread.threadId]);+ const response = await fetch(`${scenario.server.baseUrl}/api/agent-write-md`, {+ method: 'POST',+ headers: { 'Content-Type': 'application/json' },+ body: JSON.stringify({+ docName: scenario.docName,+ markdown: RECREATED,+ position: 'replace',+ agentId: 'comment-control',+ }),+ });+ expect(response.status).toBe(200);+ await expect.poll(() => client.ytext.toString()).toBe(RECREATED);+ expect(await scenario.snapshot(client)).toMatchObject({+ disk: RECREATED,+ residentPresent: true,+ threads: [thread.threadId],+ });+});++test('an older write completion cannot adopt the unmodified resident of a recreated page', async () => {+ const scenario = await startDeletionScenario('classified', registerCleanup);+ const client = await scenario.open();+ expect(client.ytext.toString()).toBe(ORIGINAL);+ const oldResident = scenario.server.instance.hocuspocus.documents.get(scenario.docName);+ expect(oldResident).toBeDefined();+ const oldWritten = '# Accepted before recreation\n';+ const replacement = '# Replacement without resident adoption\n';+ const templateDir = join(scenario.server.contentDir, '.ok', 'templates');+ mkdirSync(templateDir, { recursive: true });+ writeFileSync(join(templateDir, 'replacement.md'), replacement, { flag: 'wx' });+ const reached = Promise.withResolvers<void>();+ const release = Promise.withResolvers<void>();+ const durability = scenario.server.instance.durabilityState;+ const originalOutcome = durability.awaitDeferredStoreOutcome.bind(durability);+ let captured = false;+ const spy = vi.spyOn(durability, 'awaitDeferredStoreOutcome').mockImplementation(async (name) => {+ const result = await originalOutcome(name);+ if (name === scenario.docName && !captured) {+ captured = true;+ reached.resolve();+ await release.promise;+ }+ return result;+ });+ const canonicalWriteConsumed = Promise.withResolvers<void>();+ void canonicalWriteConsumed.promise.catch(() => {});+ nativeWriteObservation.current = {+ directory: scenario.server.contentDir,+ path: join(scenario.server.contentDir, `${scenario.docName}.md`),+ consumed: canonicalWriteConsumed,+ };+ const pendingWrite = fetch(`${scenario.server.baseUrl}/api/agent-write-md`, {+ method: 'POST',+ headers: { 'Content-Type': 'application/json' },+ body: JSON.stringify({+ docName: scenario.docName,+ markdown: oldWritten,+ position: 'replace',+ agentId: 'obsolete-writer',+ }),+ });+ registerCleanup(async () => {+ release.resolve();+ await pendingWrite;+ spy.mockRestore();+ });+ await reached.promise;+ const filePath = join(scenario.server.contentDir, `${scenario.docName}.md`);+ expect(readFileSync(filePath, 'utf8')).toBe(oldWritten);+ expect(oldResident?.getText('source').toString()).toBe(oldWritten);+ await canonicalWriteConsumed.promise;+ scenario.deleteExternally();+ await scenario.waitUntilHeld();+ const created = await fetch(`${scenario.server.baseUrl}/api/create-page`, {+ method: 'POST',+ headers: { 'Content-Type': 'application/json' },+ body: JSON.stringify({ path: `${scenario.docName}.md`, template: 'replacement' }),+ });+ expect(created.status).toBe(200);+ expect(readFileSync(filePath, 'utf8')).toBe(replacement);+ expect(oldResident?.getText('source').toString()).toBe(oldWritten);+ release.resolve();+ expect((await pendingWrite).status).toBe(200);+ await scenario.finishDeletion();+ expect(scenario.server.instance.hocuspocus.documents.get(scenario.docName)).not.toBe(oldResident);+ expect(readFileSync(filePath, 'utf8')).toBe(replacement);+ expect((await scenario.readDocument()).content).toBe(replacement);+});diff --git a/packages/app/tests/integration/stale-removal-completion.test.ts b/packages/app/tests/integration/stale-removal-completion.test.tsnew file mode 100644index 000000000..90823dc70--- /dev/null+++ b/packages/app/tests/integration/stale-removal-completion.test.ts@@ -0,0 +1,219 @@+import { existsSync, readFileSync, renameSync, writeFileSync } from 'node:fs';+import { join } from 'node:path';+import { afterEach, expect, test, vi } from 'vitest';+import { ConflictAuthority } from '../../../server/src/conflict-authority.ts';+import { DerivedDocumentIndex } from '../../../server/src/derived-document-index.ts';+import { runTeardownPhases } from '../stress/_helpers/teardown-fs.ts';+import {+ ORIGINAL,+ RECREATED,+ startDeletionScenario,+} from './stale-delete-recreation.test-helper.ts';++const cleanups: Array<() => Promise<void> | void> = [];+const registerCleanup = (cleanup: () => Promise<void> | void) => {+ cleanups.push(cleanup);+};+const renameIndex = DerivedDocumentIndex.prototype.recordDiskRename;+const deleteIndex = DerivedDocumentIndex.prototype.recordDiskDelete;+const raiseConflict = ConflictAuthority.prototype.raise;++afterEach(async () => {+ await runTeardownPhases(...cleanups.splice(0).reverse());+});++test('a completed rename preserves a recreation admitted at its old name', async () => {+ const scenario = await startDeletionScenario('none', registerCleanup);+ const original = await scenario.open();+ expect(original.ytext.toString()).toBe(ORIGINAL);+ const destination = `moved-${scenario.docName}`;+ const sourcePath = join(scenario.server.contentDir, `${scenario.docName}.md`);+ const destinationPath = join(scenario.server.contentDir, `${destination}.md`);+ const reached = Promise.withResolvers<void>();+ const release = Promise.withResolvers<void>();+ const indexSpy = vi+ .spyOn(DerivedDocumentIndex.prototype, 'recordDiskRename')+ .mockImplementation(async function (this: DerivedDocumentIndex, oldName, newName, content) {+ await renameIndex.call(this, oldName, newName, content);+ if (oldName === scenario.docName && newName === destination) {+ reached.resolve();+ await release.promise;+ }+ });+ registerCleanup(() => indexSpy.mockRestore());+ registerCleanup(() => release.resolve());++ renameSync(sourcePath, destinationPath);+ await reached.promise;+ expect(existsSync(sourcePath)).toBe(false);+ expect(readFileSync(destinationPath, 'utf-8')).toBe(ORIGINAL);++ const fresh = await scenario.create();+ const thread = await scenario.comment('Recreated page');+ expect(await scenario.snapshot(fresh)).toMatchObject({+ disk: RECREATED,+ residentPresent: true,+ residentContent: RECREATED,+ lifecycleStatus: null,+ clientContent: RECREATED,+ clientLifecycleStatus: null,+ clientClosed: false,+ removedCount: 0,+ listed: true,+ forwardLinks: ['target'],+ });++ release.resolve();+ await scenario.finishDeletion();+ const after = await scenario.snapshot(fresh);+ expect(readFileSync(destinationPath, 'utf-8')).toBe(ORIGINAL);+ expect(after.threads).toContain(thread.threadId);+ expect(after).toMatchObject({+ disk: RECREATED,+ residentPresent: true,+ residentContent: RECREATED,+ lifecycleStatus: null,+ clientContent: RECREATED,+ clientLifecycleStatus: null,+ clientClosed: false,+ removedCount: 0,+ listed: true,+ forwardLinks: ['target'],+ });+});++test('a completed conflict deletion preserves an accepted same-name creation', async () => {+ const scenario = await startDeletionScenario('none', registerCleanup);+ await scenario.open();+ const file = `${scenario.docName}.md`;+ const path = join(scenario.server.contentDir, file);+ const post = (route: string, body: unknown) =>+ fetch(`${scenario.server.baseUrl}${route}`, {+ method: 'POST',+ headers: { 'Content-Type': 'application/json' },+ body: JSON.stringify(body),+ });+ const raised = Promise.withResolvers<void>();+ const conflictSpy = vi.spyOn(ConflictAuthority.prototype, 'raise').mockImplementation(function (+ this: ConflictAuthority,+ input,+ ) {+ raiseConflict.call(this, input);+ if (input.kind === 'reconcile' && input.file === file) raised.resolve();+ });+ registerCleanup(() => conflictSpy.mockRestore());+ writeFileSync(path, '# Doc\n\n<<<<<<< ours\nalpha-ours\n=======\nalpha-theirs\n>>>>>>> theirs\n');+ await raised.promise;++ const reached = Promise.withResolvers<void>();+ const release = Promise.withResolvers<void>();+ const indexSpy = vi+ .spyOn(DerivedDocumentIndex.prototype, 'recordDiskDelete')+ .mockImplementation(async function (this: DerivedDocumentIndex, name) {+ await deleteIndex.call(this, name);+ if (name === scenario.docName) {+ reached.resolve();+ await release.promise;+ }+ });+ registerCleanup(() => indexSpy.mockRestore());+ const resolving = post('/api/sync/resolve-conflict', { file, strategy: 'delete' });+ registerCleanup(async () => {+ release.resolve();+ await resolving;+ });+ await reached.promise;+ expect(existsSync(path)).toBe(false);++ const created = await post('/api/create-page', { path: file });+ expect(created.status).toBe(200);+ expect(await created.json()).toEqual({ docName: scenario.docName });+ const fresh = await scenario.open();+ const writing = await post('/api/agent-write-md', {+ docName: scenario.docName,+ markdown: RECREATED,+ position: 'replace',+ agentId: 'recreation-during-resolution',+ });+ expect(writing.status).toBe(409);+ expect(await writing.json()).toMatchObject({ type: 'urn:ok:error:doc-in-conflict' });+ expect(await scenario.snapshot(fresh)).toMatchObject({+ disk: '',+ residentPresent: true,+ lifecycleStatus: null,+ clientLifecycleStatus: null,+ clientClosed: false,+ removedCount: 0,+ listed: true,+ });++ release.resolve();+ expect((await resolving).status).toBe(200);+ const conflicts = await fetch(`${scenario.server.baseUrl}/api/sync/conflicts`);+ expect(conflicts.status).toBe(200);+ expect(await conflicts.json()).toEqual({ conflicts: [] });+ expect(await scenario.snapshot(fresh)).toMatchObject({+ disk: '',+ residentPresent: true,+ lifecycleStatus: null,+ clientLifecycleStatus: null,+ clientClosed: false,+ removedCount: 0,+ listed: true,+ });+});++test('a completed rename retires an unadopted resident loaded at its old name during the rename', async () => {+ const scenario = await startDeletionScenario('none', registerCleanup);+ const destination = `moved-${scenario.docName}`;+ const sourcePath = join(scenario.server.contentDir, `${scenario.docName}.md`);+ const destinationPath = join(scenario.server.contentDir, `${destination}.md`);+ const reached = Promise.withResolvers<void>();+ const release = Promise.withResolvers<void>();+ const indexSpy = vi+ .spyOn(DerivedDocumentIndex.prototype, 'recordDiskRename')+ .mockImplementation(async function (this: DerivedDocumentIndex, oldName, newName, content) {+ await renameIndex.call(this, oldName, newName, content);+ if (oldName === scenario.docName && newName === destination) {+ reached.resolve();+ await release.promise;+ }+ });+ registerCleanup(() => indexSpy.mockRestore());+ registerCleanup(() => release.resolve());++ const hocuspocus = scenario.server.instance.hocuspocus;+ const load = hocuspocus.loadDocument.bind(hocuspocus);+ const loading = Promise.withResolvers<void>();+ const loaded = Promise.withResolvers<void>();+ const loadSpy = vi+ .spyOn(hocuspocus, 'loadDocument')+ .mockImplementation(async (...args: Parameters<typeof hocuspocus.loadDocument>) => {+ const document = await load(...args);+ if (args[0] === scenario.docName) {+ loading.resolve();+ await loaded.promise;+ }+ return document;+ });+ registerCleanup(() => loadSpy.mockRestore());+ registerCleanup(() => loaded.resolve());++ const opening = scenario.open();+ await loading.promise;+ expect(hocuspocus.documents.get(scenario.docName)).toBeUndefined();+ renameSync(sourcePath, destinationPath);+ await reached.promise;+ loaded.resolve();+ const late = await opening;+ expect(hocuspocus.documents.get(scenario.docName)).toBeDefined();++ release.resolve();+ await scenario.finishDeletion();+ expect(readFileSync(destinationPath, 'utf-8')).toBe(ORIGINAL);+ expect(await scenario.snapshot(late)).toMatchObject({+ lifecycleStatus: 'renamed',+ clientClosed: true,+ });+ expect(existsSync(sourcePath)).toBe(false);+});diff --git a/packages/app/tests/integration/stale-removal-coverage.test-helper.ts b/packages/app/tests/integration/stale-removal-coverage.test-helper.tsnew file mode 100644index 000000000..04bb9d82e--- /dev/null+++ b/packages/app/tests/integration/stale-removal-coverage.test-helper.ts@@ -0,0 +1,148 @@+import {+ existsSync,+ mkdirSync,+ mkdtempSync,+ readdirSync,+ readFileSync,+ realpathSync,+ rmSync,+ writeFileSync,+} from 'node:fs';+import { tmpdir } from 'node:os';+import { join } from 'node:path';+import { PagesSuccessSchema } from '@inkeep/open-knowledge-core';+import { expect, vi } from 'vitest';+import * as fileWatcher from '../../../server/src/file-watcher.ts';+import * as parcelSubscription from '../../../server/src/parcel-subscription.ts';+import { runTeardownPhases } from '../stress/_helpers/teardown-fs.ts';+import { createTestServer, type TestServer } from './test-harness.ts';++type BatchKind = 'delete' | 'write';+interface HeldBatch {+ kind: BatchKind;+ claimed: boolean;+ release: () => void;+ completed: Promise<void>;+}++export async function startAssetAdmissionScenario(+ registerCleanup: (cleanup: () => Promise<void>) => void,+) {+ const contentDir = realpathSync(mkdtempSync(join(tmpdir(), 'ok-removal-assets-')));+ const docName = 'child/note';+ const filePath = join(contentDir, `${docName}.md`);+ const assetPath = 'child/image.png';+ mkdirSync(join(contentDir, 'child'));+ writeFileSync(filePath, '# Only Markdown\n');+ writeFileSync(join(contentDir, assetPath), new Uint8Array([1, 2, 3]));+ const batches: HeldBatch[] = [];+ const waiting: Array<{ kind: BatchKind; resolve: (batch: HeldBatch) => void }> = [];+ let tail = Promise.resolve();+ let closing = false;+ let server: TestServer | undefined;+ const subscribe = parcelSubscription.subscribeParcel;+ const subscriptionSpy = vi+ .spyOn(parcelSubscription, 'subscribeParcel')+ .mockImplementation((parcel, dir, onEvents, options, platform) =>+ subscribe(+ parcel,+ dir,+ (error, events) => {+ if (dir !== contentDir) return onEvents(error, events);+ const selected = events.some((event) => event.path === filePath);+ const released = Promise.withResolvers<void>();+ const completed = Promise.withResolvers<void>();+ if (selected && !closing) {+ const kind = events.some((event) => event.path === filePath && event.type === 'delete')+ ? 'delete'+ : 'write';+ const batch: HeldBatch = {+ kind,+ claimed: false,+ release: released.resolve,+ completed: completed.promise,+ };+ batches.push(batch);+ const waiterIndex = waiting.findIndex((waiter) => waiter.kind === kind);+ if (waiterIndex !== -1) {+ batch.claimed = true;+ waiting.splice(waiterIndex, 1)[0].resolve(batch);+ }+ } else {+ released.resolve();+ }+ const work = tail.then(async () => {+ await released.promise;+ await onEvents(error, events);+ completed.resolve();+ });+ tail = work;+ return work;+ },+ options,+ platform,+ ),+ );+ const startWatcher = fileWatcher.startWatcher;+ const watcherSpy = vi+ .spyOn(fileWatcher, 'startWatcher')+ .mockImplementation((root, onDiskEvent, filter, options) =>+ startWatcher(root, onDiskEvent, filter, { ...options, forceBackend: 'parcel' }),+ );+ registerCleanup(async () => {+ closing = true;+ for (const batch of batches) batch.release();+ await runTeardownPhases(+ () => tail,+ async () => {+ await server?.cleanup();+ },+ () => {+ watcherSpy.mockRestore();+ subscriptionSpy.mockRestore();+ },+ () => rmSync(contentDir, { recursive: true, force: true }),+ );+ });+ server = await createTestServer({ contentDir });+ const running = server;+ expect(readdirSync(join(contentDir, 'child')).filter((name) => /\.mdx?$/.test(name))).toEqual([+ 'note.md',+ ]);++ const nextBatch = (kind: BatchKind): Promise<HeldBatch> => {+ const batch = batches.find((candidate) => candidate.kind === kind && !candidate.claimed);+ if (batch) {+ batch.claimed = true;+ return Promise.resolve(batch);+ }+ return new Promise((resolve) => waiting.push({ kind, resolve }));+ };+ const post = (route: string, body: unknown) =>+ fetch(`${running.baseUrl}${route}`, {+ method: 'POST',+ headers: { 'Content-Type': 'application/json' },+ body: JSON.stringify(body),+ });+ return {+ filePath,+ nextBatch,+ async snapshot() {+ const response = await fetch(`${running.baseUrl}/api/pages`);+ expect(response.status).toBe(200);+ const pages = PagesSuccessSchema.parse(await response.json());+ expect(existsSync(join(contentDir, assetPath))).toBe(true);+ return {+ disk: existsSync(filePath) ? readFileSync(filePath, 'utf-8') : null,+ indexed: pages.pages.some((page) => page.docName === docName),+ assetAdmitted: !running.instance.contentFilter.isExcluded(assetPath),+ };+ },+ async deleteThroughApi() {+ expect((await post('/api/delete-path', { kind: 'file', path: docName })).status).toBe(200);+ },+ async createThroughApi() {+ expect((await post('/api/create-page', { path: `${docName}.md` })).status).toBe(200);+ },+ };+}diff --git a/packages/app/tests/integration/stale-removal-coverage.test.ts b/packages/app/tests/integration/stale-removal-coverage.test.tsnew file mode 100644index 000000000..ce56dae61--- /dev/null+++ b/packages/app/tests/integration/stale-removal-coverage.test.ts@@ -0,0 +1,114 @@+import { unlinkSync, writeFileSync } from 'node:fs';+import type { Extension } from '@hocuspocus/server';+import { afterEach, expect, test } from 'vitest';+import { runTeardownPhases } from '../stress/_helpers/teardown-fs.ts';+import { startDeletionScenario } from './stale-delete-recreation.test-helper.ts';+import { startAssetAdmissionScenario } from './stale-removal-coverage.test-helper.ts';++const cleanups: Array<() => Promise<void> | void> = [];+const registerCleanup = (cleanup: () => Promise<void> | void) => {+ cleanups.push(cleanup);+};++afterEach(async () => {+ await runTeardownPhases(...cleanups.splice(0).reverse());+});++test('a stale native deletion preserves sibling asset admission after external recreation', async () => {+ const scenario = await startAssetAdmissionScenario(registerCleanup);+ expect((await scenario.snapshot()).assetAdmitted).toBe(true);+ unlinkSync(scenario.filePath);+ const deletion = await scenario.nextBatch('delete');+ writeFileSync(scenario.filePath, '# Recreated Markdown\n');+ deletion.release();+ await deletion.completed;+ const after = await scenario.snapshot();+ expect(after.assetAdmitted).toBe(true);+ expect(after).toMatchObject({ disk: '# Recreated Markdown\n', indexed: true });+});++test('deleting the last Markdown removes sibling asset admission', async () => {+ const scenario = await startAssetAdmissionScenario(registerCleanup);+ expect((await scenario.snapshot()).assetAdmitted).toBe(true);+ unlinkSync(scenario.filePath);+ const deletion = await scenario.nextBatch('delete');+ deletion.release();+ await deletion.completed;+ expect(await scenario.snapshot()).toEqual({ disk: null, indexed: false, assetAdmitted: false });+});++test('an API recreation and final deletion leave sibling asset counts balanced', async () => {+ const scenario = await startAssetAdmissionScenario(registerCleanup);+ expect((await scenario.snapshot()).assetAdmitted).toBe(true);+ await scenario.deleteThroughApi();+ const oldDeletion = await scenario.nextBatch('delete');+ await scenario.createThroughApi();+ const recreation = await scenario.nextBatch('write');+ oldDeletion.release();+ await oldDeletion.completed;+ expect((await scenario.snapshot()).assetAdmitted).toBe(true);+ recreation.release();+ await recreation.completed;+ expect(await scenario.snapshot()).toEqual({ disk: '', indexed: true, assetAdmitted: true });++ await scenario.deleteThroughApi();+ const finalDeletion = await scenario.nextBatch('delete');+ expect(finalDeletion).not.toBe(oldDeletion);+ finalDeletion.release();+ await finalDeletion.completed;+ expect(await scenario.snapshot()).toEqual({ disk: null, indexed: false, assetAdmitted: false });+});++test('a recreation admitted during the final unload hook survives the older removal', async () => {+ const scenario = await startDeletionScenario('none', registerCleanup);+ await scenario.open();+ const reached = Promise.withResolvers<void>();+ const release = Promise.withResolvers<void>();+ let held = false;+ const extension: Extension = {+ async beforeUnloadDocument({ documentName }) {+ if (documentName !== scenario.docName || held) return;+ held = true;+ reached.resolve();+ await release.promise;+ },+ };+ const extensions = scenario.server.instance.hocuspocus.configuration.extensions;+ extensions.push(extension);+ registerCleanup(() => {+ const index = extensions.indexOf(extension);+ if (index !== -1) extensions.splice(index, 1);+ });+ registerCleanup(() => release.resolve());+ scenario.deleteExternally();+ await reached.promise;+ const response = await fetch(`${scenario.server.baseUrl}/api/create-page`, {+ method: 'POST',+ headers: { 'Content-Type': 'application/json' },+ body: JSON.stringify({ path: `${scenario.docName}.md` }),+ });+ expect(response.status).toBe(200);+ expect(await response.json()).toEqual({ docName: scenario.docName });+ const fresh = await scenario.open();+ expect(await scenario.snapshot(fresh)).toMatchObject({+ disk: '',+ residentPresent: true,+ lifecycleStatus: null,+ clientLifecycleStatus: null,+ clientClosed: false,+ removedCount: 0,+ listed: true,+ });++ release.resolve();+ await scenario.finishDeletion();+ expect(await scenario.snapshot(fresh)).toMatchObject({+ disk: '',+ residentPresent: true,+ lifecycleStatus: null,+ clientLifecycleStatus: null,+ clientClosed: false,+ removedCount: 0,+ listed: true,+ });+});diff --git a/packages/app/tests/integration/stale-resolved-delete-retirement.test.ts b/packages/app/tests/integration/stale-resolved-delete-retirement.test.tsnew file mode 100644index 000000000..3022b4856--- /dev/null+++ b/packages/app/tests/integration/stale-resolved-delete-retirement.test.ts@@ -0,0 +1,82 @@+import { existsSync, readFileSync, writeFileSync } from 'node:fs';+import { join } from 'node:path';+import { afterEach, expect, test, vi } from 'vitest';+import { ConflictAuthority } from '../../../server/src/conflict-authority.ts';+import { DerivedDocumentIndex } from '../../../server/src/derived-document-index.ts';+import { runTeardownPhases } from '../stress/_helpers/teardown-fs.ts';+import { startDeletionScenario } from './stale-delete-recreation.test-helper.ts';++const deleteIndex = DerivedDocumentIndex.prototype.recordDiskDelete;+const cleanups: Array<() => Promise<void> | void> = [];+const registerCleanup = (cleanup: () => Promise<void> | void) => {+ cleanups.push(cleanup);+};++afterEach(async () => {+ await runTeardownPhases(...cleanups.splice(0).reverse());+});++test('a resolved deletion retires its unadopted old resident while preserving a recreated path', async () => {+ const scenario = await startDeletionScenario('none', registerCleanup);+ await scenario.open();+ const resident = scenario.server.instance.hocuspocus.documents.get(scenario.docName);+ if (!resident) throw new Error('Expected the original resident');+ const file = `${scenario.docName}.md`;+ const path = join(scenario.server.contentDir, file);+ const post = (route: string, body: unknown) =>+ fetch(`${scenario.server.baseUrl}${route}`, {+ method: 'POST',+ headers: { 'Content-Type': 'application/json' },+ body: JSON.stringify(body),+ });+ const raised = Promise.withResolvers<void>();+ const raiseConflict = ConflictAuthority.prototype.raise;+ const conflictSpy = vi.spyOn(ConflictAuthority.prototype, 'raise').mockImplementation(function (+ this: ConflictAuthority,+ input,+ ) {+ raiseConflict.call(this, input);+ if (input.kind === 'reconcile' && input.file === file) raised.resolve();+ });+ registerCleanup(() => conflictSpy.mockRestore());+ writeFileSync(path, '# Doc\n\n<<<<<<< ours\nalpha-ours\n=======\nalpha-theirs\n>>>>>>> theirs\n');+ await raised.promise;++ const reached = Promise.withResolvers<void>();+ const release = Promise.withResolvers<void>();+ const indexSpy = vi+ .spyOn(DerivedDocumentIndex.prototype, 'recordDiskDelete')+ .mockImplementation(async function (this: DerivedDocumentIndex, name) {+ await deleteIndex.call(this, name);+ if (name === scenario.docName) {+ reached.resolve();+ await release.promise;+ }+ });+ registerCleanup(() => indexSpy.mockRestore());+ const resolving = post('/api/sync/resolve-conflict', { file, strategy: 'delete' });+ registerCleanup(async () => {+ release.resolve();+ await resolving;+ });+ await reached.promise;+ expect(existsSync(path)).toBe(false);+ const created = await post('/api/create-page', { path: file });+ expect(created.status).toBe(200);+ expect(await created.json()).toEqual({ docName: scenario.docName });+ expect(readFileSync(path, 'utf8')).toBe('');+ expect(scenario.server.instance.hocuspocus.documents.get(scenario.docName)).toBe(resident);+ expect(resident.getMap('lifecycle').get('status')).toBeUndefined();+ expect(resident.getText('source').toString()).not.toBe('');++ release.resolve();+ expect((await resolving).status).toBe(200);+ expect(resident.isDestroyed).toBe(true);+ expect(scenario.server.instance.hocuspocus.documents.get(scenario.docName)).not.toBe(resident);+ expect(readFileSync(path, 'utf8')).toBe('');+ const response = await fetch(+ `${scenario.server.baseUrl}/api/document?docName=${scenario.docName}`,+ );+ expect(response.status).toBe(200);+ expect(await response.json()).toMatchObject({ content: '' });+});diff --git a/packages/app/tests/integration/stale-template-move.test.ts b/packages/app/tests/integration/stale-template-move.test.tsnew file mode 100644index 000000000..2ac9046f5--- /dev/null+++ b/packages/app/tests/integration/stale-template-move.test.ts@@ -0,0 +1,278 @@+import {+ existsSync,+ mkdirSync,+ mkdtempSync,+ readFileSync,+ realpathSync,+ rmSync,+ unlinkSync,+ writeFileSync,+} from 'node:fs';+import { tmpdir } from 'node:os';+import { join } from 'node:path';+import type { Document } from '@hocuspocus/server';+import { afterEach, expect, test, vi } from 'vitest';+import * as templateWrites from '../../../server/src/content/templates-write.ts';+import * as fileWatcher from '../../../server/src/file-watcher.ts';+import * as parcelSubscription from '../../../server/src/parcel-subscription.ts';+import { runTeardownPhases } from '../stress/_helpers/teardown-fs.ts';+import {+ createTestClient,+ createTestServer,+ type TestClient,+ type TestServer,+} from './test-harness.ts';++const SOURCE = '---\ntemplate:\n title: Before move\n---\n\n# Template body\n';+const UPDATED = '---\ntemplate:\n title: After move\n---\n\n# Template body\n';+const OLD_DESTINATION = '# Old destination\n';+const cleanups: Array<() => Promise<void>> = [];++afterEach(async () => {+ await runTeardownPhases(...cleanups.splice(0).reverse());+});++type MetadataChange = 'updated' | 'absent' | 'refused';+type DestinationState = 'fresh' | 'obsolete';+type ResidentPair = { source: Document | undefined; destination: Document | undefined };++async function startTemplateMoveScenario(+ destinationState: DestinationState,+ metadataChange: MetadataChange,+) {+ const contentDir = realpathSync(mkdtempSync(join(tmpdir(), 'ok-stale-template-move-')));+ const fromName = 'source/.ok/templates/before';+ const toName = 'destination/.ok/templates/after';+ const fromPath = join(contentDir, `${fromName}.md`);+ const toPath = join(contentDir, `${toName}.md`);+ mkdirSync(join(contentDir, 'source', '.ok', 'templates'), { recursive: true });+ mkdirSync(join(contentDir, 'destination', '.ok', 'templates'), { recursive: true });+ writeFileSync(fromPath, SOURCE, { flag: 'wx' });+ writeFileSync(toPath, OLD_DESTINATION, { flag: 'wx' });+ const deleted = Promise.withResolvers<void>();+ const releaseDelete = Promise.withResolvers<void>();+ const deleteCompleted = Promise.withResolvers<void>();+ const followers = Promise.withResolvers<void>();+ const moved = Promise.withResolvers<void>();+ const releaseMove = Promise.withResolvers<void>();+ const clients: TestClient[] = [];+ const restores: Array<() => void> = [];+ let sawDelete = false;+ let rawTail = Promise.resolve();+ let server: TestServer | undefined;+ let pendingMove: Promise<Response> | undefined;+ let moveStart: ResidentPair | undefined;+ let moveCompleted: ResidentPair | undefined;+ let clientClosed = false;++ const subscribe = parcelSubscription.subscribeParcel;+ const subscriptionSpy = vi+ .spyOn(parcelSubscription, 'subscribeParcel')+ .mockImplementation((parcel, dir, onEvents, options, platform) =>+ subscribe(+ parcel,+ dir,+ (error, events) => {+ if (!dir.startsWith(contentDir)) return onEvents(error, events);+ const selected =+ !sawDelete && events.some((event) => event.type === 'delete' && event.path === toPath);+ const follower = sawDelete;+ if (selected) sawDelete = true;+ const work = rawTail.then(async () => {+ if (follower) await followers.promise;+ await onEvents(error, events);+ if (selected) deleteCompleted.resolve();+ });+ rawTail = work;+ return work;+ },+ options,+ platform,+ ),+ );+ restores.push(() => subscriptionSpy.mockRestore());+ const startWatcher = fileWatcher.startWatcher;+ const watcherSpy = vi+ .spyOn(fileWatcher, 'startWatcher')+ .mockImplementation((root, onDiskEvent, filter, options) =>+ startWatcher(+ root,+ async (event) => {+ if (event.kind === 'delete' && event.docName === toName) {+ deleted.resolve();+ await releaseDelete.promise;+ }+ await onDiskEvent(event);+ },+ filter,+ { ...options, forceBackend: 'parcel' },+ ),+ );+ restores.push(() => watcherSpy.mockRestore());+ const move = templateWrites.applyTemplateMove;+ const moveSpy = vi+ .spyOn(templateWrites, 'applyTemplateMove')+ .mockImplementation(async (input) => {+ if (input.projectDir !== contentDir) return move(input);+ moveStart = {+ source: server?.instance.hocuspocus.documents.get(fromName),+ destination: server?.instance.hocuspocus.documents.get(toName),+ };+ const result = await move(input);+ if (result.ok) {+ moveCompleted = {+ source: server?.instance.hocuspocus.documents.get(fromName),+ destination: server?.instance.hocuspocus.documents.get(toName),+ };+ moved.resolve();+ await releaseMove.promise;+ }+ return result;+ });+ restores.push(() => moveSpy.mockRestore());++ cleanups.push(async () => {+ releaseDelete.resolve();+ releaseMove.resolve();+ followers.resolve();+ await runTeardownPhases(+ async () => {+ await pendingMove;+ },+ () => rawTail,+ ...clients.map((client) => () => client.cleanup()),+ async () => {+ await server?.cleanup();+ },+ () => {+ for (const restore of restores.reverse()) restore();+ },+ () => rmSync(contentDir, { recursive: true, force: true }),+ );+ });++ server = await createTestServer({ contentDir });+ const running = server;+ const source = await createTestClient(running.port, fromName, { resetOnCleanup: false });+ clients.push(source);+ const sourceResident = running.instance.hocuspocus.documents.get(fromName);+ expect(sourceResident).toBeDefined();+ expect(source.ytext.toString()).toBe(SOURCE);+ let destination: TestClient | undefined;+ if (destinationState === 'obsolete') {+ destination = await createTestClient(running.port, toName, { resetOnCleanup: false });+ clients.push(destination);+ expect(destination.ytext.toString()).toBe(OLD_DESTINATION);+ }+ const priorDestination = running.instance.hocuspocus.documents.get(toName);+ if (destinationState === 'fresh') expect(priorDestination).toBeUndefined();+ else expect(priorDestination).toBeDefined();+ unlinkSync(toPath);+ await deleted.promise;++ const metadata =+ metadataChange === 'absent'+ ? {}+ : { frontmatter: { title: metadataChange === 'updated' ? 'After move' : '' } };+ pendingMove = fetch(`${running.baseUrl}/api/template`, {+ method: 'POST',+ headers: { 'Content-Type': 'application/json' },+ body: JSON.stringify({+ fromFolder: 'source',+ fromName: 'before',+ toFolder: 'destination',+ toName: 'after',+ ...metadata,+ }),+ });+ await moved.promise;+ expect(moveStart).toBeDefined();+ expect(moveCompleted).toBeDefined();+ expect(moveStart?.source).toBeUndefined();+ expect(moveCompleted?.source).toBeUndefined();+ expect(moveStart?.destination).toBe(priorDestination);+ expect(moveCompleted?.destination).toBe(priorDestination);+ expect(running.instance.hocuspocus.documents.has(fromName)).toBe(false);+ expect(existsSync(fromPath)).toBe(false);+ expect(readFileSync(toPath, 'utf8')).toBe(SOURCE);++ if (destinationState === 'fresh') {+ destination = await createTestClient(running.port, toName, { resetOnCleanup: false });+ clients.push(destination);+ }+ if (!destination) throw new Error('Destination client was not admitted');+ destination.provider.on('close', () => {+ clientClosed = true;+ });+ const destinationResident = running.instance.hocuspocus.documents.get(toName);+ expect(destinationResident).toBeDefined();+ expect(destinationResident).not.toBe(sourceResident);+ if (destinationState === 'fresh') expect(destinationResident).not.toBe(priorDestination);+ else expect(destinationResident).toBe(priorDestination);+ const residentBytes = destinationState === 'fresh' ? SOURCE : OLD_DESTINATION;+ expect(destination.ytext.toString()).toBe(residentBytes);+ releaseMove.resolve();+ const response = await pendingMove;+ const responseBody: unknown = await response.json();+ if (metadataChange === 'refused') {+ expect(response.status).toBe(400);+ expect(responseBody).toMatchObject({+ type: 'urn:ok:error:invalid-request',+ detail: 'TEMPLATE_TITLE_REQUIRED',+ });+ } else {+ expect(response.status, JSON.stringify(responseBody)).toBe(200);+ expect(responseBody).toMatchObject({+ from: 'source/.ok/templates/before.md',+ to: 'destination/.ok/templates/after.md',+ });+ }+ const published = metadataChange === 'updated' ? UPDATED : SOURCE;+ expect(readFileSync(toPath, 'utf8')).toBe(published);+ expect(running.instance.hocuspocus.documents.get(toName)).toBe(destinationResident);+ expect(destinationResident?.getText('source').toString()).toBe(residentBytes);+ expect(clientClosed).toBe(false);++ return {+ async finishDeletion() {+ releaseDelete.resolve();+ await deleteCompleted.promise;+ return {+ residentPreserved:+ running.instance.hocuspocus.documents.get(toName) === destinationResident,+ lifecycleStatus: destinationResident?.getMap('lifecycle').get('status') ?? null,+ residentContent: destinationResident?.getText('source').toString() ?? null,+ clientClosed,+ disk: readFileSync(toPath, 'utf8'),+ };+ },+ };+}++test.each([+ ['metadata succeeds', 'updated', UPDATED],+ ['metadata is omitted', 'absent', SOURCE],+ ['metadata is refused after placement', 'refused', SOURCE],+] as const)(+ 'a moved template keeps its newly admitted destination after an older delete when %s',+ async (_label, metadataChange, expectedDisk) => {+ const scenario = await startTemplateMoveScenario('fresh', metadataChange);+ expect(await scenario.finishDeletion()).toEqual({+ residentPreserved: true,+ lifecycleStatus: null,+ residentContent: SOURCE,+ clientClosed: false,+ disk: expectedDisk,+ });+ },+);++test('a moved template retires the unchanged old destination resident after an older delete', async () => {+ const scenario = await startTemplateMoveScenario('obsolete', 'updated');+ expect(await scenario.finishDeletion()).toMatchObject({+ residentPreserved: false,+ lifecycleStatus: 'deleted-upstream',+ residentContent: OLD_DESTINATION,+ disk: UPDATED,+ });+});diff --git a/packages/server/src/api-extension.ts b/packages/server/src/api-extension.tsindex 9f134cd06..6127ff30a 100644--- a/packages/server/src/api-extension.ts+++ b/packages/server/src/api-extension.ts@@ -1190,10 +1190,11 @@ function createCaseOnlyRenameTempPath(sourcePath: string): string {throw new Error('Unable to allocate temporary path for case-only rename');}-function writeFileIfContentDiffers(filePath: string, content: string): void {+function writeFileIfContentDiffers(filePath: string, content: string): boolean {const current = existsSync(filePath) ? readFileSync(filePath, 'utf-8') : null;- if (current === content) return;+ if (current === content) return false;tracedWriteFileSync(filePath, content, 'utf-8');+ return true;}function renamePathOnDisk(sourcePath: string, destinationPath: string): void {@@ -1348,6 +1349,10 @@ interface ApiExtensionBaseOptions {contentFilter?: ContentFilter;installedAgentsProbe?: (scheme: InstalledAgentScheme) => Promise<boolean>;forceUnloadDocument?: (document: Document) => Promise<void>;+ onNamespaceCreateAccepted?: (docName: string) => void;+ captureNamespaceMove?: (docName: string) => () => void;+ capturePublicationReceipt?: (document: Document, docName: string) => (() => void) | undefined;+ onResidentAdopted?: (document: Document, docName: string) => void;ready?: Promise<void>;recentlyRemovedDocs?: RecentlyRemovedDocs;serializeDoc?: (docName: string) => string | null;@@ -1576,6 +1581,10 @@ export function createApiExtension(options: ApiExtensionOptions): Extension & {contentFilter,installedAgentsProbe,forceUnloadDocument,+ onNamespaceCreateAccepted,+ captureNamespaceMove,+ capturePublicationReceipt,+ onResidentAdopted,ready,recentlyRemovedDocs,serializeDoc,@@ -2378,7 +2387,8 @@ export function createApiExtension(options: ApiExtensionOptions): Extension & {}function reviveFrozenResident(docName: string, content: string): void {- reviveFrozenResidentDoc(+ const document = hocuspocus.documents.get(docName);+ const result = reviveFrozenResidentDoc(durabilityState,hocuspocus,docName,@@ -2387,6 +2397,7 @@ export function createApiExtension(options: ApiExtensionOptions): Extension & {undefined,getBridgeLossReporter?.(),);+ if (result === 'revived' && document) onResidentAdopted?.(document, docName);}function syncRenamedDocsToDisk(@@ -2449,12 +2460,20 @@ export function createApiExtension(options: ApiExtensionOptions): Extension & {return readFileSync(filePath, 'utf-8');}- function writeManagedRenameDocumentToDisk(docName: string, markdown: string): void {+ function writeManagedRenameDocumentToDisk(+ docName: string,+ markdown: string,+ document: Document | undefined,+ ): void {+ const publicationReceipt = document+ ? capturePublicationReceipt?.(document, docName)+ : undefined;const filePath = resolveContentEntryPath(contentDir, 'file', docName);tracedMkdirSync(dirname(filePath), { recursive: true });- writeFileIfContentDiffers(filePath, markdown);+ const published = writeFileIfContentDiffers(filePath, markdown);registerWrite(filePath, contentHash(markdown));durabilityState.setReconciledBase(docName, markdown);+ if (published) publicationReceipt?.();mutateFileIndex?.({ kind: 'update', path: filePath, docName, content: markdown });}@@ -2582,7 +2601,7 @@ export function createApiExtension(options: ApiExtensionOptions): Extension & {? applyAssetRenamesToLoadedDocument(pending.docName, renamedAssets): pending;if (rewritten.rewrites === 0) continue;- writeManagedRenameDocumentToDisk(pending.docName, rewritten.markdown);+ writeManagedRenameDocumentToDisk(pending.docName, rewritten.markdown, document);derivedMutations.push({kind: 'link-rewrite',documentName: pending.docName,@@ -3160,7 +3179,7 @@ export function createApiExtension(options: ApiExtensionOptions): Extension & {);if (rewritten.rewrites > 0) {- writeManagedRenameDocumentToDisk(docName, rewritten.markdown);+ writeManagedRenameDocumentToDisk(docName, rewritten.markdown, document);rewrittenDocs.push({ docName, rewrites: rewritten.rewrites });}@@ -3189,6 +3208,7 @@ export function createApiExtension(options: ApiExtensionOptions): Extension & {const rootSourcePath = resolveContentEntryPath(contentDir, kind, fromPath);const rootDestinationPath = resolveContentEntryPath(contentDir, kind, toPath);+ const acceptedMoves = affectedDocs.map(({ to }) => captureNamespaceMove?.(to));const renamedWithGit = await renameTrackedPathInGit(projectDir,rootSourcePath,@@ -3197,6 +3217,7 @@ export function createApiExtension(options: ApiExtensionOptions): Extension & {if (!renamedWithGit) {renamePathOnDisk(rootSourcePath, rootDestinationPath);}+ for (const acceptMove of acceptedMoves) acceptMove?.();if (kind === 'folder') {renameFolderIndexEntries(fromPath, toPath);}@@ -4161,7 +4182,10 @@ export function createApiExtension(options: ApiExtensionOptions): Extension & {? (docName) => recentlyRemovedDocs.delete(docName): undefined,mutateFileIndexCreate: mutateFileIndex- ? ({ path, docName, content }) => mutateFileIndex({ kind: 'create', path, docName, content })+ ? ({ path, docName, content }) => {+ onNamespaceCreateAccepted?.(docName);+ mutateFileIndex({ kind: 'create', path, docName, content });+ }: undefined,recordDerivedDocumentBestEffort,recordDerivedMutationsBestEffort,@@ -5385,6 +5409,7 @@ export function createApiExtension(options: ApiExtensionOptions): Extension & {respondStaleExternalWrite,registerWrittenDocInFileIndex,captureAndCloseDocuments,+ captureNamespaceMove,renameTrackedPathInGit,renamePathOnDisk,splitContentPath,diff --git a/packages/server/src/file-watcher.ts b/packages/server/src/file-watcher.tsindex af8a72172..2130e15b5 100644--- a/packages/server/src/file-watcher.ts+++ b/packages/server/src/file-watcher.ts@@ -25,7 +25,7 @@ import {SUPPORTED_DOC_EXTENSIONS,stripDocExtension,} from './doc-extensions.ts';-import { resolvesIntoPrivateState } from './fs-safety.ts';+import { canonicalContentPathIsRefused, resolvesIntoPrivateState } from './fs-safety.ts';import { classifyFsPath, normalizeFsPath } from './fs-traced.ts';import { errnoCode } from './http/handler-utils.ts';import { getLogger } from './logger.ts';@@ -637,6 +637,95 @@ type KnownDeletion = {type RecoveredFileEvent = { type: 'create' | 'update'; path: string } | KnownDeletion;type InternalRawFileEvent = RawFileEvent | KnownDeletion;+type MarkdownRemovalOrigin =+ | { kind: 'native'; docName: string; path: string }+ | {+ kind: 'recovery';+ docName: string;+ previousPath: string;+ isCurrent: () => boolean;+ };++type MarkdownRemovalObservation = {+ isCurrent: () => boolean;+ discarded: boolean;+};++const removalOrigins = new WeakMap<RawFileEvent, MarkdownRemovalOrigin>();+const removalObservations = new WeakMap<MarkdownDiskEvent, MarkdownRemovalObservation>();++export function getMarkdownRemovalCurrentness(+ event: Extract<DiskEvent, { kind: 'delete' | 'rename' }>,+): (() => boolean) | undefined {+ return removalObservations.get(event)?.isCurrent;+}++export function hasAdmittedMarkdownFile(+ docName: string,+ filePath: string,+ contentDir: string,+ contentFilter: Pick<ContentFilter, 'isExcluded'> | undefined,+): boolean {+ try {+ if (!statSync(filePath).isFile()) return false;+ if (!readdirSync(dirname(filePath)).includes(basename(filePath))) return false;+ if (contentFilter?.isExcluded(toPosix(relative(contentDir, filePath))) === true) return false;+ return !canonicalContentPathIsRefused(filePath, realpathSync(filePath), contentDir);+ } catch (err) {+ const code = errnoCode(err);+ if (code === undefined) {+ log.error(+ { err, docName, path: filePath },+ `[file-watcher] markdown binding for ${docName} could not be checked; treating it as absent`,+ );+ } else if (code !== 'ENOENT' && code !== 'ENOTDIR') {+ log.warn(+ { err, docName, path: filePath, code },+ `[file-watcher] markdown binding for ${docName} could not be inspected (${code}); treating it as absent`,+ );+ }+ return false;+ }+}++function observeMarkdownRemoval(+ origin: MarkdownRemovalOrigin,+ contentDir: string,+ fileIndex: ReadonlyMap<string, FileIndexEntry>,+ contentFilter: ContentFilter | undefined,+ aliasPaths: ReadonlyMap<string, string> | undefined,+): MarkdownRemovalObservation {+ const observation: MarkdownRemovalObservation = {+ discarded: false,+ isCurrent: () => {+ if (observation.discarded) return false;+ const owner = fileIndex.get(origin.docName);+ if (origin.kind === 'recovery') {+ const current =+ origin.isCurrent() &&+ (owner === undefined || owner.canonicalPath === origin.previousPath);+ observation.discarded = !current;+ return current;+ }+ const effectivePath = join(contentDir, `${origin.docName}${getDocExtension(origin.docName)}`);+ const currentPath =+ owner?.kind === 'markdown'+ ? owner.canonicalPath+ : (aliasPaths?.get(origin.docName) ??+ (existsSync(effectivePath) ? effectivePath : origin.path));+ const current = !hasAdmittedMarkdownFile(+ origin.docName,+ currentPath,+ contentDir,+ contentFilter,+ );+ observation.discarded ||= !current;+ return current;+ },+ };+ return observation;+}+export async function classifyEvents(rawEvents: RawFileEvent[],contentDir: string,@@ -667,6 +756,7 @@ async function classifyEventsInternal(fileIndex: ReadonlyMap<string, FileIndexEntry>;aliasPaths: ReadonlyMap<string, string>;canonicalRecords: Set<RawFileEvent>;+ removals: ReadonlyMap<RawFileEvent, MarkdownRemovalObservation>;},): Promise<MarkdownDiskEvent[]> {const deletes: RawFileEvent[] = [];@@ -817,6 +907,7 @@ async function classifyEventsInternal(const pairedDeletes = new Set<string>();for (const del of deletes) {+ if (aliasContext?.removals.get(del)?.isCurrent() === false) continue;const deletedHash = removeLastKnownHash(del.path);if (!deletedHash) continue;@@ -829,14 +920,17 @@ async function classifyEventsInternal(pairedCreates.add(create.path);pairedDeletes.add(del.path);updateLastKnownHash(create.path, hash);- results.push({+ const renamed: MarkdownDiskEvent = {kind: 'rename',oldPath: del.path,newPath: create.path,oldDocName: resolveDocName(del),newDocName: resolveDocName(create),content,- });+ };+ const observation = aliasContext?.removals.get(del);+ if (observation) removalObservations.set(renamed, observation);+ results.push(renamed);break;}}@@ -844,12 +938,16 @@ async function classifyEventsInternal(for (const del of deletes) {if (pairedDeletes.has(del.path)) continue;- removeLastKnownHash(del.path);- results.push({+ const observation = aliasContext?.removals.get(del);+ if (observation?.isCurrent() !== false) removeLastKnownHash(del.path);+ const origin = removalOrigins.get(del);+ const event: MarkdownDiskEvent = {kind: 'delete',path: del.path,- docName: resolveDocName(del),- });+ docName: origin?.kind === 'recovery' ? origin.docName : resolveDocName(del),+ };+ if (observation) removalObservations.set(event, observation);+ results.push(event);}for (const create of creates) {@@ -2515,6 +2613,19 @@ async function handleRawEventsInternal(}const canonicalAliasRecords = new Set<RawFileEvent>();+ const removals = new Map<RawFileEvent, MarkdownRemovalObservation>();+ for (const event of mdEvents) {+ if (event.type !== 'delete') continue;+ const origin = removalOrigins.get(event) ?? {+ kind: 'native',+ docName: pathToDocName(event.path, contentDir),+ path: event.path,+ };+ removals.set(+ event,+ observeMarkdownRemoval(origin, contentDir, fileIndex, contentFilter, aliasPaths),+ );+ }const observations = new Map<string, OwnWriteObservation>();const admittedMarkdownPaths = new Set<string>();try {@@ -2526,9 +2637,12 @@ async function handleRawEventsInternal(observations,contentFilter,aliasMap,- aliasPaths- ? { fileIndex, aliasPaths, canonicalRecords: canonicalAliasRecords }- : undefined,+ {+ fileIndex,+ aliasPaths: aliasPaths ?? new Map(),+ canonicalRecords: canonicalAliasRecords,+ removals,+ },): [];@@ -2584,10 +2698,24 @@ async function handleRawEventsInternal(event.kind === 'update' && previousIndexedFields !== undefined? { ...event, previousIndexedFields }: event;+ const staleNamespace =+ (event.kind === 'delete' || event.kind === 'rename') &&+ getMarkdownRemovalCurrentness(event)?.() === false;- updateFileIndex(indexEvent, fileIndex);+ if (!staleNamespace) updateFileIndex(indexEvent, fileIndex);+ else if (indexEvent.kind === 'rename') {+ updateFileIndex(+ {+ kind: 'create',+ path: indexEvent.newPath,+ docName: indexEvent.newDocName,+ content: indexEvent.content,+ },+ fileIndex,+ );+ }- if (contentFilter && !isSelf) {+ if (contentFilter && !isSelf && !staleNamespace) {switch (event.kind) {case 'create':contentFilter.incrementMdDir(dirname(event.docName));@@ -2630,6 +2758,7 @@ async function handleRawEventsInternal({kind: event.kind,path: event.kind === 'rename' ? event.newPath : event.path,+ staleNamespace,},`[file-watcher] Dispatching: ${event.kind}`,);@@ -2650,6 +2779,9 @@ async function handleRawEventsInternal(}} finally {releaseOwnWriteObservations(observations);+ if ([...removals.values()].some((observation) => observation.discarded)) {+ contentFilter?.rebuildDirCount();+ }}for (const event of folderEvents) {@@ -2948,7 +3080,20 @@ async function recoverDiskChanges()continue;if (current?.canonicalPath !== previous.canonicalPath) {- rawEvents.push({ type: 'delete', path: previous.canonicalPath, entryKind: 'file' });+ const removal: KnownDeletion = {+ type: 'delete',+ path: previous.canonicalPath,+ entryKind: 'file',+ };+ removalOrigins.set(removal, {+ kind: 'recovery',+ docName,+ previousPath: previous.canonicalPath,+ isCurrent: () =>+ !markdownPublicMutationFor(docName, previous) &&+ !(current && markdownPublicMutationFor(docName, current)),+ });+ rawEvents.push(removal);} else if (previousHashes.get(previous.canonicalPath) !== observed.hashes.get(current.canonicalPath)) {diff --git a/packages/server/src/http/folder-template-routes.ts b/packages/server/src/http/folder-template-routes.tsindex 8c10d1713..e67ab36be 100644--- a/packages/server/src/http/folder-template-routes.ts+++ b/packages/server/src/http/folder-template-routes.ts@@ -118,6 +118,7 @@ export interface FolderTemplateRouteDeps {docNames: string[],lifecycleStatus: RemovalLifecycleStatus,) => Promise<Map<string, string>>;+ captureNamespaceMove?: (docName: string) => () => void;renameTrackedPathInGit: (projectDir: string | undefined,sourcePath: string,@@ -162,6 +163,7 @@ export function createFolderTemplateRoutes(deps: FolderTemplateRouteDeps): ApiRorespondStaleExternalWrite,registerWrittenDocInFileIndex,captureAndCloseDocuments,+ captureNamespaceMove,renameTrackedPathInGit,renamePathOnDisk,splitContentPath,@@ -872,6 +874,9 @@ export function createFolderTemplateRoutes(deps: FolderTemplateRouteDeps): ApiRo'renamed',);+ const acceptMove = captureNamespaceMove?.(+ templateDocNameFor(toValidated.folderRel, body.toName),+ );const result = await applyTemplateMove({projectDir: fromValidated.resolvedContentDir,fromFolder: fromValidated.folderRel,@@ -951,6 +956,7 @@ export function createFolderTemplateRoutes(deps: FolderTemplateRouteDeps): ApiRoreturn;}+ acceptMove?.();recentlyRemovedDocs?.setDeleted(templateDocNameFor(fromValidated.folderRel, body.fromName));const movedAbs = resolve(toValidated.resolvedContentDir, result.toPath);diff --git a/packages/server/src/persistence-publication-receipt.test-helper.ts b/packages/server/src/persistence-publication-receipt.test-helper.tsnew file mode 100644index 000000000..4ef65429b--- /dev/null+++ b/packages/server/src/persistence-publication-receipt.test-helper.ts@@ -0,0 +1,156 @@+import {+ existsSync,+ mkdtempSync,+ readFileSync,+ realpathSync,+ rmSync,+ writeFileSync,+} from 'node:fs';+import { tmpdir } from 'node:os';+import { join } from 'node:path';+import { Document } from '@hocuspocus/server';+import { vi } from 'vitest';+import type * as Y from 'yjs';+import { DocumentDurabilityState } from './document-durability-state.ts';+import * as fsTraced from './fs-traced.ts';+import { createPersistenceExtension, type PersistenceOptions } from './persistence.ts';+import { BROWSER_ORIGIN, replaceDocParagraphs } from './persistence-store-overlap.test-helper.ts';++interface PublicationOccurrence {+ readonly document: Y.Doc;+ readonly documentName: string;+ publication: { readonly snapshotSV: Uint8Array } | null;+}++class PublicationAuthority {+ private readonly occurrences = new WeakMap<Y.Doc, PublicationOccurrence>();++ begin(document: Y.Doc, documentName: string): PublicationOccurrence {+ const occurrence: PublicationOccurrence = { document, documentName, publication: null };+ this.occurrences.set(document, occurrence);+ return occurrence;+ }++ canRetire(occurrence: PublicationOccurrence): boolean {+ return occurrence.publication === null;+ }++ capturePublicationReceipt = (+ document: Y.Doc,+ documentName: string,+ snapshotSV: Uint8Array,+ ): (() => void) | undefined => {+ const occurrence = this.occurrences.get(document);+ if (!occurrence || occurrence.documentName !== documentName) return undefined;+ const capturedSV = Uint8Array.from(snapshotSV);+ return () => {+ occurrence.publication = { snapshotSV: capturedSV };+ };+ };+}++export interface ReceiptOptions extends PersistenceOptions {+ capturePublicationReceipt?: PublicationAuthority['capturePublicationReceipt'];+}++export function replaceParagraphs(document: Y.Doc, paragraphs: string[]): void {+ document.transact(() => replaceDocParagraphs(document, paragraphs), BROWSER_ORIGIN);+}++export async function createReceiptFixture(initial = 'initial\n') {+ const contentDir = realpathSync(mkdtempSync(join(tmpdir(), 'ok-publication-receipt-')));+ const docName = 'receipt';+ const docPath = join(contentDir, `${docName}.md`);+ const documents = new Set<Document>();+ const stores = new Set<Promise<void>>();+ const releases = new Set<() => void>();+ const durability = new DocumentDurabilityState();+ const authority = new PublicationAuthority();+ durability.switchReconciledBaseScope('main');+ writeFileSync(docPath, initial, { flag: 'wx' });+ const options: ReceiptOptions = {+ contentDir,+ projectDir: contentDir,+ gitEnabled: false,+ durabilityState: durability,+ capturePublicationReceipt: authority.capturePublicationReceipt,+ };+ const persistence = createPersistenceExtension(options);++ function document(name = docName): Document {+ const result = new Document(name);+ documents.add(result);+ return result;+ }++ const original = document();+ await persistence.extension.onLoadDocument?.({+ document: original,+ documentName: docName,+ context: {},+ } as never);++ function store(target = original, name = docName): Promise<void> {+ const work = persistence.forceStore(target, name);+ stores.add(work);+ void work.then(+ () => stores.delete(work),+ () => stores.delete(work),+ );+ return work;+ }++ function holdNextPublication() {+ const entered = Promise.withResolvers<void>();+ const release = Promise.withResolvers<void>();+ releases.add(release.resolve);+ const write = fsTraced.tracedWriteFile;+ let held = false;+ vi.spyOn(fsTraced, 'tracedWriteFile').mockImplementation(async (...args) => {+ await write(...args);+ if (held || !String(args[0]).startsWith(`${docPath}.tmp.`)) return;+ held = true;+ entered.resolve();+ await release.promise;+ });+ return {+ async waitUntilHeld(work: Promise<void>): Promise<void> {+ await Promise.race([+ entered.promise,+ work.then(() => {+ throw new Error('Store completed before reaching the publication hold');+ }),+ ]);+ },+ release: release.resolve,+ };+ }++ return {+ contentDir,+ docName,+ docPath,+ original,+ persistence,+ durability,+ authority,+ document,+ store,+ holdNextPublication,+ holdWriteAdmission() {+ const release = durability.holdWriteAdmission();+ releases.add(release);+ return release;+ },+ disk: () => readFileSync(docPath, 'utf8'),+ async cleanup() {+ for (const release of releases) release();+ await Promise.allSettled(stores);+ await persistence.flushDeferredStores('discard-stale');+ vi.restoreAllMocks();+ for (const item of documents) item.destroy();+ rmSync(contentDir, { recursive: true, force: true });+ if (existsSync(contentDir)) throw new Error(`Fixture remains: ${contentDir}`);+ },+ };+}diff --git a/packages/server/src/persistence-publication-receipt.test.ts b/packages/server/src/persistence-publication-receipt.test.tsnew file mode 100644index 000000000..8f5f35b95--- /dev/null+++ b/packages/server/src/persistence-publication-receipt.test.ts@@ -0,0 +1,315 @@+import { existsSync, mkdirSync, readdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs';+import { join } from 'node:path';+import { afterEach, beforeEach, describe, expect, test } from 'vitest';+import * as Y from 'yjs';+import { createPersistenceExtension } from './persistence.ts';+import {+ createReceiptFixture,+ type ReceiptOptions,+ replaceParagraphs,+} from './persistence-publication-receipt.test-helper.ts';+import { BROWSER_ORIGIN } from './persistence-store-overlap.test-helper.ts';++describe('successful publication receipt ownership', () => {+ let fixture: Awaited<ReturnType<typeof createReceiptFixture>>;++ beforeEach(async () => {+ fixture = await createReceiptFixture();+ });++ afterEach(async () => {+ await fixture.cleanup();+ });++ test('ordinary publication remains usable without a receipt consumer', async () => {+ const persistence = createPersistenceExtension({+ contentDir: fixture.contentDir,+ projectDir: fixture.contentDir,+ gitEnabled: false,+ durabilityState: fixture.durability,+ });+ replaceParagraphs(fixture.original, ['ordinary publication']);+ await persistence.forceStore(fixture.original, fixture.docName);+ expect(fixture.disk()).toBe('ordinary publication\n');+ expect(fixture.durability.getReconciledBase(fixture.docName)).toBe('ordinary publication\n');+ });++ test.each(['capture', 'completion'] as const)(+ 'a receipt consumer failure during %s does not change publication',+ async (phase) => {+ const fail = () => {+ throw new Error('Receipt consumer cannot accept the occurrence');+ };+ const options: ReceiptOptions = {+ contentDir: fixture.contentDir,+ projectDir: fixture.contentDir,+ gitEnabled: false,+ durabilityState: fixture.durability,+ capturePublicationReceipt: phase === 'capture' ? fail : () => fail,+ };+ const persistence = createPersistenceExtension(options);+ replaceParagraphs(fixture.original, ['published despite consumer failure']);+ await persistence.forceStore(fixture.original, fixture.docName);+ expect(fixture.disk()).toBe('published despite consumer failure\n');+ expect(fixture.durability.getReconciledBase(fixture.docName)).toBe(+ 'published despite consumer failure\n',+ );+ },+ );++ test('real publication retains its captured occurrence only after the bytes land', async () => {+ const { original, authority, docName } = fixture;+ replaceParagraphs(original, ['published']);+ const occurrence = authority.begin(original, docName);+ const expectedSV = Y.encodeStateVector(original);+ const hold = fixture.holdNextPublication();+ const store = fixture.store();+ await hold.waitUntilHeld(store);+ expect(fixture.disk()).toBe('initial\n');+ expect(authority.canRetire(occurrence)).toBe(true);+ hold.release();+ await store;+ expect(fixture.disk()).toBe('published\n');+ expect(authority.canRetire(occurrence)).toBe(false);+ expect(occurrence.publication?.snapshotSV).toEqual(expectedSV);+ });++ test('a reused Document receives a fresh receipt for each successful occurrence', async () => {+ const { original, authority, docName } = fixture;+ replaceParagraphs(original, ['first publication']);+ const first = authority.begin(original, docName);+ await fixture.store();+ expect(fixture.disk()).toBe('first publication\n');+ expect(authority.canRetire(first)).toBe(false);++ replaceParagraphs(original, ['second publication']);+ const second = authority.begin(original, docName);+ expect(second.document).toBe(first.document);+ expect(authority.canRetire(second)).toBe(true);+ await fixture.store();+ expect(fixture.disk()).toBe('second publication\n');+ expect(authority.canRetire(second)).toBe(false);+ expect(second.publication?.snapshotSV).toEqual(Y.encodeStateVector(original));+ });++ test('completion from before same-object reuse cannot retain the replacement occurrence', async () => {+ const { original, authority, docName } = fixture;+ replaceParagraphs(original, ['older publication']);+ const older = authority.begin(original, docName);+ const snapshotSV = Y.encodeStateVector(original);+ const hold = fixture.holdNextPublication();+ const store = fixture.store();+ await hold.waitUntilHeld(store);++ const replacement = authority.begin(original, docName);+ expect(replacement.document).toBe(older.document);+ expect(Y.encodeStateVector(original)).toEqual(snapshotSV);+ hold.release();+ await store;++ expect(fixture.disk()).toBe('older publication\n');+ expect(authority.canRetire(older)).toBe(false);+ expect(authority.canRetire(replacement)).toBe(true);+ expect(older.publication?.snapshotSV).toEqual(snapshotSV);+ });++ test('completion for a replaced Document retains only the original object even with equal vectors', async () => {+ const { original, authority, docName } = fixture;+ replaceParagraphs(original, ['original object publication']);+ const older = authority.begin(original, docName);+ const hold = fixture.holdNextPublication();+ const store = fixture.store();+ await hold.waitUntilHeld(store);++ const replacementDocument = fixture.document();+ Y.applyUpdate(replacementDocument, Y.encodeStateAsUpdate(original));+ const replacement = authority.begin(replacementDocument, docName);+ expect(replacementDocument).not.toBe(original);+ expect(Y.encodeStateVector(replacementDocument)).toEqual(Y.encodeStateVector(original));+ expect(replacementDocument.getText('source').toString()).toBe(+ original.getText('source').toString(),+ );+ hold.release();+ await store;++ expect(fixture.disk()).toBe('original object publication\n');+ expect(authority.canRetire(older)).toBe(false);+ expect(authority.canRetire(replacement)).toBe(true);+ });++ test('an earlier completion cannot retain a later deletion-only mutation with the same vector', async () => {+ const { original, authority, docName } = fixture;+ replaceParagraphs(original, ['published suffix']);+ const older = authority.begin(original, docName);+ const snapshotSV = Y.encodeStateVector(original);+ const hold = fixture.holdNextPublication();+ const store = fixture.store();+ await hold.waitUntilHeld(store);++ original.transact(() => {+ original.getText('source').delete('published'.length, ' suffix'.length);+ const paragraph = original.getXmlFragment('default').get(0);+ if (!(paragraph instanceof Y.XmlElement)) throw new Error('Expected a paragraph');+ const text = paragraph.get(0);+ if (!(text instanceof Y.XmlText)) throw new Error('Expected paragraph text');+ text.delete('published'.length, ' suffix'.length);+ }, BROWSER_ORIGIN);+ const deletion = authority.begin(original, docName);+ expect(original.getText('source').toString()).toBe('published\n');+ expect(Y.encodeStateVector(original)).toEqual(snapshotSV);+ hold.release();+ await store;++ expect(fixture.disk()).toBe('published suffix\n');+ expect(authority.canRetire(older)).toBe(false);+ expect(authority.canRetire(deletion)).toBe(true);+ await fixture.store();+ expect(fixture.disk()).toBe('published\n');+ expect(authority.canRetire(deletion)).toBe(false);+ });++ test('a superseded store cannot retain the occurrence whose bytes never publish', async () => {+ const { original, authority, docName } = fixture;+ replaceParagraphs(original, ['older snapshot']);+ const older = authority.begin(original, docName);+ const hold = fixture.holdNextPublication();+ const firstStore = fixture.store();+ await hold.waitUntilHeld(firstStore);+ expect(fixture.disk()).toBe('initial\n');+ expect(authority.canRetire(older)).toBe(true);++ replaceParagraphs(original, ['newer snapshot']);+ const newer = authority.begin(original, docName);+ await fixture.store();+ expect(fixture.disk()).toBe('newer snapshot\n');+ expect(authority.canRetire(newer)).toBe(false);+ hold.release();+ await firstStore;+ expect(fixture.disk()).toBe('newer snapshot\n');+ expect(authority.canRetire(older)).toBe(true);+ expect(authority.canRetire(newer)).toBe(false);+ });++ test('semantically unchanged Markdown leaves retirement allowed until an actual publication', async () => {+ const { original, authority, docName } = fixture;+ original.transact(() => original.getText('source').delete('initial'.length, 1), BROWSER_ORIGIN);+ const unchanged = authority.begin(original, docName);+ expect(original.getText('source').toString()).toBe('initial');+ await fixture.store();+ expect(fixture.disk()).toBe('initial\n');+ expect(authority.canRetire(unchanged)).toBe(true);++ replaceParagraphs(original, ['changed']);+ const changed = authority.begin(original, docName);+ await fixture.store();+ expect(fixture.disk()).toBe('changed\n');+ expect(authority.canRetire(changed)).toBe(false);+ expect(authority.canRetire(unchanged)).toBe(true);+ });++ test('a phantom write leaves retirement allowed until nonempty content is published', async () => {+ const { authority } = fixture;+ const docName = 'phantom';+ const document = fixture.document(docName);+ const docPath = join(fixture.contentDir, `${docName}.md`);+ const phantom = authority.begin(document, docName);+ expect(document.getText('source').toString()).toBe('');+ await fixture.store(document, docName);+ expect(existsSync(docPath)).toBe(false);+ expect(authority.canRetire(phantom)).toBe(true);++ replaceParagraphs(document, ['now present']);+ const content = authority.begin(document, docName);+ await fixture.store(document, docName);+ expect(readFileSync(docPath, 'utf8')).toBe('now present\n');+ expect(authority.canRetire(content)).toBe(false);+ expect(authority.canRetire(phantom)).toBe(true);+ });++ test('an invalid path refusal cannot retain an occurrence', async () => {+ const { original, authority, durability } = fixture;+ const invalidName = 'invalid\0path';+ replaceParagraphs(original, ['refused bytes']);+ durability.setReconciledBase(invalidName, 'initial\n');+ const refused = authority.begin(original, invalidName);+ const entries = readdirSync(fixture.contentDir).sort();+ await expect(fixture.store(original, invalidName)).rejects.toThrow('Invalid document name');+ expect(authority.canRetire(refused)).toBe(true);+ expect(readdirSync(fixture.contentDir).sort()).toEqual(entries);+ expect(fixture.disk()).toBe('initial\n');++ const valid = authority.begin(original, fixture.docName);+ await fixture.store();+ expect(fixture.disk()).toBe('refused bytes\n');+ expect(authority.canRetire(valid)).toBe(false);+ expect(authority.canRetire(refused)).toBe(true);+ });++ test('a real filesystem failure leaves retirement allowed', async () => {+ const { original, authority, docName } = fixture;+ replaceParagraphs(original, ['unpublished']);+ const occurrence = authority.begin(original, docName);+ rmSync(fixture.docPath);+ mkdirSync(fixture.docPath);+ await expect(fixture.store()).rejects.toThrow();+ expect(readdirSync(fixture.docPath)).toEqual([]);+ expect(authority.canRetire(occurrence)).toBe(true);++ rmSync(fixture.docPath, { recursive: true });+ writeFileSync(fixture.docPath, 'initial\n');+ await fixture.store();+ expect(fixture.disk()).toBe('unpublished\n');+ expect(authority.canRetire(occurrence)).toBe(false);+ });++ test('a deferred store retains the occurrence actually published by the drain', async () => {+ const { original, authority, durability, docName, persistence } = fixture;+ const releaseAdmission = fixture.holdWriteAdmission();+ replaceParagraphs(original, ['queued']);+ const queued = authority.begin(original, docName);+ durability.setBatchInProgress(true);+ await fixture.store();+ expect(fixture.disk()).toBe('initial\n');+ expect(authority.canRetire(queued)).toBe(true);+ const outcome = durability.awaitDeferredStoreOutcome(docName);++ replaceParagraphs(original, ['published by drain']);+ const drained = authority.begin(original, docName);+ durability.setBatchInProgress(false);+ await persistence.flushDeferredStores('within-branch');+ await expect(outcome).resolves.toEqual({ kind: 'completed' });+ releaseAdmission();+ expect(fixture.disk()).toBe('published by drain\n');+ expect(authority.canRetire(drained)).toBe(false);+ expect(authority.canRetire(queued)).toBe(true);+ expect(drained.publication?.snapshotSV).toEqual(Y.encodeStateVector(original));+ });++ test('discarding a deferred store leaves its occurrence eligible for retirement', async () => {+ const { original, authority, durability, docName, persistence } = fixture;+ const releaseAdmission = fixture.holdWriteAdmission();+ replaceParagraphs(original, ['discarded branch edit']);+ const discarded = authority.begin(original, docName);+ durability.setBatchInProgress(true);+ await fixture.store();+ const outcome = durability.awaitDeferredStoreOutcome(docName);+ expect(authority.canRetire(discarded)).toBe(true);+ expect(fixture.disk()).toBe('initial\n');+ durability.switchReconciledBaseScope('other');+ writeFileSync(fixture.docPath, 'other branch\n');+ durability.setBatchInProgress(false);+ await persistence.flushDeferredStores('discard-stale');+ await expect(outcome).resolves.toEqual({ kind: 'discarded' });+ releaseAdmission();+ expect(fixture.disk()).toBe('other branch\n');+ expect(authority.canRetire(discarded)).toBe(true);++ durability.setReconciledBase(docName, 'other branch\n');+ replaceParagraphs(original, ['new branch edit']);+ const current = authority.begin(original, docName);+ await fixture.store();+ expect(fixture.disk()).toBe('new branch edit\n');+ expect(authority.canRetire(current)).toBe(false);+ expect(authority.canRetire(discarded)).toBe(true);+ });+});diff --git a/packages/server/src/persistence.ts b/packages/server/src/persistence.tsindex f815964ef..7580f7b87 100644--- a/packages/server/src/persistence.ts+++ b/packages/server/src/persistence.ts@@ -296,6 +296,11 @@ export interface PersistenceOptions {getPrincipal?: () => Principal | null;onAgentCommit?: () => void;onFlushCommit?: () => void;+ capturePublicationReceipt?: (+ document: Y.Doc,+ documentName: string,+ snapshotSV: Uint8Array,+ ) => (() => void) | undefined;onDiskFlush?: (docName: string,sv: Uint8Array,@@ -1423,6 +1428,19 @@ export function createPersistenceExtension(options?: PersistenceOptions): Persisconst { sv: stateVectorAtRead, json } = captureDocSnapshotForPersistence(document);const ytextSnapshot = document.getText('source').toString();+ let publicationReceipt: (() => void) | undefined;+ try {+ publicationReceipt = options?.capturePublicationReceipt?.(+ document,+ documentName,+ stateVectorAtRead,+ );+ } catch (err) {+ log.warn(+ { err, documentName },+ '[persistence] capturePublicationReceipt callback failed',+ );+ }const storeAttempt = durabilityState.beginStoreAttempt(documentName);const { frontmatter, body } = stripFrontmatter(ytextSnapshot);@@ -1931,6 +1949,14 @@ export function createPersistenceExtension(options?: PersistenceOptions): PersisregisterWrite(canonicalPath, contentHash(markdown));durabilityState.clearStoreFailure(documentName);incrementPersistenceDiskWrite();+ try {+ publicationReceipt?.();+ } catch (err) {+ log.warn(+ { err, documentName },+ '[persistence] capturePublicationReceipt receipt completion failed',+ );+ }try {onDiskFlush?.(documentName, stateVectorAtRead, markdown, currentBase ?? null);} catch (flushErr) {diff --git a/packages/server/src/server-factory.ts b/packages/server/src/server-factory.tsindex 41faea0c5..473e1f4ca 100644--- a/packages/server/src/server-factory.ts+++ b/packages/server/src/server-factory.ts@@ -51,6 +51,7 @@ import {resolveShadowDir,} from '@inkeep/open-knowledge-core/shadow-repo-layout';import simpleGit from 'simple-git';+import type { Doc } from 'yjs';import { AcpPermissionStore } from './acp/permissions.ts';import { AcpRegistry, loadCustomAgents } from './acp/registry.ts';import { AgentFocusBroadcaster } from './agent-focus.ts';@@ -154,6 +155,8 @@ import {assertNeverDiskEvent,contentHash,type DiskEvent,+ getMarkdownRemovalCurrentness,+ hasAdmittedMarkdownFile,pathToDocName,reconcileFileIndexAfterFilterRebuild,registerRemoval,@@ -633,6 +636,7 @@ export function createServer(options: ServerOptions): ServerInstance {throw cause;}contentFilter.decrementMdDir(dirname(docName));+ commentDocHooksRef.current?.deleted(docName);},applyResolvedContent: async (docName, absPath, bytes) => {assertContentPath(absPath);@@ -661,70 +665,110 @@ export function createServer(options: ServerOptions): ServerInstance {}durabilityState.clearStaleExternalWrite(docName);}+ if (strategy === 'delete' && hasAdmittedMarkdownBinding(docName)) {+ log.debug(+ { docName, source: 'resolve-delete', bindingRecreated: true },+ `[reconcile] delete skipped: ${docName} (binding recreated)`,+ );+ return;+ }const document = hocuspocus.documents.get(docName);const lifecycle = document?.getMap('lifecycle');if (strategy === 'delete') {- const isDirty = document- ? rescueUnflushedEditsBeforeTeardown(- docName,- headWatcher?.getLastKnownBranch() ?? 'main',- 'delete',- true,- )- : false;- await derivedDocumentIndex.recordDiskDelete(docName);- log.info(- { docName, isDirty, source: 'resolve-delete' },- `[reconcile] delete: ${docName} (dirty=${isDirty})`,- );- const priorLifecycle = lifecycle- ? {- status: lifecycle.get('status'),- reason: lifecycle.get('reason'),- detectedAt: lifecycle.get('detectedAt'),- }- : undefined;- lifecycle?.set('status', 'deleted-upstream');- lifecycle?.delete('reason');- lifecycle?.delete('detectedAt');- if (document) {- try {- hocuspocus.closeConnections(docName);- await forceUnloadDocument(document);- } catch (cause) {- if (lifecycle && priorLifecycle) {- for (const [key, value] of Object.entries(priorLifecycle)) {- if (value === undefined) lifecycle.delete(key);- else lifecycle.set(key, value);+ const removal = beginRemoval(docName);+ try {+ const isDirty =+ removal.residentCurrent() && document+ ? rescueUnflushedEditsBeforeTeardown(+ docName,+ headWatcher?.getLastKnownBranch() ?? 'main',+ 'delete',+ true,+ )+ : false;+ if (removal.namespaceCurrent()) {+ commentDocHooksRef.current?.deleted(docName);+ await derivedDocumentIndex.recordDiskDelete(docName);+ }+ if (!removal.namespaceCurrent() && !removal.residentCurrent()) {+ log.debug(+ {+ docName,+ source: 'resolve-delete',+ namespaceCurrent: false,+ residentCurrent: false,+ },+ `[reconcile] delete skipped: ${docName} (stale removal)`,+ );+ return;+ }+ log.info(+ { docName, isDirty, source: 'resolve-delete' },+ `[reconcile] delete: ${docName} (dirty=${isDirty})`,+ );+ const priorLifecycle = lifecycle+ ? {+ status: lifecycle.get('status'),+ reason: lifecycle.get('reason'),+ detectedAt: lifecycle.get('detectedAt'),}- }- if (priorStaleExternalWrite) {- try {- durabilityState.restoreStaleExternalWrite(- priorStaleExternalWrite.conflict,- priorStaleExternalWrite.branch,- );- } catch (rollbackFailure) {- log.error(- { err: rollbackFailure, docName },- `[reconcile] stale-external-write rollback failed for ${docName}; the retry may not find the conflict`,- );+ : undefined;+ if (document && removal.residentCurrent()) {+ lifecycle?.set('status', 'deleted-upstream');+ lifecycle?.delete('reason');+ lifecycle?.delete('detectedAt');+ try {+ hocuspocus.closeConnections(docName);+ await forceUnloadDocument(document, removal.residentCurrent);+ } catch (cause) {+ if (removal.residentCurrent() && lifecycle && priorLifecycle) {+ for (const [key, value] of Object.entries(priorLifecycle)) {+ if (value === undefined) lifecycle.delete(key);+ else lifecycle.set(key, value);+ }}+ if (removal.namespaceCurrent() && priorStaleExternalWrite) {+ try {+ durabilityState.restoreStaleExternalWrite(+ priorStaleExternalWrite.conflict,+ priorStaleExternalWrite.branch,+ );+ } catch (rollbackFailure) {+ log.error(+ { err: rollbackFailure, docName },+ `[reconcile] stale-external-write rollback failed for ${docName}; the retry may not find the conflict`,+ );+ }+ }+ throw cause;}- throw cause;}+ if (!removal.namespaceCurrent()) {+ log.debug(+ {+ docName,+ source: 'resolve-delete',+ namespaceCurrent: false,+ residentCurrent: removal.residentCurrent(),+ },+ `[reconcile] delete: ${docName} not recorded as removed (stale removal)`,+ );+ return;+ }+ deleteReconciledBase(docName);+ onUpstreamDelete(docName);+ scheduleIndexRegenerationAfterRemoval(docName);+ console.info(+ JSON.stringify({+ event: 'recently-removed-docs-populate',+ docName,+ kind: 'deleted',+ source: 'resolve-delete',+ }),+ );+ } finally {+ removal.dispose();}- deleteReconciledBase(docName);- onUpstreamDelete(docName);- scheduleIndexRegenerationAfterRemoval(docName);- console.info(- JSON.stringify({- event: 'recently-removed-docs-populate',- docName,- kind: 'deleted',- source: 'resolve-delete',- }),- );} else if (lifecycle?.get('reason') === entry.reason) {lifecycle.delete('status');lifecycle.delete('reason');@@ -1167,10 +1211,10 @@ export function createServer(options: ServerOptions): ServerInstance {});let loadedPrincipal: Principal | null = null;- const forceUnloadSet = new Set<Document>();+ const forceUnloadRequests = new Map<Document, Set<() => boolean>>();let shutdownAllowsUnload = false;const unregisterWorkloadProviders: Array<() => void> = [];- let forceUnloadDocument!: (document: Document) => Promise<void>;+ let forceUnloadDocument!: (document: Document, isCurrent?: () => boolean) => Promise<void>;let defaultShouldUnloadDocument!: (document: Document) => boolean;let resolveReady!: () => void;@@ -1789,6 +1833,156 @@ export function createServer(options: ServerOptions): ServerInstance {removedDocsJournalWritable = true;const commentDocHooksRef: { current: CommentDocHooks | null } = { current: null };+ type ResidentOccurrence = {+ source: string | undefined;+ mutation: { occurrence: ResidentOccurrence } | undefined;+ adoption: object | undefined;+ pendingRetirement: boolean;+ };+ let residentOccurrences = new WeakMap<Doc, ResidentOccurrence>();+ type RemovalOperation = { namespaceAllowed: boolean; residentAllowed: boolean };+ const activeRemovals = new Map<string, Set<RemovalOperation>>();++ const resolvedContentDir = resolve(contentDir);+ function resolveDocFilePath(docName: string): string | null {+ if (!isSafeDocName(docName)) return null;+ const relativePath = docNameToRelativePath(docName);+ const filePath = resolve(resolvedContentDir, relativePath);+ if (!isWithinDir(filePath, resolvedContentDir)) return null;+ return filePath;+ }++ function hasAdmittedMarkdownBinding(docName: string): boolean {+ const owner = watcher?.getFileIndex().get(docName);+ const filePath = owner?.canonicalPath ?? resolveDocFilePath(docName);+ if (filePath === null) return false;+ return hasAdmittedMarkdownFile(docName, filePath, contentDir, contentFilter);+ }++ function occurrenceFor(document: Doc): ResidentOccurrence {+ let occurrence = residentOccurrences.get(document);+ if (!occurrence) {+ occurrence = {+ source: document.getText('source').toString(),+ mutation: undefined,+ adoption: undefined,+ pendingRetirement: false,+ };+ residentOccurrences.set(document, occurrence);+ }+ return occurrence;+ }++ function revokeNamespaceRemovals(docName: string): void {+ for (const removal of activeRemovals.get(docName) ?? []) removal.namespaceAllowed = false;+ }++ function acceptResident(document: Doc, docName: string): void {+ if (+ documentAdmissionClosed ||+ document.isDestroyed ||+ hocuspocus.documents.get(docName) !== document+ )+ return;+ const occurrence = occurrenceFor(document);+ occurrence.pendingRetirement = false;+ occurrence.adoption = {};+ for (const removal of activeRemovals.get(docName) ?? []) {+ removal.namespaceAllowed = false;+ removal.residentAllowed = false;+ }+ }++ function markResidentForRetirement(document: Doc): void {+ residentOccurrences.set(document, {+ source: document.getText('source').toString(),+ mutation: undefined,+ adoption: undefined,+ pendingRetirement: true,+ });+ }++ function acceptNamespaceCreate(docName: string): void {+ if (documentAdmissionClosed) return;+ revokeNamespaceRemovals(docName);+ const document = hocuspocus.documents.get(docName);+ if (document) markResidentForRetirement(document);+ commentDocHooksRef.current?.deleted(docName);+ }++ function captureNamespaceMove(docName: string): () => void {+ const document = hocuspocus.documents.get(docName);+ const occurrence = document ? occurrenceFor(document) : undefined;+ const adoption = occurrence?.adoption;+ return () => {+ if (documentAdmissionClosed) return;+ revokeNamespaceRemovals(docName);+ if (+ document &&+ hocuspocus.documents.get(docName) === document &&+ residentOccurrences.get(document) === occurrence &&+ occurrence?.adoption === adoption+ )+ markResidentForRetirement(document);+ };+ }++ function capturePublicationReceipt(document: Doc, docName: string): (() => void) | undefined {+ const mutation = residentOccurrences.get(document)?.mutation;+ if (!mutation) return undefined;+ return () => {+ if (residentOccurrences.get(document) === mutation.occurrence)+ acceptResident(document, docName);+ };+ }++ function acceptConnectedDocument(docName: string, document: Document): void {+ if (hasAdmittedMarkdownBinding(docName)) acceptResident(document, docName);+ }++ function beginRemoval(docName: string, observedCurrent?: () => boolean) {+ const document = hocuspocus.documents.get(docName);+ const isApplicable = () =>+ (observedCurrent?.() ?? true) && !hasAdmittedMarkdownBinding(docName);+ const namespaceAllowed = !documentAdmissionClosed && isApplicable();+ const operation: RemovalOperation = {+ namespaceAllowed,+ residentAllowed:+ !documentAdmissionClosed &&+ document !== undefined &&+ (namespaceAllowed || occurrenceFor(document).pendingRetirement),+ };+ const removals = activeRemovals.get(docName) ?? new Set<RemovalOperation>();+ if (!documentAdmissionClosed) {+ removals.add(operation);+ activeRemovals.set(docName, removals);+ }+ const namespaceCurrent = () => {+ if (operation.namespaceAllowed && !isApplicable()) operation.namespaceAllowed = false;+ return operation.namespaceAllowed;+ };+ return {+ document,+ namespaceCurrent,+ residentCurrent: () => {+ if (+ operation.residentAllowed &&+ (document === undefined ||+ document.isDestroyed ||+ hocuspocus.documents.get(docName) !== document)+ )+ operation.residentAllowed = false;+ return operation.residentAllowed;+ },+ dispose: () => {+ operation.namespaceAllowed = false;+ operation.residentAllowed = false;+ removals.delete(operation);+ if (removals.size === 0) activeRemovals.delete(docName);+ },+ };+ }+const onUpstreamRename = (oldDocName: string, newDocName: string): void => {if (isReservedForUserTree(oldDocName)) return;recentlyRemovedDocs.setRenamed(oldDocName, newDocName);@@ -1806,7 +2000,6 @@ export function createServer(options: ServerOptions): ServerInstance {return;}recentlyRemovedDocs.setDeleted(docName);- commentDocHooksRef.current?.deleted(docName);};const onUpstreamAdd = (docName: string): void => {if (isReservedForUserTree(docName)) return;@@ -1948,6 +2141,7 @@ export function createServer(options: ServerOptions): ServerInstance {getPrincipal: () => loadedPrincipal,onAgentCommit: () => cc1Broadcaster?.signal('session-activity'),onFlushCommit: () => maintenanceCoordinator?.noteFlushCommit(),+ capturePublicationReceipt,onDiskFlush: (docName, sv, persistedMarkdown, previousMarkdown) => {cc1Broadcaster?.emitDiskAck(docName, sv);if (isReservedForUserTree(docName)) return;@@ -1994,6 +2188,34 @@ export function createServer(options: ServerOptions): ServerInstance {extensions: [persistence.extension],});+ const dispatchHooks = hocuspocus.hooks.bind(hocuspocus);+ hocuspocus.hooks = (name, payload, callback) => {+ if (+ (name === 'afterLoadDocument' || name === 'onChange') &&+ 'document' in payload &&+ !isReservedForUserTree(payload.document.name)+ ) {+ const document = payload.document;+ const occurrence = occurrenceFor(document);+ if (name === 'onChange' && occurrence.mutation === undefined) {+ const source = document.getText('source').toString();+ if (source !== occurrence.source) {+ occurrence.source = undefined;+ occurrence.mutation = { occurrence };+ }+ }+ }+ if (+ name === 'connected' &&+ 'connection' in payload &&+ payload.connection &&+ !isReservedForUserTree(payload.documentName)+ ) {+ acceptConnectedDocument(payload.documentName, payload.connection.document);+ }+ return dispatchHooks(name, payload, callback);+ };+const openDirect = hocuspocus.openDirectConnection.bind(hocuspocus);function assertDocumentProjectScope(documentName: string): void {if (documentAdmissionClosed) throw new ServerMutationShuttingDownError(false);@@ -2082,9 +2304,8 @@ export function createServer(options: ServerOptions): ServerInstance {defaultShouldUnloadDocument = hocuspocus.shouldUnloadDocument.bind(hocuspocus);hocuspocus.shouldUnloadDocument = (document) => {- if (forceUnloadSet.has(document)) {- return true;- }+ const requests = forceUnloadRequests.get(document);+ if (requests) return [...requests].some((isCurrent) => isCurrent());if (shutdownAllowsUnload &&!isReservedForUserTree(document.name) &&@@ -2103,12 +2324,17 @@ export function createServer(options: ServerOptions): ServerInstance {return defaultShouldUnloadDocument(document);};- forceUnloadDocument = async (document: Document): Promise<void> => {- forceUnloadSet.add(document);+ forceUnloadDocument = async (document, isCurrent): Promise<void> => {+ const request = () =>+ hocuspocus.documents.get(document.name) === document && (isCurrent?.() ?? true);+ const requests = forceUnloadRequests.get(document) ?? new Set<() => boolean>();+ requests.add(request);+ forceUnloadRequests.set(document, requests);try {await hocuspocus.unloadDocument(document);} finally {- forceUnloadSet.delete(document);+ requests.delete(request);+ if (requests.size === 0) forceUnloadRequests.delete(document);}};@@ -2237,16 +2463,6 @@ export function createServer(options: ServerOptions): ServerInstance {};hocuspocus.configuration.extensions.push(configDocAdmissionGuard);- const resolvedContentDir = resolve(contentDir);- function resolveDocFilePath(docName: string): string | null {- if (!isSafeDocName(docName)) return null;- const relativePath = docNameToRelativePath(docName);- const filePath = resolve(resolvedContentDir, relativePath);- if (!isWithinDir(filePath, resolvedContentDir)) {- return null;- }- return filePath;- }const removalRedirectGuard: Extension & { __kind: 'removal-redirect-guard' } = {__kind: 'removal-redirect-guard',async onAuthenticate(payload) {@@ -2308,6 +2524,10 @@ export function createServer(options: ServerOptions): ServerInstance {ingressPolicy,sessionManager,commentDocHooksRef,+ onNamespaceCreateAccepted: acceptNamespaceCreate,+ capturePublicationReceipt,+ captureNamespaceMove,+ onResidentAdopted: acceptResident,contentDir,assertContentPath,assertContentSubtree,@@ -2337,6 +2557,7 @@ export function createServer(options: ServerOptions): ServerInstance {scheduleIndexRegeneration(event.docName);break;case 'delete':+ commentDocHooksRef.current?.deleted(event.docName);scheduleIndexRegenerationAfterRemoval(event.docName);break;case 'rename':@@ -2507,7 +2728,8 @@ export function createServer(options: ServerOptions): ServerInstance {return serializeYDocSource(document);}- const applyToDoc = (docName: string, content: string): void =>+ const applyToDoc = (docName: string, content: string): void => {+ const document = hocuspocus.documents.get(docName);applyExternalChange(durabilityState,hocuspocus,@@ -2517,9 +2739,11 @@ export function createServer(options: ServerOptions): ServerInstance {resolveSize,bridgeLossReporter,);+ if (document && hasAdmittedMarkdownBinding(docName)) acceptResident(document, docName);+ };- const reviveFrozenResident = (docName: string, content: string): FrozenResidentRevival =>- reviveFrozenResidentDoc(+ const reviveFrozenResident = (docName: string, content: string): FrozenResidentRevival => {+ const result = reviveFrozenResidentDoc(durabilityState,hocuspocus,docName,@@ -2528,6 +2752,12 @@ export function createServer(options: ServerOptions): ServerInstance {resolveSize,bridgeLossReporter,);+ const document = hocuspocus.documents.get(docName);+ if (result === 'revived' && document && hasAdmittedMarkdownBinding(docName)) {+ acceptResident(document, docName);+ }+ return result;+ };const refreshLoadedEmbeds = (needle: string,@@ -2793,7 +3023,21 @@ export function createServer(options: ServerOptions): ServerInstance {}};- async function handleDiskEvent(event: DiskEvent): Promise<void> {+ const bufferedRemovalScopes = new Map<+ Extract<DiskEvent, { kind: 'delete' | 'rename' }>,+ ReturnType<typeof beginRemoval>+ >();++ async function handleDiskEvent(+ event: DiskEvent,+ preparedRemoval = event.kind === 'delete' || event.kind === 'rename'+ ? (bufferedRemovalScopes.get(event) ??+ beginRemoval(+ event.kind === 'delete' ? event.docName : event.oldDocName,+ getMarkdownRemovalCurrentness(event),+ ))+ : undefined,+ ): Promise<void> {try {switch (event.kind) {case 'create': {@@ -2974,11 +3218,68 @@ export function createServer(options: ServerOptions): ServerInstance {case 'delete': {const { docName } = event;- dissolveNonGitConflicts(docName);- const document = hocuspocus.documents.get(docName);- if (!document) {- deleteReconciledBase(docName);- await derivedDocumentIndex.recordDiskDelete(docName);+ const removal =+ preparedRemoval ?? beginRemoval(docName, getMarkdownRemovalCurrentness(event));+ bufferedRemovalScopes.delete(event);+ try {+ const namespaceCurrent = removal.namespaceCurrent();+ const residentCurrent = removal.residentCurrent();+ if (!namespaceCurrent && !residentCurrent) {+ log.debug(+ { docName, source: 'watcher-delete', namespaceCurrent, residentCurrent },+ `[reconcile] delete skipped: ${docName} (stale removal)`,+ );+ return;+ }+ if (namespaceCurrent) {+ dissolveNonGitConflicts(docName);+ commentDocHooksRef.current?.deleted(docName);+ }+ const { document } = removal;+ const isDirty =+ residentCurrent && document+ ? rescueUnflushedEditsBeforeTeardown(+ docName,+ getStorageBranch() ?? 'main',+ 'delete',+ true,+ )+ : false;+ if (residentCurrent) document?.getMap('lifecycle').set('status', 'deleted-upstream');+ if (namespaceCurrent) deleteReconciledBase(docName);+ if (namespaceCurrent) await derivedDocumentIndex.recordDiskDelete(docName);+ if (!removal.namespaceCurrent() && !removal.residentCurrent()) {+ log.debug(+ {+ docName,+ source: 'watcher-delete',+ namespaceCurrent: false,+ residentCurrent: false,+ },+ `[reconcile] delete skipped: ${docName} (stale removal)`,+ );+ return;+ }+ log.info(+ { docName, isDirty, source: 'watcher-delete' },+ `[reconcile] delete: ${docName} (dirty=${isDirty})`,+ );+ if (removal.residentCurrent() && document) {+ hocuspocus.closeConnections(docName);+ await forceUnloadDocument(document, removal.residentCurrent);+ }+ if (!removal.namespaceCurrent()) {+ log.debug(+ {+ docName,+ source: 'watcher-delete',+ namespaceCurrent: false,+ residentCurrent: removal.residentCurrent(),+ },+ `[reconcile] delete: ${docName} not recorded as removed (stale removal)`,+ );+ return;+ }signalChannel('files');onUpstreamDelete(docName);scheduleIndexRegenerationAfterRemoval(docName);@@ -2990,107 +3291,108 @@ export function createServer(options: ServerOptions): ServerInstance {source: 'watcher-delete',}),);- return;+ } finally {+ removal.dispose();}-- const isDirty = rescueUnflushedEditsBeforeTeardown(- docName,- getStorageBranch() ?? 'main',- 'delete',- true,- );-- const lifecycleMap = document.getMap('lifecycle');- lifecycleMap.set('status', 'deleted-upstream');-- deleteReconciledBase(docName);- await derivedDocumentIndex.recordDiskDelete(docName);- log.info(- { docName, isDirty, source: 'watcher-delete' },- `[reconcile] delete: ${docName} (dirty=${isDirty})`,- );-- hocuspocus.closeConnections(docName);- await forceUnloadDocument(document);- signalChannel('files');- onUpstreamDelete(docName);- scheduleIndexRegenerationAfterRemoval(docName);- console.info(- JSON.stringify({- event: 'recently-removed-docs-populate',- docName,- kind: 'deleted',- source: 'watcher-delete',- }),- );break;}case 'rename': {const { oldDocName, newDocName, content } = event;- dissolveNonGitConflicts(oldDocName);- const freezeAsRenamed = (doc: Document): void => {- const lifecycleMap = doc.getMap('lifecycle');- lifecycleMap.set('status', 'renamed');- lifecycleMap.set('newPath', newDocName);- };- const loadedBeforeIndex = hocuspocus.documents.get(oldDocName);- const isDirty = loadedBeforeIndex- ? rescueUnflushedEditsBeforeTeardown(oldDocName, getStorageBranch() ?? 'main', 'rename')- : false;- if (loadedBeforeIndex) freezeAsRenamed(loadedBeforeIndex);-- deleteReconciledBase(oldDocName);- persistence.forgetObservedFile(oldDocName);- setReconciledBase(newDocName, content);-- log.info(- { oldDocName, newDocName, isDirty },- `[reconcile] rename: ${oldDocName} → ${newDocName} (dirty=${isDirty})`,- );- signalChannel('files');- onUpstreamAdd(newDocName);- onUpstreamRename(oldDocName, newDocName);-+ const removal =+ preparedRemoval ?? beginRemoval(oldDocName, getMarkdownRemovalCurrentness(event));+ bufferedRemovalScopes.delete(event);try {- await derivedDocumentIndex.recordDiskRename(oldDocName, newDocName, content);- } catch (err) {- log.error(- { oldDocName, newDocName, err },- `[reconcile] rename: index update failed for ${oldDocName} → ${newDocName}; completing client teardown anyway`,- );- }-- const document = hocuspocus.documents.get(oldDocName);- if (document && document !== loadedBeforeIndex) freezeAsRenamed(document);-- reviveFrozenResident(newDocName, content);-- await sessionManager.closeAllForDoc(oldDocName);- const closedConnections = document?.getConnectionsCount() ?? 0;- /* WARN: the `delete` branch above and `captureAndCloseDocuments` pair their- close with `forceUnloadDocument`; this branch does not, so the frozen doc- stays resident under the old name and template-watcher-capabilities.test.ts- pins that residency. The destination clear above is what stops a move back- to this path from being served that stale frozen doc. */- hocuspocus.closeConnections(oldDocName);- if (closedConnections > 0) {+ const document = removal.document;+ const freezeAsRenamed = (doc: Document): void => {+ const lifecycle = doc.getMap('lifecycle');+ lifecycle.set('status', 'renamed');+ lifecycle.set('newPath', newDocName);+ };+ const isDirty =+ removal.residentCurrent() && document+ ? rescueUnflushedEditsBeforeTeardown(+ oldDocName,+ getStorageBranch() ?? 'main',+ 'rename',+ )+ : false;+ if (removal.residentCurrent() && document) freezeAsRenamed(document);+ if (removal.namespaceCurrent()) {+ dissolveNonGitConflicts(oldDocName);+ deleteReconciledBase(oldDocName);+ persistence.forgetObservedFile(oldDocName);+ onUpstreamRename(oldDocName, newDocName);+ }+ setReconciledBase(newDocName, content);log.info(- { oldDocName, newDocName, closedConnections },- `[reconcile] rename: closed ${closedConnections} connection(s) on ${oldDocName}`,+ { oldDocName, newDocName, isDirty },+ `[reconcile] rename: ${oldDocName} → ${newDocName} (dirty=${isDirty})`,+ );+ signalChannel('files');+ onUpstreamAdd(newDocName);+ try {+ if (removal.namespaceCurrent()) {+ await derivedDocumentIndex.recordDiskRename(oldDocName, newDocName, content);+ } else {+ await derivedDocumentIndex.recordDiskUpsert(newDocName, content);+ }+ } catch (err) {+ log.error(+ { oldDocName, newDocName, err },+ `[reconcile] rename: index update failed for ${oldDocName} → ${newDocName}; completing client teardown anyway`,+ );+ }+ const loadedDuringIndex = removal.namespaceCurrent()+ ? hocuspocus.documents.get(oldDocName)+ : undefined;+ const lateResident = loadedDuringIndex !== document ? loadedDuringIndex : undefined;+ if (lateResident) freezeAsRenamed(lateResident);+ reviveFrozenResident(newDocName, content);+ scheduleIndexRegeneration(newDocName);+ const retired = removal.residentCurrent() && document ? document : lateResident;+ if (retired) {+ const closedConnections = retired.getConnectionsCount();+ /* WARN: the `delete` branch above and `captureAndCloseDocuments` pair their+ close with `forceUnloadDocument`; this branch does not, so the frozen doc+ stays resident under the old name and template-watcher-capabilities.test.ts+ pins that residency. The destination clear above is what stops a move back+ to this path from being served that stale frozen doc. */+ hocuspocus.closeConnections(oldDocName);+ await sessionManager.closeAllForDoc(oldDocName);+ if (closedConnections > 0) {+ log.info(+ { oldDocName, newDocName, closedConnections },+ `[reconcile] rename: closed ${closedConnections} connection(s) on ${oldDocName}`,+ );+ }+ }+ if (!removal.namespaceCurrent()) {+ log.debug(+ {+ docName: oldDocName,+ newDocName,+ source: 'watcher-rename',+ namespaceCurrent: false,+ residentCurrent: removal.residentCurrent(),+ },+ `[reconcile] rename: ${oldDocName} not recorded as removed (stale removal)`,+ );+ break;+ }+ console.info(+ JSON.stringify({+ event: 'recently-removed-docs-populate',+ from: oldDocName,+ to: newDocName,+ kind: 'renamed',+ source: 'watcher-rename',+ }),);+ scheduleIndexRegenerationAfterRemoval(oldDocName);+ } finally {+ removal.dispose();}- console.info(- JSON.stringify({- event: 'recently-removed-docs-populate',- from: oldDocName,- to: newDocName,- kind: 'renamed',- source: 'watcher-rename',- }),- );- scheduleIndexRegenerationAfterRemoval(oldDocName);- scheduleIndexRegeneration(newDocName);break;}@@ -3166,6 +3468,16 @@ export function createServer(options: ServerOptions): ServerInstance {const eventBuffer: DiskEvent[] = [];async function onDiskEvent(event: DiskEvent): Promise<void> {+ if (documentAdmissionClosed) return;+ if ((event.kind === 'delete' || event.kind === 'rename') && !bufferedRemovalScopes.has(event)) {+ bufferedRemovalScopes.set(+ event,+ beginRemoval(+ event.kind === 'delete' ? event.docName : event.oldDocName,+ getMarkdownRemovalCurrentness(event),+ ),+ );+ }if (isBatchInProgress()) {eventBuffer.push(event);return;@@ -3180,6 +3492,11 @@ export function createServer(options: ServerOptions): ServerInstance {}}+ function disposeBufferedRemovalScopes(): void {+ for (const scope of bufferedRemovalScopes.values()) scope.dispose();+ bufferedRemovalScopes.clear();+ }+let watcher: WatcherHandle | null = null;let syncEngine: SyncEngine | null = null;let inflightDestroy: Promise<void> | null = null;@@ -3459,6 +3776,15 @@ export function createServer(options: ServerOptions): ServerInstance {log.error({ err }, '[server] shutdown mutation drain failed');}documentAdmissionClosed = true;+ disposeBufferedRemovalScopes();+ for (const removals of activeRemovals.values()) {+ for (const removal of removals) {+ removal.namespaceAllowed = false;+ removal.residentAllowed = false;+ }+ }+ activeRemovals.clear();+ residentOccurrences = new WeakMap();try {await joinPrelude(closeIndexRegeneration(), 'generated-index');@@ -4464,7 +4790,11 @@ export function createServer(options: ServerOptions): ServerInstance {if (info.batchKind === 'within-branch') {setBatchInProgress(false);try {- await drainEventBuffer();+ try {+ await drainEventBuffer();+ } finally {+ disposeBufferedRemovalScopes();+ }if (info.headMoved && info.newHead) {const changes = resolveUpstreamChanges(@@ -4502,6 +4832,7 @@ export function createServer(options: ServerOptions): ServerInstance {} else {incrementBranchSwitch();eventBuffer.splice(0, eventBuffer.length);+ disposeBufferedRemovalScopes();let deferredStoresFlushed = false;let branchTransition: DerivedDocumentIndexBranchTransition | undefined;try {