Skip to content

Beautiful, AI-native markdown IDE and LLM wiki

PRD-10140 Take in outside writes the Node server dropped (#7940)

Andrew Mikofalvyc7ee9e9dcc2d
diff --git a/.changeset/outside-edits-reach-new-pages.md b/.changeset/outside-edits-reach-new-pages.md
new file mode 100644
index 000000000..71e9c1ca0
--- /dev/null
+++ b/.changeset/outside-edits-reach-new-pages.md
@@ -0,0 +1,5 @@
+---
+'@inkeep/open-knowledge': patch
+---
+
+An edit another app makes to a new page moments after OpenKnowledge first saves it now reaches the open page, and a new `.mdx` file opened before OpenKnowledge has listed it now loads its content instead of an empty page.
diff --git a/packages/app/tests/integration/outside-write-over-first-save.test.ts b/packages/app/tests/integration/outside-write-over-first-save.test.ts
new file mode 100644
index 000000000..1259c7c6c
--- /dev/null
+++ b/packages/app/tests/integration/outside-write-over-first-save.test.ts
@@ -0,0 +1,48 @@
+import { randomUUID } from 'node:crypto';
+import { existsSync, readFileSync, writeFileSync } from 'node:fs';
+import { join } from 'node:path';
+import { afterEach, expect, test } from 'vitest';
+import { createManualClock } from '../../../server/src/watcher-clock.test-helper';
+import { createWatcherDouble } from '../../../server/src/watcher-double.test-helper';
+import { agentWriteMd, createTestClient, createTestServer, pollUntil } from './test-harness';
+
+const cleanups: Array<() => Promise<void> | void> = [];
+
+afterEach(async () => {
+ for (const cleanup of cleanups.splice(0).reverse()) await cleanup();
+});
+
+test('an outside edit reported in one create with the first save of a new page reaches the open page', async () => {
+ const double = createWatcherDouble();
+ const server = await createTestServer({
+ watcherBackend: double.backend,
+ watcherClock: createManualClock(Date.now()),
+ freshnessIntervalMs: 5_000,
+ });
+ cleanups.push(() => server.cleanup());
+ const docName = `first-save-${randomUUID()}`;
+ const file = join(server.contentDir, `${docName}.md`);
+ const saved = '# New page\n\nFirst paragraph.\n';
+ await agentWriteMd(server.port, saved, { docName, position: 'replace' });
+ await pollUntil(
+ () => existsSync(file) && readFileSync(file, 'utf-8') === saved,
+ 10_000,
+ 25,
+ 'the first save to reach disk',
+ );
+ const client = await createTestClient(server.port, docName, { skipInvariantWatcher: true });
+ cleanups.push(() => client.cleanup());
+ await pollUntil(() => client.ytext.toString() === saved, 10_000, 25, 'the page to load');
+
+ const outside = '# New page\n\nFirst paragraph, edited in another app.\n';
+ writeFileSync(file, outside);
+ double.subscriptionFor(server.contentDir).deliver([{ type: 'create', path: file }]);
+
+ await pollUntil(
+ () => client.ytext.toString().includes('edited in another app'),
+ 10_000,
+ 25,
+ 'the outside edit to reach the open page',
+ );
+ expect(readFileSync(file, 'utf-8')).toBe(outside);
+}, 60_000);
diff --git a/packages/server/src/content/file-operation-paths.ts b/packages/server/src/content/file-operation-paths.ts
new file mode 100644
index 000000000..e9efb569b
--- /dev/null
+++ b/packages/server/src/content/file-operation-paths.ts
@@ -0,0 +1,39 @@
+import { existsSync } from 'node:fs';
+import { extname, resolve, sep } from 'node:path';
+import {
+ isSupportedDocFile,
+ registerDocExtension,
+ SUPPORTED_DOC_EXTENSIONS,
+ stripDocExtension,
+} from '../doc-extensions.ts';
+import { isValidRelativeContentPath } from '../relative-content-path.ts';
+
+export function registerSourceFileExtension(contentDir: string, path: string): void {
+ if (!isValidRelativeContentPath(path)) return;
+ const resolvedContentDir = resolve(contentDir);
+ const inside = (candidate: string) =>
+ candidate === resolvedContentDir || candidate.startsWith(`${resolvedContentDir}${sep}`);
+ if (isSupportedDocFile(path)) {
+ const extensionless = stripDocExtension(path);
+ for (const ext of SUPPORTED_DOC_EXTENSIONS) {
+ const candidate = resolve(resolvedContentDir, `${extensionless}${ext}`);
+ if (inside(candidate) && existsSync(candidate)) registerDocExtension(extensionless, ext);
+ }
+ const explicitCandidate = resolve(resolvedContentDir, path);
+ if (
+ explicitCandidate !== resolvedContentDir &&
+ inside(explicitCandidate) &&
+ existsSync(explicitCandidate)
+ ) {
+ registerDocExtension(extensionless, extname(path));
+ }
+ return;
+ }
+ for (const ext of SUPPORTED_DOC_EXTENSIONS) {
+ const candidate = resolve(resolvedContentDir, `${path}${ext}`);
+ if (inside(candidate) && existsSync(candidate)) {
+ registerDocExtension(path, ext);
+ return;
+ }
+ }
+}
diff --git a/packages/server/src/file-watcher.test.ts b/packages/server/src/file-watcher.test.ts
index e18b1fc0f..349e6483e 100644
--- a/packages/server/src/file-watcher.test.ts
+++ b/packages/server/src/file-watcher.test.ts
@@ -20,6 +20,7 @@ import {
contentHash,
evictStaleTrackerEntries,
type FileIndexEntry,
+ getWatcherDecisionRingSnapshot,
handleRawEvents,
isSelfRemoval,
isSelfWrite,
@@ -29,6 +30,7 @@ import {
registerRemoval,
registerWrite,
removalTracker,
+ resetWatcherDecisionDiagnostics,
retractRemoval,
startWatcher,
toParcelIgnorePaths,
@@ -449,6 +451,92 @@ describe('isSelfRemoval', () => {
});
});
+describe('an outside write in the same batch as an own save', () => {
+ let batchDir: string;
+
+ beforeEach(async () => {
+ writeTracker.clear();
+ lastKnownHash.clear();
+ resetWatcherDecisionDiagnostics();
+ batchDir = realpathSync(await mkdtemp(resolve(tmpdir(), 'ok-merged-batch-test-')));
+ });
+
+ afterEach(async () => {
+ await rm(batchDir, { recursive: true, force: true });
+ });
+
+ test('reaches the server as an update carrying the outside bytes when the batch reads as a create', async () => {
+ const filePath = resolve(batchDir, 'merged.md');
+ const saved = '# Merged\n\nIntro paragraph.\n';
+ const outside = '# Merged\n\nIntro paragraph, edited out of band.\n';
+ writeFileSync(filePath, saved, 'utf-8');
+ registerWrite(filePath, contentHash(saved));
+ writeFileSync(filePath, outside, 'utf-8');
+
+ const dispatched: DiskEvent[] = [];
+ await handleRawEvents(
+ [{ type: 'create', path: filePath }],
+ batchDir,
+ undefined,
+ new Map(),
+ new Map(),
+ async (event) => {
+ dispatched.push(event);
+ },
+ );
+
+ expect(dispatched).toHaveLength(1);
+ expect(dispatched[0]).toMatchObject({ kind: 'update', docName: 'merged', content: outside });
+ expect(
+ getWatcherDecisionRingSnapshot().filter(
+ (record) => record.decision === 'outside-write-over-own-write',
+ ),
+ ).toMatchObject([{ kind: 'create' }]);
+ });
+
+ test('stays a create when the file was deleted after the own save', async () => {
+ const filePath = resolve(batchDir, 'recreated.md');
+ const saved = '# Recreated\n\nIntro paragraph.\n';
+ const outside = '# Recreated\n\nIntro paragraph, saved again out of band.\n';
+ writeFileSync(filePath, saved, 'utf-8');
+ registerWrite(filePath, contentHash(saved));
+ unlinkSync(filePath);
+
+ const dispatched: DiskEvent[] = [];
+ const dispatch = async (event: DiskEvent) => {
+ dispatched.push(event);
+ };
+ await handleRawEvents(
+ [{ type: 'delete', path: filePath }],
+ batchDir,
+ undefined,
+ new Map(),
+ new Map(),
+ dispatch,
+ );
+ writeFileSync(filePath, outside, 'utf-8');
+ await handleRawEvents(
+ [{ type: 'create', path: filePath }],
+ batchDir,
+ undefined,
+ new Map(),
+ new Map(),
+ dispatch,
+ );
+
+ expect(dispatched.at(-1)).toMatchObject({
+ kind: 'create',
+ docName: 'recreated',
+ content: outside,
+ });
+ expect(
+ getWatcherDecisionRingSnapshot().some(
+ (record) => record.decision === 'outside-write-over-own-write',
+ ),
+ ).toBe(false);
+ });
+});
+
describe('classifyEvents', () => {
let tmpDir: string;
let contentDir: string;
diff --git a/packages/server/src/file-watcher.ts b/packages/server/src/file-watcher.ts
index 4799bff5e..8d33237e6 100644
--- a/packages/server/src/file-watcher.ts
+++ b/packages/server/src/file-watcher.ts
@@ -322,6 +322,7 @@ export interface WatcherHandle {
export const writeTracker = new Map<string, Array<{ hash: string; timestamp: number }>>();
const WRITE_TRACKER_TTL_MS = 10_000;
const REMOVAL_TRACKER_TTL_MS = 30_000;
+const deletedOwnWrites = new WeakSet<{ hash: string; timestamp: number }>();
interface OwnWriteObservation {
key: string;
@@ -492,6 +493,7 @@ type WatcherDecision =
| 'dispatched'
| 'self-write-skip'
| 'superseded-own-write-skip'
+ | 'outside-write-over-own-write'
| 'self-removal-skip'
| `drop-${WatcherDropReason}`;
@@ -510,6 +512,7 @@ const watcherDropCounts = new Map<WatcherDropReason, number>();
let watcherDispatchedCount = 0;
let watcherSelfWriteSkipCount = 0;
let watcherSupersededOwnWriteSkipCount = 0;
+let watcherOutsideWriteOverOwnWriteCount = 0;
let watcherSelfRemovalSkipCount = 0;
let watcherDropsSinceLastSummary = 0;
@@ -536,6 +539,10 @@ function recordWatcherDecision(decision: WatcherDecision, kind: string, rawPath:
watcherSupersededOwnWriteSkipCount++;
return;
}
+ if (decision === 'outside-write-over-own-write') {
+ watcherOutsideWriteOverOwnWriteCount++;
+ return;
+ }
if (decision === 'self-removal-skip') {
watcherSelfRemovalSkipCount++;
return;
@@ -555,6 +562,14 @@ function recordSupersededOwnWriteSkip(kind: DiskEvent['kind'], filePath: string)
recordWatcherDecision('superseded-own-write-skip', kind, filePath);
}
+function recordOutsideWriteOverOwnWrite(filePath: string): void {
+ log.debug(
+ { path: filePath },
+ '[file-watcher] Read a create over a pending own write as an update',
+ );
+ recordWatcherDecision('outside-write-over-own-write', 'create', filePath);
+}
+
export function getWatcherDecisionRingSnapshot(): WatcherDecisionRecord[] {
return watcherDecisionRing.map((record) => ({ ...record }));
}
@@ -576,6 +591,7 @@ export function logWatcherDropSummary(): void {
dispatched: watcherDispatchedCount,
selfWriteSkips: watcherSelfWriteSkipCount,
supersededOwnWriteSkips: watcherSupersededOwnWriteSkipCount,
+ outsideWritesOverOwnWrites: watcherOutsideWriteOverOwnWriteCount,
selfRemovalSkips: watcherSelfRemovalSkipCount,
},
`[file-watcher] drop summary: ${droppedSinceLastSummary} event(s) dropped since last summary`,
@@ -588,6 +604,7 @@ export function resetWatcherDecisionDiagnostics(): void {
watcherDispatchedCount = 0;
watcherSelfWriteSkipCount = 0;
watcherSupersededOwnWriteSkipCount = 0;
+ watcherOutsideWriteOverOwnWriteCount = 0;
watcherSelfRemovalSkipCount = 0;
watcherDropsSinceLastSummary = 0;
}
@@ -1014,6 +1031,9 @@ async function classifyEventsInternal(
continue;
}
}
+ if (writeTracker.size > 0)
+ for (const write of writeTracker.get(observedPathKey(event.path)) ?? [])
+ deletedOwnWrites.add(write);
deletes.push(event);
break;
}
@@ -1069,6 +1089,16 @@ async function classifyEventsInternal(
}
}
+ for (const event of creates) {
+ const content = createContents.get(event.path);
+ if (content === undefined || !overwritesOwnWrite(event.path, content)) continue;
+ createContents.delete(event.path);
+ createFacts.delete(event.path);
+ updateContents.set(event.path, content);
+ updates.push(event);
+ recordOutsideWriteOverOwnWrite(event.path);
+ }
+
function resolveDocName(event: RawFileEvent): string {
const rawPath = event.path;
const raw = pathToDocName(rawPath, contentDir);
@@ -1269,6 +1299,15 @@ export function isSelfWrite(filePath: string, hash: string): boolean {
return false;
}
+function overwritesOwnWrite(filePath: string, content: string): boolean {
+ if (writeTracker.size === 0) return false;
+ const queue = writeTracker.get(observedPathKey(filePath));
+ const newest = queue?.at(-1);
+ if (queue === undefined || newest === undefined || deletedOwnWrites.has(newest)) return false;
+ const hash = contentHash(content);
+ return queue.every((write) => write.hash !== hash);
+}
+
function consumeOwnWrite(key: string, hash: string): boolean {
const queue = writeTracker.get(key);
if (!queue) return false;
diff --git a/packages/server/src/http/file-ops-routes.ts b/packages/server/src/http/file-ops-routes.ts
index 67bf71cea..73a6e7786 100644
--- a/packages/server/src/http/file-ops-routes.ts
+++ b/packages/server/src/http/file-ops-routes.ts
@@ -7,7 +7,7 @@ import {
unlinkSync,
} from 'node:fs';
import type { IncomingMessage, ServerResponse } from 'node:http';
-import { dirname, extname, resolve, sep } from 'node:path';
+import { dirname, extname, resolve } from 'node:path';
import { pipeline } from 'node:stream/promises';
import {
type AttachmentFolderUploadIssue,
@@ -42,6 +42,7 @@ import { ManagedRenameCollisionError } from '../apply-managed-rename.ts';
import { isConfigDoc, isSystemDoc } from '../cc1-broadcast.ts';
import type { ConflictAuthority } from '../conflict-authority.ts';
import { DocInConflictError, respondDocInConflict } from '../conflict-errors.ts';
+import { registerSourceFileExtension } from '../content/file-operation-paths.ts';
import { isReservedProjectStatePath } from '../content/managed-doc-enum.ts';
import { applySubstitution } from '../content/substitution.ts';
import { resolveTemplatesAvailable } from '../content/templates-resolver.ts';
@@ -51,8 +52,6 @@ import {
getDocExtension,
isSupportedAssetFile,
isSupportedDocFile,
- registerDocExtension,
- SUPPORTED_DOC_EXTENSIONS,
stripDocExtension,
} from '../doc-extensions.ts';
import { extractActorIdentity } from '../extract-actor-identity.ts';
@@ -392,44 +391,7 @@ export function createFileOpsRoutes(deps: FileOpsRouteDeps): ApiRouteGroup {
function probeAndRegisterSourceFileExtension(probeContentDir: string, fromPath: string): void {
if (!isValidRelativeContentPath(fromPath)) return;
- const resolvedContentDir = resolve(probeContentDir);
- if (isSupportedDocFile(fromPath)) {
- const extensionless = stripDocExtension(fromPath);
- for (const ext of SUPPORTED_DOC_EXTENSIONS) {
- const candidate = resolve(resolvedContentDir, `${extensionless}${ext}`);
- if (
- candidate !== resolvedContentDir &&
- !candidate.startsWith(`${resolvedContentDir}${sep}`)
- ) {
- continue;
- }
- if (existsSync(candidate)) {
- registerDocExtension(extensionless, ext);
- }
- }
- const explicitCandidate = resolve(resolvedContentDir, fromPath);
- if (
- explicitCandidate !== resolvedContentDir &&
- explicitCandidate.startsWith(`${resolvedContentDir}${sep}`) &&
- existsSync(explicitCandidate)
- ) {
- registerDocExtension(extensionless, extname(fromPath));
- }
- return;
- }
- for (const ext of SUPPORTED_DOC_EXTENSIONS) {
- const candidate = resolve(resolvedContentDir, `${fromPath}${ext}`);
- if (
- candidate !== resolvedContentDir &&
- !candidate.startsWith(`${resolvedContentDir}${sep}`)
- ) {
- continue;
- }
- if (existsSync(candidate)) {
- registerDocExtension(fromPath, ext);
- return;
- }
- }
+ registerSourceFileExtension(probeContentDir, fromPath);
}
function resolveExtensionlessAssetPath(assetPath: string): {
diff --git a/packages/server/src/persistence-load-before-watcher.test.ts b/packages/server/src/persistence-load-before-watcher.test.ts
new file mode 100644
index 000000000..9974baf9c
--- /dev/null
+++ b/packages/server/src/persistence-load-before-watcher.test.ts
@@ -0,0 +1,55 @@
+import { mkdtempSync, rmSync, writeFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { afterEach, beforeEach, expect, test } from 'vitest';
+import * as Y from 'yjs';
+import { _resetDocExtensionsForTests, isRegisteredMarkdownDocName } from './doc-extensions.ts';
+import { createPersistenceExtension } from './persistence.ts';
+
+let contentDir: string;
+
+beforeEach(() => {
+ _resetDocExtensionsForTests();
+ contentDir = mkdtempSync(join(tmpdir(), 'ok-load-before-watcher-'));
+});
+
+afterEach(() => {
+ _resetDocExtensionsForTests();
+ rmSync(contentDir, { recursive: true, force: true });
+});
+
+test('a page opened before the watcher has seen its new .mdx file loads that file', async () => {
+ const seed = '# Seed\n\n<Callout>Body.</Callout>\n';
+ writeFileSync(join(contentDir, 'fresh.mdx'), seed);
+ const persistence = createPersistenceExtension({
+ mergeHistory: null,
+ ingestDiskAtStore: null,
+ contentDir,
+ });
+ const document = new Y.Doc();
+
+ await persistence.extension.onLoadDocument?.({
+ document,
+ documentName: 'fresh',
+ context: {},
+ } as never);
+
+ expect(document.getText('source').toString()).toBe(seed);
+});
+
+test('a page name with a parent segment registers no extension when it loads', async () => {
+ writeFileSync(join(contentDir, 'fresh.mdx'), '# Seed\n');
+ const persistence = createPersistenceExtension({
+ mergeHistory: null,
+ ingestDiskAtStore: null,
+ contentDir,
+ });
+
+ await persistence.extension.onLoadDocument?.({
+ document: new Y.Doc(),
+ documentName: 'other/../fresh',
+ context: {},
+ } as never);
+
+ expect(isRegisteredMarkdownDocName('other/../fresh')).toBe(false);
+});
diff --git a/packages/server/src/persistence.ts b/packages/server/src/persistence.ts
index 10bd98060..ff31d8111 100644
--- a/packages/server/src/persistence.ts
+++ b/packages/server/src/persistence.ts
@@ -56,6 +56,7 @@ import {
type LifecycleView,
type RemovalLifecycleStatus,
} from './conflict-kinds.ts';
+import { registerSourceFileExtension } from './content/file-operation-paths.ts';
import { isWithinContentDir, safeContentPath } from './content-path.ts';
import type { ContributorEntry } from './contributor-tracker.ts';
import {
@@ -74,6 +75,7 @@ import { resolveDirectoryRoot, resolveNativePath } from './directory-root.ts';
import { applyDiskContentToDoc, FILE_WATCHER_ORIGIN } from './disk-content-intake.ts';
import type { FileObservation } from './disk-receipts.ts';
import { displayPathFor } from './display-path.ts';
+import { isRegisteredMarkdownDocName } from './doc-extensions.ts';
import {
assertNeverStorePublishOutcome,
type DeferredStoreDiscardCause,
@@ -2478,6 +2480,8 @@ export function createPersistenceExtension(options: PersistenceOptions): Persist
loadManagedArtifactDoc(document, documentName, managedArtifactCtx);
return;
}
+ if (!isRegisteredMarkdownDocName(documentName))
+ registerSourceFileExtension(contentDir, documentName);
options?.assertContentPath?.(safeContentPath(documentName, contentDir));
if (
isMermaidDoc(documentName) ||