Beautiful, AI-native markdown IDE and LLM wiki
TypeScript 2nd-brainagent-skillsclaudecodexcompany-braindocsknowledge-knowledge-baseknowledge-managementllmllm-wikillm-wiki-karpathymarkdownmarkdown-editormdmdxnotespkmsecond-brainskills
PRD-10140 Take in outside writes the Node server dropped (#7940)
Andrew Mikofalvyc7ee9e9dcc2d
diff --git a/.changeset/outside-edits-reach-new-pages.md b/.changeset/outside-edits-reach-new-pages.mdnew file mode 100644index 000000000..71e9c1ca0--- /dev/null+++ b/.changeset/outside-edits-reach-new-pages.md@@ -0,0 +1,5 @@+---+'@inkeep/open-knowledge': patch+---++An edit another app makes to a new page moments after OpenKnowledge first saves it now reaches the open page, and a new `.mdx` file opened before OpenKnowledge has listed it now loads its content instead of an empty page.diff --git a/packages/app/tests/integration/outside-write-over-first-save.test.ts b/packages/app/tests/integration/outside-write-over-first-save.test.tsnew file mode 100644index 000000000..1259c7c6c--- /dev/null+++ b/packages/app/tests/integration/outside-write-over-first-save.test.ts@@ -0,0 +1,48 @@+import { randomUUID } from 'node:crypto';+import { existsSync, readFileSync, writeFileSync } from 'node:fs';+import { join } from 'node:path';+import { afterEach, expect, test } from 'vitest';+import { createManualClock } from '../../../server/src/watcher-clock.test-helper';+import { createWatcherDouble } from '../../../server/src/watcher-double.test-helper';+import { agentWriteMd, createTestClient, createTestServer, pollUntil } from './test-harness';++const cleanups: Array<() => Promise<void> | void> = [];++afterEach(async () => {+ for (const cleanup of cleanups.splice(0).reverse()) await cleanup();+});++test('an outside edit reported in one create with the first save of a new page reaches the open page', async () => {+ const double = createWatcherDouble();+ const server = await createTestServer({+ watcherBackend: double.backend,+ watcherClock: createManualClock(Date.now()),+ freshnessIntervalMs: 5_000,+ });+ cleanups.push(() => server.cleanup());+ const docName = `first-save-${randomUUID()}`;+ const file = join(server.contentDir, `${docName}.md`);+ const saved = '# New page\n\nFirst paragraph.\n';+ await agentWriteMd(server.port, saved, { docName, position: 'replace' });+ await pollUntil(+ () => existsSync(file) && readFileSync(file, 'utf-8') === saved,+ 10_000,+ 25,+ 'the first save to reach disk',+ );+ const client = await createTestClient(server.port, docName, { skipInvariantWatcher: true });+ cleanups.push(() => client.cleanup());+ await pollUntil(() => client.ytext.toString() === saved, 10_000, 25, 'the page to load');++ const outside = '# New page\n\nFirst paragraph, edited in another app.\n';+ writeFileSync(file, outside);+ double.subscriptionFor(server.contentDir).deliver([{ type: 'create', path: file }]);++ await pollUntil(+ () => client.ytext.toString().includes('edited in another app'),+ 10_000,+ 25,+ 'the outside edit to reach the open page',+ );+ expect(readFileSync(file, 'utf-8')).toBe(outside);+}, 60_000);diff --git a/packages/server/src/content/file-operation-paths.ts b/packages/server/src/content/file-operation-paths.tsnew file mode 100644index 000000000..e9efb569b--- /dev/null+++ b/packages/server/src/content/file-operation-paths.ts@@ -0,0 +1,39 @@+import { existsSync } from 'node:fs';+import { extname, resolve, sep } from 'node:path';+import {+ isSupportedDocFile,+ registerDocExtension,+ SUPPORTED_DOC_EXTENSIONS,+ stripDocExtension,+} from '../doc-extensions.ts';+import { isValidRelativeContentPath } from '../relative-content-path.ts';++export function registerSourceFileExtension(contentDir: string, path: string): void {+ if (!isValidRelativeContentPath(path)) return;+ const resolvedContentDir = resolve(contentDir);+ const inside = (candidate: string) =>+ candidate === resolvedContentDir || candidate.startsWith(`${resolvedContentDir}${sep}`);+ if (isSupportedDocFile(path)) {+ const extensionless = stripDocExtension(path);+ for (const ext of SUPPORTED_DOC_EXTENSIONS) {+ const candidate = resolve(resolvedContentDir, `${extensionless}${ext}`);+ if (inside(candidate) && existsSync(candidate)) registerDocExtension(extensionless, ext);+ }+ const explicitCandidate = resolve(resolvedContentDir, path);+ if (+ explicitCandidate !== resolvedContentDir &&+ inside(explicitCandidate) &&+ existsSync(explicitCandidate)+ ) {+ registerDocExtension(extensionless, extname(path));+ }+ return;+ }+ for (const ext of SUPPORTED_DOC_EXTENSIONS) {+ const candidate = resolve(resolvedContentDir, `${path}${ext}`);+ if (inside(candidate) && existsSync(candidate)) {+ registerDocExtension(path, ext);+ return;+ }+ }+}diff --git a/packages/server/src/file-watcher.test.ts b/packages/server/src/file-watcher.test.tsindex e18b1fc0f..349e6483e 100644--- a/packages/server/src/file-watcher.test.ts+++ b/packages/server/src/file-watcher.test.ts@@ -20,6 +20,7 @@ import {contentHash,evictStaleTrackerEntries,type FileIndexEntry,+ getWatcherDecisionRingSnapshot,handleRawEvents,isSelfRemoval,isSelfWrite,@@ -29,6 +30,7 @@ import {registerRemoval,registerWrite,removalTracker,+ resetWatcherDecisionDiagnostics,retractRemoval,startWatcher,toParcelIgnorePaths,@@ -449,6 +451,92 @@ describe('isSelfRemoval', () => {});});+describe('an outside write in the same batch as an own save', () => {+ let batchDir: string;++ beforeEach(async () => {+ writeTracker.clear();+ lastKnownHash.clear();+ resetWatcherDecisionDiagnostics();+ batchDir = realpathSync(await mkdtemp(resolve(tmpdir(), 'ok-merged-batch-test-')));+ });++ afterEach(async () => {+ await rm(batchDir, { recursive: true, force: true });+ });++ test('reaches the server as an update carrying the outside bytes when the batch reads as a create', async () => {+ const filePath = resolve(batchDir, 'merged.md');+ const saved = '# Merged\n\nIntro paragraph.\n';+ const outside = '# Merged\n\nIntro paragraph, edited out of band.\n';+ writeFileSync(filePath, saved, 'utf-8');+ registerWrite(filePath, contentHash(saved));+ writeFileSync(filePath, outside, 'utf-8');++ const dispatched: DiskEvent[] = [];+ await handleRawEvents(+ [{ type: 'create', path: filePath }],+ batchDir,+ undefined,+ new Map(),+ new Map(),+ async (event) => {+ dispatched.push(event);+ },+ );++ expect(dispatched).toHaveLength(1);+ expect(dispatched[0]).toMatchObject({ kind: 'update', docName: 'merged', content: outside });+ expect(+ getWatcherDecisionRingSnapshot().filter(+ (record) => record.decision === 'outside-write-over-own-write',+ ),+ ).toMatchObject([{ kind: 'create' }]);+ });++ test('stays a create when the file was deleted after the own save', async () => {+ const filePath = resolve(batchDir, 'recreated.md');+ const saved = '# Recreated\n\nIntro paragraph.\n';+ const outside = '# Recreated\n\nIntro paragraph, saved again out of band.\n';+ writeFileSync(filePath, saved, 'utf-8');+ registerWrite(filePath, contentHash(saved));+ unlinkSync(filePath);++ const dispatched: DiskEvent[] = [];+ const dispatch = async (event: DiskEvent) => {+ dispatched.push(event);+ };+ await handleRawEvents(+ [{ type: 'delete', path: filePath }],+ batchDir,+ undefined,+ new Map(),+ new Map(),+ dispatch,+ );+ writeFileSync(filePath, outside, 'utf-8');+ await handleRawEvents(+ [{ type: 'create', path: filePath }],+ batchDir,+ undefined,+ new Map(),+ new Map(),+ dispatch,+ );++ expect(dispatched.at(-1)).toMatchObject({+ kind: 'create',+ docName: 'recreated',+ content: outside,+ });+ expect(+ getWatcherDecisionRingSnapshot().some(+ (record) => record.decision === 'outside-write-over-own-write',+ ),+ ).toBe(false);+ });+});+describe('classifyEvents', () => {let tmpDir: string;let contentDir: string;diff --git a/packages/server/src/file-watcher.ts b/packages/server/src/file-watcher.tsindex 4799bff5e..8d33237e6 100644--- a/packages/server/src/file-watcher.ts+++ b/packages/server/src/file-watcher.ts@@ -322,6 +322,7 @@ export interface WatcherHandle {export const writeTracker = new Map<string, Array<{ hash: string; timestamp: number }>>();const WRITE_TRACKER_TTL_MS = 10_000;const REMOVAL_TRACKER_TTL_MS = 30_000;+const deletedOwnWrites = new WeakSet<{ hash: string; timestamp: number }>();interface OwnWriteObservation {key: string;@@ -492,6 +493,7 @@ type WatcherDecision =| 'dispatched'| 'self-write-skip'| 'superseded-own-write-skip'+ | 'outside-write-over-own-write'| 'self-removal-skip'| `drop-${WatcherDropReason}`;@@ -510,6 +512,7 @@ const watcherDropCounts = new Map<WatcherDropReason, number>();let watcherDispatchedCount = 0;let watcherSelfWriteSkipCount = 0;let watcherSupersededOwnWriteSkipCount = 0;+let watcherOutsideWriteOverOwnWriteCount = 0;let watcherSelfRemovalSkipCount = 0;let watcherDropsSinceLastSummary = 0;@@ -536,6 +539,10 @@ function recordWatcherDecision(decision: WatcherDecision, kind: string, rawPath:watcherSupersededOwnWriteSkipCount++;return;}+ if (decision === 'outside-write-over-own-write') {+ watcherOutsideWriteOverOwnWriteCount++;+ return;+ }if (decision === 'self-removal-skip') {watcherSelfRemovalSkipCount++;return;@@ -555,6 +562,14 @@ function recordSupersededOwnWriteSkip(kind: DiskEvent['kind'], filePath: string)recordWatcherDecision('superseded-own-write-skip', kind, filePath);}+function recordOutsideWriteOverOwnWrite(filePath: string): void {+ log.debug(+ { path: filePath },+ '[file-watcher] Read a create over a pending own write as an update',+ );+ recordWatcherDecision('outside-write-over-own-write', 'create', filePath);+}+export function getWatcherDecisionRingSnapshot(): WatcherDecisionRecord[] {return watcherDecisionRing.map((record) => ({ ...record }));}@@ -576,6 +591,7 @@ export function logWatcherDropSummary(): void {dispatched: watcherDispatchedCount,selfWriteSkips: watcherSelfWriteSkipCount,supersededOwnWriteSkips: watcherSupersededOwnWriteSkipCount,+ outsideWritesOverOwnWrites: watcherOutsideWriteOverOwnWriteCount,selfRemovalSkips: watcherSelfRemovalSkipCount,},`[file-watcher] drop summary: ${droppedSinceLastSummary} event(s) dropped since last summary`,@@ -588,6 +604,7 @@ export function resetWatcherDecisionDiagnostics(): void {watcherDispatchedCount = 0;watcherSelfWriteSkipCount = 0;watcherSupersededOwnWriteSkipCount = 0;+ watcherOutsideWriteOverOwnWriteCount = 0;watcherSelfRemovalSkipCount = 0;watcherDropsSinceLastSummary = 0;}@@ -1014,6 +1031,9 @@ async function classifyEventsInternal(continue;}}+ if (writeTracker.size > 0)+ for (const write of writeTracker.get(observedPathKey(event.path)) ?? [])+ deletedOwnWrites.add(write);deletes.push(event);break;}@@ -1069,6 +1089,16 @@ async function classifyEventsInternal(}}+ for (const event of creates) {+ const content = createContents.get(event.path);+ if (content === undefined || !overwritesOwnWrite(event.path, content)) continue;+ createContents.delete(event.path);+ createFacts.delete(event.path);+ updateContents.set(event.path, content);+ updates.push(event);+ recordOutsideWriteOverOwnWrite(event.path);+ }+function resolveDocName(event: RawFileEvent): string {const rawPath = event.path;const raw = pathToDocName(rawPath, contentDir);@@ -1269,6 +1299,15 @@ export function isSelfWrite(filePath: string, hash: string): boolean {return false;}+function overwritesOwnWrite(filePath: string, content: string): boolean {+ if (writeTracker.size === 0) return false;+ const queue = writeTracker.get(observedPathKey(filePath));+ const newest = queue?.at(-1);+ if (queue === undefined || newest === undefined || deletedOwnWrites.has(newest)) return false;+ const hash = contentHash(content);+ return queue.every((write) => write.hash !== hash);+}+function consumeOwnWrite(key: string, hash: string): boolean {const queue = writeTracker.get(key);if (!queue) return false;diff --git a/packages/server/src/http/file-ops-routes.ts b/packages/server/src/http/file-ops-routes.tsindex 67bf71cea..73a6e7786 100644--- a/packages/server/src/http/file-ops-routes.ts+++ b/packages/server/src/http/file-ops-routes.ts@@ -7,7 +7,7 @@ import {unlinkSync,} from 'node:fs';import type { IncomingMessage, ServerResponse } from 'node:http';-import { dirname, extname, resolve, sep } from 'node:path';+import { dirname, extname, resolve } from 'node:path';import { pipeline } from 'node:stream/promises';import {type AttachmentFolderUploadIssue,@@ -42,6 +42,7 @@ import { ManagedRenameCollisionError } from '../apply-managed-rename.ts';import { isConfigDoc, isSystemDoc } from '../cc1-broadcast.ts';import type { ConflictAuthority } from '../conflict-authority.ts';import { DocInConflictError, respondDocInConflict } from '../conflict-errors.ts';+import { registerSourceFileExtension } from '../content/file-operation-paths.ts';import { isReservedProjectStatePath } from '../content/managed-doc-enum.ts';import { applySubstitution } from '../content/substitution.ts';import { resolveTemplatesAvailable } from '../content/templates-resolver.ts';@@ -51,8 +52,6 @@ import {getDocExtension,isSupportedAssetFile,isSupportedDocFile,- registerDocExtension,- SUPPORTED_DOC_EXTENSIONS,stripDocExtension,} from '../doc-extensions.ts';import { extractActorIdentity } from '../extract-actor-identity.ts';@@ -392,44 +391,7 @@ export function createFileOpsRoutes(deps: FileOpsRouteDeps): ApiRouteGroup {function probeAndRegisterSourceFileExtension(probeContentDir: string, fromPath: string): void {if (!isValidRelativeContentPath(fromPath)) return;- const resolvedContentDir = resolve(probeContentDir);- if (isSupportedDocFile(fromPath)) {- const extensionless = stripDocExtension(fromPath);- for (const ext of SUPPORTED_DOC_EXTENSIONS) {- const candidate = resolve(resolvedContentDir, `${extensionless}${ext}`);- if (- candidate !== resolvedContentDir &&- !candidate.startsWith(`${resolvedContentDir}${sep}`)- ) {- continue;- }- if (existsSync(candidate)) {- registerDocExtension(extensionless, ext);- }- }- const explicitCandidate = resolve(resolvedContentDir, fromPath);- if (- explicitCandidate !== resolvedContentDir &&- explicitCandidate.startsWith(`${resolvedContentDir}${sep}`) &&- existsSync(explicitCandidate)- ) {- registerDocExtension(extensionless, extname(fromPath));- }- return;- }- for (const ext of SUPPORTED_DOC_EXTENSIONS) {- const candidate = resolve(resolvedContentDir, `${fromPath}${ext}`);- if (- candidate !== resolvedContentDir &&- !candidate.startsWith(`${resolvedContentDir}${sep}`)- ) {- continue;- }- if (existsSync(candidate)) {- registerDocExtension(fromPath, ext);- return;- }- }+ registerSourceFileExtension(probeContentDir, fromPath);}function resolveExtensionlessAssetPath(assetPath: string): {diff --git a/packages/server/src/persistence-load-before-watcher.test.ts b/packages/server/src/persistence-load-before-watcher.test.tsnew file mode 100644index 000000000..9974baf9c--- /dev/null+++ b/packages/server/src/persistence-load-before-watcher.test.ts@@ -0,0 +1,55 @@+import { mkdtempSync, rmSync, writeFileSync } from 'node:fs';+import { tmpdir } from 'node:os';+import { join } from 'node:path';+import { afterEach, beforeEach, expect, test } from 'vitest';+import * as Y from 'yjs';+import { _resetDocExtensionsForTests, isRegisteredMarkdownDocName } from './doc-extensions.ts';+import { createPersistenceExtension } from './persistence.ts';++let contentDir: string;++beforeEach(() => {+ _resetDocExtensionsForTests();+ contentDir = mkdtempSync(join(tmpdir(), 'ok-load-before-watcher-'));+});++afterEach(() => {+ _resetDocExtensionsForTests();+ rmSync(contentDir, { recursive: true, force: true });+});++test('a page opened before the watcher has seen its new .mdx file loads that file', async () => {+ const seed = '# Seed\n\n<Callout>Body.</Callout>\n';+ writeFileSync(join(contentDir, 'fresh.mdx'), seed);+ const persistence = createPersistenceExtension({+ mergeHistory: null,+ ingestDiskAtStore: null,+ contentDir,+ });+ const document = new Y.Doc();++ await persistence.extension.onLoadDocument?.({+ document,+ documentName: 'fresh',+ context: {},+ } as never);++ expect(document.getText('source').toString()).toBe(seed);+});++test('a page name with a parent segment registers no extension when it loads', async () => {+ writeFileSync(join(contentDir, 'fresh.mdx'), '# Seed\n');+ const persistence = createPersistenceExtension({+ mergeHistory: null,+ ingestDiskAtStore: null,+ contentDir,+ });++ await persistence.extension.onLoadDocument?.({+ document: new Y.Doc(),+ documentName: 'other/../fresh',+ context: {},+ } as never);++ expect(isRegisteredMarkdownDocName('other/../fresh')).toBe(false);+});diff --git a/packages/server/src/persistence.ts b/packages/server/src/persistence.tsindex 10bd98060..ff31d8111 100644--- a/packages/server/src/persistence.ts+++ b/packages/server/src/persistence.ts@@ -56,6 +56,7 @@ import {type LifecycleView,type RemovalLifecycleStatus,} from './conflict-kinds.ts';+import { registerSourceFileExtension } from './content/file-operation-paths.ts';import { isWithinContentDir, safeContentPath } from './content-path.ts';import type { ContributorEntry } from './contributor-tracker.ts';import {@@ -74,6 +75,7 @@ import { resolveDirectoryRoot, resolveNativePath } from './directory-root.ts';import { applyDiskContentToDoc, FILE_WATCHER_ORIGIN } from './disk-content-intake.ts';import type { FileObservation } from './disk-receipts.ts';import { displayPathFor } from './display-path.ts';+import { isRegisteredMarkdownDocName } from './doc-extensions.ts';import {assertNeverStorePublishOutcome,type DeferredStoreDiscardCause,@@ -2478,6 +2480,8 @@ export function createPersistenceExtension(options: PersistenceOptions): PersistloadManagedArtifactDoc(document, documentName, managedArtifactCtx);return;}+ if (!isRegisteredMarkdownDocName(documentName))+ registerSourceFileExtension(contentDir, documentName);options?.assertContentPath?.(safeContentPath(documentName, contentDir));if (isMermaidDoc(documentName) ||