Beautiful, AI-native markdown IDE and LLM wiki
TypeScript 2nd-brainagent-skillsclaudecodexcompany-braindocsknowledge-knowledge-baseknowledge-managementllmllm-wikillm-wiki-karpathymarkdownmarkdown-editormdmdxnotespkmsecond-brainskills
fix(ok): cut the per-keystroke Observer A drain on large documents (#4162)
Nick Gomez1311e2e14193
diff --git a/.changeset/lucky-pandas-repeat.md b/.changeset/lucky-pandas-repeat.mdnew file mode 100644index 000000000..0cf9290ee--- /dev/null+++ b/.changeset/lucky-pandas-repeat.md@@ -0,0 +1,10 @@+---+"@inkeep/open-knowledge": patch+---++Typing in a large document now costs the server less per keystroke.++On a 231 KB document the per-keystroke cost drops by roughly 1.7x to 2x, measured across+several cursor positions. The gain is position-dependent: an edit near the end of a document+gains more than one at the very top. Editing inside a list, a blockquote, or a list item keeps+part of the old cost. Nothing changes about what gets written to your files.diff --git a/package.json b/package.jsonindex 449f72092..9fc734217 100644--- a/package.json+++ b/package.json@@ -36,6 +36,7 @@"measure:stress": "cd packages/app && pnpm run measure:stress","measure:sweep": "cd packages/app && pnpm run measure:sweep","measure:generated-index-sweep": "pnpm --dir packages/server run measure:generated-index-sweep",+ "measure:observer-a-drain": "pnpm --dir packages/server run measure:observer-a-drain","prepare": "bash scripts/husky-prepare.sh","changeset": "changeset",diff --git a/packages/app/tests/integration/c3-mixed-mode.test.ts b/packages/app/tests/integration/c3-mixed-mode.test.tsindex 4cfc948e2..9a6bafc0c 100644--- a/packages/app/tests/integration/c3-mixed-mode.test.ts+++ b/packages/app/tests/integration/c3-mixed-mode.test.ts@@ -1,4 +1,5 @@import { setTimeout as wait } from 'node:timers/promises';+import { getMetrics } from '@inkeep/open-knowledge-server';import { afterAll, beforeAll, describe, expect, test } from 'vitest';import * as Y from 'yjs';import { HARNESS_BOOT_TIMEOUT_MS } from './harness-boot-timeout';@@ -7,6 +8,7 @@ import {assertBridgeInvariant,createTestClients,createTestServer,+ getServerState,pollUntil,serializeFragment,type TestClient,@@ -74,6 +76,40 @@ describe('C3: mixed-mode concurrent edits', () => {}});+ test('a remote source edit followed by local typing exercises the spliced-body memo', async () => {+ const clients = await createTestClients(server.port, {+ count: 2,+ perClientOptions: { skipInvariantWatcher: true },+ });+ try {+ appendParagraph(clients[0], 'C3-MEMO-SEED');+ await assertConverged(clients, ['C3-MEMO-SEED']);++ clients[1].doc.transact(() => {+ clients[1].ytext.insert(0, 'C3-REMOTE-SOURCE\n\n');+ });+ await assertConverged(clients, ['C3-MEMO-SEED', 'C3-REMOTE-SOURCE']);++ const before = getMetrics();+ for (let i = 0; i < 4; i++) {+ appendParagraph(clients[0], `C3-MEMO-TYPED-${i}`);+ await wait(120);+ }+ const after = getMetrics();++ await assertConverged(clients, ['C3-REMOTE-SOURCE', 'C3-MEMO-TYPED-3']);+ expect(after.mapDrivenSpliceMemoHits).toBeGreaterThan(before.mapDrivenSpliceMemoHits);+ expect(+ (after.mapDrivenSpliceMemoSkips['entry-already-current'] ?? 0) -+ (before.mapDrivenSpliceMemoSkips['entry-already-current'] ?? 0),+ ).toBeGreaterThan(0);+ const serverBody = getServerState(server, clients[0].docName)?.fullMd ?? '';+ expect(serverBody).toContain('C3-REMOTE-SOURCE');+ } finally {+ for (const c of clients) await c.cleanup();+ }+ });+test('WYSIWYG + source with seed content — no content loss', async () => {const docName = `c3-seed-${crypto.randomUUID()}`;const clients = await createTestClients(server.port, {diff --git a/packages/app/tests/stress/bridge-convergence.fuzz.test.ts b/packages/app/tests/stress/bridge-convergence.fuzz.test.tsindex b174fdd36..cbb762cc4 100644--- a/packages/app/tests/stress/bridge-convergence.fuzz.test.ts+++ b/packages/app/tests/stress/bridge-convergence.fuzz.test.ts@@ -422,7 +422,7 @@ function parseIntegerEnv(name: string, raw: string): number {if (!Number.isFinite(parsed) || !Number.isInteger(parsed)) {throw new Error(`${name} must be a finite integer, got ${JSON.stringify(raw)}. ` +- `Example: ${name}=42 bun test tests/stress/bridge-convergence.fuzz.test.ts`,+ `Example: ${name}=42 pnpm exec vitest run tests/stress/bridge-convergence.fuzz.test.ts`,);}return parsed;diff --git a/packages/app/tests/stress/crdt-stress.e2e.ts b/packages/app/tests/stress/crdt-stress.e2e.tsindex e96112898..ff195f3b1 100644--- a/packages/app/tests/stress/crdt-stress.e2e.ts+++ b/packages/app/tests/stress/crdt-stress.e2e.ts@@ -3,6 +3,8 @@ import { loadLargeRealistic } from '../../../core/src/markdown/fixtures/index.tsimport { expect, filterCriticalErrors, test } from './_helpers';const FIXTURE = loadLargeRealistic();+const AGENT_WRITE_PROPAGATION_BUDGET_MS = 30_000;+const KEYSTROKE_TO_YTEXT_DRAIN_BUDGET_MS = 30_000;test('S6: multi-turn stress — large content + user edits', async ({ page, api, baseURL }) => {const logs: Array<{ type: string; text: string; url?: string; line?: number }> = [];@@ -44,7 +46,7 @@ test('S6: multi-turn stress — large content + user edits', async ({ page, api,(expected: number) =>(window.__activeProvider?.document?.getText('source')?.toString()?.length ?? 0) >= expected,lengthBeforeWrite + FIXTURE.length - 200,- { timeout: 30_000 },+ { timeout: AGENT_WRITE_PROPAGATION_BUDGET_MS },);await page.locator('.ProseMirror:not(.composer-prosemirror)').focus();@@ -53,7 +55,7 @@ test('S6: multi-turn stress — large content + user edits', async ({ page, api,await page.waitForFunction((m: string) => window.__activeProvider?.document?.getText('source')?.toString()?.includes(m),marker,- { timeout: 30_000 },+ { timeout: KEYSTROKE_TO_YTEXT_DRAIN_BUDGET_MS },);const turnState = await page.evaluate(() => {diff --git a/packages/app/tests/stress/server-authoritative-stress.test.ts b/packages/app/tests/stress/server-authoritative-stress.test.tsindex f22cac824..6f5019910 100644--- a/packages/app/tests/stress/server-authoritative-stress.test.ts+++ b/packages/app/tests/stress/server-authoritative-stress.test.ts@@ -114,7 +114,7 @@ describe('server-authoritative stress (US-013)', () => {if (!Number.isFinite(parsed) || !Number.isInteger(parsed)) {throw new Error(`STRESS_SEED must be a finite integer, got ${JSON.stringify(raw)}. ` +- `Example: STRESS_SEED=42 bun test tests/stress/server-authoritative-stress.test.ts`,+ `Example: STRESS_SEED=42 pnpm exec vitest run tests/stress/server-authoritative-stress.test.ts`,);}seed = parsed;diff --git a/packages/app/tests/stress/stress-api.ts b/packages/app/tests/stress/stress-api.tsindex a199075d9..2a83411f5 100644--- a/packages/app/tests/stress/stress-api.ts+++ b/packages/app/tests/stress/stress-api.ts@@ -197,7 +197,7 @@ async function main(): Promise<void> {try {await fetch(`${BASE}/api/document`);} catch {- console.error('[stress-api] ERROR: Dev server not reachable. Run `bun run dev` first.');+ console.error('[stress-api] ERROR: Dev server not reachable. Run `pnpm dev` first.');process.exit(1);}diff --git a/packages/server/measure-observer-a-drain.test.ts b/packages/server/measure-observer-a-drain.test.tsnew file mode 100644index 000000000..49c47e986--- /dev/null+++ b/packages/server/measure-observer-a-drain.test.ts@@ -0,0 +1,75 @@+import { describe, expect, test } from 'vitest';+import { parseDrainMeasurementArgs } from './measure-observer-a-drain.ts';++describe('measure-observer-a-drain argument parsing', () => {+ test('defaults cover all three carets at the S6 scale', () => {+ expect(parseDrainMeasurementArgs([])).toEqual({+ carets: ['start', 'middle', 'end'],+ fixtureMultiple: 3,+ keystrokes: 15,+ });+ });++ test('a caret list selects a subset in the order given', () => {+ expect(parseDrainMeasurementArgs(['--caret', 'end,start']).carets).toEqual(['end', 'start']);+ });++ test('fixture multiple and keystroke count are read as integers', () => {+ const args = parseDrainMeasurementArgs(['--fixture-multiple', '1', '--keystrokes', '4']);+ expect(args.fixtureMultiple).toBe(1);+ expect(args.keystrokes).toBe(4);+ });++ test('a misspelled flag is rejected rather than silently ignored', () => {+ expect(() => parseDrainMeasurementArgs(['--carets', 'start'])).toThrow(/unknown flag --carets/);+ });++ test('a misspelled flag in trailing position is reported as unknown, not as missing a value', () => {+ expect(() => parseDrainMeasurementArgs(['--carets'])).toThrow(/unknown flag --carets/);+ });++ test('a leading -- separator is normalized away', () => {+ expect(parseDrainMeasurementArgs(['--', '--caret', 'end']).carets).toEqual(['end']);+ });++ test('a -- separator anywhere but the leading position is rejected', () => {+ expect(() => parseDrainMeasurementArgs(['--', '--', '--caret', 'end'])).toThrow(+ /unknown flag --/,+ );+ expect(() => parseDrainMeasurementArgs(['--caret', 'end', '--'])).toThrow(/unknown flag --/);+ });++ test('a flag with no value is rejected rather than keeping the default', () => {+ expect(() => parseDrainMeasurementArgs(['--caret'])).toThrow(/--caret needs a value/);+ expect(() => parseDrainMeasurementArgs(['--keystrokes'])).toThrow(/--keystrokes needs a value/);+ });++ test('a non-decimal numeric literal is rejected rather than coerced', () => {+ expect(() => parseDrainMeasurementArgs(['--fixture-multiple', '0x10'])).toThrow(+ /--fixture-multiple/,+ );+ expect(() => parseDrainMeasurementArgs(['--keystrokes', '1e4'])).toThrow(/--keystrokes/);+ });++ test('a fractional count is rejected rather than truncated', () => {+ expect(() => parseDrainMeasurementArgs(['--fixture-multiple', '3.5'])).toThrow(+ /--fixture-multiple/,+ );+ expect(() => parseDrainMeasurementArgs(['--keystrokes', '2.5'])).toThrow(/--keystrokes/);+ });++ test('an unrecognized caret name is rejected even when a valid one accompanies it', () => {+ expect(() => parseDrainMeasurementArgs(['--caret', 'middleish'])).toThrow(/middleish/);+ expect(() => parseDrainMeasurementArgs(['--caret', 'start,middleish'])).toThrow(/middleish/);+ });++ test('a non-positive fixture multiple is rejected', () => {+ expect(() => parseDrainMeasurementArgs(['--fixture-multiple', '0'])).toThrow(+ /--fixture-multiple/,+ );+ });++ test('a non-positive keystroke count is rejected', () => {+ expect(() => parseDrainMeasurementArgs(['--keystrokes', 'x'])).toThrow(/--keystrokes/);+ });+});diff --git a/packages/server/measure-observer-a-drain.ts b/packages/server/measure-observer-a-drain.tsnew file mode 100644index 000000000..a55775576--- /dev/null+++ b/packages/server/measure-observer-a-drain.ts@@ -0,0 +1,181 @@+import { arch, cpus, platform } from 'node:os';+import { resolve } from 'node:path';+import { pathToFileURL } from 'node:url';+import { sharedExtensions } from '@inkeep/open-knowledge-core';+import { getSchema } from '@tiptap/core';+import * as Y from 'yjs';+import { loadLargeRealistic } from '../core/src/markdown/fixtures/index.ts';+import { composeAndWriteRawBody } from './src/bridge-intake.ts';+import { mdManager } from './src/md-manager.ts';+import { setupServerObservers } from './src/server-observers.ts';++export type CaretPosition = 'start' | 'middle' | 'end';++export interface DrainMeasurementArgs {+ readonly carets: CaretPosition[];+ readonly fixtureMultiple: number;+ readonly keystrokes: number;+}++export interface DrainMeasurement {+ readonly caret: CaretPosition;+ readonly capturedAt: string;+ readonly host: { readonly platform: string; readonly arch: string; readonly cpus: number };+ readonly nodeVersion: string;+ readonly bodyBytes: number;+ readonly keystrokes: number;+ readonly totalMs: number;+ readonly perKeystrokeMedianMs: number;+ readonly perKeystrokeMaxMs: number;+ readonly serializeCallsPerDrain: number;+ readonly parseCallsPerDrain: number;+ readonly markerLanded: boolean;+}++const RESULT_PREFIX = 'OBSERVER_A_DRAIN_RESULT ';+const ALL_CARETS: CaretPosition[] = ['start', 'middle', 'end'];++export function parseDrainMeasurementArgs(argv: readonly string[]): DrainMeasurementArgs {+ let carets = ALL_CARETS;+ let fixtureMultiple = 3;+ let keystrokes = 15;+ const positiveInteger = (flag: string, raw: string): number => {+ if (!/^[0-9]+$/.test(raw)) throw new Error(`${flag} must be a positive integer, got ${raw}`);+ const parsed = Number(raw);+ if (parsed < 1) throw new Error(`${flag} must be a positive integer, got ${raw}`);+ return parsed;+ };+ const normalized = argv[0] === '--' ? argv.slice(1) : argv;+ for (let i = 0; i < normalized.length; i++) {+ const flag = normalized[i];+ if (flag !== '--caret' && flag !== '--fixture-multiple' && flag !== '--keystrokes') {+ throw new Error(`unknown flag ${flag}; use --caret, --fixture-multiple or --keystrokes`);+ }+ const value = normalized[i + 1];+ if (value === undefined) throw new Error(`${flag} needs a value`);+ if (flag === '--caret') {+ const requested = value.split(',');+ const unknown = requested.filter((c) => !ALL_CARETS.includes(c as CaretPosition));+ if (unknown.length > 0) {+ throw new Error(`--caret does not accept ${unknown.join(',')}; use start, middle or end`);+ }+ carets = requested as CaretPosition[];+ } else if (flag === '--fixture-multiple') {+ fixtureMultiple = positiveInteger(flag, value);+ } else if (flag === '--keystrokes') {+ keystrokes = positiveInteger(flag, value);+ } else {+ const unreachable: never = flag;+ throw new Error(`unknown flag ${String(unreachable)}`);+ }+ i++;+ }+ return { carets, fixtureMultiple, keystrokes };+}++function collectTextNodes(+ node: Y.XmlFragment | Y.XmlElement,+ found: Y.XmlText[] = [],+): Y.XmlText[] {+ for (let i = 0; i < node.length; i++) {+ const child = node.get(i);+ if (child instanceof Y.XmlText) found.push(child);+ else if (child instanceof Y.XmlElement) collectTextNodes(child, found);+ }+ return found;+}++function median(values: readonly number[]): number {+ const sorted = [...values].sort((a, b) => a - b);+ return Number((sorted[Math.floor(sorted.length / 2)] ?? 0).toFixed(1));+}++export function measureDrain(caret: CaretPosition, args: DrainMeasurementArgs): DrainMeasurement {+ const base = loadLargeRealistic();+ const raw = Array.from({ length: args.fixtureMultiple }, () => base).join('\n');+ const doc = new Y.Doc();+ const xmlFragment = doc.getXmlFragment('default');+ const ytext = doc.getText('source');++ doc.transact(() => {+ composeAndWriteRawBody(doc, raw, 'agent');+ });+ const cleanup = setupServerObservers({+ doc,+ xmlFragment,+ ytext,+ mdManager,+ schema: getSchema(sharedExtensions),+ docName: 'measure-observer-a-drain',+ });++ const texts = collectTextNodes(xmlFragment);+ const target =+ caret === 'start'+ ? texts[0]+ : caret === 'middle'+ ? texts[Math.floor(texts.length / 2)]+ : texts[texts.length - 1];+ if (!target) throw new Error('fixture produced no editable text node');++ const originalSerialize = mdManager.serialize.bind(mdManager);+ const originalParse = mdManager.parseToEditorMdast.bind(mdManager);+ let serializeCalls = 0;+ let parseCalls = 0;+ mdManager.serialize = (json, opts) => {+ serializeCalls++;+ return originalSerialize(json, opts);+ };+ mdManager.parseToEditorMdast = (markdown) => {+ parseCalls++;+ return originalParse(markdown);+ };++ const marker = 'MEASURE'.repeat(Math.ceil(args.keystrokes / 7)).slice(0, args.keystrokes);+ const perKeystroke: number[] = [];+ try {+ const startedAt = performance.now();+ for (const char of marker) {+ const at = performance.now();+ doc.transact(() => {+ target.insert(target.length, char);+ });+ perKeystroke.push(performance.now() - at);+ }+ const totalMs = performance.now() - startedAt;+ return {+ caret,+ capturedAt: new Date().toISOString(),+ host: { platform: platform(), arch: arch(), cpus: cpus().length },+ nodeVersion: process.version,+ bodyBytes: raw.length,+ keystrokes: marker.length,+ totalMs: Number(totalMs.toFixed(1)),+ perKeystrokeMedianMs: median(perKeystroke),+ perKeystrokeMaxMs: Number(Math.max(...perKeystroke).toFixed(1)),+ serializeCallsPerDrain: Number((serializeCalls / marker.length).toFixed(2)),+ parseCallsPerDrain: Number((parseCalls / marker.length).toFixed(2)),+ markerLanded: ytext.toString().includes(marker),+ };+ } finally {+ mdManager.serialize = originalSerialize;+ mdManager.parseToEditorMdast = originalParse;+ cleanup?.();+ }+}++function runCampaign(args: DrainMeasurementArgs): void {+ for (const caret of args.carets) {+ const measurement = measureDrain(caret, args);+ process.stdout.write(`${RESULT_PREFIX}${JSON.stringify(measurement)}\n`);+ }+}++if (process.argv[1] && pathToFileURL(resolve(process.argv[1])).href === import.meta.url) {+ try {+ runCampaign(parseDrainMeasurementArgs(process.argv.slice(2)));+ } catch (err) {+ process.stderr.write(`${err instanceof Error ? err.message : String(err)}\n`);+ process.exitCode = 1;+ }+}diff --git a/packages/server/package.json b/packages/server/package.jsonindex d931dde82..3d6f6a6af 100644--- a/packages/server/package.json+++ b/packages/server/package.json@@ -74,6 +74,7 @@"typecheck": "pnpm run typecheck:bash && tsc","test": "vitest run","measure:generated-index-sweep": "node --import tsx --conditions=development measure-generated-index-sweep.ts",+ "measure:observer-a-drain": "node --import tsx --conditions=development measure-observer-a-drain.ts","typecheck:bash": "tsc -p tsconfig.bash.json"},"devDependencies": {diff --git a/packages/server/src/map-driven-observer-a.test.ts b/packages/server/src/map-driven-observer-a.test.tsindex 390a65951..52764f05e 100644--- a/packages/server/src/map-driven-observer-a.test.ts+++ b/packages/server/src/map-driven-observer-a.test.ts@@ -1,5 +1,9 @@-import { MarkdownManager, sharedExtensions } from '@inkeep/open-knowledge-core';-import { getSchema } from '@tiptap/core';+import {+ MarkdownManager,+ type SerializeCallOptions,+ sharedExtensions,+} from '@inkeep/open-knowledge-core';+import { getSchema, type JSONContent } from '@tiptap/core';import { updateYFragment } from '@tiptap/y-tiptap';import { describe, expect, test, vi } from 'vitest';import * as Y from 'yjs';@@ -11,6 +15,7 @@ import { getMetrics } from './metrics.ts';import { createCountingManager } from './parse-counting.test-helper.ts';import {__resetMapDrivenParseErrorWarnForTests,+ __resetMemoComposeFailureWarnForTests,OBSERVER_SYNC_ORIGIN,setupServerObservers,} from './server-observers.ts';@@ -329,14 +334,11 @@ describe('map-driven Observer A — default Path A behavior', () => {} as unknown as MarkdownManager;const reasons: string[] = [];- const splice = computeMapDrivenBodySplice(- 'A.\n',- mdManager.parse('A.\n'),- throwingManager,- (reason) => {+ const splice = computeMapDrivenBodySplice('A.\n', mdManager.parse('A.\n'), throwingManager, {+ onFallback: (reason) => {reasons.push(reason);},- );+ });expect(splice).toBeNull();expect(reasons).toEqual(['parse-error']);@@ -391,14 +393,69 @@ describe('map-driven Observer A — default Path A behavior', () => {cleanup();});+ test('a memo composition failure warns once with the error, then stays counter-only', () => {+ const raw = '# Heading\n\npreserved \n\nFirst.\n\nSecond.\n\nThird.\n';+ const { doc, xmlFragment, ytext } = createTestDoc();+ const uncloneableManager = new Proxy(mdManager, {+ get(target, prop) {+ if (prop === 'parseToEditorMdast') {+ return (markdown: string) => {+ const tree = target.parseToEditorMdast(markdown);+ for (const child of tree.children) {+ (child as { data?: Record<string, unknown> }).data = {+ uncloneable: () => undefined,+ };+ }+ return tree;+ };+ }+ const value = Reflect.get(target, prop, target);+ return typeof value === 'function' ? value.bind(target) : value;+ },+ });+ __resetMemoComposeFailureWarnForTests();+ const warnSpy = vi.spyOn(getLogger('server-observers'), 'warn');+ const cleanup = setupServerObservers({+ doc,+ xmlFragment,+ ytext,+ mdManager: uncloneableManager,+ schema,+ });+ doc.transact(() => {+ composeAndWriteRawBody(doc, raw, 'agent');+ }, AGENT_WRITE_ORIGIN);++ const before = getMetrics();+ populateFragment(doc, xmlFragment, raw.replace('First.', 'First EDITED.'));+ populateFragment(doc, xmlFragment, raw.replace('First.', 'First EDITED TWICE.'));+ const after = getMetrics();++ expect(ytext.toString()).toContain('First EDITED TWICE.');+ expect(+ (after.mapDrivenSpliceMemoSkips['compose-failed'] ?? 0) -+ (before.mapDrivenSpliceMemoSkips['compose-failed'] ?? 0),+ ).toBeGreaterThanOrEqual(2);+ const memoWarns = warnSpy.mock.calls.filter((args) =>+ String(args[1]).includes('Spliced-body memo composition threw'),+ );+ expect(memoWarns).toHaveLength(1);+ expect((memoWarns[0]?.[0] as { err?: Error }).err?.name).toBe('DataCloneError');++ warnSpy.mockRestore();+ cleanup();+ });+test('an offset-less block reports missing-position through the pure computer', () => {const reasons: string[] = [];const splice = computeMapDrivenBodySplice('<!-- note -->\n',mdManager.parse('<!-- note -->\n\nX.\n'),mdManager,- (reason) => {- reasons.push(reason);+ {+ onFallback: (reason) => {+ reasons.push(reason);+ },},);@@ -448,4 +505,77 @@ describe('typing-burst parse economy (PRD-8273)', () => {cleanup();});++ test('a keystroke drain serializes the fragment JSON once, not once per consumer', () => {+ const { manager: counted, serializes } = createCountingManager();+ const { doc, xmlFragment, ytext } = createTestDoc();+ const cleanup = setupServerObservers({+ doc,+ xmlFragment,+ ytext,+ mdManager: counted,+ schema,+ });++ doc.transact(() => {+ const pmNode = schema.nodeFromJSON(counted.parse('# H\n\nalpha\n'));+ updateYFragment(doc, xmlFragment, pmNode, { mapping: new Map(), isOMark: new Map() });+ });++ const before = serializes();+ for (const ch of 'XYZ') typeChar(doc, xmlFragment, ch);++ expect(ytext.toString()).toContain('alphaXYZ');+ expect(serializes() - before).toBe(3);++ cleanup();+ });++ test('a drain inside the freshness window serializes for itself instead of reusing the suppressed body', () => {+ const real = new MarkdownManager({+ extensions: sharedExtensions,+ deriveStructuralFreshness: true,+ });+ const serializeOpts: Array<SerializeCallOptions | undefined> = [];+ const recording = new Proxy(real, {+ get(target, prop, receiver) {+ if (prop === 'serialize') {+ return (json: JSONContent, opts?: SerializeCallOptions) => {+ serializeOpts.push(opts);+ return target.serialize(json, opts);+ };+ }+ const value = Reflect.get(target, prop, receiver);+ return typeof value === 'function' ? value.bind(target) : value;+ },+ });+ const { doc, xmlFragment, ytext } = createTestDoc();+ const cleanup = setupServerObservers({+ doc,+ xmlFragment,+ ytext,+ mdManager: recording,+ schema,+ docName: 'freshness-window',+ });++ doc.transact(() => {+ const pmNode = schema.nodeFromJSON(real.parse('# H\n\nalpha\n\nbeta\n'));+ updateYFragment(doc, xmlFragment, pmNode, { mapping: new Map(), isOMark: new Map() });+ });++ doc.transact(() => {+ ytext.insert(ytext.length, 'Source tail.\n');+ });++ const before = serializeOpts.length;+ typeChar(doc, xmlFragment, 'X');+ const drainCalls = serializeOpts.slice(before);++ expect(drainCalls.some((o) => o?.skipFreshnessDerive === true)).toBe(true);+ expect(drainCalls.some((o) => o?.skipFreshnessDerive !== true)).toBe(true);+ expect(ytext.toString()).toContain('X');++ cleanup();+ });});diff --git a/packages/server/src/map-driven-splice.test.ts b/packages/server/src/map-driven-splice.test.tsindex 58a850939..f14719b2a 100644--- a/packages/server/src/map-driven-splice.test.ts+++ b/packages/server/src/map-driven-splice.test.ts@@ -1,7 +1,12 @@import { MarkdownManager, sharedExtensions } from '@inkeep/open-knowledge-core';import type { JSONContent } from '@tiptap/core';import { describe, expect, test } from 'vitest';-import { computeMapDrivenBodySplice, createEditorMdastMemo } from './map-driven-splice.ts';+import {+ computeMapDrivenBodySplice,+ createEditorMdastMemo,+ type EditorMdastMemo,+} from './map-driven-splice.ts';+import type { MapDrivenSpliceMemoSkipReason } from './metrics.ts';import { createCountingManager } from './parse-counting.test-helper.ts';const mdManager = new MarkdownManager({ extensions: sharedExtensions });@@ -212,19 +217,15 @@ describe('editor-mdast parse memo (PRD-8273)', () => {const memo = createEditorMdastMemo();const bodyA = '# H\n\nalpha\n';- const first = computeMapDrivenBodySplice(- bodyA,- counted.parse('# H\n\nalphaX\n'),- counted,- undefined,+ const first = computeMapDrivenBodySplice(bodyA, counted.parse('# H\n\nalphaX\n'), counted, {memo,- );+ });expect(first).not.toBeNull();if (!first) return;const bodyB = applySplice(bodyA, first);const afterFirst = parses();- computeMapDrivenBodySplice(bodyB, counted.parse('# H\n\nalphaXY\n'), counted, undefined, memo);+ computeMapDrivenBodySplice(bodyB, counted.parse('# H\n\nalphaXY\n'), counted, { memo });expect(parses() - afterFirst).toBe(1);});@@ -234,7 +235,7 @@ describe('editor-mdast parse memo (PRD-8273)', () => {const memo = createEditorMdastMemo();const primed = '# H\n\nalpha\n';- computeMapDrivenBodySplice(primed, counted.parse('# H\n\nalphaX\n'), counted, undefined, memo);+ computeMapDrivenBodySplice(primed, counted.parse('# H\n\nalphaX\n'), counted, { memo });const afterPrime = parses();const external = '# DIFFERENT\n\nomega\n\ntail\n';@@ -242,8 +243,7 @@ describe('editor-mdast parse memo (PRD-8273)', () => {external,counted.parse('# DIFFERENT\n\nomega EDITED\n\ntail\n'),counted,- undefined,- memo,+ { memo },);expect(splice).not.toBeNull();if (!splice) return;@@ -265,8 +265,7 @@ describe('editor-mdast parse memo (PRD-8273)', () => {bodyA,counted.parse('# H\n\none EDITED\n\ntwo\n\nthree\n'),counted,- undefined,- memo,+ { memo },);expect(first).not.toBeNull();if (!first) return;@@ -274,7 +273,7 @@ describe('editor-mdast parse memo (PRD-8273)', () => {const newPm = counted.parse('# H\n\none EDITED\n\ntwo CHANGED\n\nthree\n');const before = parses();- const fromHit = computeMapDrivenBodySplice(bodyB, newPm, counted, undefined, memo);+ const fromHit = computeMapDrivenBodySplice(bodyB, newPm, counted, { memo });expect(parses() - before).toBe(1);const fromFreshParse = computeMapDrivenBodySplice(bodyB, newPm, counted);@@ -282,7 +281,7 @@ describe('editor-mdast parse memo (PRD-8273)', () => {expect(fromHit).not.toBeNull();});- test('preserved bytes the serializer would not emit cause the next drain to miss', () => {+ test('preserved bytes the serializer would not emit still hit the next drain', () => {const { manager: counted, parses } = createCountingManager();const memo = createEditorMdastMemo();const bodyA = 'one \n\ntwo\n\nthree\n';@@ -291,22 +290,19 @@ describe('editor-mdast parse memo (PRD-8273)', () => {bodyA,counted.parse('one \n\ntwo A\n\nthree\n'),counted,- undefined,- memo,+ { memo },);expect(first).not.toBeNull();if (!first) return;const bodyB = applySplice(bodyA, first);+ expect(bodyB).not.toBe(counted.serialize(counted.parse(bodyB)));+ expect(memo.entry?.body).toBe(bodyB);+ const newPm = counted.parse('one \n\ntwo B\n\nthree\n');const before = parses();- computeMapDrivenBodySplice(- bodyB,- counted.parse('one \n\ntwo B\n\nthree\n'),- counted,- undefined,- memo,- );- expect(parses() - before).toBe(2);+ const fromHit = computeMapDrivenBodySplice(bodyB, newPm, counted, { memo });+ expect(parses() - before).toBe(1);+ expect(fromHit).toEqual(computeMapDrivenBodySplice(bodyB, newPm, counted));});test('a same-length body with different content misses — the key is bytes, not length', () => {@@ -318,11 +314,11 @@ describe('editor-mdast parse memo (PRD-8273)', () => {expect(actual.length).toBe(primed.length);expect(actual).not.toBe(primed);- computeMapDrivenBodySplice('zzz\n', counted.parse(primed), counted, undefined, memo);+ computeMapDrivenBodySplice('zzz\n', counted.parse(primed), counted, { memo });const newPm = counted.parse('one two\n\nthreeY\n');const before = parses();- const withMemo = computeMapDrivenBodySplice(actual, newPm, counted, undefined, memo);+ const withMemo = computeMapDrivenBodySplice(actual, newPm, counted, { memo });expect(parses() - before).toBe(2);const withoutMemo = computeMapDrivenBodySplice(actual, newPm, counted);@@ -330,3 +326,212 @@ describe('editor-mdast parse memo (PRD-8273)', () => {expect(withMemo).not.toBeNull();});});++describe('caller-supplied serialization', () => {+ test('a body serialized from the same PM JSON is reused instead of re-serialized', () => {+ const { manager: counted, serializes } = createCountingManager();+ const memo = createEditorMdastMemo();+ const oldBody = '# H\n\none\n\ntwo\n';+ const newPm = counted.parse('# H\n\none EDITED\n\ntwo\n');+ const body = counted.serialize(newPm);++ const before = serializes();+ const reused = computeMapDrivenBodySplice(oldBody, newPm, counted, {+ memo,+ serializedNewPm: { json: newPm, body, opts: undefined },+ });+ expect(serializes() - before).toBe(0);+ expect(reused).toEqual(computeMapDrivenBodySplice(oldBody, newPm, counted));+ });++ test('a body carried from different PM JSON is ignored and the splice serializes itself', () => {+ const { manager: counted, serializes } = createCountingManager();+ const oldBody = '# H\n\none\n\ntwo\n';+ const newPm = counted.parse('# H\n\none EDITED\n\ntwo\n');+ const stale = counted.parse('# H\n\nSTALE\n\ntwo\n');+ const staleBody = counted.serialize(stale);++ const before = serializes();+ const splice = computeMapDrivenBodySplice(oldBody, newPm, counted, {+ serializedNewPm: { json: stale, body: staleBody, opts: undefined },+ });+ const selfSerializes = serializes() - before;+ expect(selfSerializes).toBe(1);+ expect(splice).toEqual(computeMapDrivenBodySplice(oldBody, newPm, counted));+ });+});++describe('spliced-body memo fidelity', () => {+ const DIRTY_PREFIX = 'preserved \n\n';++ const NARROWABLE_SHAPES = new Set(['list', 'blockquote', 'loose-list']);++ const CORPUS: Array<[string, string]> = [+ ['paragraphs', 'one\n\ntwo\n\nthree\n\nfour\n'],+ ['headings', '# H\n\npara\n\n## H2\n\npara two\n\ntail\n'],+ ['list', '# H\n\n- one\n- two\n- three\n\npara\n\ntail\n'],+ ['blockquote', '# H\n\n> one\n>\n> two\n\npara\n\ntail\n'],+ ['table', '# H\n\n| a | b |\n|---|---|\n| 1 | 2 |\n\npara\n\ntail\n'],+ ['code-fence', '# H\n\n```js\nconst a = 1;\n```\n\npara\n\ntail\n'],+ ['setext', 'Title\n=====\n\npara\n\ntail\n'],+ ['definitions', '# H\n\n[a]: http://example.com\n\nsee [a]\n\npara two\n\ntail\n'],+ ['footnote', '# H\n\ntext[^1]\n\n[^1]: note\n\npara two\n\ntail\n'],+ ['dirty-bytes', 'one \n\ntwo\n\nthree\n\nfour\n'],+ ['blank-runs', 'one\n\n\n\ntwo\n\nthree\n\nfour\n'],+ ['thematic-break', 'one\n\n---\n\ntwo\n\nthree\n'],+ ['loose-list', '# H\n\n- one\n\n- two\n\npara\n\ntail\n'],+ [+ 'many-blocks',+ `${Array.from({ length: 60 }, (_, i) => `Paragraph ${i} body text.`).join('\n\n')}\n`,+ ],+ ];++ const EDITS: Array<[string, (text: string) => string]> = [+ ['append-char', (text) => `${text}Z`],+ ['prepend-char', (text) => `Z${text}`],+ ['emphasis-marker', (text) => `${text} *em*`],+ ['dash-run', (text) => `${text} ---`],+ ];++ function editFirstLeaf(+ json: JSONContent,+ blockIndex: number,+ edit: (text: string) => string,+ ): JSONContent | null {+ const clone = structuredClone(json) as JSONContent;+ const block = clone.content?.[blockIndex];+ if (!block) return null;+ const stack: JSONContent[] = [block];+ while (stack.length > 0) {+ const node = stack.pop();+ if (!node) continue;+ if (typeof node.text === 'string') {+ node.text = edit(node.text);+ return clone;+ }+ for (const child of node.content ?? []) stack.push(child);+ }+ return null;+ }++ function appendToFirstLeaf(json: JSONContent, blockIndex: number): JSONContent | null {+ return editFirstLeaf(json, blockIndex, (text) => `${text}Z`);+ }++ for (const [label, canonicalDoc] of CORPUS) {+ for (const [form, doc] of [+ ['canonical', canonicalDoc],+ ['non-canonical', `${DIRTY_PREFIX}${canonicalDoc}`],+ ] as const) {+ test(`${label} (${form}): every composed entry equals a fresh parse of the spliced body`, () => {+ const pm = mdManager.parse(doc);+ const blockCount = pm.content?.length ?? 0;+ expect(blockCount).toBeGreaterThan(0);+ const firstEditable = form === 'canonical' ? 0 : 1;+ let composed = 0;+ let attempted = 0;+ const skips: MapDrivenSpliceMemoSkipReason[] = [];++ for (let index = firstEditable; index < blockCount; index++) {+ for (const [, edit] of EDITS) {+ const edited = editFirstLeaf(pm, index, edit);+ if (!edited) continue;+ attempted++;+ const memo = createEditorMdastMemo();+ const newBody = mdManager.serialize(edited);+ const splice = computeMapDrivenBodySplice(doc, edited, mdManager, {+ memo,+ onMemoSkip: (reason) => skips.push(reason),+ });+ expect(splice).not.toBeNull();+ if (!splice) continue;+ const applied = applySplice(doc, splice);+ expect(memo.entry).not.toBeNull();+ if (applied === newBody) {+ expect(memo.entry?.body).toBe(newBody);+ continue;+ }+ if (memo.entry?.body !== applied) continue;+ composed++;+ expect(JSON.stringify(memo.entry.children)).toBe(+ JSON.stringify(mdManager.parseToEditorMdast(applied).children),+ );+ }+ }++ const narrowedSkips = skips.filter((reason) => reason === 'narrowed').length;+ const alreadyCurrentSkips = skips.filter(+ (reason) => reason === 'entry-already-current',+ ).length;++ expect(attempted).toBeGreaterThan(0);+ expect(skips.filter((r) => r !== 'narrowed' && r !== 'entry-already-current')).toEqual([]);+ expect(composed).toBe(attempted - narrowedSkips - alreadyCurrentSkips);+ if (form === 'canonical') {+ expect(alreadyCurrentSkips).toBeGreaterThan(0);+ } else {+ expect(composed).toBeGreaterThan(0);+ }+ if (NARROWABLE_SHAPES.has(label)) {+ expect(narrowedSkips).toBeGreaterThan(0);+ } else {+ expect(narrowedSkips).toBe(0);+ }+ });+ }+ }++ test('a container-narrowed splice leaves the constructed entry unwritten', () => {+ const doc = 'para \n\n- one\n- two\n- three\n\ntail\n';+ const pm = mdManager.parse(doc);+ const edited = appendToFirstLeaf(pm, 1);+ expect(edited).not.toBeNull();+ if (!edited) return;+ const memo = createEditorMdastMemo();+ const splice = computeMapDrivenBodySplice(doc, edited, mdManager, { memo });+ expect(splice).not.toBeNull();+ if (!splice) return;+ const applied = applySplice(doc, splice);+ const newBody = mdManager.serialize(edited);+ expect(applied).not.toBe(newBody);+ expect(applied.startsWith('para ')).toBe(true);+ expect(splice.spliceStart).toBeGreaterThan(doc.indexOf('- one'));+ expect(memo.entry?.body).toBe(newBody);+ });+});++describe('spliced-body memo failure isolation', () => {+ test('a throwing memo write costs the optimization, not the splice', () => {+ const oldBody = 'preserved \n\none\n\ntwo\n\nthree\n';+ const newPm = mdManager.parse('preserved \n\none EDITED\n\ntwo\n\nthree\n');+ const reasons: MapDrivenSpliceMemoSkipReason[] = [];+ const errors: unknown[] = [];+ const composeFailure = new Error('synthetic composition regression');+ let stored: EditorMdastMemo['entry'] = null;+ let writes = 0;+ const hostileMemo = {+ get entry() {+ return stored;+ },+ set entry(value: EditorMdastMemo['entry']) {+ writes++;+ if (writes > 2) throw composeFailure;+ stored = value;+ },+ } as EditorMdastMemo;++ const splice = computeMapDrivenBodySplice(oldBody, newPm, mdManager, {+ memo: hostileMemo,+ onMemoSkip: (reason, err) => {+ reasons.push(reason);+ errors.push(err);+ },+ });++ expect(splice).not.toBeNull();+ expect(splice).toEqual(computeMapDrivenBodySplice(oldBody, newPm, mdManager));+ expect(writes).toBe(3);+ expect(reasons).toEqual(['compose-failed']);+ expect(errors).toEqual([composeFailure]);+ });+});diff --git a/packages/server/src/map-driven-splice.ts b/packages/server/src/map-driven-splice.tsindex 9c09efbcb..5227f250f 100644--- a/packages/server/src/map-driven-splice.ts+++ b/packages/server/src/map-driven-splice.ts@@ -1,6 +1,7 @@-import type { MarkdownManager } from '@inkeep/open-knowledge-core';+import type { MarkdownManager, SerializeCallOptions } from '@inkeep/open-knowledge-core';import type { JSONContent } from '@tiptap/core';-import type { Root, RootContent } from 'mdast';+import type { RootContent } from 'mdast';+import type { MapDrivenSpliceMemoSkipReason } from './metrics.ts';export interface MapDrivenSplice {readonly spliceStart: number;@@ -8,39 +9,75 @@ export interface MapDrivenSplice {readonly newSlice: string;}+export interface SerializedEditorBody {+ readonly json: JSONContent;+ readonly body: string;+ readonly opts: SerializeCallOptions | undefined;+}++export function serializeEditorBody(+ mdManager: MarkdownManager,+ json: JSONContent,+ opts?: SerializeCallOptions,+): SerializedEditorBody {+ return { json, body: mdManager.serialize(json, opts), opts };+}++function reusableBodyFrom(+ serializedNewPm: SerializedEditorBody | undefined,+ newPmJson: JSONContent,+): string | undefined {+ if (serializedNewPm === undefined) return undefined;+ if (serializedNewPm.json !== newPmJson) return undefined;+ if (serializedNewPm.opts?.skipFreshnessDerive === true) return undefined;+ return serializedNewPm.body;+}++export interface MapDrivenSpliceOptions {+ readonly onFallback?: (reason: 'parse-error' | 'missing-position', err?: unknown) => void;+ readonly onMemoHit?: () => void;+ readonly onMemoSkip?: (reason: MapDrivenSpliceMemoSkipReason, err?: unknown) => void;+ readonly memo?: EditorMdastMemo;+ readonly serializedNewPm?: SerializedEditorBody;+}+export interface EditorMdastMemo {- entry: { readonly body: string; readonly tree: Root } | null;+ entry: { readonly body: string; readonly children: readonly RootContent[] } | null;}export function createEditorMdastMemo(): EditorMdastMemo {return { entry: null };}-function parseEditorMdastMemoized(+function editorMdastChildren(mdManager: MarkdownManager,body: string,memo: EditorMdastMemo | undefined,-): Root {- if (memo?.entry?.body === body) return memo.entry.tree;- const tree = mdManager.parseToEditorMdast(body);- if (memo !== undefined) memo.entry = { body, tree };- return tree;+ onHit?: () => void,+): readonly RootContent[] {+ if (memo?.entry?.body === body) {+ onHit?.();+ return memo.entry.children;+ }+ const children = mdManager.parseToEditorMdast(body).children;+ if (memo !== undefined) memo.entry = { body, children };+ return children;}export function computeMapDrivenBodySplice(oldBody: string,newPmJson: JSONContent,mdManager: MarkdownManager,- onFallback?: (reason: 'parse-error' | 'missing-position', err?: unknown) => void,- memo?: EditorMdastMemo,+ options: MapDrivenSpliceOptions = {},): MapDrivenSplice | null {+ const { onFallback, onMemoHit, onMemoSkip, memo, serializedNewPm } = options;let oldChildren: readonly RootContent[];let newBody: string;let newChildren: readonly RootContent[];try {- oldChildren = parseEditorMdastMemoized(mdManager, oldBody, memo).children;- newBody = mdManager.serialize(newPmJson);- newChildren = parseEditorMdastMemoized(mdManager, newBody, memo).children;+ oldChildren = editorMdastChildren(mdManager, oldBody, memo, onMemoHit);+ newBody = reusableBodyFrom(serializedNewPm, newPmJson) ?? mdManager.serialize(newPmJson);+ newChildren = editorMdastChildren(mdManager, newBody, memo);} catch (err) {onFallback?.('parse-error', err);return null;@@ -51,8 +88,9 @@ export function computeMapDrivenBodySplice(return null;}+ let walked: ChildrenSplice;try {- return computeChildrenSplice(+ walked = computeChildrenSplice(oldChildren,newChildren,{@@ -69,6 +107,15 @@ export function computeMapDrivenBodySplice(onFallback?.('missing-position', err);return null;}++ if (memo !== undefined) {+ try {+ memoizeSplicedBody(memo, { oldBody, oldChildren, newBody, newChildren, walked }, onMemoSkip);+ } catch (err) {+ onMemoSkip?.('compose-failed', err);+ }+ }+ return walked.splice;}interface ByteRegion {@@ -76,6 +123,16 @@ interface ByteRegion {readonly end: number;}+type ChildrenSplice =+ | { readonly narrowed: true; readonly splice: MapDrivenSplice }+ | {+ readonly narrowed: false;+ readonly splice: MapDrivenSplice;+ readonly prefixLen: number;+ readonly suffixLen: number;+ readonly newSliceStart: number;+ };+const NARROWABLE_CONTAINER_TYPES = new Set(['blockquote', 'list', 'listItem']);function computeChildrenSplice(@@ -84,7 +141,7 @@ function computeChildrenSplice(oldRegion: ByteRegion,newRegion: ByteRegion,newBody: string,-): MapDrivenSplice {+): ChildrenSplice {let prefixLen = 0;while (prefixLen < oldChildren.length &&@@ -113,7 +170,7 @@ function computeChildrenSplice(const oldChanged = oldChildren[prefixLen];const newChanged = newChildren[prefixLen];const narrowed = tryNarrowIntoContainer(oldChanged, newChanged, newBody);- if (narrowed) return narrowed;+ if (narrowed) return { narrowed: true, splice: narrowed.splice };}const spliceStart = prefixLen > 0 ? blockEndOffset(oldChildren[prefixLen - 1]) : oldRegion.start;@@ -126,9 +183,15 @@ function computeChildrenSplice(suffixLen > 0 ? blockStartOffset(newChildren[newChildren.length - suffixLen]) : newRegion.end;return {- spliceStart,- spliceEnd,- newSlice: newBody.slice(newSliceStart, newSliceEnd),+ splice: {+ spliceStart,+ spliceEnd,+ newSlice: newBody.slice(newSliceStart, newSliceEnd),+ },+ prefixLen,+ suffixLen,+ newSliceStart,+ narrowed: false,};}@@ -136,7 +199,7 @@ function tryNarrowIntoContainer(oldNode: RootContent,newNode: RootContent,newBody: string,-): MapDrivenSplice | null {+): ChildrenSplice | null {if (oldNode.type !== newNode.type || !NARROWABLE_CONTAINER_TYPES.has(oldNode.type)) return null;if (!('children' in oldNode) || !('children' in newNode)) return null;const oldKids = oldNode.children as readonly RootContent[];@@ -158,6 +221,105 @@ function tryNarrowIntoContainer();}+interface SplicedBodyInputs {+ readonly oldBody: string;+ readonly oldChildren: readonly RootContent[];+ readonly newBody: string;+ readonly newChildren: readonly RootContent[];+ readonly walked: ChildrenSplice;+}++function memoizeSplicedBody(+ memo: EditorMdastMemo,+ inputs: SplicedBodyInputs,+ onSkip: ((reason: MapDrivenSpliceMemoSkipReason, err?: unknown) => void) | undefined,+): void {+ const { oldBody, oldChildren, newBody, newChildren, walked } = inputs;+ if (walked.narrowed) {+ onSkip?.('narrowed');+ return;+ }+ if (oldChildren.length === 0 || newChildren.length === 0) {+ onSkip?.('empty-children');+ return;+ }++ const { splice, prefixLen, suffixLen, newSliceStart } = walked;+ const applied =+ oldBody.slice(0, splice.spliceStart) + splice.newSlice + oldBody.slice(splice.spliceEnd);+ if (applied === newBody) {+ onSkip?.('entry-already-current');+ return;+ }++ const mid = cloneShifted(+ newChildren.slice(prefixLen, newChildren.length - suffixLen),+ splice.spliceStart - newSliceStart,+ countNewlines(oldBody, 0, splice.spliceStart) - countNewlines(newBody, 0, newSliceStart),+ );+ if (mid === null) {+ onSkip?.('position-not-numeric');+ return;+ }++ const tail = cloneShifted(+ oldChildren.slice(oldChildren.length - suffixLen),+ applied.length - oldBody.length,+ countNewlines(splice.newSlice, 0, splice.newSlice.length) -+ countNewlines(oldBody, splice.spliceStart, splice.spliceEnd),+ );+ if (tail === null) {+ onSkip?.('position-not-numeric');+ return;+ }++ memo.entry = { body: applied, children: [...oldChildren.slice(0, prefixLen), ...mid, ...tail] };+}++function cloneShifted(+ nodes: readonly RootContent[],+ byteDelta: number,+ lineDelta: number,+): RootContent[] | null {+ const cloned = structuredClone(nodes) as RootContent[];+ return shiftPositions(cloned, byteDelta, lineDelta) ? cloned : null;+}++function shiftPositions(value: unknown, byteDelta: number, lineDelta: number): boolean {+ if (Array.isArray(value)) {+ for (const entry of value) {+ if (!shiftPositions(entry, byteDelta, lineDelta)) return false;+ }+ return true;+ }+ if (value === null || typeof value !== 'object') return true;+ const node = value as Record<string, unknown>;+ const position = node.position as+ | { start?: { offset?: number; line?: number }; end?: { offset?: number; line?: number } }+ | undefined;+ if (position !== undefined) {+ for (const point of [position.start, position.end]) {+ if (point === undefined) continue;+ if (typeof point.offset !== 'number' || typeof point.line !== 'number') return false;+ point.offset += byteDelta;+ point.line += lineDelta;+ }+ }+ for (const key of Object.keys(node)) {+ if (key === 'position') continue;+ if (!shiftPositions(node[key], byteDelta, lineDelta)) return false;+ }+ return true;+}++function countNewlines(text: string, from: number, to: number): number {+ let count = 0;+ for (let index = from; index < to; index++) {+ if (text.charCodeAt(index) === 10) count++;+ }+ return count;+}+function allBlocksCarryPositions(children: readonly RootContent[]): boolean {for (const child of children) {const start = child.position?.start?.offset;diff --git a/packages/server/src/metrics.test.ts b/packages/server/src/metrics.test.tsindex debdbce27..05021bf86 100644--- a/packages/server/src/metrics.test.ts+++ b/packages/server/src/metrics.test.ts@@ -10,6 +10,8 @@ import {incrementConflict,incrementMapDrivenSpliceApplied,incrementMapDrivenSpliceFallback,+ incrementMapDrivenSpliceMemoHit,+ incrementMapDrivenSpliceMemoSkip,incrementPark,incrementReconcile,incrementRescueBuffer,@@ -96,6 +98,29 @@ describe('reconciliation metrics', () => {expect(getMetrics().reconcileCount).toBe(2);});+ test('the map-driven splice memo counters accumulate and snapshot independently', () => {+ resetMetrics();+ incrementMapDrivenSpliceMemoHit();+ incrementMapDrivenSpliceMemoSkip('narrowed');+ incrementMapDrivenSpliceMemoSkip('narrowed');+ incrementMapDrivenSpliceMemoSkip('entry-already-current');+ const first = getMetrics();+ expect(first.mapDrivenSpliceMemoHits).toBe(1);+ expect(first.mapDrivenSpliceMemoSkips).toEqual({ narrowed: 2, 'entry-already-current': 1 });++ incrementMapDrivenSpliceMemoHit();+ incrementMapDrivenSpliceMemoSkip('position-not-numeric');+ const second = getMetrics();+ expect(second.mapDrivenSpliceMemoHits).toBe(2);+ expect(second.mapDrivenSpliceMemoSkips).toEqual({+ narrowed: 2,+ 'entry-already-current': 1,+ 'position-not-numeric': 1,+ });+ expect(first.mapDrivenSpliceMemoHits).toBe(1);+ expect(first.mapDrivenSpliceMemoSkips).toEqual({ narrowed: 2, 'entry-already-current': 1 });+ });+test('resetMetrics clears all counters', () => {incrementReconcile();incrementConflict();@@ -108,7 +133,8 @@ describe('reconciliation metrics', () => {if (key === 'cc1LastSeq' ||key === 'bridgeToleranceApplied' ||- key === 'mapDrivenSpliceFallback'+ key === 'mapDrivenSpliceFallback' ||+ key === 'mapDrivenSpliceMemoSkips') {expect(value).toEqual({});} else {diff --git a/packages/server/src/metrics.ts b/packages/server/src/metrics.tsindex 3a47a300d..548b83f35 100644--- a/packages/server/src/metrics.ts+++ b/packages/server/src/metrics.ts@@ -6,6 +6,13 @@ export type MapDrivenSpliceFallbackReason =| 'parse-error'| 'missing-position';+export type MapDrivenSpliceMemoSkipReason =+ | 'narrowed'+ | 'empty-children'+ | 'entry-already-current'+ | 'position-not-numeric'+ | 'compose-failed';+export interface ReconciliationMetrics {reconcileCount: number;conflictCount: number;@@ -56,6 +63,8 @@ export interface ReconciliationMetrics {observerAPathBFiresSuppressed: number;mapDrivenSpliceApplied: number;mapDrivenSpliceFallback: Partial<Record<MapDrivenSpliceFallbackReason, number>>;+ mapDrivenSpliceMemoHits: number;+ mapDrivenSpliceMemoSkips: Partial<Record<MapDrivenSpliceMemoSkipReason, number>>;observerAResidualMergeRuns: number;/** Count of Observer A duplication-gate recoveries — a substantive body* line materialized more times in the fragment than clean Y.Text justified,@@ -166,6 +175,8 @@ const counters: ReconciliationMetrics = {observerAPathBFires: 0,observerAPathBFiresSuppressed: 0,mapDrivenSpliceApplied: 0,+ mapDrivenSpliceMemoHits: 0,+ mapDrivenSpliceMemoSkips: {},mapDrivenSpliceFallback: {},observerAResidualMergeRuns: 0,observerADuplicationRederives: 0,@@ -366,6 +377,14 @@ export function incrementMapDrivenSpliceFallback(reason: MapDrivenSpliceFallbackcounters.mapDrivenSpliceFallback[reason] = (counters.mapDrivenSpliceFallback[reason] ?? 0) + 1;}+export function incrementMapDrivenSpliceMemoHit(): void {+ counters.mapDrivenSpliceMemoHits++;+}++export function incrementMapDrivenSpliceMemoSkip(reason: MapDrivenSpliceMemoSkipReason): void {+ counters.mapDrivenSpliceMemoSkips[reason] = (counters.mapDrivenSpliceMemoSkips[reason] ?? 0) + 1;+}+export function incrementObserverAResidualMergeRuns(): void {counters.observerAResidualMergeRuns++;}@@ -590,6 +609,7 @@ export function getMetrics(): ReconciliationMetrics {cc1LastSeq: { ...counters.cc1LastSeq },bridgeToleranceApplied: { ...counters.bridgeToleranceApplied },mapDrivenSpliceFallback: { ...counters.mapDrivenSpliceFallback },+ mapDrivenSpliceMemoSkips: { ...counters.mapDrivenSpliceMemoSkips },};}@@ -643,6 +663,8 @@ export function resetMetrics(): void {counters.observerAPathBFiresSuppressed = 0;counters.mapDrivenSpliceApplied = 0;counters.mapDrivenSpliceFallback = {};+ counters.mapDrivenSpliceMemoHits = 0;+ counters.mapDrivenSpliceMemoSkips = {};counters.observerAResidualMergeRuns = 0;counters.observerADuplicationRederives = 0;counters.observerADuplicationCheckpointCreated = 0;diff --git a/packages/server/src/parse-counting.test-helper.ts b/packages/server/src/parse-counting.test-helper.tsindex d20cb23bb..cad9d3219 100644--- a/packages/server/src/parse-counting.test-helper.ts+++ b/packages/server/src/parse-counting.test-helper.ts@@ -3,15 +3,22 @@ import { MarkdownManager, sharedExtensions } from '@inkeep/open-knowledge-core';export interface CountingManager {readonly manager: MarkdownManager;readonly parses: () => number;+ readonly serializes: () => number;}export function createCountingManager(): CountingManager {const manager = new MarkdownManager({ extensions: sharedExtensions });let calls = 0;+ let serializeCalls = 0;const original = manager.parseToEditorMdast.bind(manager);manager.parseToEditorMdast = (markdown: string) => {calls += 1;return original(markdown);};- return { manager, parses: () => calls };+ const originalSerialize = manager.serialize.bind(manager);+ manager.serialize = (json, opts) => {+ serializeCalls += 1;+ return originalSerialize(json, opts);+ };+ return { manager, parses: () => calls, serializes: () => serializeCalls };}diff --git a/packages/server/src/server-observers.ts b/packages/server/src/server-observers.tsindex 4293642d2..c1c2fe76d 100644--- a/packages/server/src/server-observers.ts+++ b/packages/server/src/server-observers.ts@@ -51,6 +51,7 @@ import {splitFmBoundarySlot,stripFrontmatter,} from '@inkeep/open-knowledge-core';+import type { JSONContent } from '@tiptap/core';import type { Schema } from '@tiptap/pm/model';import { updateYFragment, yXmlFragmentToProseMirrorRootNode } from '@tiptap/y-tiptap';import * as Y from 'yjs';@@ -77,6 +78,8 @@ import {computeMapDrivenBodySplice,createEditorMdastMemo,type EditorMdastMemo,+ type SerializedEditorBody,+ serializeEditorBody,} from './map-driven-splice.ts';import {incrementBridgeMergeCheckpointCreated,@@ -86,6 +89,8 @@ import {incrementDeriveTimingDeferForceResolved,incrementMapDrivenSpliceApplied,incrementMapDrivenSpliceFallback,+ incrementMapDrivenSpliceMemoHit,+ incrementMapDrivenSpliceMemoSkip,incrementObserverAApplyLoss,incrementObserverAApplyLossCheckpointCreated,incrementObserverADuplicationCheckpointCreated,@@ -265,10 +270,10 @@ interface YTextMapDrivenSplice {interface TryComputeMapDrivenSpliceArgs {readonly currentText: string;readonly lastSyncedXmlMd: string;- readonly json: unknown;readonly mdManager: MarkdownManager;readonly docName: string | undefined;readonly mdastMemo: EditorMdastMemo;+ readonly serializedNewPm: SerializedEditorBody;}let mapDrivenParseErrorWarned = false;@@ -286,10 +291,26 @@ function warnOnceMapDrivenParseError(docName: string | undefined, err: unknown):);}+let memoComposeFailureWarned = false;++export function __resetMemoComposeFailureWarnForTests(): void {+ memoComposeFailureWarned = false;+}++function warnOnceMemoComposeFailure(docName: string | undefined, err: unknown): void {+ if (memoComposeFailureWarned) return;+ memoComposeFailureWarned = true;+ log.warn(+ { docName: docName ?? 'unknown', err: err instanceof Error ? err : new Error(String(err)) },+ `[Server Observer A] Spliced-body memo composition threw (doc: ${docName ?? 'unknown'}); the splice is unaffected and the next drain re-parses (warned once; further failures count in mapDrivenSpliceMemoSkips only)`,+ );+}+function tryComputeMapDrivenSplice(args: TryComputeMapDrivenSpliceArgs,): YTextMapDrivenSplice | null {- const { currentText, lastSyncedXmlMd, json, mdManager, docName, mdastMemo } = args;+ const { currentText, lastSyncedXmlMd, mdManager, docName, mdastMemo, serializedNewPm } = args;+ const newPmJson = serializedNewPm.json;if (currentText !== lastSyncedXmlMd) {incrementMapDrivenSpliceFallback('text-mismatch');return null;@@ -301,16 +322,19 @@ function tryComputeMapDrivenSplice(const { body: oldBody } = stripFrontmatter(currentText);const bodyOffset = currentText.length - oldBody.length;- const splice = computeMapDrivenBodySplice(- oldBody,- json as Parameters<typeof computeMapDrivenBodySplice>[1],- mdManager,- (reason, err) => {+ const splice = computeMapDrivenBodySplice(oldBody, newPmJson, mdManager, {+ onFallback: (reason, err) => {incrementMapDrivenSpliceFallback(reason);if (reason === 'parse-error') warnOnceMapDrivenParseError(docName, err);},- mdastMemo,- );+ onMemoHit: incrementMapDrivenSpliceMemoHit,+ onMemoSkip: (reason, err) => {+ incrementMapDrivenSpliceMemoSkip(reason);+ if (reason === 'compose-failed') warnOnceMemoComposeFailure(docName, err);+ },+ memo: mdastMemo,+ serializedNewPm,+ });if (!splice) return null;return {@@ -1115,7 +1139,10 @@ export function setupServerObservers(opts: SetupServerObserversOpts): () => voidconst rawWitnessCoherent = ytext.toString() === lastSyncedYTextBytes;const ytextQuiescent = Date.now() - lastExternalYtextChangeMs >= FRESHNESS_QUIESCENCE_MS;const freshnessSafe = rawWitnessCoherent && ytextQuiescent;- const body = mdManager.serialize(json, { skipFreshnessDerive: !freshnessSafe });+ const serializedNewPm = serializeEditorBody(mdManager, json as JSONContent, {+ skipFreshnessDerive: !freshnessSafe,+ });+ const body = serializedNewPm.body;if (freshnessSafe) runProducerGuard(json as PmStructuralNode, body);const frontmatter = readCurrentFm();const composition = composeDerivedBodyMd(frontmatter, body);@@ -1199,10 +1226,10 @@ export function setupServerObservers(opts: SetupServerObserversOpts): () => void: tryComputeMapDrivenSplice({currentText,lastSyncedXmlMd: lastSyncedYTextBytes,- json,mdManager,docName: opts.docName,mdastMemo,+ serializedNewPm,});if (mapDrivenSplice) {setActiveSpanAttributes({