Skip to content

Beautiful, AI-native markdown IDE and LLM wiki

fix(ok): cut the per-keystroke Observer A drain on large documents (#4162)

Nick Gomez1311e2e14193
diff --git a/.changeset/lucky-pandas-repeat.md b/.changeset/lucky-pandas-repeat.md
new file mode 100644
index 000000000..0cf9290ee
--- /dev/null
+++ b/.changeset/lucky-pandas-repeat.md
@@ -0,0 +1,10 @@
+---
+"@inkeep/open-knowledge": patch
+---
+
+Typing in a large document now costs the server less per keystroke.
+
+On a 231 KB document the per-keystroke cost drops by roughly 1.7x to 2x, measured across
+several cursor positions. The gain is position-dependent: an edit near the end of a document
+gains more than one at the very top. Editing inside a list, a blockquote, or a list item keeps
+part of the old cost. Nothing changes about what gets written to your files.
diff --git a/package.json b/package.json
index 449f72092..9fc734217 100644
--- a/package.json
+++ b/package.json
@@ -36,6 +36,7 @@
"measure:stress": "cd packages/app && pnpm run measure:stress",
"measure:sweep": "cd packages/app && pnpm run measure:sweep",
"measure:generated-index-sweep": "pnpm --dir packages/server run measure:generated-index-sweep",
+ "measure:observer-a-drain": "pnpm --dir packages/server run measure:observer-a-drain",
"prepare": "bash scripts/husky-prepare.sh",
"changeset": "changeset",
diff --git a/packages/app/tests/integration/c3-mixed-mode.test.ts b/packages/app/tests/integration/c3-mixed-mode.test.ts
index 4cfc948e2..9a6bafc0c 100644
--- a/packages/app/tests/integration/c3-mixed-mode.test.ts
+++ b/packages/app/tests/integration/c3-mixed-mode.test.ts
@@ -1,4 +1,5 @@
import { setTimeout as wait } from 'node:timers/promises';
+import { getMetrics } from '@inkeep/open-knowledge-server';
import { afterAll, beforeAll, describe, expect, test } from 'vitest';
import * as Y from 'yjs';
import { HARNESS_BOOT_TIMEOUT_MS } from './harness-boot-timeout';
@@ -7,6 +8,7 @@ import {
assertBridgeInvariant,
createTestClients,
createTestServer,
+ getServerState,
pollUntil,
serializeFragment,
type TestClient,
@@ -74,6 +76,40 @@ describe('C3: mixed-mode concurrent edits', () => {
}
});
+ test('a remote source edit followed by local typing exercises the spliced-body memo', async () => {
+ const clients = await createTestClients(server.port, {
+ count: 2,
+ perClientOptions: { skipInvariantWatcher: true },
+ });
+ try {
+ appendParagraph(clients[0], 'C3-MEMO-SEED');
+ await assertConverged(clients, ['C3-MEMO-SEED']);
+
+ clients[1].doc.transact(() => {
+ clients[1].ytext.insert(0, 'C3-REMOTE-SOURCE\n\n');
+ });
+ await assertConverged(clients, ['C3-MEMO-SEED', 'C3-REMOTE-SOURCE']);
+
+ const before = getMetrics();
+ for (let i = 0; i < 4; i++) {
+ appendParagraph(clients[0], `C3-MEMO-TYPED-${i}`);
+ await wait(120);
+ }
+ const after = getMetrics();
+
+ await assertConverged(clients, ['C3-REMOTE-SOURCE', 'C3-MEMO-TYPED-3']);
+ expect(after.mapDrivenSpliceMemoHits).toBeGreaterThan(before.mapDrivenSpliceMemoHits);
+ expect(
+ (after.mapDrivenSpliceMemoSkips['entry-already-current'] ?? 0) -
+ (before.mapDrivenSpliceMemoSkips['entry-already-current'] ?? 0),
+ ).toBeGreaterThan(0);
+ const serverBody = getServerState(server, clients[0].docName)?.fullMd ?? '';
+ expect(serverBody).toContain('C3-REMOTE-SOURCE');
+ } finally {
+ for (const c of clients) await c.cleanup();
+ }
+ });
+
test('WYSIWYG + source with seed content — no content loss', async () => {
const docName = `c3-seed-${crypto.randomUUID()}`;
const clients = await createTestClients(server.port, {
diff --git a/packages/app/tests/stress/bridge-convergence.fuzz.test.ts b/packages/app/tests/stress/bridge-convergence.fuzz.test.ts
index b174fdd36..cbb762cc4 100644
--- a/packages/app/tests/stress/bridge-convergence.fuzz.test.ts
+++ b/packages/app/tests/stress/bridge-convergence.fuzz.test.ts
@@ -422,7 +422,7 @@ function parseIntegerEnv(name: string, raw: string): number {
if (!Number.isFinite(parsed) || !Number.isInteger(parsed)) {
throw new Error(
`${name} must be a finite integer, got ${JSON.stringify(raw)}. ` +
- `Example: ${name}=42 bun test tests/stress/bridge-convergence.fuzz.test.ts`,
+ `Example: ${name}=42 pnpm exec vitest run tests/stress/bridge-convergence.fuzz.test.ts`,
);
}
return parsed;
diff --git a/packages/app/tests/stress/crdt-stress.e2e.ts b/packages/app/tests/stress/crdt-stress.e2e.ts
index e96112898..ff195f3b1 100644
--- a/packages/app/tests/stress/crdt-stress.e2e.ts
+++ b/packages/app/tests/stress/crdt-stress.e2e.ts
@@ -3,6 +3,8 @@ import { loadLargeRealistic } from '../../../core/src/markdown/fixtures/index.ts
import { expect, filterCriticalErrors, test } from './_helpers';
const FIXTURE = loadLargeRealistic();
+const AGENT_WRITE_PROPAGATION_BUDGET_MS = 30_000;
+const KEYSTROKE_TO_YTEXT_DRAIN_BUDGET_MS = 30_000;
test('S6: multi-turn stress — large content + user edits', async ({ page, api, baseURL }) => {
const logs: Array<{ type: string; text: string; url?: string; line?: number }> = [];
@@ -44,7 +46,7 @@ test('S6: multi-turn stress — large content + user edits', async ({ page, api,
(expected: number) =>
(window.__activeProvider?.document?.getText('source')?.toString()?.length ?? 0) >= expected,
lengthBeforeWrite + FIXTURE.length - 200,
- { timeout: 30_000 },
+ { timeout: AGENT_WRITE_PROPAGATION_BUDGET_MS },
);
await page.locator('.ProseMirror:not(.composer-prosemirror)').focus();
@@ -53,7 +55,7 @@ test('S6: multi-turn stress — large content + user edits', async ({ page, api,
await page.waitForFunction(
(m: string) => window.__activeProvider?.document?.getText('source')?.toString()?.includes(m),
marker,
- { timeout: 30_000 },
+ { timeout: KEYSTROKE_TO_YTEXT_DRAIN_BUDGET_MS },
);
const turnState = await page.evaluate(() => {
diff --git a/packages/app/tests/stress/server-authoritative-stress.test.ts b/packages/app/tests/stress/server-authoritative-stress.test.ts
index f22cac824..6f5019910 100644
--- a/packages/app/tests/stress/server-authoritative-stress.test.ts
+++ b/packages/app/tests/stress/server-authoritative-stress.test.ts
@@ -114,7 +114,7 @@ describe('server-authoritative stress (US-013)', () => {
if (!Number.isFinite(parsed) || !Number.isInteger(parsed)) {
throw new Error(
`STRESS_SEED must be a finite integer, got ${JSON.stringify(raw)}. ` +
- `Example: STRESS_SEED=42 bun test tests/stress/server-authoritative-stress.test.ts`,
+ `Example: STRESS_SEED=42 pnpm exec vitest run tests/stress/server-authoritative-stress.test.ts`,
);
}
seed = parsed;
diff --git a/packages/app/tests/stress/stress-api.ts b/packages/app/tests/stress/stress-api.ts
index a199075d9..2a83411f5 100644
--- a/packages/app/tests/stress/stress-api.ts
+++ b/packages/app/tests/stress/stress-api.ts
@@ -197,7 +197,7 @@ async function main(): Promise<void> {
try {
await fetch(`${BASE}/api/document`);
} catch {
- console.error('[stress-api] ERROR: Dev server not reachable. Run `bun run dev` first.');
+ console.error('[stress-api] ERROR: Dev server not reachable. Run `pnpm dev` first.');
process.exit(1);
}
diff --git a/packages/server/measure-observer-a-drain.test.ts b/packages/server/measure-observer-a-drain.test.ts
new file mode 100644
index 000000000..49c47e986
--- /dev/null
+++ b/packages/server/measure-observer-a-drain.test.ts
@@ -0,0 +1,75 @@
+import { describe, expect, test } from 'vitest';
+import { parseDrainMeasurementArgs } from './measure-observer-a-drain.ts';
+
+describe('measure-observer-a-drain argument parsing', () => {
+ test('defaults cover all three carets at the S6 scale', () => {
+ expect(parseDrainMeasurementArgs([])).toEqual({
+ carets: ['start', 'middle', 'end'],
+ fixtureMultiple: 3,
+ keystrokes: 15,
+ });
+ });
+
+ test('a caret list selects a subset in the order given', () => {
+ expect(parseDrainMeasurementArgs(['--caret', 'end,start']).carets).toEqual(['end', 'start']);
+ });
+
+ test('fixture multiple and keystroke count are read as integers', () => {
+ const args = parseDrainMeasurementArgs(['--fixture-multiple', '1', '--keystrokes', '4']);
+ expect(args.fixtureMultiple).toBe(1);
+ expect(args.keystrokes).toBe(4);
+ });
+
+ test('a misspelled flag is rejected rather than silently ignored', () => {
+ expect(() => parseDrainMeasurementArgs(['--carets', 'start'])).toThrow(/unknown flag --carets/);
+ });
+
+ test('a misspelled flag in trailing position is reported as unknown, not as missing a value', () => {
+ expect(() => parseDrainMeasurementArgs(['--carets'])).toThrow(/unknown flag --carets/);
+ });
+
+ test('a leading -- separator is normalized away', () => {
+ expect(parseDrainMeasurementArgs(['--', '--caret', 'end']).carets).toEqual(['end']);
+ });
+
+ test('a -- separator anywhere but the leading position is rejected', () => {
+ expect(() => parseDrainMeasurementArgs(['--', '--', '--caret', 'end'])).toThrow(
+ /unknown flag --/,
+ );
+ expect(() => parseDrainMeasurementArgs(['--caret', 'end', '--'])).toThrow(/unknown flag --/);
+ });
+
+ test('a flag with no value is rejected rather than keeping the default', () => {
+ expect(() => parseDrainMeasurementArgs(['--caret'])).toThrow(/--caret needs a value/);
+ expect(() => parseDrainMeasurementArgs(['--keystrokes'])).toThrow(/--keystrokes needs a value/);
+ });
+
+ test('a non-decimal numeric literal is rejected rather than coerced', () => {
+ expect(() => parseDrainMeasurementArgs(['--fixture-multiple', '0x10'])).toThrow(
+ /--fixture-multiple/,
+ );
+ expect(() => parseDrainMeasurementArgs(['--keystrokes', '1e4'])).toThrow(/--keystrokes/);
+ });
+
+ test('a fractional count is rejected rather than truncated', () => {
+ expect(() => parseDrainMeasurementArgs(['--fixture-multiple', '3.5'])).toThrow(
+ /--fixture-multiple/,
+ );
+ expect(() => parseDrainMeasurementArgs(['--keystrokes', '2.5'])).toThrow(/--keystrokes/);
+ });
+
+ test('an unrecognized caret name is rejected even when a valid one accompanies it', () => {
+ expect(() => parseDrainMeasurementArgs(['--caret', 'middleish'])).toThrow(/middleish/);
+ expect(() => parseDrainMeasurementArgs(['--caret', 'start,middleish'])).toThrow(/middleish/);
+ });
+
+ test('a non-positive fixture multiple is rejected', () => {
+ expect(() => parseDrainMeasurementArgs(['--fixture-multiple', '0'])).toThrow(
+ /--fixture-multiple/,
+ );
+ });
+
+ test('a non-positive keystroke count is rejected', () => {
+ expect(() => parseDrainMeasurementArgs(['--keystrokes', 'x'])).toThrow(/--keystrokes/);
+ });
+});
diff --git a/packages/server/measure-observer-a-drain.ts b/packages/server/measure-observer-a-drain.ts
new file mode 100644
index 000000000..a55775576
--- /dev/null
+++ b/packages/server/measure-observer-a-drain.ts
@@ -0,0 +1,181 @@
+import { arch, cpus, platform } from 'node:os';
+import { resolve } from 'node:path';
+import { pathToFileURL } from 'node:url';
+import { sharedExtensions } from '@inkeep/open-knowledge-core';
+import { getSchema } from '@tiptap/core';
+import * as Y from 'yjs';
+import { loadLargeRealistic } from '../core/src/markdown/fixtures/index.ts';
+import { composeAndWriteRawBody } from './src/bridge-intake.ts';
+import { mdManager } from './src/md-manager.ts';
+import { setupServerObservers } from './src/server-observers.ts';
+
+export type CaretPosition = 'start' | 'middle' | 'end';
+
+export interface DrainMeasurementArgs {
+ readonly carets: CaretPosition[];
+ readonly fixtureMultiple: number;
+ readonly keystrokes: number;
+}
+
+export interface DrainMeasurement {
+ readonly caret: CaretPosition;
+ readonly capturedAt: string;
+ readonly host: { readonly platform: string; readonly arch: string; readonly cpus: number };
+ readonly nodeVersion: string;
+ readonly bodyBytes: number;
+ readonly keystrokes: number;
+ readonly totalMs: number;
+ readonly perKeystrokeMedianMs: number;
+ readonly perKeystrokeMaxMs: number;
+ readonly serializeCallsPerDrain: number;
+ readonly parseCallsPerDrain: number;
+ readonly markerLanded: boolean;
+}
+
+const RESULT_PREFIX = 'OBSERVER_A_DRAIN_RESULT ';
+const ALL_CARETS: CaretPosition[] = ['start', 'middle', 'end'];
+
+export function parseDrainMeasurementArgs(argv: readonly string[]): DrainMeasurementArgs {
+ let carets = ALL_CARETS;
+ let fixtureMultiple = 3;
+ let keystrokes = 15;
+ const positiveInteger = (flag: string, raw: string): number => {
+ if (!/^[0-9]+$/.test(raw)) throw new Error(`${flag} must be a positive integer, got ${raw}`);
+ const parsed = Number(raw);
+ if (parsed < 1) throw new Error(`${flag} must be a positive integer, got ${raw}`);
+ return parsed;
+ };
+ const normalized = argv[0] === '--' ? argv.slice(1) : argv;
+ for (let i = 0; i < normalized.length; i++) {
+ const flag = normalized[i];
+ if (flag !== '--caret' && flag !== '--fixture-multiple' && flag !== '--keystrokes') {
+ throw new Error(`unknown flag ${flag}; use --caret, --fixture-multiple or --keystrokes`);
+ }
+ const value = normalized[i + 1];
+ if (value === undefined) throw new Error(`${flag} needs a value`);
+ if (flag === '--caret') {
+ const requested = value.split(',');
+ const unknown = requested.filter((c) => !ALL_CARETS.includes(c as CaretPosition));
+ if (unknown.length > 0) {
+ throw new Error(`--caret does not accept ${unknown.join(',')}; use start, middle or end`);
+ }
+ carets = requested as CaretPosition[];
+ } else if (flag === '--fixture-multiple') {
+ fixtureMultiple = positiveInteger(flag, value);
+ } else if (flag === '--keystrokes') {
+ keystrokes = positiveInteger(flag, value);
+ } else {
+ const unreachable: never = flag;
+ throw new Error(`unknown flag ${String(unreachable)}`);
+ }
+ i++;
+ }
+ return { carets, fixtureMultiple, keystrokes };
+}
+
+function collectTextNodes(
+ node: Y.XmlFragment | Y.XmlElement,
+ found: Y.XmlText[] = [],
+): Y.XmlText[] {
+ for (let i = 0; i < node.length; i++) {
+ const child = node.get(i);
+ if (child instanceof Y.XmlText) found.push(child);
+ else if (child instanceof Y.XmlElement) collectTextNodes(child, found);
+ }
+ return found;
+}
+
+function median(values: readonly number[]): number {
+ const sorted = [...values].sort((a, b) => a - b);
+ return Number((sorted[Math.floor(sorted.length / 2)] ?? 0).toFixed(1));
+}
+
+export function measureDrain(caret: CaretPosition, args: DrainMeasurementArgs): DrainMeasurement {
+ const base = loadLargeRealistic();
+ const raw = Array.from({ length: args.fixtureMultiple }, () => base).join('\n');
+ const doc = new Y.Doc();
+ const xmlFragment = doc.getXmlFragment('default');
+ const ytext = doc.getText('source');
+
+ doc.transact(() => {
+ composeAndWriteRawBody(doc, raw, 'agent');
+ });
+ const cleanup = setupServerObservers({
+ doc,
+ xmlFragment,
+ ytext,
+ mdManager,
+ schema: getSchema(sharedExtensions),
+ docName: 'measure-observer-a-drain',
+ });
+
+ const texts = collectTextNodes(xmlFragment);
+ const target =
+ caret === 'start'
+ ? texts[0]
+ : caret === 'middle'
+ ? texts[Math.floor(texts.length / 2)]
+ : texts[texts.length - 1];
+ if (!target) throw new Error('fixture produced no editable text node');
+
+ const originalSerialize = mdManager.serialize.bind(mdManager);
+ const originalParse = mdManager.parseToEditorMdast.bind(mdManager);
+ let serializeCalls = 0;
+ let parseCalls = 0;
+ mdManager.serialize = (json, opts) => {
+ serializeCalls++;
+ return originalSerialize(json, opts);
+ };
+ mdManager.parseToEditorMdast = (markdown) => {
+ parseCalls++;
+ return originalParse(markdown);
+ };
+
+ const marker = 'MEASURE'.repeat(Math.ceil(args.keystrokes / 7)).slice(0, args.keystrokes);
+ const perKeystroke: number[] = [];
+ try {
+ const startedAt = performance.now();
+ for (const char of marker) {
+ const at = performance.now();
+ doc.transact(() => {
+ target.insert(target.length, char);
+ });
+ perKeystroke.push(performance.now() - at);
+ }
+ const totalMs = performance.now() - startedAt;
+ return {
+ caret,
+ capturedAt: new Date().toISOString(),
+ host: { platform: platform(), arch: arch(), cpus: cpus().length },
+ nodeVersion: process.version,
+ bodyBytes: raw.length,
+ keystrokes: marker.length,
+ totalMs: Number(totalMs.toFixed(1)),
+ perKeystrokeMedianMs: median(perKeystroke),
+ perKeystrokeMaxMs: Number(Math.max(...perKeystroke).toFixed(1)),
+ serializeCallsPerDrain: Number((serializeCalls / marker.length).toFixed(2)),
+ parseCallsPerDrain: Number((parseCalls / marker.length).toFixed(2)),
+ markerLanded: ytext.toString().includes(marker),
+ };
+ } finally {
+ mdManager.serialize = originalSerialize;
+ mdManager.parseToEditorMdast = originalParse;
+ cleanup?.();
+ }
+}
+
+function runCampaign(args: DrainMeasurementArgs): void {
+ for (const caret of args.carets) {
+ const measurement = measureDrain(caret, args);
+ process.stdout.write(`${RESULT_PREFIX}${JSON.stringify(measurement)}\n`);
+ }
+}
+
+if (process.argv[1] && pathToFileURL(resolve(process.argv[1])).href === import.meta.url) {
+ try {
+ runCampaign(parseDrainMeasurementArgs(process.argv.slice(2)));
+ } catch (err) {
+ process.stderr.write(`${err instanceof Error ? err.message : String(err)}\n`);
+ process.exitCode = 1;
+ }
+}
diff --git a/packages/server/package.json b/packages/server/package.json
index d931dde82..3d6f6a6af 100644
--- a/packages/server/package.json
+++ b/packages/server/package.json
@@ -74,6 +74,7 @@
"typecheck": "pnpm run typecheck:bash && tsc",
"test": "vitest run",
"measure:generated-index-sweep": "node --import tsx --conditions=development measure-generated-index-sweep.ts",
+ "measure:observer-a-drain": "node --import tsx --conditions=development measure-observer-a-drain.ts",
"typecheck:bash": "tsc -p tsconfig.bash.json"
},
"devDependencies": {
diff --git a/packages/server/src/map-driven-observer-a.test.ts b/packages/server/src/map-driven-observer-a.test.ts
index 390a65951..52764f05e 100644
--- a/packages/server/src/map-driven-observer-a.test.ts
+++ b/packages/server/src/map-driven-observer-a.test.ts
@@ -1,5 +1,9 @@
-import { MarkdownManager, sharedExtensions } from '@inkeep/open-knowledge-core';
-import { getSchema } from '@tiptap/core';
+import {
+ MarkdownManager,
+ type SerializeCallOptions,
+ sharedExtensions,
+} from '@inkeep/open-knowledge-core';
+import { getSchema, type JSONContent } from '@tiptap/core';
import { updateYFragment } from '@tiptap/y-tiptap';
import { describe, expect, test, vi } from 'vitest';
import * as Y from 'yjs';
@@ -11,6 +15,7 @@ import { getMetrics } from './metrics.ts';
import { createCountingManager } from './parse-counting.test-helper.ts';
import {
__resetMapDrivenParseErrorWarnForTests,
+ __resetMemoComposeFailureWarnForTests,
OBSERVER_SYNC_ORIGIN,
setupServerObservers,
} from './server-observers.ts';
@@ -329,14 +334,11 @@ describe('map-driven Observer A — default Path A behavior', () => {
} as unknown as MarkdownManager;
const reasons: string[] = [];
- const splice = computeMapDrivenBodySplice(
- 'A.\n',
- mdManager.parse('A.\n'),
- throwingManager,
- (reason) => {
+ const splice = computeMapDrivenBodySplice('A.\n', mdManager.parse('A.\n'), throwingManager, {
+ onFallback: (reason) => {
reasons.push(reason);
},
- );
+ });
expect(splice).toBeNull();
expect(reasons).toEqual(['parse-error']);
@@ -391,14 +393,69 @@ describe('map-driven Observer A — default Path A behavior', () => {
cleanup();
});
+ test('a memo composition failure warns once with the error, then stays counter-only', () => {
+ const raw = '# Heading\n\npreserved \n\nFirst.\n\nSecond.\n\nThird.\n';
+ const { doc, xmlFragment, ytext } = createTestDoc();
+ const uncloneableManager = new Proxy(mdManager, {
+ get(target, prop) {
+ if (prop === 'parseToEditorMdast') {
+ return (markdown: string) => {
+ const tree = target.parseToEditorMdast(markdown);
+ for (const child of tree.children) {
+ (child as { data?: Record<string, unknown> }).data = {
+ uncloneable: () => undefined,
+ };
+ }
+ return tree;
+ };
+ }
+ const value = Reflect.get(target, prop, target);
+ return typeof value === 'function' ? value.bind(target) : value;
+ },
+ });
+ __resetMemoComposeFailureWarnForTests();
+ const warnSpy = vi.spyOn(getLogger('server-observers'), 'warn');
+ const cleanup = setupServerObservers({
+ doc,
+ xmlFragment,
+ ytext,
+ mdManager: uncloneableManager,
+ schema,
+ });
+ doc.transact(() => {
+ composeAndWriteRawBody(doc, raw, 'agent');
+ }, AGENT_WRITE_ORIGIN);
+
+ const before = getMetrics();
+ populateFragment(doc, xmlFragment, raw.replace('First.', 'First EDITED.'));
+ populateFragment(doc, xmlFragment, raw.replace('First.', 'First EDITED TWICE.'));
+ const after = getMetrics();
+
+ expect(ytext.toString()).toContain('First EDITED TWICE.');
+ expect(
+ (after.mapDrivenSpliceMemoSkips['compose-failed'] ?? 0) -
+ (before.mapDrivenSpliceMemoSkips['compose-failed'] ?? 0),
+ ).toBeGreaterThanOrEqual(2);
+ const memoWarns = warnSpy.mock.calls.filter((args) =>
+ String(args[1]).includes('Spliced-body memo composition threw'),
+ );
+ expect(memoWarns).toHaveLength(1);
+ expect((memoWarns[0]?.[0] as { err?: Error }).err?.name).toBe('DataCloneError');
+
+ warnSpy.mockRestore();
+ cleanup();
+ });
+
test('an offset-less block reports missing-position through the pure computer', () => {
const reasons: string[] = [];
const splice = computeMapDrivenBodySplice(
'<!-- note -->\n',
mdManager.parse('<!-- note -->\n\nX.\n'),
mdManager,
- (reason) => {
- reasons.push(reason);
+ {
+ onFallback: (reason) => {
+ reasons.push(reason);
+ },
},
);
@@ -448,4 +505,77 @@ describe('typing-burst parse economy (PRD-8273)', () => {
cleanup();
});
+
+ test('a keystroke drain serializes the fragment JSON once, not once per consumer', () => {
+ const { manager: counted, serializes } = createCountingManager();
+ const { doc, xmlFragment, ytext } = createTestDoc();
+ const cleanup = setupServerObservers({
+ doc,
+ xmlFragment,
+ ytext,
+ mdManager: counted,
+ schema,
+ });
+
+ doc.transact(() => {
+ const pmNode = schema.nodeFromJSON(counted.parse('# H\n\nalpha\n'));
+ updateYFragment(doc, xmlFragment, pmNode, { mapping: new Map(), isOMark: new Map() });
+ });
+
+ const before = serializes();
+ for (const ch of 'XYZ') typeChar(doc, xmlFragment, ch);
+
+ expect(ytext.toString()).toContain('alphaXYZ');
+ expect(serializes() - before).toBe(3);
+
+ cleanup();
+ });
+
+ test('a drain inside the freshness window serializes for itself instead of reusing the suppressed body', () => {
+ const real = new MarkdownManager({
+ extensions: sharedExtensions,
+ deriveStructuralFreshness: true,
+ });
+ const serializeOpts: Array<SerializeCallOptions | undefined> = [];
+ const recording = new Proxy(real, {
+ get(target, prop, receiver) {
+ if (prop === 'serialize') {
+ return (json: JSONContent, opts?: SerializeCallOptions) => {
+ serializeOpts.push(opts);
+ return target.serialize(json, opts);
+ };
+ }
+ const value = Reflect.get(target, prop, receiver);
+ return typeof value === 'function' ? value.bind(target) : value;
+ },
+ });
+ const { doc, xmlFragment, ytext } = createTestDoc();
+ const cleanup = setupServerObservers({
+ doc,
+ xmlFragment,
+ ytext,
+ mdManager: recording,
+ schema,
+ docName: 'freshness-window',
+ });
+
+ doc.transact(() => {
+ const pmNode = schema.nodeFromJSON(real.parse('# H\n\nalpha\n\nbeta\n'));
+ updateYFragment(doc, xmlFragment, pmNode, { mapping: new Map(), isOMark: new Map() });
+ });
+
+ doc.transact(() => {
+ ytext.insert(ytext.length, 'Source tail.\n');
+ });
+
+ const before = serializeOpts.length;
+ typeChar(doc, xmlFragment, 'X');
+ const drainCalls = serializeOpts.slice(before);
+
+ expect(drainCalls.some((o) => o?.skipFreshnessDerive === true)).toBe(true);
+ expect(drainCalls.some((o) => o?.skipFreshnessDerive !== true)).toBe(true);
+ expect(ytext.toString()).toContain('X');
+
+ cleanup();
+ });
});
diff --git a/packages/server/src/map-driven-splice.test.ts b/packages/server/src/map-driven-splice.test.ts
index 58a850939..f14719b2a 100644
--- a/packages/server/src/map-driven-splice.test.ts
+++ b/packages/server/src/map-driven-splice.test.ts
@@ -1,7 +1,12 @@
import { MarkdownManager, sharedExtensions } from '@inkeep/open-knowledge-core';
import type { JSONContent } from '@tiptap/core';
import { describe, expect, test } from 'vitest';
-import { computeMapDrivenBodySplice, createEditorMdastMemo } from './map-driven-splice.ts';
+import {
+ computeMapDrivenBodySplice,
+ createEditorMdastMemo,
+ type EditorMdastMemo,
+} from './map-driven-splice.ts';
+import type { MapDrivenSpliceMemoSkipReason } from './metrics.ts';
import { createCountingManager } from './parse-counting.test-helper.ts';
const mdManager = new MarkdownManager({ extensions: sharedExtensions });
@@ -212,19 +217,15 @@ describe('editor-mdast parse memo (PRD-8273)', () => {
const memo = createEditorMdastMemo();
const bodyA = '# H\n\nalpha\n';
- const first = computeMapDrivenBodySplice(
- bodyA,
- counted.parse('# H\n\nalphaX\n'),
- counted,
- undefined,
+ const first = computeMapDrivenBodySplice(bodyA, counted.parse('# H\n\nalphaX\n'), counted, {
memo,
- );
+ });
expect(first).not.toBeNull();
if (!first) return;
const bodyB = applySplice(bodyA, first);
const afterFirst = parses();
- computeMapDrivenBodySplice(bodyB, counted.parse('# H\n\nalphaXY\n'), counted, undefined, memo);
+ computeMapDrivenBodySplice(bodyB, counted.parse('# H\n\nalphaXY\n'), counted, { memo });
expect(parses() - afterFirst).toBe(1);
});
@@ -234,7 +235,7 @@ describe('editor-mdast parse memo (PRD-8273)', () => {
const memo = createEditorMdastMemo();
const primed = '# H\n\nalpha\n';
- computeMapDrivenBodySplice(primed, counted.parse('# H\n\nalphaX\n'), counted, undefined, memo);
+ computeMapDrivenBodySplice(primed, counted.parse('# H\n\nalphaX\n'), counted, { memo });
const afterPrime = parses();
const external = '# DIFFERENT\n\nomega\n\ntail\n';
@@ -242,8 +243,7 @@ describe('editor-mdast parse memo (PRD-8273)', () => {
external,
counted.parse('# DIFFERENT\n\nomega EDITED\n\ntail\n'),
counted,
- undefined,
- memo,
+ { memo },
);
expect(splice).not.toBeNull();
if (!splice) return;
@@ -265,8 +265,7 @@ describe('editor-mdast parse memo (PRD-8273)', () => {
bodyA,
counted.parse('# H\n\none EDITED\n\ntwo\n\nthree\n'),
counted,
- undefined,
- memo,
+ { memo },
);
expect(first).not.toBeNull();
if (!first) return;
@@ -274,7 +273,7 @@ describe('editor-mdast parse memo (PRD-8273)', () => {
const newPm = counted.parse('# H\n\none EDITED\n\ntwo CHANGED\n\nthree\n');
const before = parses();
- const fromHit = computeMapDrivenBodySplice(bodyB, newPm, counted, undefined, memo);
+ const fromHit = computeMapDrivenBodySplice(bodyB, newPm, counted, { memo });
expect(parses() - before).toBe(1);
const fromFreshParse = computeMapDrivenBodySplice(bodyB, newPm, counted);
@@ -282,7 +281,7 @@ describe('editor-mdast parse memo (PRD-8273)', () => {
expect(fromHit).not.toBeNull();
});
- test('preserved bytes the serializer would not emit cause the next drain to miss', () => {
+ test('preserved bytes the serializer would not emit still hit the next drain', () => {
const { manager: counted, parses } = createCountingManager();
const memo = createEditorMdastMemo();
const bodyA = 'one \n\ntwo\n\nthree\n';
@@ -291,22 +290,19 @@ describe('editor-mdast parse memo (PRD-8273)', () => {
bodyA,
counted.parse('one \n\ntwo A\n\nthree\n'),
counted,
- undefined,
- memo,
+ { memo },
);
expect(first).not.toBeNull();
if (!first) return;
const bodyB = applySplice(bodyA, first);
+ expect(bodyB).not.toBe(counted.serialize(counted.parse(bodyB)));
+ expect(memo.entry?.body).toBe(bodyB);
+ const newPm = counted.parse('one \n\ntwo B\n\nthree\n');
const before = parses();
- computeMapDrivenBodySplice(
- bodyB,
- counted.parse('one \n\ntwo B\n\nthree\n'),
- counted,
- undefined,
- memo,
- );
- expect(parses() - before).toBe(2);
+ const fromHit = computeMapDrivenBodySplice(bodyB, newPm, counted, { memo });
+ expect(parses() - before).toBe(1);
+ expect(fromHit).toEqual(computeMapDrivenBodySplice(bodyB, newPm, counted));
});
test('a same-length body with different content misses — the key is bytes, not length', () => {
@@ -318,11 +314,11 @@ describe('editor-mdast parse memo (PRD-8273)', () => {
expect(actual.length).toBe(primed.length);
expect(actual).not.toBe(primed);
- computeMapDrivenBodySplice('zzz\n', counted.parse(primed), counted, undefined, memo);
+ computeMapDrivenBodySplice('zzz\n', counted.parse(primed), counted, { memo });
const newPm = counted.parse('one two\n\nthreeY\n');
const before = parses();
- const withMemo = computeMapDrivenBodySplice(actual, newPm, counted, undefined, memo);
+ const withMemo = computeMapDrivenBodySplice(actual, newPm, counted, { memo });
expect(parses() - before).toBe(2);
const withoutMemo = computeMapDrivenBodySplice(actual, newPm, counted);
@@ -330,3 +326,212 @@ describe('editor-mdast parse memo (PRD-8273)', () => {
expect(withMemo).not.toBeNull();
});
});
+
+describe('caller-supplied serialization', () => {
+ test('a body serialized from the same PM JSON is reused instead of re-serialized', () => {
+ const { manager: counted, serializes } = createCountingManager();
+ const memo = createEditorMdastMemo();
+ const oldBody = '# H\n\none\n\ntwo\n';
+ const newPm = counted.parse('# H\n\none EDITED\n\ntwo\n');
+ const body = counted.serialize(newPm);
+
+ const before = serializes();
+ const reused = computeMapDrivenBodySplice(oldBody, newPm, counted, {
+ memo,
+ serializedNewPm: { json: newPm, body, opts: undefined },
+ });
+ expect(serializes() - before).toBe(0);
+ expect(reused).toEqual(computeMapDrivenBodySplice(oldBody, newPm, counted));
+ });
+
+ test('a body carried from different PM JSON is ignored and the splice serializes itself', () => {
+ const { manager: counted, serializes } = createCountingManager();
+ const oldBody = '# H\n\none\n\ntwo\n';
+ const newPm = counted.parse('# H\n\none EDITED\n\ntwo\n');
+ const stale = counted.parse('# H\n\nSTALE\n\ntwo\n');
+ const staleBody = counted.serialize(stale);
+
+ const before = serializes();
+ const splice = computeMapDrivenBodySplice(oldBody, newPm, counted, {
+ serializedNewPm: { json: stale, body: staleBody, opts: undefined },
+ });
+ const selfSerializes = serializes() - before;
+ expect(selfSerializes).toBe(1);
+ expect(splice).toEqual(computeMapDrivenBodySplice(oldBody, newPm, counted));
+ });
+});
+
+describe('spliced-body memo fidelity', () => {
+ const DIRTY_PREFIX = 'preserved \n\n';
+
+ const NARROWABLE_SHAPES = new Set(['list', 'blockquote', 'loose-list']);
+
+ const CORPUS: Array<[string, string]> = [
+ ['paragraphs', 'one\n\ntwo\n\nthree\n\nfour\n'],
+ ['headings', '# H\n\npara\n\n## H2\n\npara two\n\ntail\n'],
+ ['list', '# H\n\n- one\n- two\n- three\n\npara\n\ntail\n'],
+ ['blockquote', '# H\n\n> one\n>\n> two\n\npara\n\ntail\n'],
+ ['table', '# H\n\n| a | b |\n|---|---|\n| 1 | 2 |\n\npara\n\ntail\n'],
+ ['code-fence', '# H\n\n```js\nconst a = 1;\n```\n\npara\n\ntail\n'],
+ ['setext', 'Title\n=====\n\npara\n\ntail\n'],
+ ['definitions', '# H\n\n[a]: http://example.com\n\nsee [a]\n\npara two\n\ntail\n'],
+ ['footnote', '# H\n\ntext[^1]\n\n[^1]: note\n\npara two\n\ntail\n'],
+ ['dirty-bytes', 'one \n\ntwo\n\nthree\n\nfour\n'],
+ ['blank-runs', 'one\n\n\n\ntwo\n\nthree\n\nfour\n'],
+ ['thematic-break', 'one\n\n---\n\ntwo\n\nthree\n'],
+ ['loose-list', '# H\n\n- one\n\n- two\n\npara\n\ntail\n'],
+ [
+ 'many-blocks',
+ `${Array.from({ length: 60 }, (_, i) => `Paragraph ${i} body text.`).join('\n\n')}\n`,
+ ],
+ ];
+
+ const EDITS: Array<[string, (text: string) => string]> = [
+ ['append-char', (text) => `${text}Z`],
+ ['prepend-char', (text) => `Z${text}`],
+ ['emphasis-marker', (text) => `${text} *em*`],
+ ['dash-run', (text) => `${text} ---`],
+ ];
+
+ function editFirstLeaf(
+ json: JSONContent,
+ blockIndex: number,
+ edit: (text: string) => string,
+ ): JSONContent | null {
+ const clone = structuredClone(json) as JSONContent;
+ const block = clone.content?.[blockIndex];
+ if (!block) return null;
+ const stack: JSONContent[] = [block];
+ while (stack.length > 0) {
+ const node = stack.pop();
+ if (!node) continue;
+ if (typeof node.text === 'string') {
+ node.text = edit(node.text);
+ return clone;
+ }
+ for (const child of node.content ?? []) stack.push(child);
+ }
+ return null;
+ }
+
+ function appendToFirstLeaf(json: JSONContent, blockIndex: number): JSONContent | null {
+ return editFirstLeaf(json, blockIndex, (text) => `${text}Z`);
+ }
+
+ for (const [label, canonicalDoc] of CORPUS) {
+ for (const [form, doc] of [
+ ['canonical', canonicalDoc],
+ ['non-canonical', `${DIRTY_PREFIX}${canonicalDoc}`],
+ ] as const) {
+ test(`${label} (${form}): every composed entry equals a fresh parse of the spliced body`, () => {
+ const pm = mdManager.parse(doc);
+ const blockCount = pm.content?.length ?? 0;
+ expect(blockCount).toBeGreaterThan(0);
+ const firstEditable = form === 'canonical' ? 0 : 1;
+ let composed = 0;
+ let attempted = 0;
+ const skips: MapDrivenSpliceMemoSkipReason[] = [];
+
+ for (let index = firstEditable; index < blockCount; index++) {
+ for (const [, edit] of EDITS) {
+ const edited = editFirstLeaf(pm, index, edit);
+ if (!edited) continue;
+ attempted++;
+ const memo = createEditorMdastMemo();
+ const newBody = mdManager.serialize(edited);
+ const splice = computeMapDrivenBodySplice(doc, edited, mdManager, {
+ memo,
+ onMemoSkip: (reason) => skips.push(reason),
+ });
+ expect(splice).not.toBeNull();
+ if (!splice) continue;
+ const applied = applySplice(doc, splice);
+ expect(memo.entry).not.toBeNull();
+ if (applied === newBody) {
+ expect(memo.entry?.body).toBe(newBody);
+ continue;
+ }
+ if (memo.entry?.body !== applied) continue;
+ composed++;
+ expect(JSON.stringify(memo.entry.children)).toBe(
+ JSON.stringify(mdManager.parseToEditorMdast(applied).children),
+ );
+ }
+ }
+
+ const narrowedSkips = skips.filter((reason) => reason === 'narrowed').length;
+ const alreadyCurrentSkips = skips.filter(
+ (reason) => reason === 'entry-already-current',
+ ).length;
+
+ expect(attempted).toBeGreaterThan(0);
+ expect(skips.filter((r) => r !== 'narrowed' && r !== 'entry-already-current')).toEqual([]);
+ expect(composed).toBe(attempted - narrowedSkips - alreadyCurrentSkips);
+ if (form === 'canonical') {
+ expect(alreadyCurrentSkips).toBeGreaterThan(0);
+ } else {
+ expect(composed).toBeGreaterThan(0);
+ }
+ if (NARROWABLE_SHAPES.has(label)) {
+ expect(narrowedSkips).toBeGreaterThan(0);
+ } else {
+ expect(narrowedSkips).toBe(0);
+ }
+ });
+ }
+ }
+
+ test('a container-narrowed splice leaves the constructed entry unwritten', () => {
+ const doc = 'para \n\n- one\n- two\n- three\n\ntail\n';
+ const pm = mdManager.parse(doc);
+ const edited = appendToFirstLeaf(pm, 1);
+ expect(edited).not.toBeNull();
+ if (!edited) return;
+ const memo = createEditorMdastMemo();
+ const splice = computeMapDrivenBodySplice(doc, edited, mdManager, { memo });
+ expect(splice).not.toBeNull();
+ if (!splice) return;
+ const applied = applySplice(doc, splice);
+ const newBody = mdManager.serialize(edited);
+ expect(applied).not.toBe(newBody);
+ expect(applied.startsWith('para ')).toBe(true);
+ expect(splice.spliceStart).toBeGreaterThan(doc.indexOf('- one'));
+ expect(memo.entry?.body).toBe(newBody);
+ });
+});
+
+describe('spliced-body memo failure isolation', () => {
+ test('a throwing memo write costs the optimization, not the splice', () => {
+ const oldBody = 'preserved \n\none\n\ntwo\n\nthree\n';
+ const newPm = mdManager.parse('preserved \n\none EDITED\n\ntwo\n\nthree\n');
+ const reasons: MapDrivenSpliceMemoSkipReason[] = [];
+ const errors: unknown[] = [];
+ const composeFailure = new Error('synthetic composition regression');
+ let stored: EditorMdastMemo['entry'] = null;
+ let writes = 0;
+ const hostileMemo = {
+ get entry() {
+ return stored;
+ },
+ set entry(value: EditorMdastMemo['entry']) {
+ writes++;
+ if (writes > 2) throw composeFailure;
+ stored = value;
+ },
+ } as EditorMdastMemo;
+
+ const splice = computeMapDrivenBodySplice(oldBody, newPm, mdManager, {
+ memo: hostileMemo,
+ onMemoSkip: (reason, err) => {
+ reasons.push(reason);
+ errors.push(err);
+ },
+ });
+
+ expect(splice).not.toBeNull();
+ expect(splice).toEqual(computeMapDrivenBodySplice(oldBody, newPm, mdManager));
+ expect(writes).toBe(3);
+ expect(reasons).toEqual(['compose-failed']);
+ expect(errors).toEqual([composeFailure]);
+ });
+});
diff --git a/packages/server/src/map-driven-splice.ts b/packages/server/src/map-driven-splice.ts
index 9c09efbcb..5227f250f 100644
--- a/packages/server/src/map-driven-splice.ts
+++ b/packages/server/src/map-driven-splice.ts
@@ -1,6 +1,7 @@
-import type { MarkdownManager } from '@inkeep/open-knowledge-core';
+import type { MarkdownManager, SerializeCallOptions } from '@inkeep/open-knowledge-core';
import type { JSONContent } from '@tiptap/core';
-import type { Root, RootContent } from 'mdast';
+import type { RootContent } from 'mdast';
+import type { MapDrivenSpliceMemoSkipReason } from './metrics.ts';
export interface MapDrivenSplice {
readonly spliceStart: number;
@@ -8,39 +9,75 @@ export interface MapDrivenSplice {
readonly newSlice: string;
}
+export interface SerializedEditorBody {
+ readonly json: JSONContent;
+ readonly body: string;
+ readonly opts: SerializeCallOptions | undefined;
+}
+
+export function serializeEditorBody(
+ mdManager: MarkdownManager,
+ json: JSONContent,
+ opts?: SerializeCallOptions,
+): SerializedEditorBody {
+ return { json, body: mdManager.serialize(json, opts), opts };
+}
+
+function reusableBodyFrom(
+ serializedNewPm: SerializedEditorBody | undefined,
+ newPmJson: JSONContent,
+): string | undefined {
+ if (serializedNewPm === undefined) return undefined;
+ if (serializedNewPm.json !== newPmJson) return undefined;
+ if (serializedNewPm.opts?.skipFreshnessDerive === true) return undefined;
+ return serializedNewPm.body;
+}
+
+export interface MapDrivenSpliceOptions {
+ readonly onFallback?: (reason: 'parse-error' | 'missing-position', err?: unknown) => void;
+ readonly onMemoHit?: () => void;
+ readonly onMemoSkip?: (reason: MapDrivenSpliceMemoSkipReason, err?: unknown) => void;
+ readonly memo?: EditorMdastMemo;
+ readonly serializedNewPm?: SerializedEditorBody;
+}
+
export interface EditorMdastMemo {
- entry: { readonly body: string; readonly tree: Root } | null;
+ entry: { readonly body: string; readonly children: readonly RootContent[] } | null;
}
export function createEditorMdastMemo(): EditorMdastMemo {
return { entry: null };
}
-function parseEditorMdastMemoized(
+function editorMdastChildren(
mdManager: MarkdownManager,
body: string,
memo: EditorMdastMemo | undefined,
-): Root {
- if (memo?.entry?.body === body) return memo.entry.tree;
- const tree = mdManager.parseToEditorMdast(body);
- if (memo !== undefined) memo.entry = { body, tree };
- return tree;
+ onHit?: () => void,
+): readonly RootContent[] {
+ if (memo?.entry?.body === body) {
+ onHit?.();
+ return memo.entry.children;
+ }
+ const children = mdManager.parseToEditorMdast(body).children;
+ if (memo !== undefined) memo.entry = { body, children };
+ return children;
}
export function computeMapDrivenBodySplice(
oldBody: string,
newPmJson: JSONContent,
mdManager: MarkdownManager,
- onFallback?: (reason: 'parse-error' | 'missing-position', err?: unknown) => void,
- memo?: EditorMdastMemo,
+ options: MapDrivenSpliceOptions = {},
): MapDrivenSplice | null {
+ const { onFallback, onMemoHit, onMemoSkip, memo, serializedNewPm } = options;
let oldChildren: readonly RootContent[];
let newBody: string;
let newChildren: readonly RootContent[];
try {
- oldChildren = parseEditorMdastMemoized(mdManager, oldBody, memo).children;
- newBody = mdManager.serialize(newPmJson);
- newChildren = parseEditorMdastMemoized(mdManager, newBody, memo).children;
+ oldChildren = editorMdastChildren(mdManager, oldBody, memo, onMemoHit);
+ newBody = reusableBodyFrom(serializedNewPm, newPmJson) ?? mdManager.serialize(newPmJson);
+ newChildren = editorMdastChildren(mdManager, newBody, memo);
} catch (err) {
onFallback?.('parse-error', err);
return null;
@@ -51,8 +88,9 @@ export function computeMapDrivenBodySplice(
return null;
}
+ let walked: ChildrenSplice;
try {
- return computeChildrenSplice(
+ walked = computeChildrenSplice(
oldChildren,
newChildren,
{
@@ -69,6 +107,15 @@ export function computeMapDrivenBodySplice(
onFallback?.('missing-position', err);
return null;
}
+
+ if (memo !== undefined) {
+ try {
+ memoizeSplicedBody(memo, { oldBody, oldChildren, newBody, newChildren, walked }, onMemoSkip);
+ } catch (err) {
+ onMemoSkip?.('compose-failed', err);
+ }
+ }
+ return walked.splice;
}
interface ByteRegion {
@@ -76,6 +123,16 @@ interface ByteRegion {
readonly end: number;
}
+type ChildrenSplice =
+ | { readonly narrowed: true; readonly splice: MapDrivenSplice }
+ | {
+ readonly narrowed: false;
+ readonly splice: MapDrivenSplice;
+ readonly prefixLen: number;
+ readonly suffixLen: number;
+ readonly newSliceStart: number;
+ };
+
const NARROWABLE_CONTAINER_TYPES = new Set(['blockquote', 'list', 'listItem']);
function computeChildrenSplice(
@@ -84,7 +141,7 @@ function computeChildrenSplice(
oldRegion: ByteRegion,
newRegion: ByteRegion,
newBody: string,
-): MapDrivenSplice {
+): ChildrenSplice {
let prefixLen = 0;
while (
prefixLen < oldChildren.length &&
@@ -113,7 +170,7 @@ function computeChildrenSplice(
const oldChanged = oldChildren[prefixLen];
const newChanged = newChildren[prefixLen];
const narrowed = tryNarrowIntoContainer(oldChanged, newChanged, newBody);
- if (narrowed) return narrowed;
+ if (narrowed) return { narrowed: true, splice: narrowed.splice };
}
const spliceStart = prefixLen > 0 ? blockEndOffset(oldChildren[prefixLen - 1]) : oldRegion.start;
@@ -126,9 +183,15 @@ function computeChildrenSplice(
suffixLen > 0 ? blockStartOffset(newChildren[newChildren.length - suffixLen]) : newRegion.end;
return {
- spliceStart,
- spliceEnd,
- newSlice: newBody.slice(newSliceStart, newSliceEnd),
+ splice: {
+ spliceStart,
+ spliceEnd,
+ newSlice: newBody.slice(newSliceStart, newSliceEnd),
+ },
+ prefixLen,
+ suffixLen,
+ newSliceStart,
+ narrowed: false,
};
}
@@ -136,7 +199,7 @@ function tryNarrowIntoContainer(
oldNode: RootContent,
newNode: RootContent,
newBody: string,
-): MapDrivenSplice | null {
+): ChildrenSplice | null {
if (oldNode.type !== newNode.type || !NARROWABLE_CONTAINER_TYPES.has(oldNode.type)) return null;
if (!('children' in oldNode) || !('children' in newNode)) return null;
const oldKids = oldNode.children as readonly RootContent[];
@@ -158,6 +221,105 @@ function tryNarrowIntoContainer(
);
}
+interface SplicedBodyInputs {
+ readonly oldBody: string;
+ readonly oldChildren: readonly RootContent[];
+ readonly newBody: string;
+ readonly newChildren: readonly RootContent[];
+ readonly walked: ChildrenSplice;
+}
+
+function memoizeSplicedBody(
+ memo: EditorMdastMemo,
+ inputs: SplicedBodyInputs,
+ onSkip: ((reason: MapDrivenSpliceMemoSkipReason, err?: unknown) => void) | undefined,
+): void {
+ const { oldBody, oldChildren, newBody, newChildren, walked } = inputs;
+ if (walked.narrowed) {
+ onSkip?.('narrowed');
+ return;
+ }
+ if (oldChildren.length === 0 || newChildren.length === 0) {
+ onSkip?.('empty-children');
+ return;
+ }
+
+ const { splice, prefixLen, suffixLen, newSliceStart } = walked;
+ const applied =
+ oldBody.slice(0, splice.spliceStart) + splice.newSlice + oldBody.slice(splice.spliceEnd);
+ if (applied === newBody) {
+ onSkip?.('entry-already-current');
+ return;
+ }
+
+ const mid = cloneShifted(
+ newChildren.slice(prefixLen, newChildren.length - suffixLen),
+ splice.spliceStart - newSliceStart,
+ countNewlines(oldBody, 0, splice.spliceStart) - countNewlines(newBody, 0, newSliceStart),
+ );
+ if (mid === null) {
+ onSkip?.('position-not-numeric');
+ return;
+ }
+
+ const tail = cloneShifted(
+ oldChildren.slice(oldChildren.length - suffixLen),
+ applied.length - oldBody.length,
+ countNewlines(splice.newSlice, 0, splice.newSlice.length) -
+ countNewlines(oldBody, splice.spliceStart, splice.spliceEnd),
+ );
+ if (tail === null) {
+ onSkip?.('position-not-numeric');
+ return;
+ }
+
+ memo.entry = { body: applied, children: [...oldChildren.slice(0, prefixLen), ...mid, ...tail] };
+}
+
+function cloneShifted(
+ nodes: readonly RootContent[],
+ byteDelta: number,
+ lineDelta: number,
+): RootContent[] | null {
+ const cloned = structuredClone(nodes) as RootContent[];
+ return shiftPositions(cloned, byteDelta, lineDelta) ? cloned : null;
+}
+
+function shiftPositions(value: unknown, byteDelta: number, lineDelta: number): boolean {
+ if (Array.isArray(value)) {
+ for (const entry of value) {
+ if (!shiftPositions(entry, byteDelta, lineDelta)) return false;
+ }
+ return true;
+ }
+ if (value === null || typeof value !== 'object') return true;
+ const node = value as Record<string, unknown>;
+ const position = node.position as
+ | { start?: { offset?: number; line?: number }; end?: { offset?: number; line?: number } }
+ | undefined;
+ if (position !== undefined) {
+ for (const point of [position.start, position.end]) {
+ if (point === undefined) continue;
+ if (typeof point.offset !== 'number' || typeof point.line !== 'number') return false;
+ point.offset += byteDelta;
+ point.line += lineDelta;
+ }
+ }
+ for (const key of Object.keys(node)) {
+ if (key === 'position') continue;
+ if (!shiftPositions(node[key], byteDelta, lineDelta)) return false;
+ }
+ return true;
+}
+
+function countNewlines(text: string, from: number, to: number): number {
+ let count = 0;
+ for (let index = from; index < to; index++) {
+ if (text.charCodeAt(index) === 10) count++;
+ }
+ return count;
+}
+
function allBlocksCarryPositions(children: readonly RootContent[]): boolean {
for (const child of children) {
const start = child.position?.start?.offset;
diff --git a/packages/server/src/metrics.test.ts b/packages/server/src/metrics.test.ts
index debdbce27..05021bf86 100644
--- a/packages/server/src/metrics.test.ts
+++ b/packages/server/src/metrics.test.ts
@@ -10,6 +10,8 @@ import {
incrementConflict,
incrementMapDrivenSpliceApplied,
incrementMapDrivenSpliceFallback,
+ incrementMapDrivenSpliceMemoHit,
+ incrementMapDrivenSpliceMemoSkip,
incrementPark,
incrementReconcile,
incrementRescueBuffer,
@@ -96,6 +98,29 @@ describe('reconciliation metrics', () => {
expect(getMetrics().reconcileCount).toBe(2);
});
+ test('the map-driven splice memo counters accumulate and snapshot independently', () => {
+ resetMetrics();
+ incrementMapDrivenSpliceMemoHit();
+ incrementMapDrivenSpliceMemoSkip('narrowed');
+ incrementMapDrivenSpliceMemoSkip('narrowed');
+ incrementMapDrivenSpliceMemoSkip('entry-already-current');
+ const first = getMetrics();
+ expect(first.mapDrivenSpliceMemoHits).toBe(1);
+ expect(first.mapDrivenSpliceMemoSkips).toEqual({ narrowed: 2, 'entry-already-current': 1 });
+
+ incrementMapDrivenSpliceMemoHit();
+ incrementMapDrivenSpliceMemoSkip('position-not-numeric');
+ const second = getMetrics();
+ expect(second.mapDrivenSpliceMemoHits).toBe(2);
+ expect(second.mapDrivenSpliceMemoSkips).toEqual({
+ narrowed: 2,
+ 'entry-already-current': 1,
+ 'position-not-numeric': 1,
+ });
+ expect(first.mapDrivenSpliceMemoHits).toBe(1);
+ expect(first.mapDrivenSpliceMemoSkips).toEqual({ narrowed: 2, 'entry-already-current': 1 });
+ });
+
test('resetMetrics clears all counters', () => {
incrementReconcile();
incrementConflict();
@@ -108,7 +133,8 @@ describe('reconciliation metrics', () => {
if (
key === 'cc1LastSeq' ||
key === 'bridgeToleranceApplied' ||
- key === 'mapDrivenSpliceFallback'
+ key === 'mapDrivenSpliceFallback' ||
+ key === 'mapDrivenSpliceMemoSkips'
) {
expect(value).toEqual({});
} else {
diff --git a/packages/server/src/metrics.ts b/packages/server/src/metrics.ts
index 3a47a300d..548b83f35 100644
--- a/packages/server/src/metrics.ts
+++ b/packages/server/src/metrics.ts
@@ -6,6 +6,13 @@ export type MapDrivenSpliceFallbackReason =
| 'parse-error'
| 'missing-position';
+export type MapDrivenSpliceMemoSkipReason =
+ | 'narrowed'
+ | 'empty-children'
+ | 'entry-already-current'
+ | 'position-not-numeric'
+ | 'compose-failed';
+
export interface ReconciliationMetrics {
reconcileCount: number;
conflictCount: number;
@@ -56,6 +63,8 @@ export interface ReconciliationMetrics {
observerAPathBFiresSuppressed: number;
mapDrivenSpliceApplied: number;
mapDrivenSpliceFallback: Partial<Record<MapDrivenSpliceFallbackReason, number>>;
+ mapDrivenSpliceMemoHits: number;
+ mapDrivenSpliceMemoSkips: Partial<Record<MapDrivenSpliceMemoSkipReason, number>>;
observerAResidualMergeRuns: number;
/** Count of Observer A duplication-gate recoveries — a substantive body
* line materialized more times in the fragment than clean Y.Text justified,
@@ -166,6 +175,8 @@ const counters: ReconciliationMetrics = {
observerAPathBFires: 0,
observerAPathBFiresSuppressed: 0,
mapDrivenSpliceApplied: 0,
+ mapDrivenSpliceMemoHits: 0,
+ mapDrivenSpliceMemoSkips: {},
mapDrivenSpliceFallback: {},
observerAResidualMergeRuns: 0,
observerADuplicationRederives: 0,
@@ -366,6 +377,14 @@ export function incrementMapDrivenSpliceFallback(reason: MapDrivenSpliceFallback
counters.mapDrivenSpliceFallback[reason] = (counters.mapDrivenSpliceFallback[reason] ?? 0) + 1;
}
+export function incrementMapDrivenSpliceMemoHit(): void {
+ counters.mapDrivenSpliceMemoHits++;
+}
+
+export function incrementMapDrivenSpliceMemoSkip(reason: MapDrivenSpliceMemoSkipReason): void {
+ counters.mapDrivenSpliceMemoSkips[reason] = (counters.mapDrivenSpliceMemoSkips[reason] ?? 0) + 1;
+}
+
export function incrementObserverAResidualMergeRuns(): void {
counters.observerAResidualMergeRuns++;
}
@@ -590,6 +609,7 @@ export function getMetrics(): ReconciliationMetrics {
cc1LastSeq: { ...counters.cc1LastSeq },
bridgeToleranceApplied: { ...counters.bridgeToleranceApplied },
mapDrivenSpliceFallback: { ...counters.mapDrivenSpliceFallback },
+ mapDrivenSpliceMemoSkips: { ...counters.mapDrivenSpliceMemoSkips },
};
}
@@ -643,6 +663,8 @@ export function resetMetrics(): void {
counters.observerAPathBFiresSuppressed = 0;
counters.mapDrivenSpliceApplied = 0;
counters.mapDrivenSpliceFallback = {};
+ counters.mapDrivenSpliceMemoHits = 0;
+ counters.mapDrivenSpliceMemoSkips = {};
counters.observerAResidualMergeRuns = 0;
counters.observerADuplicationRederives = 0;
counters.observerADuplicationCheckpointCreated = 0;
diff --git a/packages/server/src/parse-counting.test-helper.ts b/packages/server/src/parse-counting.test-helper.ts
index d20cb23bb..cad9d3219 100644
--- a/packages/server/src/parse-counting.test-helper.ts
+++ b/packages/server/src/parse-counting.test-helper.ts
@@ -3,15 +3,22 @@ import { MarkdownManager, sharedExtensions } from '@inkeep/open-knowledge-core';
export interface CountingManager {
readonly manager: MarkdownManager;
readonly parses: () => number;
+ readonly serializes: () => number;
}
export function createCountingManager(): CountingManager {
const manager = new MarkdownManager({ extensions: sharedExtensions });
let calls = 0;
+ let serializeCalls = 0;
const original = manager.parseToEditorMdast.bind(manager);
manager.parseToEditorMdast = (markdown: string) => {
calls += 1;
return original(markdown);
};
- return { manager, parses: () => calls };
+ const originalSerialize = manager.serialize.bind(manager);
+ manager.serialize = (json, opts) => {
+ serializeCalls += 1;
+ return originalSerialize(json, opts);
+ };
+ return { manager, parses: () => calls, serializes: () => serializeCalls };
}
diff --git a/packages/server/src/server-observers.ts b/packages/server/src/server-observers.ts
index 4293642d2..c1c2fe76d 100644
--- a/packages/server/src/server-observers.ts
+++ b/packages/server/src/server-observers.ts
@@ -51,6 +51,7 @@ import {
splitFmBoundarySlot,
stripFrontmatter,
} from '@inkeep/open-knowledge-core';
+import type { JSONContent } from '@tiptap/core';
import type { Schema } from '@tiptap/pm/model';
import { updateYFragment, yXmlFragmentToProseMirrorRootNode } from '@tiptap/y-tiptap';
import * as Y from 'yjs';
@@ -77,6 +78,8 @@ import {
computeMapDrivenBodySplice,
createEditorMdastMemo,
type EditorMdastMemo,
+ type SerializedEditorBody,
+ serializeEditorBody,
} from './map-driven-splice.ts';
import {
incrementBridgeMergeCheckpointCreated,
@@ -86,6 +89,8 @@ import {
incrementDeriveTimingDeferForceResolved,
incrementMapDrivenSpliceApplied,
incrementMapDrivenSpliceFallback,
+ incrementMapDrivenSpliceMemoHit,
+ incrementMapDrivenSpliceMemoSkip,
incrementObserverAApplyLoss,
incrementObserverAApplyLossCheckpointCreated,
incrementObserverADuplicationCheckpointCreated,
@@ -265,10 +270,10 @@ interface YTextMapDrivenSplice {
interface TryComputeMapDrivenSpliceArgs {
readonly currentText: string;
readonly lastSyncedXmlMd: string;
- readonly json: unknown;
readonly mdManager: MarkdownManager;
readonly docName: string | undefined;
readonly mdastMemo: EditorMdastMemo;
+ readonly serializedNewPm: SerializedEditorBody;
}
let mapDrivenParseErrorWarned = false;
@@ -286,10 +291,26 @@ function warnOnceMapDrivenParseError(docName: string | undefined, err: unknown):
);
}
+let memoComposeFailureWarned = false;
+
+export function __resetMemoComposeFailureWarnForTests(): void {
+ memoComposeFailureWarned = false;
+}
+
+function warnOnceMemoComposeFailure(docName: string | undefined, err: unknown): void {
+ if (memoComposeFailureWarned) return;
+ memoComposeFailureWarned = true;
+ log.warn(
+ { docName: docName ?? 'unknown', err: err instanceof Error ? err : new Error(String(err)) },
+ `[Server Observer A] Spliced-body memo composition threw (doc: ${docName ?? 'unknown'}); the splice is unaffected and the next drain re-parses (warned once; further failures count in mapDrivenSpliceMemoSkips only)`,
+ );
+}
+
function tryComputeMapDrivenSplice(
args: TryComputeMapDrivenSpliceArgs,
): YTextMapDrivenSplice | null {
- const { currentText, lastSyncedXmlMd, json, mdManager, docName, mdastMemo } = args;
+ const { currentText, lastSyncedXmlMd, mdManager, docName, mdastMemo, serializedNewPm } = args;
+ const newPmJson = serializedNewPm.json;
if (currentText !== lastSyncedXmlMd) {
incrementMapDrivenSpliceFallback('text-mismatch');
return null;
@@ -301,16 +322,19 @@ function tryComputeMapDrivenSplice(
const { body: oldBody } = stripFrontmatter(currentText);
const bodyOffset = currentText.length - oldBody.length;
- const splice = computeMapDrivenBodySplice(
- oldBody,
- json as Parameters<typeof computeMapDrivenBodySplice>[1],
- mdManager,
- (reason, err) => {
+ const splice = computeMapDrivenBodySplice(oldBody, newPmJson, mdManager, {
+ onFallback: (reason, err) => {
incrementMapDrivenSpliceFallback(reason);
if (reason === 'parse-error') warnOnceMapDrivenParseError(docName, err);
},
- mdastMemo,
- );
+ onMemoHit: incrementMapDrivenSpliceMemoHit,
+ onMemoSkip: (reason, err) => {
+ incrementMapDrivenSpliceMemoSkip(reason);
+ if (reason === 'compose-failed') warnOnceMemoComposeFailure(docName, err);
+ },
+ memo: mdastMemo,
+ serializedNewPm,
+ });
if (!splice) return null;
return {
@@ -1115,7 +1139,10 @@ export function setupServerObservers(opts: SetupServerObserversOpts): () => void
const rawWitnessCoherent = ytext.toString() === lastSyncedYTextBytes;
const ytextQuiescent = Date.now() - lastExternalYtextChangeMs >= FRESHNESS_QUIESCENCE_MS;
const freshnessSafe = rawWitnessCoherent && ytextQuiescent;
- const body = mdManager.serialize(json, { skipFreshnessDerive: !freshnessSafe });
+ const serializedNewPm = serializeEditorBody(mdManager, json as JSONContent, {
+ skipFreshnessDerive: !freshnessSafe,
+ });
+ const body = serializedNewPm.body;
if (freshnessSafe) runProducerGuard(json as PmStructuralNode, body);
const frontmatter = readCurrentFm();
const composition = composeDerivedBodyMd(frontmatter, body);
@@ -1199,10 +1226,10 @@ export function setupServerObservers(opts: SetupServerObserversOpts): () => void
: tryComputeMapDrivenSplice({
currentText,
lastSyncedXmlMd: lastSyncedYTextBytes,
- json,
mdManager,
docName: opts.docName,
mdastMemo,
+ serializedNewPm,
});
if (mapDrivenSplice) {
setActiveSpanAttributes({