Skip to content

Beautiful, AI-native markdown IDE and LLM wiki

Keep CRLF line endings when editing (PRD-9139) (#5928)

Andrew Mikofalvy61de57742512
diff --git a/.changeset/crlf-edit-keeps-line-endings.md b/.changeset/crlf-edit-keeps-line-endings.md
new file mode 100644
index 000000000..b624f2be2
--- /dev/null
+++ b/.changeset/crlf-edit-keeps-line-endings.md
@@ -0,0 +1,5 @@
+---
+"@inkeep/open-knowledge": patch
+---
+
+Typing in a Markdown file that uses Windows (CRLF) line endings no longer converts the edited lines to LF.
diff --git a/knip.config.ts b/knip.config.ts
index c33b744b2..e59ddff92 100644
--- a/knip.config.ts
+++ b/knip.config.ts
@@ -148,15 +148,11 @@ export default {
'packages/server': {
entry: ['src/**/*.test.ts', 'src/parse-worker.ts'],
project: 'src/**',
- ignoreDependencies: [
- '@types/shell-quote',
- ],
+ ignoreDependencies: ['@types/shell-quote'],
},
'packages/cli': {
entry: ['src/**/*.test.ts', 'scripts/*.ts', 'tests/**/*.ts', 'src/parse-worker.ts'],
- ignoreDependencies: [
- 'yjs',
- ],
+ ignoreDependencies: ['yjs'],
},
'packages/desktop': {
entry: [
diff --git a/packages/app/tests/integration/bug3-source-mode-writeback.test.ts b/packages/app/tests/integration/bug3-source-mode-writeback.test.ts
index 4549c74b3..a50b00123 100644
--- a/packages/app/tests/integration/bug3-source-mode-writeback.test.ts
+++ b/packages/app/tests/integration/bug3-source-mode-writeback.test.ts
@@ -1,4 +1,3 @@
-
import { readFileSync } from 'node:fs';
import { join } from 'node:path';
import { setTimeout as wait } from 'node:timers/promises';
diff --git a/packages/app/tests/integration/crlf-line-endings.test.ts b/packages/app/tests/integration/crlf-line-endings.test.ts
new file mode 100644
index 000000000..43b16f186
--- /dev/null
+++ b/packages/app/tests/integration/crlf-line-endings.test.ts
@@ -0,0 +1,102 @@
+import { writeFileSync } from 'node:fs';
+import { join } from 'node:path';
+import { afterAll, beforeAll, describe, expect, test } from 'vitest';
+import * as Y from 'yjs';
+import { HARNESS_BOOT_TIMEOUT_MS } from './harness-boot-timeout';
+import {
+ createTestClients,
+ createTestServer,
+ pollDiskContentStable,
+ pollUntil,
+ type TestServer,
+} from './test-harness';
+
+let server: TestServer;
+
+beforeAll(async () => {
+ server = await createTestServer();
+}, HARNESS_BOOT_TIMEOUT_MS);
+
+afterAll(async () => {
+ await server.cleanup();
+});
+
+const MISSION_LINES = [
+ '# Mission',
+ '',
+ 'Notre __mission__ est simple.',
+ '',
+ '## Objectifs',
+ '',
+ '+ un',
+ '+ deux',
+ '',
+];
+
+function findTextNodeContaining(
+ node: Y.XmlFragment | Y.XmlElement,
+ needle: string,
+): Y.XmlText | null {
+ for (let i = 0; i < node.length; i++) {
+ const child = node.get(i);
+ if (child instanceof Y.XmlText && child.toString().includes(needle)) return child;
+ if (child instanceof Y.XmlElement) {
+ const found = findTextNodeContaining(child, needle);
+ if (found) return found;
+ }
+ }
+ return null;
+}
+
+const FRONTMATTER_LINES = ['---', 'title: Mission', '---', ''];
+
+async function typeIntoHeadingFromDisk(
+ eol: '\n' | '\r\n',
+ withFrontmatter: boolean,
+): Promise<string> {
+ const docName = `crlf-line-endings-${crypto.randomUUID()}`;
+ const filePath = join(server.contentDir, `${docName}.md`);
+ const raw = [...(withFrontmatter ? FRONTMATTER_LINES : []), ...MISSION_LINES].join(eol);
+ writeFileSync(filePath, raw, 'utf-8');
+
+ const clients = await createTestClients(server.port, { count: 2, docName });
+ try {
+ await pollUntil(() => clients.every((c) => c.ytext.toString() === raw), 5000);
+ const [typist, peer] = clients;
+
+ for (const [offset, ch] of [
+ [0, 'X'],
+ [1, 'Y'],
+ ] as const) {
+ typist.doc.transact(() => {
+ const heading = findTextNodeContaining(typist.fragment, 'Objectifs');
+ if (!heading) throw new Error('no fragment text node containing "Objectifs"');
+ heading.insert(offset, ch);
+ });
+ }
+
+ const expected = raw.replace('## Objectifs', '## XYObjectifs');
+ await pollUntil(() => peer.ytext.toString() === expected, 5000);
+ return await pollDiskContentStable(filePath, (content) => content === expected, {
+ timeoutMs: 10_000,
+ });
+ } finally {
+ for (const client of clients) await client.cleanup();
+ }
+}
+
+describe('CRLF documents keep their line endings from disk, through an edit, back to disk (PRD-9139)', () => {
+ test.each([
+ { label: 'without frontmatter', withFrontmatter: false },
+ { label: 'with frontmatter', withFrontmatter: true },
+ ])(
+ 'a WYSIWYG keystroke in a CRLF file persists only the typed characters, every line still CRLF ($label)',
+ async ({ withFrontmatter }) => {
+ const lf = await typeIntoHeadingFromDisk('\n', withFrontmatter);
+ const crlf = await typeIntoHeadingFromDisk('\r\n', withFrontmatter);
+
+ expect(crlf).toBe(lf.replaceAll('\n', '\r\n'));
+ expect(crlf.split('\r\n').join('')).not.toContain('\n');
+ },
+ );
+});
diff --git a/packages/app/tests/integration/indented-jsx-byte-sacred.test.ts b/packages/app/tests/integration/indented-jsx-byte-sacred.test.ts
index 4fc2c11f9..6fcdc23f2 100644
--- a/packages/app/tests/integration/indented-jsx-byte-sacred.test.ts
+++ b/packages/app/tests/integration/indented-jsx-byte-sacred.test.ts
@@ -1,4 +1,3 @@
-
import { setTimeout as wait } from 'node:timers/promises';
import { normalizeBridge } from '@inkeep/open-knowledge-core';
import { updateYFragment } from '@tiptap/y-tiptap';
diff --git a/packages/app/tests/integration/interior-edit-freshness.test.ts b/packages/app/tests/integration/interior-edit-freshness.test.ts
index 9f2f6feb6..40b5cdfd6 100644
--- a/packages/app/tests/integration/interior-edit-freshness.test.ts
+++ b/packages/app/tests/integration/interior-edit-freshness.test.ts
@@ -1,4 +1,3 @@
-
import { setTimeout as wait } from 'node:timers/promises';
import { updateYFragment, yXmlFragmentToProseMirrorRootNode } from '@tiptap/y-tiptap';
import { afterAll, beforeAll, describe, expect, test } from 'vitest';
diff --git a/packages/app/tests/integration/keystroke-cadence-danger-space.test.ts b/packages/app/tests/integration/keystroke-cadence-danger-space.test.ts
index 2a085742b..93666af70 100644
--- a/packages/app/tests/integration/keystroke-cadence-danger-space.test.ts
+++ b/packages/app/tests/integration/keystroke-cadence-danger-space.test.ts
@@ -1,4 +1,3 @@
-
import { setTimeout as wait } from 'node:timers/promises';
import { updateYFragment, yXmlFragmentToProseMirrorRootNode } from '@tiptap/y-tiptap';
import { afterAll, beforeAll, describe, expect, test } from 'vitest';
diff --git a/packages/app/tests/integration/producer-guard-wiring.test.ts b/packages/app/tests/integration/producer-guard-wiring.test.ts
index 878d5196b..867598d34 100644
--- a/packages/app/tests/integration/producer-guard-wiring.test.ts
+++ b/packages/app/tests/integration/producer-guard-wiring.test.ts
@@ -1,4 +1,3 @@
-
import { setTimeout as wait } from 'node:timers/promises';
import { updateYFragment, yXmlFragmentToProseMirrorRootNode } from '@tiptap/y-tiptap';
import { afterAll, beforeAll, describe, expect, test } from 'vitest';
diff --git a/packages/app/tests/integration/qa-039-reconciliation-counters-http.test.ts b/packages/app/tests/integration/qa-039-reconciliation-counters-http.test.ts
index 4ada3cf26..0970ac12f 100644
--- a/packages/app/tests/integration/qa-039-reconciliation-counters-http.test.ts
+++ b/packages/app/tests/integration/qa-039-reconciliation-counters-http.test.ts
@@ -1,4 +1,3 @@
-
import { randomUUID } from 'node:crypto';
import { afterEach, describe, expect, test } from 'vitest';
import { agentWriteMd, createTestServer, type TestServer } from './test-harness';
diff --git a/packages/app/tests/integration/qa-045-restore-after-gc.test.ts b/packages/app/tests/integration/qa-045-restore-after-gc.test.ts
index cbe2b705f..b632185e1 100644
--- a/packages/app/tests/integration/qa-045-restore-after-gc.test.ts
+++ b/packages/app/tests/integration/qa-045-restore-after-gc.test.ts
@@ -1,4 +1,3 @@
-
import { execFileSync } from 'node:child_process';
import { randomUUID } from 'node:crypto';
import { join } from 'node:path';
diff --git a/packages/app/tests/integration/qa-050-rollback-undo-shipped-path.test.ts b/packages/app/tests/integration/qa-050-rollback-undo-shipped-path.test.ts
index 89bc9b4d9..c530ce4ba 100644
--- a/packages/app/tests/integration/qa-050-rollback-undo-shipped-path.test.ts
+++ b/packages/app/tests/integration/qa-050-rollback-undo-shipped-path.test.ts
@@ -1,4 +1,3 @@
-
import { randomUUID } from 'node:crypto';
import { afterAll, afterEach, beforeAll, describe, expect, test } from 'vitest';
import type * as Y from 'yjs';
diff --git a/packages/app/tests/integration/undo-after-rollback.test.ts b/packages/app/tests/integration/undo-after-rollback.test.ts
index a78aeb0e4..d3767adc9 100644
--- a/packages/app/tests/integration/undo-after-rollback.test.ts
+++ b/packages/app/tests/integration/undo-after-rollback.test.ts
@@ -1,4 +1,3 @@
-
import { ROLLBACK_ORIGIN } from '@inkeep/open-knowledge-server';
import { afterAll, beforeAll, describe, expect, test } from 'vitest';
import * as Y from 'yjs';
diff --git a/packages/app/tests/integration/undo-recycle-reset.test.ts b/packages/app/tests/integration/undo-recycle-reset.test.ts
index e7cc69c3f..b076f955c 100644
--- a/packages/app/tests/integration/undo-recycle-reset.test.ts
+++ b/packages/app/tests/integration/undo-recycle-reset.test.ts
@@ -1,4 +1,3 @@
-
import { writeFileSync } from 'node:fs';
import { join } from 'node:path';
import { setTimeout as wait } from 'node:timers/promises';
diff --git a/packages/app/tests/integration/undo-rederive-survival.test.ts b/packages/app/tests/integration/undo-rederive-survival.test.ts
index 9ddcd0730..7935e35f8 100644
--- a/packages/app/tests/integration/undo-rederive-survival.test.ts
+++ b/packages/app/tests/integration/undo-rederive-survival.test.ts
@@ -1,4 +1,3 @@
-
import { prependFrontmatter } from '@inkeep/open-knowledge-core';
import { Plugin, PluginKey } from '@tiptap/pm/state';
import { updateYFragment, yXmlFragmentToProseMirrorRootNode } from '@tiptap/y-tiptap';
diff --git a/packages/core/src/bridge/compose.ts b/packages/core/src/bridge/compose.ts
index 2a597ec3d..f4cc8a286 100644
--- a/packages/core/src/bridge/compose.ts
+++ b/packages/core/src/bridge/compose.ts
@@ -1,4 +1,3 @@
-
import { prependFrontmatter, stripFrontmatter } from '../extensions/frontmatter.ts';
export type ComposeAdjustment = 'none' | 'body' | 'frontmatter' | 'unresolved';
diff --git a/packages/core/src/bridge/doc-boundary-space.ts b/packages/core/src/bridge/doc-boundary-space.ts
index ffd8fce06..e7e63f43d 100644
--- a/packages/core/src/bridge/doc-boundary-space.ts
+++ b/packages/core/src/bridge/doc-boundary-space.ts
@@ -1,4 +1,3 @@
-
import { FRONTMATTER_RE, stripFrontmatter } from '../extensions/frontmatter.ts';
import { carriedEdgeEmpties, type DocEdgeEmpties } from '../markdown/doc-edge-blank-runs.ts';
diff --git a/packages/core/src/bridge/index.ts b/packages/core/src/bridge/index.ts
index f5f91826b..b110b6306 100644
--- a/packages/core/src/bridge/index.ts
+++ b/packages/core/src/bridge/index.ts
@@ -71,6 +71,14 @@ export {
fragmentHoldsPendingContent,
pendingContentLines,
} from './intraline-predicate.ts';
+export {
+ documentBodyLineEnding,
+ dominantLineEnding,
+ type LfProjection,
+ type LineEnding,
+ projectToLf,
+ spellLineEndings,
+} from './line-endings.ts';
export {
assertContentPreservation,
BridgeMergeContentLossError,
diff --git a/packages/core/src/bridge/line-endings.ts b/packages/core/src/bridge/line-endings.ts
new file mode 100644
index 000000000..bb4ed7446
--- /dev/null
+++ b/packages/core/src/bridge/line-endings.ts
@@ -0,0 +1,60 @@
+import { stripFrontmatter } from '../extensions/frontmatter.ts';
+
+export type LineEnding = '\n' | '\r\n';
+
+export interface LfProjection {
+ readonly text: string;
+ readonly lineEnding: LineEnding;
+ toRawOffset(offset: number): number;
+}
+
+const CARRIAGE_RETURN = 13;
+
+function crlfPositionsInLfSpace(raw: string): { positions: number[]; bareLfCount: number } {
+ const positions: number[] = [];
+ let bareLfCount = 0;
+ for (let i = raw.indexOf('\n'); i !== -1; i = raw.indexOf('\n', i + 1)) {
+ if (i > 0 && raw.charCodeAt(i - 1) === CARRIAGE_RETURN) {
+ positions.push(i - 1 - positions.length);
+ } else {
+ bareLfCount++;
+ }
+ }
+ return { positions, bareLfCount };
+}
+
+function countBelow(sorted: readonly number[], value: number): number {
+ let lo = 0;
+ let hi = sorted.length;
+ while (lo < hi) {
+ const mid = (lo + hi) >>> 1;
+ if ((sorted[mid] ?? 0) < value) lo = mid + 1;
+ else hi = mid;
+ }
+ return lo;
+}
+
+export function dominantLineEnding(text: string, whenNone: LineEnding = '\n'): LineEnding {
+ const { positions, bareLfCount } = crlfPositionsInLfSpace(text);
+ if (positions.length === 0 && bareLfCount === 0) return whenNone;
+ return positions.length > bareLfCount ? '\r\n' : '\n';
+}
+
+export function documentBodyLineEnding(doc: string): LineEnding {
+ const { frontmatter, body } = stripFrontmatter(doc);
+ return dominantLineEnding(body, dominantLineEnding(frontmatter));
+}
+
+export function spellLineEndings(text: string, lineEnding: LineEnding): string {
+ return lineEnding === '\r\n' ? text.replace(/\r?\n/g, '\r\n') : text.replaceAll('\r\n', '\n');
+}
+
+export function projectToLf(raw: string): LfProjection {
+ const { positions, bareLfCount } = crlfPositionsInLfSpace(raw);
+ if (positions.length === 0) return { text: raw, lineEnding: '\n', toRawOffset: (o) => o };
+ return {
+ text: raw.replaceAll('\r\n', '\n'),
+ lineEnding: positions.length > bareLfCount ? '\r\n' : '\n',
+ toRawOffset: (offset) => offset + countBelow(positions, offset),
+ };
+}
diff --git a/packages/core/src/bridge/parse-equivalence.ts b/packages/core/src/bridge/parse-equivalence.ts
index 72ce4f79e..4da3b444c 100644
--- a/packages/core/src/bridge/parse-equivalence.ts
+++ b/packages/core/src/bridge/parse-equivalence.ts
@@ -1,4 +1,5 @@
import { stripFrontmatter } from '../extensions/frontmatter.ts';
+import { spellLineEndings } from './line-endings.ts';
import type { BridgeToleranceClass } from './normalize.ts';
import { isSubsequence } from './subsequence.ts';
@@ -6,6 +7,10 @@ export const PARSE_EQUIVALENCE_TOLERANCE = 'parse-equivalence' as const;
export type BridgeToleranceSignal = BridgeToleranceClass | typeof PARSE_EQUIVALENCE_TOLERANCE;
+function comparableBody(body: string): string {
+ return stripDocBoundary(stripTrailingLineWhitespace(spellLineEndings(body, '\n')));
+}
+
function stripDocBoundary(body: string): string {
return body.replace(/^\n+/, '').replace(/\n+$/, '');
}
@@ -55,11 +60,6 @@ export function isParseEquivalentBridge(
}
return false;
}
- if (
- stripDocBoundary(stripTrailingLineWhitespace(canonicalLeftBody)) !==
- stripDocBoundary(stripTrailingLineWhitespace(rightSplit.body))
- ) {
- return false;
- }
+ if (comparableBody(canonicalLeftBody) !== comparableBody(rightSplit.body)) return false;
return isSubsequence(contentSkeleton(leftSplit.body), contentSkeleton(canonicalLeftBody));
}
diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts
index ac5f500c3..3998f83d1 100644
--- a/packages/core/src/index.ts
+++ b/packages/core/src/index.ts
@@ -49,6 +49,8 @@ export {
detectFmRegion,
diffLinesFast,
docEdgeRunsDiffer,
+ documentBodyLineEnding,
+ dominantLineEnding,
emitToleranceFire,
type FmBoundarySlotSplit,
type FmEditError,
@@ -73,6 +75,8 @@ export {
fragmentHoldsPendingContent,
type InvariantViolation,
isParseEquivalentBridge,
+ type LfProjection,
+ type LineEnding,
locateBridgeDivergence,
MAX_FM_REGION_BYTES,
type MergeBoundarySpace,
@@ -85,6 +89,7 @@ export {
parseFencedFmRegion,
parseFmRegion,
pendingContentLines,
+ projectToLf,
readFmKeys,
readFmMap,
readFmRegionWithError,
@@ -97,6 +102,7 @@ export {
type SynthesisedConflictMarkers,
type SynthesisedConflictRegion,
setToleranceTelemetryHook,
+ spellLineEndings,
splitFmBoundarySlot,
structuralDivergence,
synthesiseConflictMarkers,
diff --git a/packages/core/src/markdown/comment-promoter.ts b/packages/core/src/markdown/comment-promoter.ts
index fb0ba5a7c..fc267cbea 100644
--- a/packages/core/src/markdown/comment-promoter.ts
+++ b/packages/core/src/markdown/comment-promoter.ts
@@ -1,4 +1,3 @@
-
import type { Nodes, Paragraph, PhrasingContent, Root, RootContent, Text } from 'mdast';
import { SKIP, visit } from 'unist-util-visit';
import type { VFile } from 'vfile';
diff --git a/packages/core/src/markdown/merged-walker.ts b/packages/core/src/markdown/merged-walker.ts
index fa3cfcd86..bfd7e3109 100644
--- a/packages/core/src/markdown/merged-walker.ts
+++ b/packages/core/src/markdown/merged-walker.ts
@@ -1,4 +1,3 @@
-
import type { Nodes, Parent, Root } from 'mdast';
import { SKIP, visit } from 'unist-util-visit';
import type { VFile } from 'vfile';
diff --git a/packages/core/src/markdown/safe-url.ts b/packages/core/src/markdown/safe-url.ts
index 8be632ba4..4d3250d3a 100644
--- a/packages/core/src/markdown/safe-url.ts
+++ b/packages/core/src/markdown/safe-url.ts
@@ -1,4 +1,3 @@
-
export const SAFE_URL_SCHEMES = ['https', 'http', 'mailto', 'tel', 'ftp', 'sms'] as const;
const SCHEME_ALT = SAFE_URL_SCHEMES.map((s) => `${s}:`).join('|');
diff --git a/packages/server/src/map-driven-splice.test.ts b/packages/server/src/map-driven-splice.test.ts
index f14719b2a..3353cebed 100644
--- a/packages/server/src/map-driven-splice.test.ts
+++ b/packages/server/src/map-driven-splice.test.ts
@@ -211,6 +211,59 @@ describe('computeMapDrivenBodySplice', () => {
});
});
+describe('CRLF bodies keep their line endings through the splice (PRD-9139)', () => {
+ const crlf = (lf: string): string => lf.replaceAll('\n', '\r\n');
+
+ test('a uniformly CRLF body stays uniformly CRLF', () => {
+ const oldBody = crlf('# Heading\n\nFirst paragraph.\n\n## Second\n\n+ un\n+ deux\n');
+ const newPm = pmFromMd('# Heading\n\nFirst paragraph.\n\n## XSecond\n\n+ un\n+ deux\n');
+
+ const splice = computeMapDrivenBodySplice(oldBody, newPm, mdManager);
+ expect(splice).not.toBeNull();
+ if (!splice) return;
+
+ expect(applySplice(oldBody, splice)).toBe(
+ crlf('# Heading\n\nFirst paragraph.\n\n## XSecond\n\n+ un\n+ deux\n'),
+ );
+ expect(oldBody.slice(0, splice.spliceStart)).not.toContain('Second');
+ });
+
+ test('a mixed body keeps every byte outside the splice and spells the slice in its majority ending', () => {
+ const oldBody = 'Alpha.\r\n\r\nBeta.\r\n\r\nGamma.\n';
+ const splice = computeMapDrivenBodySplice(
+ oldBody,
+ pmFromMd('Alpha.\n\nBeta edited.\n\nGamma.\n'),
+ mdManager,
+ );
+ expect(splice).not.toBeNull();
+ if (!splice) return;
+
+ expect(applySplice(oldBody, splice)).toBe('Alpha.\r\n\r\nBeta edited.\r\n\r\nGamma.\n');
+ });
+
+ test('consecutive CRLF keystrokes still parse each body once', () => {
+ const { manager: counted, parses } = createCountingManager();
+ const memo = createEditorMdastMemo();
+ const bodyA = crlf('# H\n\nalpha\n');
+
+ const first = computeMapDrivenBodySplice(bodyA, counted.parse('# H\n\nalphaX\n'), counted, {
+ memo,
+ });
+ expect(first).not.toBeNull();
+ if (!first) return;
+ const bodyB = applySplice(bodyA, first);
+ expect(bodyB).toBe(crlf('# H\n\nalphaX\n'));
+ const afterFirst = parses();
+
+ const second = computeMapDrivenBodySplice(bodyB, counted.parse('# H\n\nalphaXY\n'), counted, {
+ memo,
+ });
+
+ expect(parses() - afterFirst).toBe(1);
+ expect(second && applySplice(bodyB, second)).toBe(crlf('# H\n\nalphaXY\n'));
+ });
+});
+
describe('editor-mdast parse memo (PRD-8273)', () => {
test('a repeated body is parsed once, not once per call', () => {
const { manager: counted, parses } = createCountingManager();
diff --git a/packages/server/src/map-driven-splice.ts b/packages/server/src/map-driven-splice.ts
index 5227f250f..a0f1143b7 100644
--- a/packages/server/src/map-driven-splice.ts
+++ b/packages/server/src/map-driven-splice.ts
@@ -1,4 +1,11 @@
-import type { MarkdownManager, SerializeCallOptions } from '@inkeep/open-knowledge-core';
+import {
+ type LfProjection,
+ type LineEnding,
+ type MarkdownManager,
+ projectToLf,
+ type SerializeCallOptions,
+ spellLineEndings,
+} from '@inkeep/open-knowledge-core';
import type { JSONContent } from '@tiptap/core';
import type { RootContent } from 'mdast';
import type { MapDrivenSpliceMemoSkipReason } from './metrics.ts';
@@ -39,6 +46,7 @@ export interface MapDrivenSpliceOptions {
readonly onMemoSkip?: (reason: MapDrivenSpliceMemoSkipReason, err?: unknown) => void;
readonly memo?: EditorMdastMemo;
readonly serializedNewPm?: SerializedEditorBody;
+ readonly lineEnding?: LineEnding;
}
export interface EditorMdastMemo {
@@ -65,10 +73,34 @@ function editorMdastChildren(
}
export function computeMapDrivenBodySplice(
- oldBody: string,
+ rawOldBody: string,
newPmJson: JSONContent,
mdManager: MarkdownManager,
options: MapDrivenSpliceOptions = {},
+): MapDrivenSplice | null {
+ const projection = projectToLf(rawOldBody);
+ const splice = computeLfBodySplice(projection.text, newPmJson, mdManager, options);
+ if (splice === null) return null;
+ return toRawSplice(splice, projection, options.lineEnding ?? projection.lineEnding);
+}
+
+function toRawSplice(
+ splice: MapDrivenSplice,
+ projection: LfProjection,
+ lineEnding: LineEnding,
+): MapDrivenSplice {
+ return {
+ spliceStart: projection.toRawOffset(splice.spliceStart),
+ spliceEnd: projection.toRawOffset(splice.spliceEnd),
+ newSlice: spellLineEndings(splice.newSlice, lineEnding),
+ };
+}
+
+function computeLfBodySplice(
+ oldBody: string,
+ newPmJson: JSONContent,
+ mdManager: MarkdownManager,
+ options: MapDrivenSpliceOptions,
): MapDrivenSplice | null {
const { onFallback, onMemoHit, onMemoSkip, memo, serializedNewPm } = options;
let oldChildren: readonly RootContent[];
diff --git a/packages/server/src/pre-drain-corpus.test-helper.ts b/packages/server/src/pre-drain-corpus.test-helper.ts
index 9fdc7e6d1..c9bda9bc3 100644
--- a/packages/server/src/pre-drain-corpus.test-helper.ts
+++ b/packages/server/src/pre-drain-corpus.test-helper.ts
@@ -1,5 +1,6 @@
import type { Document } from '@hocuspocus/server';
import {
+ documentBodyLineEnding,
type MarkdownManager,
sharedExtensions,
stripFrontmatter,
@@ -128,6 +129,7 @@ export async function createDiscriminatorRig(baseMd: string): Promise<Discrimina
fmPrefixLen,
op: { kind: 'agent-undo', ytext, stackItem },
mdManager: productionMdManager,
+ lineEnding: documentBodyLineEnding(ytext.toString()),
}).verdict;
},
undoInputs: () => {
@@ -151,6 +153,7 @@ export async function createDiscriminatorRig(baseMd: string): Promise<Discrimina
fmPrefixLen,
op: { kind: 'agent-write', writeKind: position },
mdManager: productionMdManager,
+ lineEnding: documentBodyLineEnding(ytext.toString()),
}).verdict;
},
cleanup: () => manager.closeAll(),
diff --git a/packages/server/src/pre-drain-discriminator.test.ts b/packages/server/src/pre-drain-discriminator.test.ts
index fe91ecf9b..a5d6ef73b 100644
--- a/packages/server/src/pre-drain-discriminator.test.ts
+++ b/packages/server/src/pre-drain-discriminator.test.ts
@@ -238,6 +238,7 @@ describe('cost gate: cheap fail-closed guards short-circuit before the localizer
fragmentPmJson: {},
fmPrefixLen: 0,
mdManager: throwingMdManager,
+ lineEnding: '\n',
op: { kind: 'agent-write', composedBody: 'x', writeKind: 'prepend' } as const,
};
@@ -267,6 +268,50 @@ describe('cost gate: cheap fail-closed guards short-circuit before the localizer
});
expect(verdict).toEqual({ preDrain: false, reason: 'checkpoint-no-target' });
});
+
+ it('flushes a pending keystroke into a CRLF body in CRLF (PRD-9139)', () => {
+ const body = 'para one\r\n\r\npara two\r\n\r\npara three\r\n';
+ const plan = planPreDrain({
+ pendingDirty: true,
+ witnessMatched: true,
+ body,
+ fragmentPmJson: mdManager.parse('para one!\n\npara two\n\npara three\n'),
+ fmPrefixLen: 0,
+ mdManager,
+ lineEnding: '\r\n',
+ op: { kind: 'agent-write', writeKind: 'append' },
+ });
+
+ expect(plan.preDrain).toBe(true);
+ if (!plan.preDrain) return;
+ expect(
+ body.slice(0, plan.splice.spliceStart) +
+ plan.splice.newSlice +
+ body.slice(plan.splice.spliceEnd),
+ ).toBe('para one!\r\n\r\npara two\r\n\r\npara three\r\n');
+ });
+
+ it('flushes added lines into a body with no line ending in the ending it is given (PRD-9139)', () => {
+ const body = 'Alpha.';
+ const plan = planPreDrain({
+ pendingDirty: true,
+ witnessMatched: true,
+ body,
+ fragmentPmJson: mdManager.parse('Alpha.\n\nBeta.\n'),
+ fmPrefixLen: '---\r\ntitle: t\r\n---\r\n'.length,
+ mdManager,
+ lineEnding: '\r\n',
+ op: { kind: 'agent-write', writeKind: 'append' },
+ });
+
+ expect(plan.preDrain).toBe(true);
+ if (!plan.preDrain) return;
+ expect(
+ body.slice(0, plan.splice.spliceStart) +
+ plan.splice.newSlice +
+ body.slice(plan.splice.spliceEnd),
+ ).toBe('Alpha.\r\n\r\nBeta.\r\n');
+ });
});
describe('same-block discriminator corpus (splice+graze)', () => {
@@ -387,6 +432,7 @@ describe('full-body-overwrite positions are structurally inert', () => {
fmPrefixLen: 0,
op: { kind: 'agent-write', writeKind: position },
mdManager,
+ lineEnding: '\n',
});
expect(plan.verdict.preDrain).toBe(false);
expect(plan.verdict.reason).toBe('checkpoint-full-overwrite');
@@ -403,6 +449,7 @@ describe('full-body-overwrite positions are structurally inert', () => {
fmPrefixLen: 0,
op: { kind: 'agent-write', writeKind: 'append' },
mdManager,
+ lineEnding: '\n',
});
expect(plan.verdict.preDrain).toBe(true);
expect(plan.splice).not.toBeNull();
diff --git a/packages/server/src/pre-drain-discriminator.ts b/packages/server/src/pre-drain-discriminator.ts
index 3cc696405..918ea5652 100644
--- a/packages/server/src/pre-drain-discriminator.ts
+++ b/packages/server/src/pre-drain-discriminator.ts
@@ -1,4 +1,4 @@
-import type { MarkdownManager } from '@inkeep/open-knowledge-core';
+import type { LineEnding, MarkdownManager } from '@inkeep/open-knowledge-core';
import type { JSONContent } from '@tiptap/core';
import type * as Y from 'yjs';
import { ContentString } from 'yjs';
@@ -147,6 +147,7 @@ export interface DiscriminatePreDrainInput {
readonly fmPrefixLen: number;
readonly op: PreDrainOp;
readonly mdManager: MarkdownManager;
+ readonly lineEnding: LineEnding;
}
export type PreDrainOpInput =
@@ -180,7 +181,9 @@ export function planPreDrain(input: DiscriminatePreDrainInput): PreDrainPlan {
: extractComposeTargetSpan(input.body, input.op.writeKind);
if (targetSpan === null) return decline('checkpoint-no-target');
- const splice = computeMapDrivenBodySplice(input.body, input.fragmentPmJson, input.mdManager);
+ const splice = computeMapDrivenBodySplice(input.body, input.fragmentPmJson, input.mdManager, {
+ lineEnding: input.lineEnding,
+ });
if (splice === null) return decline('checkpoint-null-splice');
if (splice.newSlice === input.body.slice(splice.spliceStart, splice.spliceEnd)) {
diff --git a/packages/server/src/server-observers.test.ts b/packages/server/src/server-observers.test.ts
index 7a275b112..a2a89ad62 100644
--- a/packages/server/src/server-observers.test.ts
+++ b/packages/server/src/server-observers.test.ts
@@ -1515,4 +1515,164 @@ describe('Observer A routing — Path B fires iff Y.Text holds unabsorbed change
});
test.todo('each of several sequential remote writes refreshes the baseline');
+
+ describe('CRLF documents keep their line endings through fragment edits (PRD-9139)', () => {
+ const MISSION_LINES = [
+ '# Mission',
+ '',
+ 'Notre __mission__ est simple.',
+ '',
+ '## Objectifs',
+ '',
+ '+ un',
+ '+ deux',
+ '',
+ ];
+
+ const FRONTMATTER_LINES = ['---', 'title: Mission', '---', ''];
+
+ type Eol = '\n' | '\r\n';
+ const source = (eol: Eol, withFrontmatter: boolean): string =>
+ [...(withFrontmatter ? FRONTMATTER_LINES : []), ...MISSION_LINES].join(eol);
+
+ function editTwice(eol: Eol, withFrontmatter: boolean, docName: string) {
+ __resetBridgeWatchdogForTests();
+ resetMetrics();
+ const raw = source(eol, withFrontmatter);
+ const { doc, xmlFragment, ytext, cleanup } = seedThenAttach(raw, docName);
+ expect(ytext.toString()).toBe(raw);
+
+ const events = capturePathBEvents(() => {
+ populateFragment(
+ doc,
+ xmlFragment,
+ serializeFragmentBody(xmlFragment).replace('## Objectifs', '## XObjectifs'),
+ );
+ populateFragment(
+ doc,
+ xmlFragment,
+ serializeFragmentBody(xmlFragment).replace('est simple.', 'est simple!'),
+ );
+ });
+ const metrics = getMetrics();
+ const outcome = {
+ text: ytext.toString(),
+ routes: {
+ pathBEvents: events.length,
+ pathBFires: totalPathBFires(),
+ residualMergeRuns: metrics.observerAResidualMergeRuns,
+ splicesApplied: metrics.mapDrivenSpliceApplied,
+ },
+ };
+ cleanup();
+ return outcome;
+ }
+
+ const edited = (eol: Eol, withFrontmatter: boolean): string =>
+ source(eol, withFrontmatter)
+ .replace('## Objectifs', '## XObjectifs')
+ .replace('est simple.', 'est simple!');
+
+ describe.each([
+ { label: 'without frontmatter', withFrontmatter: false },
+ { label: 'with frontmatter', withFrontmatter: true },
+ ])('$label', ({ withFrontmatter }) => {
+ test('keystrokes in a uniformly CRLF document leave every line CRLF', () => {
+ expect(editTwice('\r\n', withFrontmatter, 'routing-crlf-keystrokes').text).toBe(
+ edited('\r\n', withFrontmatter),
+ );
+ });
+
+ test('an LF document is byte-identical, and a CRLF one takes the same routes with no Path B or residual merge', () => {
+ const lf = editTwice('\n', withFrontmatter, 'routing-lf-keystrokes-control');
+ expect(lf.text).toBe(edited('\n', withFrontmatter));
+ const crlf = editTwice('\r\n', withFrontmatter, 'routing-crlf-keystrokes-routes');
+ expect(crlf.routes).toEqual(lf.routes);
+ expect(lf.routes).toEqual({
+ pathBEvents: 0,
+ pathBFires: 0,
+ residualMergeRuns: 0,
+ splicesApplied: 2,
+ });
+ });
+ });
+
+ test('the incremental-diff fallback keeps a CRLF document CRLF', () => {
+ __resetBridgeWatchdogForTests();
+ resetMetrics();
+ const raw = '<!-- note -->\r\n\r\nOriginal.\r\n';
+ const { doc, xmlFragment, ytext, cleanup } = seedThenAttach(raw, 'routing-crlf-fallback');
+ expect(ytext.toString()).toBe(raw);
+
+ populateFragment(doc, xmlFragment, `${serializeFragmentBody(xmlFragment)}\nAdded.\n`);
+
+ expect(getMetrics().mapDrivenSpliceApplied).toBe(0);
+ expect(ytext.toString()).toBe('<!-- note -->\r\n\r\nOriginal.\r\n\r\nAdded.\r\n');
+ cleanup();
+ });
+
+ test.each([
+ {
+ label: 'an empty body, splice route',
+ raw: '---\r\ntitle: t\r\n---\r\n',
+ fragment: 'First.\n\nSecond.\n\nThird.\n',
+ expectedBody: 'First.\r\n\r\nSecond.\r\n\r\nThird.\r\n',
+ splicesApplied: 1,
+ },
+ {
+ label: 'one unterminated line, splice route',
+ raw: '---\r\ntitle: t\r\n---\r\nFirst.',
+ fragment: 'First.\n\nSecond.\n\nThird.\n',
+ expectedBody: 'First.\r\n\r\nSecond.\r\n\r\nThird.\r\n',
+ splicesApplied: 1,
+ },
+ {
+ label: 'one unterminated line, incremental-diff fallback',
+ raw: '---\r\ntitle: t\r\n---\r\n<!-- note -->',
+ fragment: '<!-- note -->\n\nSecond.\n\nThird.\n',
+ expectedBody: '<!-- note -->\r\n\r\nSecond.\r\n\r\nThird.\r\n',
+ splicesApplied: 0,
+ },
+ ])(
+ 'a CRLF document whose body has no line ending yet stays CRLF when it gains lines ($label)',
+ ({ raw, fragment, expectedBody, splicesApplied }) => {
+ __resetBridgeWatchdogForTests();
+ resetMetrics();
+ const { doc, xmlFragment, ytext, cleanup } = seedThenAttach(
+ raw,
+ 'routing-crlf-no-body-eol',
+ );
+ expect(ytext.toString()).toBe(raw);
+
+ populateFragment(doc, xmlFragment, fragment);
+
+ expect(getMetrics().mapDrivenSpliceApplied).toBe(splicesApplied);
+ expect(ytext.toString()).toBe(`---\r\ntitle: t\r\n---\r\n${expectedBody}`);
+ cleanup();
+ },
+ );
+
+ test.each([
+ { frontmatter: '---\ntitle: t\n---\n', bodyEol: '\r\n' },
+ { frontmatter: '---\r\ntitle: t\r\n---\r\n', bodyEol: '\n' },
+ ] as const)(
+ 'the incremental-diff fallback leaves frontmatter bytes alone and spells the body in its own ending (frontmatter $frontmatter)',
+ ({ frontmatter, bodyEol }) => {
+ __resetBridgeWatchdogForTests();
+ resetMetrics();
+ const body = (lines: string[]): string => lines.join(bodyEol);
+ const raw = frontmatter + body(['', '<!-- note -->', '', 'Original.', '']);
+ const { doc, xmlFragment, ytext, cleanup } = seedThenAttach(raw, 'routing-fm-eol-fallback');
+ expect(ytext.toString()).toBe(raw);
+
+ populateFragment(doc, xmlFragment, `${serializeFragmentBody(xmlFragment)}\nAdded.\n`);
+
+ expect(getMetrics().mapDrivenSpliceApplied).toBe(0);
+ expect(ytext.toString()).toBe(
+ frontmatter + body(['', '<!-- note -->', '', 'Original.', '', 'Added.', '']),
+ );
+ cleanup();
+ },
+ );
+ });
});
diff --git a/packages/server/src/server-observers.ts b/packages/server/src/server-observers.ts
index b301f311c..ab8b60c33 100644
--- a/packages/server/src/server-observers.ts
+++ b/packages/server/src/server-observers.ts
@@ -22,6 +22,7 @@ import {
createMergeBoundarySpace,
DUPLICATION_GATE_MIN_LINE_LENGTH,
docEdgeRunsDiffer,
+ documentBodyLineEnding,
findDroppedSourceContentInRange,
fnv1aDigest,
fragmentHoldsPendingContent,
@@ -31,6 +32,7 @@ import {
overMultipliedBodyLines,
pendingContentLines,
prependFrontmatter,
+ spellLineEndings,
splitFmBoundarySlot,
stripFrontmatter,
} from '@inkeep/open-knowledge-core';
@@ -280,6 +282,7 @@ function tryComputeMapDrivenSplice(
},
memo: mdastMemo,
serializedNewPm,
+ lineEnding: documentBodyLineEnding(currentText),
});
if (!splice) return null;
@@ -882,6 +885,10 @@ export function setupServerObservers(opts: SetupServerObserversOpts): () => void
return composeWithDerivedBody(frontmatter, slot + derivedBody);
};
+ const spellDerivedBody = (composition: BridgeComposition, currentText: string): string =>
+ composition.frontmatter +
+ spellLineEndings(composition.body, documentBodyLineEnding(currentText));
+
const observerParseOpts =
opts.resolveEmbed && opts.docName
? {
@@ -1203,7 +1210,7 @@ export function setupServerObservers(opts: SetupServerObserversOpts): () => void
if (mapDrivenSplice) {
applyMapDrivenSplice(ytext, mapDrivenSplice);
} else if (ytextInSync && !residualMergeEligible) {
- applyIncrementalDiff(ytext, currentText, md);
+ applyIncrementalDiff(ytext, currentText, spellDerivedBody(composition, currentText));
} else {
const mergeBase = ytextInSync ? lastSyncedCanonicalMd : preMergeBaseline;
const boundarySpace = createMergeBoundarySpace(body);
@@ -1600,6 +1607,7 @@ export function setupServerObservers(opts: SetupServerObserversOpts): () => void
fragmentPmJson: json,
witnessMatched: fullMd === lastSyncedYTextBytes,
fmPrefixLen,
+ lineEnding: documentBodyLineEnding(fullMd),
op:
op.kind === 'agent-undo'
? { kind: 'agent-undo', ytext, stackItem: op.stackItem }