Beautiful, AI-native markdown IDE and LLM wiki
TypeScript 2nd-brainagent-skillsclaudecodexcompany-braindocsknowledge-knowledge-baseknowledge-managementllmllm-wikillm-wiki-karpathymarkdownmarkdown-editormdmdxnotespkmsecond-brainskills
Keep CRLF line endings when editing (PRD-9139) (#5928)
Andrew Mikofalvy61de57742512
diff --git a/.changeset/crlf-edit-keeps-line-endings.md b/.changeset/crlf-edit-keeps-line-endings.mdnew file mode 100644index 000000000..b624f2be2--- /dev/null+++ b/.changeset/crlf-edit-keeps-line-endings.md@@ -0,0 +1,5 @@+---+"@inkeep/open-knowledge": patch+---++Typing in a Markdown file that uses Windows (CRLF) line endings no longer converts the edited lines to LF.diff --git a/knip.config.ts b/knip.config.tsindex c33b744b2..e59ddff92 100644--- a/knip.config.ts+++ b/knip.config.ts@@ -148,15 +148,11 @@ export default {'packages/server': {entry: ['src/**/*.test.ts', 'src/parse-worker.ts'],project: 'src/**',- ignoreDependencies: [- '@types/shell-quote',- ],+ ignoreDependencies: ['@types/shell-quote'],},'packages/cli': {entry: ['src/**/*.test.ts', 'scripts/*.ts', 'tests/**/*.ts', 'src/parse-worker.ts'],- ignoreDependencies: [- 'yjs',- ],+ ignoreDependencies: ['yjs'],},'packages/desktop': {entry: [diff --git a/packages/app/tests/integration/bug3-source-mode-writeback.test.ts b/packages/app/tests/integration/bug3-source-mode-writeback.test.tsindex 4549c74b3..a50b00123 100644--- a/packages/app/tests/integration/bug3-source-mode-writeback.test.ts+++ b/packages/app/tests/integration/bug3-source-mode-writeback.test.ts@@ -1,4 +1,3 @@-import { readFileSync } from 'node:fs';import { join } from 'node:path';import { setTimeout as wait } from 'node:timers/promises';diff --git a/packages/app/tests/integration/crlf-line-endings.test.ts b/packages/app/tests/integration/crlf-line-endings.test.tsnew file mode 100644index 000000000..43b16f186--- /dev/null+++ b/packages/app/tests/integration/crlf-line-endings.test.ts@@ -0,0 +1,102 @@+import { writeFileSync } from 'node:fs';+import { join } from 'node:path';+import { afterAll, beforeAll, describe, expect, test } from 'vitest';+import * as Y from 'yjs';+import { HARNESS_BOOT_TIMEOUT_MS } from './harness-boot-timeout';+import {+ createTestClients,+ createTestServer,+ pollDiskContentStable,+ pollUntil,+ type TestServer,+} from './test-harness';++let server: TestServer;++beforeAll(async () => {+ server = await createTestServer();+}, HARNESS_BOOT_TIMEOUT_MS);++afterAll(async () => {+ await server.cleanup();+});++const MISSION_LINES = [+ '# Mission',+ '',+ 'Notre __mission__ est simple.',+ '',+ '## Objectifs',+ '',+ '+ un',+ '+ deux',+ '',+];++function findTextNodeContaining(+ node: Y.XmlFragment | Y.XmlElement,+ needle: string,+): Y.XmlText | null {+ for (let i = 0; i < node.length; i++) {+ const child = node.get(i);+ if (child instanceof Y.XmlText && child.toString().includes(needle)) return child;+ if (child instanceof Y.XmlElement) {+ const found = findTextNodeContaining(child, needle);+ if (found) return found;+ }+ }+ return null;+}++const FRONTMATTER_LINES = ['---', 'title: Mission', '---', ''];++async function typeIntoHeadingFromDisk(+ eol: '\n' | '\r\n',+ withFrontmatter: boolean,+): Promise<string> {+ const docName = `crlf-line-endings-${crypto.randomUUID()}`;+ const filePath = join(server.contentDir, `${docName}.md`);+ const raw = [...(withFrontmatter ? FRONTMATTER_LINES : []), ...MISSION_LINES].join(eol);+ writeFileSync(filePath, raw, 'utf-8');++ const clients = await createTestClients(server.port, { count: 2, docName });+ try {+ await pollUntil(() => clients.every((c) => c.ytext.toString() === raw), 5000);+ const [typist, peer] = clients;++ for (const [offset, ch] of [+ [0, 'X'],+ [1, 'Y'],+ ] as const) {+ typist.doc.transact(() => {+ const heading = findTextNodeContaining(typist.fragment, 'Objectifs');+ if (!heading) throw new Error('no fragment text node containing "Objectifs"');+ heading.insert(offset, ch);+ });+ }++ const expected = raw.replace('## Objectifs', '## XYObjectifs');+ await pollUntil(() => peer.ytext.toString() === expected, 5000);+ return await pollDiskContentStable(filePath, (content) => content === expected, {+ timeoutMs: 10_000,+ });+ } finally {+ for (const client of clients) await client.cleanup();+ }+}++describe('CRLF documents keep their line endings from disk, through an edit, back to disk (PRD-9139)', () => {+ test.each([+ { label: 'without frontmatter', withFrontmatter: false },+ { label: 'with frontmatter', withFrontmatter: true },+ ])(+ 'a WYSIWYG keystroke in a CRLF file persists only the typed characters, every line still CRLF ($label)',+ async ({ withFrontmatter }) => {+ const lf = await typeIntoHeadingFromDisk('\n', withFrontmatter);+ const crlf = await typeIntoHeadingFromDisk('\r\n', withFrontmatter);++ expect(crlf).toBe(lf.replaceAll('\n', '\r\n'));+ expect(crlf.split('\r\n').join('')).not.toContain('\n');+ },+ );+});diff --git a/packages/app/tests/integration/indented-jsx-byte-sacred.test.ts b/packages/app/tests/integration/indented-jsx-byte-sacred.test.tsindex 4fc2c11f9..6fcdc23f2 100644--- a/packages/app/tests/integration/indented-jsx-byte-sacred.test.ts+++ b/packages/app/tests/integration/indented-jsx-byte-sacred.test.ts@@ -1,4 +1,3 @@-import { setTimeout as wait } from 'node:timers/promises';import { normalizeBridge } from '@inkeep/open-knowledge-core';import { updateYFragment } from '@tiptap/y-tiptap';diff --git a/packages/app/tests/integration/interior-edit-freshness.test.ts b/packages/app/tests/integration/interior-edit-freshness.test.tsindex 9f2f6feb6..40b5cdfd6 100644--- a/packages/app/tests/integration/interior-edit-freshness.test.ts+++ b/packages/app/tests/integration/interior-edit-freshness.test.ts@@ -1,4 +1,3 @@-import { setTimeout as wait } from 'node:timers/promises';import { updateYFragment, yXmlFragmentToProseMirrorRootNode } from '@tiptap/y-tiptap';import { afterAll, beforeAll, describe, expect, test } from 'vitest';diff --git a/packages/app/tests/integration/keystroke-cadence-danger-space.test.ts b/packages/app/tests/integration/keystroke-cadence-danger-space.test.tsindex 2a085742b..93666af70 100644--- a/packages/app/tests/integration/keystroke-cadence-danger-space.test.ts+++ b/packages/app/tests/integration/keystroke-cadence-danger-space.test.ts@@ -1,4 +1,3 @@-import { setTimeout as wait } from 'node:timers/promises';import { updateYFragment, yXmlFragmentToProseMirrorRootNode } from '@tiptap/y-tiptap';import { afterAll, beforeAll, describe, expect, test } from 'vitest';diff --git a/packages/app/tests/integration/producer-guard-wiring.test.ts b/packages/app/tests/integration/producer-guard-wiring.test.tsindex 878d5196b..867598d34 100644--- a/packages/app/tests/integration/producer-guard-wiring.test.ts+++ b/packages/app/tests/integration/producer-guard-wiring.test.ts@@ -1,4 +1,3 @@-import { setTimeout as wait } from 'node:timers/promises';import { updateYFragment, yXmlFragmentToProseMirrorRootNode } from '@tiptap/y-tiptap';import { afterAll, beforeAll, describe, expect, test } from 'vitest';diff --git a/packages/app/tests/integration/qa-039-reconciliation-counters-http.test.ts b/packages/app/tests/integration/qa-039-reconciliation-counters-http.test.tsindex 4ada3cf26..0970ac12f 100644--- a/packages/app/tests/integration/qa-039-reconciliation-counters-http.test.ts+++ b/packages/app/tests/integration/qa-039-reconciliation-counters-http.test.ts@@ -1,4 +1,3 @@-import { randomUUID } from 'node:crypto';import { afterEach, describe, expect, test } from 'vitest';import { agentWriteMd, createTestServer, type TestServer } from './test-harness';diff --git a/packages/app/tests/integration/qa-045-restore-after-gc.test.ts b/packages/app/tests/integration/qa-045-restore-after-gc.test.tsindex cbe2b705f..b632185e1 100644--- a/packages/app/tests/integration/qa-045-restore-after-gc.test.ts+++ b/packages/app/tests/integration/qa-045-restore-after-gc.test.ts@@ -1,4 +1,3 @@-import { execFileSync } from 'node:child_process';import { randomUUID } from 'node:crypto';import { join } from 'node:path';diff --git a/packages/app/tests/integration/qa-050-rollback-undo-shipped-path.test.ts b/packages/app/tests/integration/qa-050-rollback-undo-shipped-path.test.tsindex 89bc9b4d9..c530ce4ba 100644--- a/packages/app/tests/integration/qa-050-rollback-undo-shipped-path.test.ts+++ b/packages/app/tests/integration/qa-050-rollback-undo-shipped-path.test.ts@@ -1,4 +1,3 @@-import { randomUUID } from 'node:crypto';import { afterAll, afterEach, beforeAll, describe, expect, test } from 'vitest';import type * as Y from 'yjs';diff --git a/packages/app/tests/integration/undo-after-rollback.test.ts b/packages/app/tests/integration/undo-after-rollback.test.tsindex a78aeb0e4..d3767adc9 100644--- a/packages/app/tests/integration/undo-after-rollback.test.ts+++ b/packages/app/tests/integration/undo-after-rollback.test.ts@@ -1,4 +1,3 @@-import { ROLLBACK_ORIGIN } from '@inkeep/open-knowledge-server';import { afterAll, beforeAll, describe, expect, test } from 'vitest';import * as Y from 'yjs';diff --git a/packages/app/tests/integration/undo-recycle-reset.test.ts b/packages/app/tests/integration/undo-recycle-reset.test.tsindex e7cc69c3f..b076f955c 100644--- a/packages/app/tests/integration/undo-recycle-reset.test.ts+++ b/packages/app/tests/integration/undo-recycle-reset.test.ts@@ -1,4 +1,3 @@-import { writeFileSync } from 'node:fs';import { join } from 'node:path';import { setTimeout as wait } from 'node:timers/promises';diff --git a/packages/app/tests/integration/undo-rederive-survival.test.ts b/packages/app/tests/integration/undo-rederive-survival.test.tsindex 9ddcd0730..7935e35f8 100644--- a/packages/app/tests/integration/undo-rederive-survival.test.ts+++ b/packages/app/tests/integration/undo-rederive-survival.test.ts@@ -1,4 +1,3 @@-import { prependFrontmatter } from '@inkeep/open-knowledge-core';import { Plugin, PluginKey } from '@tiptap/pm/state';import { updateYFragment, yXmlFragmentToProseMirrorRootNode } from '@tiptap/y-tiptap';diff --git a/packages/core/src/bridge/compose.ts b/packages/core/src/bridge/compose.tsindex 2a597ec3d..f4cc8a286 100644--- a/packages/core/src/bridge/compose.ts+++ b/packages/core/src/bridge/compose.ts@@ -1,4 +1,3 @@-import { prependFrontmatter, stripFrontmatter } from '../extensions/frontmatter.ts';export type ComposeAdjustment = 'none' | 'body' | 'frontmatter' | 'unresolved';diff --git a/packages/core/src/bridge/doc-boundary-space.ts b/packages/core/src/bridge/doc-boundary-space.tsindex ffd8fce06..e7e63f43d 100644--- a/packages/core/src/bridge/doc-boundary-space.ts+++ b/packages/core/src/bridge/doc-boundary-space.ts@@ -1,4 +1,3 @@-import { FRONTMATTER_RE, stripFrontmatter } from '../extensions/frontmatter.ts';import { carriedEdgeEmpties, type DocEdgeEmpties } from '../markdown/doc-edge-blank-runs.ts';diff --git a/packages/core/src/bridge/index.ts b/packages/core/src/bridge/index.tsindex f5f91826b..b110b6306 100644--- a/packages/core/src/bridge/index.ts+++ b/packages/core/src/bridge/index.ts@@ -71,6 +71,14 @@ export {fragmentHoldsPendingContent,pendingContentLines,} from './intraline-predicate.ts';+export {+ documentBodyLineEnding,+ dominantLineEnding,+ type LfProjection,+ type LineEnding,+ projectToLf,+ spellLineEndings,+} from './line-endings.ts';export {assertContentPreservation,BridgeMergeContentLossError,diff --git a/packages/core/src/bridge/line-endings.ts b/packages/core/src/bridge/line-endings.tsnew file mode 100644index 000000000..bb4ed7446--- /dev/null+++ b/packages/core/src/bridge/line-endings.ts@@ -0,0 +1,60 @@+import { stripFrontmatter } from '../extensions/frontmatter.ts';++export type LineEnding = '\n' | '\r\n';++export interface LfProjection {+ readonly text: string;+ readonly lineEnding: LineEnding;+ toRawOffset(offset: number): number;+}++const CARRIAGE_RETURN = 13;++function crlfPositionsInLfSpace(raw: string): { positions: number[]; bareLfCount: number } {+ const positions: number[] = [];+ let bareLfCount = 0;+ for (let i = raw.indexOf('\n'); i !== -1; i = raw.indexOf('\n', i + 1)) {+ if (i > 0 && raw.charCodeAt(i - 1) === CARRIAGE_RETURN) {+ positions.push(i - 1 - positions.length);+ } else {+ bareLfCount++;+ }+ }+ return { positions, bareLfCount };+}++function countBelow(sorted: readonly number[], value: number): number {+ let lo = 0;+ let hi = sorted.length;+ while (lo < hi) {+ const mid = (lo + hi) >>> 1;+ if ((sorted[mid] ?? 0) < value) lo = mid + 1;+ else hi = mid;+ }+ return lo;+}++export function dominantLineEnding(text: string, whenNone: LineEnding = '\n'): LineEnding {+ const { positions, bareLfCount } = crlfPositionsInLfSpace(text);+ if (positions.length === 0 && bareLfCount === 0) return whenNone;+ return positions.length > bareLfCount ? '\r\n' : '\n';+}++export function documentBodyLineEnding(doc: string): LineEnding {+ const { frontmatter, body } = stripFrontmatter(doc);+ return dominantLineEnding(body, dominantLineEnding(frontmatter));+}++export function spellLineEndings(text: string, lineEnding: LineEnding): string {+ return lineEnding === '\r\n' ? text.replace(/\r?\n/g, '\r\n') : text.replaceAll('\r\n', '\n');+}++export function projectToLf(raw: string): LfProjection {+ const { positions, bareLfCount } = crlfPositionsInLfSpace(raw);+ if (positions.length === 0) return { text: raw, lineEnding: '\n', toRawOffset: (o) => o };+ return {+ text: raw.replaceAll('\r\n', '\n'),+ lineEnding: positions.length > bareLfCount ? '\r\n' : '\n',+ toRawOffset: (offset) => offset + countBelow(positions, offset),+ };+}diff --git a/packages/core/src/bridge/parse-equivalence.ts b/packages/core/src/bridge/parse-equivalence.tsindex 72ce4f79e..4da3b444c 100644--- a/packages/core/src/bridge/parse-equivalence.ts+++ b/packages/core/src/bridge/parse-equivalence.ts@@ -1,4 +1,5 @@import { stripFrontmatter } from '../extensions/frontmatter.ts';+import { spellLineEndings } from './line-endings.ts';import type { BridgeToleranceClass } from './normalize.ts';import { isSubsequence } from './subsequence.ts';@@ -6,6 +7,10 @@ export const PARSE_EQUIVALENCE_TOLERANCE = 'parse-equivalence' as const;export type BridgeToleranceSignal = BridgeToleranceClass | typeof PARSE_EQUIVALENCE_TOLERANCE;+function comparableBody(body: string): string {+ return stripDocBoundary(stripTrailingLineWhitespace(spellLineEndings(body, '\n')));+}+function stripDocBoundary(body: string): string {return body.replace(/^\n+/, '').replace(/\n+$/, '');}@@ -55,11 +60,6 @@ export function isParseEquivalentBridge(}return false;}- if (- stripDocBoundary(stripTrailingLineWhitespace(canonicalLeftBody)) !==- stripDocBoundary(stripTrailingLineWhitespace(rightSplit.body))- ) {- return false;- }+ if (comparableBody(canonicalLeftBody) !== comparableBody(rightSplit.body)) return false;return isSubsequence(contentSkeleton(leftSplit.body), contentSkeleton(canonicalLeftBody));}diff --git a/packages/core/src/index.ts b/packages/core/src/index.tsindex ac5f500c3..3998f83d1 100644--- a/packages/core/src/index.ts+++ b/packages/core/src/index.ts@@ -49,6 +49,8 @@ export {detectFmRegion,diffLinesFast,docEdgeRunsDiffer,+ documentBodyLineEnding,+ dominantLineEnding,emitToleranceFire,type FmBoundarySlotSplit,type FmEditError,@@ -73,6 +75,8 @@ export {fragmentHoldsPendingContent,type InvariantViolation,isParseEquivalentBridge,+ type LfProjection,+ type LineEnding,locateBridgeDivergence,MAX_FM_REGION_BYTES,type MergeBoundarySpace,@@ -85,6 +89,7 @@ export {parseFencedFmRegion,parseFmRegion,pendingContentLines,+ projectToLf,readFmKeys,readFmMap,readFmRegionWithError,@@ -97,6 +102,7 @@ export {type SynthesisedConflictMarkers,type SynthesisedConflictRegion,setToleranceTelemetryHook,+ spellLineEndings,splitFmBoundarySlot,structuralDivergence,synthesiseConflictMarkers,diff --git a/packages/core/src/markdown/comment-promoter.ts b/packages/core/src/markdown/comment-promoter.tsindex fb0ba5a7c..fc267cbea 100644--- a/packages/core/src/markdown/comment-promoter.ts+++ b/packages/core/src/markdown/comment-promoter.ts@@ -1,4 +1,3 @@-import type { Nodes, Paragraph, PhrasingContent, Root, RootContent, Text } from 'mdast';import { SKIP, visit } from 'unist-util-visit';import type { VFile } from 'vfile';diff --git a/packages/core/src/markdown/merged-walker.ts b/packages/core/src/markdown/merged-walker.tsindex fa3cfcd86..bfd7e3109 100644--- a/packages/core/src/markdown/merged-walker.ts+++ b/packages/core/src/markdown/merged-walker.ts@@ -1,4 +1,3 @@-import type { Nodes, Parent, Root } from 'mdast';import { SKIP, visit } from 'unist-util-visit';import type { VFile } from 'vfile';diff --git a/packages/core/src/markdown/safe-url.ts b/packages/core/src/markdown/safe-url.tsindex 8be632ba4..4d3250d3a 100644--- a/packages/core/src/markdown/safe-url.ts+++ b/packages/core/src/markdown/safe-url.ts@@ -1,4 +1,3 @@-export const SAFE_URL_SCHEMES = ['https', 'http', 'mailto', 'tel', 'ftp', 'sms'] as const;const SCHEME_ALT = SAFE_URL_SCHEMES.map((s) => `${s}:`).join('|');diff --git a/packages/server/src/map-driven-splice.test.ts b/packages/server/src/map-driven-splice.test.tsindex f14719b2a..3353cebed 100644--- a/packages/server/src/map-driven-splice.test.ts+++ b/packages/server/src/map-driven-splice.test.ts@@ -211,6 +211,59 @@ describe('computeMapDrivenBodySplice', () => {});});+describe('CRLF bodies keep their line endings through the splice (PRD-9139)', () => {+ const crlf = (lf: string): string => lf.replaceAll('\n', '\r\n');++ test('a uniformly CRLF body stays uniformly CRLF', () => {+ const oldBody = crlf('# Heading\n\nFirst paragraph.\n\n## Second\n\n+ un\n+ deux\n');+ const newPm = pmFromMd('# Heading\n\nFirst paragraph.\n\n## XSecond\n\n+ un\n+ deux\n');++ const splice = computeMapDrivenBodySplice(oldBody, newPm, mdManager);+ expect(splice).not.toBeNull();+ if (!splice) return;++ expect(applySplice(oldBody, splice)).toBe(+ crlf('# Heading\n\nFirst paragraph.\n\n## XSecond\n\n+ un\n+ deux\n'),+ );+ expect(oldBody.slice(0, splice.spliceStart)).not.toContain('Second');+ });++ test('a mixed body keeps every byte outside the splice and spells the slice in its majority ending', () => {+ const oldBody = 'Alpha.\r\n\r\nBeta.\r\n\r\nGamma.\n';+ const splice = computeMapDrivenBodySplice(+ oldBody,+ pmFromMd('Alpha.\n\nBeta edited.\n\nGamma.\n'),+ mdManager,+ );+ expect(splice).not.toBeNull();+ if (!splice) return;++ expect(applySplice(oldBody, splice)).toBe('Alpha.\r\n\r\nBeta edited.\r\n\r\nGamma.\n');+ });++ test('consecutive CRLF keystrokes still parse each body once', () => {+ const { manager: counted, parses } = createCountingManager();+ const memo = createEditorMdastMemo();+ const bodyA = crlf('# H\n\nalpha\n');++ const first = computeMapDrivenBodySplice(bodyA, counted.parse('# H\n\nalphaX\n'), counted, {+ memo,+ });+ expect(first).not.toBeNull();+ if (!first) return;+ const bodyB = applySplice(bodyA, first);+ expect(bodyB).toBe(crlf('# H\n\nalphaX\n'));+ const afterFirst = parses();++ const second = computeMapDrivenBodySplice(bodyB, counted.parse('# H\n\nalphaXY\n'), counted, {+ memo,+ });++ expect(parses() - afterFirst).toBe(1);+ expect(second && applySplice(bodyB, second)).toBe(crlf('# H\n\nalphaXY\n'));+ });+});+describe('editor-mdast parse memo (PRD-8273)', () => {test('a repeated body is parsed once, not once per call', () => {const { manager: counted, parses } = createCountingManager();diff --git a/packages/server/src/map-driven-splice.ts b/packages/server/src/map-driven-splice.tsindex 5227f250f..a0f1143b7 100644--- a/packages/server/src/map-driven-splice.ts+++ b/packages/server/src/map-driven-splice.ts@@ -1,4 +1,11 @@-import type { MarkdownManager, SerializeCallOptions } from '@inkeep/open-knowledge-core';+import {+ type LfProjection,+ type LineEnding,+ type MarkdownManager,+ projectToLf,+ type SerializeCallOptions,+ spellLineEndings,+} from '@inkeep/open-knowledge-core';import type { JSONContent } from '@tiptap/core';import type { RootContent } from 'mdast';import type { MapDrivenSpliceMemoSkipReason } from './metrics.ts';@@ -39,6 +46,7 @@ export interface MapDrivenSpliceOptions {readonly onMemoSkip?: (reason: MapDrivenSpliceMemoSkipReason, err?: unknown) => void;readonly memo?: EditorMdastMemo;readonly serializedNewPm?: SerializedEditorBody;+ readonly lineEnding?: LineEnding;}export interface EditorMdastMemo {@@ -65,10 +73,34 @@ function editorMdastChildren(}export function computeMapDrivenBodySplice(- oldBody: string,+ rawOldBody: string,newPmJson: JSONContent,mdManager: MarkdownManager,options: MapDrivenSpliceOptions = {},+): MapDrivenSplice | null {+ const projection = projectToLf(rawOldBody);+ const splice = computeLfBodySplice(projection.text, newPmJson, mdManager, options);+ if (splice === null) return null;+ return toRawSplice(splice, projection, options.lineEnding ?? projection.lineEnding);+}++function toRawSplice(+ splice: MapDrivenSplice,+ projection: LfProjection,+ lineEnding: LineEnding,+): MapDrivenSplice {+ return {+ spliceStart: projection.toRawOffset(splice.spliceStart),+ spliceEnd: projection.toRawOffset(splice.spliceEnd),+ newSlice: spellLineEndings(splice.newSlice, lineEnding),+ };+}++function computeLfBodySplice(+ oldBody: string,+ newPmJson: JSONContent,+ mdManager: MarkdownManager,+ options: MapDrivenSpliceOptions,): MapDrivenSplice | null {const { onFallback, onMemoHit, onMemoSkip, memo, serializedNewPm } = options;let oldChildren: readonly RootContent[];diff --git a/packages/server/src/pre-drain-corpus.test-helper.ts b/packages/server/src/pre-drain-corpus.test-helper.tsindex 9fdc7e6d1..c9bda9bc3 100644--- a/packages/server/src/pre-drain-corpus.test-helper.ts+++ b/packages/server/src/pre-drain-corpus.test-helper.ts@@ -1,5 +1,6 @@import type { Document } from '@hocuspocus/server';import {+ documentBodyLineEnding,type MarkdownManager,sharedExtensions,stripFrontmatter,@@ -128,6 +129,7 @@ export async function createDiscriminatorRig(baseMd: string): Promise<DiscriminafmPrefixLen,op: { kind: 'agent-undo', ytext, stackItem },mdManager: productionMdManager,+ lineEnding: documentBodyLineEnding(ytext.toString()),}).verdict;},undoInputs: () => {@@ -151,6 +153,7 @@ export async function createDiscriminatorRig(baseMd: string): Promise<DiscriminafmPrefixLen,op: { kind: 'agent-write', writeKind: position },mdManager: productionMdManager,+ lineEnding: documentBodyLineEnding(ytext.toString()),}).verdict;},cleanup: () => manager.closeAll(),diff --git a/packages/server/src/pre-drain-discriminator.test.ts b/packages/server/src/pre-drain-discriminator.test.tsindex fe91ecf9b..a5d6ef73b 100644--- a/packages/server/src/pre-drain-discriminator.test.ts+++ b/packages/server/src/pre-drain-discriminator.test.ts@@ -238,6 +238,7 @@ describe('cost gate: cheap fail-closed guards short-circuit before the localizerfragmentPmJson: {},fmPrefixLen: 0,mdManager: throwingMdManager,+ lineEnding: '\n',op: { kind: 'agent-write', composedBody: 'x', writeKind: 'prepend' } as const,};@@ -267,6 +268,50 @@ describe('cost gate: cheap fail-closed guards short-circuit before the localizer});expect(verdict).toEqual({ preDrain: false, reason: 'checkpoint-no-target' });});++ it('flushes a pending keystroke into a CRLF body in CRLF (PRD-9139)', () => {+ const body = 'para one\r\n\r\npara two\r\n\r\npara three\r\n';+ const plan = planPreDrain({+ pendingDirty: true,+ witnessMatched: true,+ body,+ fragmentPmJson: mdManager.parse('para one!\n\npara two\n\npara three\n'),+ fmPrefixLen: 0,+ mdManager,+ lineEnding: '\r\n',+ op: { kind: 'agent-write', writeKind: 'append' },+ });++ expect(plan.preDrain).toBe(true);+ if (!plan.preDrain) return;+ expect(+ body.slice(0, plan.splice.spliceStart) ++ plan.splice.newSlice ++ body.slice(plan.splice.spliceEnd),+ ).toBe('para one!\r\n\r\npara two\r\n\r\npara three\r\n');+ });++ it('flushes added lines into a body with no line ending in the ending it is given (PRD-9139)', () => {+ const body = 'Alpha.';+ const plan = planPreDrain({+ pendingDirty: true,+ witnessMatched: true,+ body,+ fragmentPmJson: mdManager.parse('Alpha.\n\nBeta.\n'),+ fmPrefixLen: '---\r\ntitle: t\r\n---\r\n'.length,+ mdManager,+ lineEnding: '\r\n',+ op: { kind: 'agent-write', writeKind: 'append' },+ });++ expect(plan.preDrain).toBe(true);+ if (!plan.preDrain) return;+ expect(+ body.slice(0, plan.splice.spliceStart) ++ plan.splice.newSlice ++ body.slice(plan.splice.spliceEnd),+ ).toBe('Alpha.\r\n\r\nBeta.\r\n');+ });});describe('same-block discriminator corpus (splice+graze)', () => {@@ -387,6 +432,7 @@ describe('full-body-overwrite positions are structurally inert', () => {fmPrefixLen: 0,op: { kind: 'agent-write', writeKind: position },mdManager,+ lineEnding: '\n',});expect(plan.verdict.preDrain).toBe(false);expect(plan.verdict.reason).toBe('checkpoint-full-overwrite');@@ -403,6 +449,7 @@ describe('full-body-overwrite positions are structurally inert', () => {fmPrefixLen: 0,op: { kind: 'agent-write', writeKind: 'append' },mdManager,+ lineEnding: '\n',});expect(plan.verdict.preDrain).toBe(true);expect(plan.splice).not.toBeNull();diff --git a/packages/server/src/pre-drain-discriminator.ts b/packages/server/src/pre-drain-discriminator.tsindex 3cc696405..918ea5652 100644--- a/packages/server/src/pre-drain-discriminator.ts+++ b/packages/server/src/pre-drain-discriminator.ts@@ -1,4 +1,4 @@-import type { MarkdownManager } from '@inkeep/open-knowledge-core';+import type { LineEnding, MarkdownManager } from '@inkeep/open-knowledge-core';import type { JSONContent } from '@tiptap/core';import type * as Y from 'yjs';import { ContentString } from 'yjs';@@ -147,6 +147,7 @@ export interface DiscriminatePreDrainInput {readonly fmPrefixLen: number;readonly op: PreDrainOp;readonly mdManager: MarkdownManager;+ readonly lineEnding: LineEnding;}export type PreDrainOpInput =@@ -180,7 +181,9 @@ export function planPreDrain(input: DiscriminatePreDrainInput): PreDrainPlan {: extractComposeTargetSpan(input.body, input.op.writeKind);if (targetSpan === null) return decline('checkpoint-no-target');- const splice = computeMapDrivenBodySplice(input.body, input.fragmentPmJson, input.mdManager);+ const splice = computeMapDrivenBodySplice(input.body, input.fragmentPmJson, input.mdManager, {+ lineEnding: input.lineEnding,+ });if (splice === null) return decline('checkpoint-null-splice');if (splice.newSlice === input.body.slice(splice.spliceStart, splice.spliceEnd)) {diff --git a/packages/server/src/server-observers.test.ts b/packages/server/src/server-observers.test.tsindex 7a275b112..a2a89ad62 100644--- a/packages/server/src/server-observers.test.ts+++ b/packages/server/src/server-observers.test.ts@@ -1515,4 +1515,164 @@ describe('Observer A routing — Path B fires iff Y.Text holds unabsorbed change});test.todo('each of several sequential remote writes refreshes the baseline');++ describe('CRLF documents keep their line endings through fragment edits (PRD-9139)', () => {+ const MISSION_LINES = [+ '# Mission',+ '',+ 'Notre __mission__ est simple.',+ '',+ '## Objectifs',+ '',+ '+ un',+ '+ deux',+ '',+ ];++ const FRONTMATTER_LINES = ['---', 'title: Mission', '---', ''];++ type Eol = '\n' | '\r\n';+ const source = (eol: Eol, withFrontmatter: boolean): string =>+ [...(withFrontmatter ? FRONTMATTER_LINES : []), ...MISSION_LINES].join(eol);++ function editTwice(eol: Eol, withFrontmatter: boolean, docName: string) {+ __resetBridgeWatchdogForTests();+ resetMetrics();+ const raw = source(eol, withFrontmatter);+ const { doc, xmlFragment, ytext, cleanup } = seedThenAttach(raw, docName);+ expect(ytext.toString()).toBe(raw);++ const events = capturePathBEvents(() => {+ populateFragment(+ doc,+ xmlFragment,+ serializeFragmentBody(xmlFragment).replace('## Objectifs', '## XObjectifs'),+ );+ populateFragment(+ doc,+ xmlFragment,+ serializeFragmentBody(xmlFragment).replace('est simple.', 'est simple!'),+ );+ });+ const metrics = getMetrics();+ const outcome = {+ text: ytext.toString(),+ routes: {+ pathBEvents: events.length,+ pathBFires: totalPathBFires(),+ residualMergeRuns: metrics.observerAResidualMergeRuns,+ splicesApplied: metrics.mapDrivenSpliceApplied,+ },+ };+ cleanup();+ return outcome;+ }++ const edited = (eol: Eol, withFrontmatter: boolean): string =>+ source(eol, withFrontmatter)+ .replace('## Objectifs', '## XObjectifs')+ .replace('est simple.', 'est simple!');++ describe.each([+ { label: 'without frontmatter', withFrontmatter: false },+ { label: 'with frontmatter', withFrontmatter: true },+ ])('$label', ({ withFrontmatter }) => {+ test('keystrokes in a uniformly CRLF document leave every line CRLF', () => {+ expect(editTwice('\r\n', withFrontmatter, 'routing-crlf-keystrokes').text).toBe(+ edited('\r\n', withFrontmatter),+ );+ });++ test('an LF document is byte-identical, and a CRLF one takes the same routes with no Path B or residual merge', () => {+ const lf = editTwice('\n', withFrontmatter, 'routing-lf-keystrokes-control');+ expect(lf.text).toBe(edited('\n', withFrontmatter));+ const crlf = editTwice('\r\n', withFrontmatter, 'routing-crlf-keystrokes-routes');+ expect(crlf.routes).toEqual(lf.routes);+ expect(lf.routes).toEqual({+ pathBEvents: 0,+ pathBFires: 0,+ residualMergeRuns: 0,+ splicesApplied: 2,+ });+ });+ });++ test('the incremental-diff fallback keeps a CRLF document CRLF', () => {+ __resetBridgeWatchdogForTests();+ resetMetrics();+ const raw = '<!-- note -->\r\n\r\nOriginal.\r\n';+ const { doc, xmlFragment, ytext, cleanup } = seedThenAttach(raw, 'routing-crlf-fallback');+ expect(ytext.toString()).toBe(raw);++ populateFragment(doc, xmlFragment, `${serializeFragmentBody(xmlFragment)}\nAdded.\n`);++ expect(getMetrics().mapDrivenSpliceApplied).toBe(0);+ expect(ytext.toString()).toBe('<!-- note -->\r\n\r\nOriginal.\r\n\r\nAdded.\r\n');+ cleanup();+ });++ test.each([+ {+ label: 'an empty body, splice route',+ raw: '---\r\ntitle: t\r\n---\r\n',+ fragment: 'First.\n\nSecond.\n\nThird.\n',+ expectedBody: 'First.\r\n\r\nSecond.\r\n\r\nThird.\r\n',+ splicesApplied: 1,+ },+ {+ label: 'one unterminated line, splice route',+ raw: '---\r\ntitle: t\r\n---\r\nFirst.',+ fragment: 'First.\n\nSecond.\n\nThird.\n',+ expectedBody: 'First.\r\n\r\nSecond.\r\n\r\nThird.\r\n',+ splicesApplied: 1,+ },+ {+ label: 'one unterminated line, incremental-diff fallback',+ raw: '---\r\ntitle: t\r\n---\r\n<!-- note -->',+ fragment: '<!-- note -->\n\nSecond.\n\nThird.\n',+ expectedBody: '<!-- note -->\r\n\r\nSecond.\r\n\r\nThird.\r\n',+ splicesApplied: 0,+ },+ ])(+ 'a CRLF document whose body has no line ending yet stays CRLF when it gains lines ($label)',+ ({ raw, fragment, expectedBody, splicesApplied }) => {+ __resetBridgeWatchdogForTests();+ resetMetrics();+ const { doc, xmlFragment, ytext, cleanup } = seedThenAttach(+ raw,+ 'routing-crlf-no-body-eol',+ );+ expect(ytext.toString()).toBe(raw);++ populateFragment(doc, xmlFragment, fragment);++ expect(getMetrics().mapDrivenSpliceApplied).toBe(splicesApplied);+ expect(ytext.toString()).toBe(`---\r\ntitle: t\r\n---\r\n${expectedBody}`);+ cleanup();+ },+ );++ test.each([+ { frontmatter: '---\ntitle: t\n---\n', bodyEol: '\r\n' },+ { frontmatter: '---\r\ntitle: t\r\n---\r\n', bodyEol: '\n' },+ ] as const)(+ 'the incremental-diff fallback leaves frontmatter bytes alone and spells the body in its own ending (frontmatter $frontmatter)',+ ({ frontmatter, bodyEol }) => {+ __resetBridgeWatchdogForTests();+ resetMetrics();+ const body = (lines: string[]): string => lines.join(bodyEol);+ const raw = frontmatter + body(['', '<!-- note -->', '', 'Original.', '']);+ const { doc, xmlFragment, ytext, cleanup } = seedThenAttach(raw, 'routing-fm-eol-fallback');+ expect(ytext.toString()).toBe(raw);++ populateFragment(doc, xmlFragment, `${serializeFragmentBody(xmlFragment)}\nAdded.\n`);++ expect(getMetrics().mapDrivenSpliceApplied).toBe(0);+ expect(ytext.toString()).toBe(+ frontmatter + body(['', '<!-- note -->', '', 'Original.', '', 'Added.', '']),+ );+ cleanup();+ },+ );+ });});diff --git a/packages/server/src/server-observers.ts b/packages/server/src/server-observers.tsindex b301f311c..ab8b60c33 100644--- a/packages/server/src/server-observers.ts+++ b/packages/server/src/server-observers.ts@@ -22,6 +22,7 @@ import {createMergeBoundarySpace,DUPLICATION_GATE_MIN_LINE_LENGTH,docEdgeRunsDiffer,+ documentBodyLineEnding,findDroppedSourceContentInRange,fnv1aDigest,fragmentHoldsPendingContent,@@ -31,6 +32,7 @@ import {overMultipliedBodyLines,pendingContentLines,prependFrontmatter,+ spellLineEndings,splitFmBoundarySlot,stripFrontmatter,} from '@inkeep/open-knowledge-core';@@ -280,6 +282,7 @@ function tryComputeMapDrivenSplice(},memo: mdastMemo,serializedNewPm,+ lineEnding: documentBodyLineEnding(currentText),});if (!splice) return null;@@ -882,6 +885,10 @@ export function setupServerObservers(opts: SetupServerObserversOpts): () => voidreturn composeWithDerivedBody(frontmatter, slot + derivedBody);};+ const spellDerivedBody = (composition: BridgeComposition, currentText: string): string =>+ composition.frontmatter ++ spellLineEndings(composition.body, documentBodyLineEnding(currentText));+const observerParseOpts =opts.resolveEmbed && opts.docName? {@@ -1203,7 +1210,7 @@ export function setupServerObservers(opts: SetupServerObserversOpts): () => voidif (mapDrivenSplice) {applyMapDrivenSplice(ytext, mapDrivenSplice);} else if (ytextInSync && !residualMergeEligible) {- applyIncrementalDiff(ytext, currentText, md);+ applyIncrementalDiff(ytext, currentText, spellDerivedBody(composition, currentText));} else {const mergeBase = ytextInSync ? lastSyncedCanonicalMd : preMergeBaseline;const boundarySpace = createMergeBoundarySpace(body);@@ -1600,6 +1607,7 @@ export function setupServerObservers(opts: SetupServerObserversOpts): () => voidfragmentPmJson: json,witnessMatched: fullMd === lastSyncedYTextBytes,fmPrefixLen,+ lineEnding: documentBodyLineEnding(fullMd),op:op.kind === 'agent-undo'? { kind: 'agent-undo', ytext, stackItem: op.stackItem }