Skip to content

Beautiful, AI-native markdown IDE and LLM wiki

Keep line-start escaped markers distinct in the bridge comparator (#7660)

Dimitri POSTOLOVa75331225333
diff --git a/.changeset/empty-block-marker-source.md b/.changeset/empty-block-marker-source.md
new file mode 100644
index 000000000..32244d5ab
--- /dev/null
+++ b/.changeset/empty-block-marker-source.md
@@ -0,0 +1,6 @@
+---
+"@inkeep/open-knowledge": patch
+"@inkeep/open-knowledge-core": patch
+---
+
+An empty bullet, numbered item, heading or quote made by typing its marker now saves as that block, not as an escaped literal such as `\*`.
diff --git a/packages/app/tests/stress/typed-block-marker-source.e2e.ts b/packages/app/tests/stress/typed-block-marker-source.e2e.ts
new file mode 100644
index 000000000..c32f67de7
--- /dev/null
+++ b/packages/app/tests/stress/typed-block-marker-source.e2e.ts
@@ -0,0 +1,64 @@
+import { randomUUID } from 'node:crypto';
+import { readFileSync } from 'node:fs';
+import { join } from 'node:path';
+import type { Page } from '@playwright/test';
+import { expect, placeCaretAtEndOfText, test, waitForActiveProviderSynced } from './_helpers';
+
+const EDITOR = '.ProseMirror:not(.composer-prosemirror)';
+const FIRST_LINE = 'Start here:';
+
+interface MarkerCase {
+ marker: string;
+ literalLine: string;
+ block: string;
+ blockLine: string;
+}
+
+const CASES: MarkerCase[] = [
+ { marker: '*', literalLine: '\\*', block: 'ul > li', blockLine: '* ' },
+ { marker: '-', literalLine: '\\-', block: 'ul > li', blockLine: '- ' },
+ { marker: '+', literalLine: '\\+', block: 'ul > li', blockLine: '+ ' },
+ { marker: '1.', literalLine: '1\\.', block: 'ol > li', blockLine: '1. ' },
+ { marker: '#', literalLine: '\\#', block: 'h1', blockLine: '#' },
+ { marker: '>', literalLine: '\\>', block: 'blockquote', blockLine: '>' },
+];
+
+const sourceWith = (secondLine: string): string => `${FIRST_LINE}\n\n${secondLine}\n`;
+
+const readSource = (page: Page): Promise<string> =>
+ page.evaluate(() => window.__activeProvider?.document.getText('source').toString() ?? '');
+
+const nextFrame = (page: Page): Promise<void> =>
+ page.evaluate(
+ () => new Promise<void>((resolve) => requestAnimationFrame(() => setTimeout(resolve, 0))),
+ );
+
+test.describe('a block marker typed into a new paragraph', () => {
+ for (const { marker, literalLine, block, blockLine } of CASES) {
+ test(`"${marker} " writes the empty block's markdown to the source and the saved file`, async ({
+ page,
+ api,
+ workerServer,
+ }) => {
+ const docName = `typed-block-marker-${randomUUID()}`;
+ await api.seedDocs([{ name: docName, markdown: `${FIRST_LINE}\n` }]);
+ await page.goto(`/#/${docName}`);
+ await waitForActiveProviderSynced(page);
+ await expect(page.locator(EDITOR)).toContainText(FIRST_LINE);
+ await placeCaretAtEndOfText(page, FIRST_LINE);
+
+ await page.keyboard.press('Enter');
+ await nextFrame(page);
+ await page.keyboard.type(marker);
+ await expect.poll(() => readSource(page)).toBe(sourceWith(literalLine));
+
+ await page.keyboard.type(' ');
+ await expect(page.locator(EDITOR).locator(block)).toHaveCount(1);
+
+ await expect.poll(() => readSource(page)).toBe(sourceWith(blockLine));
+ await expect
+ .poll(() => readFileSync(join(workerServer.contentDir, `${docName}.md`), 'utf8'))
+ .toBe(sourceWith(blockLine));
+ });
+ }
+});
diff --git a/packages/core/src/bridge/normalize.ts b/packages/core/src/bridge/normalize.ts
index 8125970e9..f055086c8 100644
--- a/packages/core/src/bridge/normalize.ts
+++ b/packages/core/src/bridge/normalize.ts
@@ -3,6 +3,61 @@ import { bodyEdgeEmpties } from './doc-boundary-space.ts';
const COMMONMARK_ESCAPE_RE = /\\([!"#$%&'()*+,\-./:;<=>?@[\]^_`{|}~])/g;
+const LINE_START_ESCAPE_RE =
+ /(?<![^\n])((?:[ \t]*(?:>|(?:[-+*]|\d{1,9}[.)])[ \t]))*)[ \t]*(\d{1,9}\\[.)]|\\[-+*#>_=])/g;
+
+const BLOCK_START_OPENER_RE =
+ /^(?:[-+*](?:[ \t]|$)|\d{1,9}[.)](?:[ \t]|$)|#{1,6}(?:[ \t]|$)|>|(?:\*[ \t]*){3,}$|(?:-[ \t]*){3,}$|(?:_[ \t]*){3,}$)/;
+
+const PARAGRAPH_INTERRUPTER_RE =
+ /^(?:[-+*][ \t]+\S|1[.)][ \t]+\S|#{1,6}(?:[ \t]|$)|>|(?:\*[ \t]*){3,}$|(?:_[ \t]*){3,}$|(?:-[ \t]*){3,}$|-+[ \t]*$|=+[ \t]*$)/;
+
+const BLOCK_BOUNDARY_LINE_RE =
+ /^(?:$|#{1,6}(?:[ \t]|$)|`{3,}|~{3,}|(?:\*[ \t]*){3,}$|(?:-[ \t]*){3,}$|(?:_[ \t]*){3,}$)/;
+
+const JSX_CONTAINER_TAG_LINE_RE = /^<\/?[A-Z][A-Za-z0-9.]*(?:\s[^>]*)?>$/;
+
+function previousLine(s: string, lineStart: number): string {
+ return s.slice(s.lastIndexOf('\n', lineStart - 2) + 1, lineStart - 1);
+}
+
+function isBlockBoundaryLine(line: string): boolean {
+ const content = line.replace(/^[ \t>]*/, '');
+ return (
+ BLOCK_BOUNDARY_LINE_RE.test(content) ||
+ JSX_CONTAINER_TAG_LINE_RE.test(content.replace(/[ \t]+$/, ''))
+ );
+}
+
+function structuralEscapeOffsets(s: string): Set<number> {
+ const offsets = new Set<number>();
+ if (!s.includes('\\')) return offsets;
+ for (const m of s.matchAll(LINE_START_ESCAPE_RE)) {
+ const containerPrefix = m[1] ?? '';
+ const marker = m[2] ?? '';
+ const lineStart = m.index;
+ const markerStart = lineStart + m[0].length - marker.length;
+ const lineEnd = s.indexOf('\n', markerStart);
+ const unescapedContent = s
+ .slice(markerStart, lineEnd === -1 ? s.length : lineEnd)
+ .replace(COMMONMARK_ESCAPE_RE, '$1');
+ const opensListItem = /[^ \t>]/.test(containerPrefix);
+ const followsBlockBoundary = lineStart === 0 || isBlockBoundaryLine(previousLine(s, lineStart));
+ const opener =
+ opensListItem || followsBlockBoundary ? BLOCK_START_OPENER_RE : PARAGRAPH_INTERRUPTER_RE;
+ if (opener.test(unescapedContent)) offsets.add(markerStart + marker.indexOf('\\'));
+ }
+ return offsets;
+}
+
+function collapseCommonMarkEscapes(s: string): string {
+ const structural = structuralEscapeOffsets(s);
+ if (structural.size === 0) return s.replace(COMMONMARK_ESCAPE_RE, '$1');
+ return s.replace(COMMONMARK_ESCAPE_RE, (escaped: string, char: string, offset: number) =>
+ structural.has(offset) ? escaped : char,
+ );
+}
+
const TABLE_ALIGN_ROW_RE = /^\s*\|?\s*:?-+:?\s*(\|\s*:?-+:?\s*)+\|?\s*$/;
const LIST_ITEM_INDENT_RE = /^[ \t]+([-+*]|\d+[.)]|[a-zA-Z][.)])\s/;
@@ -75,7 +130,6 @@ function findFenceInteriorLines(lines: readonly string[]): Set<number> {
return interior;
}
-const JSX_CONTAINER_TAG_LINE_RE = /^<\/?[A-Z][A-Za-z0-9.]*(?:\s[^>]*)?>$/;
const JSX_CONTAINER_OPEN_RE = /^<[A-Z][A-Za-z0-9.]*(?:\s[^>]*)?>$/;
const JSX_CONTAINER_CLOSE_RE = /^<\/[A-Z][A-Za-z0-9.]*\s*>$/;
const CAPITALIZED_JSX_TAG_RE = /<\/?[A-Z]/;
@@ -146,10 +200,7 @@ function foldJsxContainerBoundaryBlanks(lines: readonly string[]): string[] {
export function normalizeBridge(s: string): string {
const lines = foldJsxContainerBoundaryBlanks(
- s
- .replace(/^/, '')
- .replace(/\r/g, '')
- .replace(COMMONMARK_ESCAPE_RE, '$1')
+ collapseCommonMarkEscapes(s.replace(/^/, '').replace(/\r/g, ''))
.replace(EMPHASIS_AROUND_CODE_RE, '$1')
.replace(/^\n+/, '')
.replace(/^[*-]{3,}(?=\n|$)/, '---')