Skip to content

armgit / orca

Public Imported from GitHub
0

Orca is the place for working with a fleet of agents (e.g. Grok Bot, Claude Code). Run any coding agent with your own subscription. Available on desktop, mobile and remote runtime.

Reuse Markdown tokenizer and details start searches across paragraph suffixes (#35374)

OrcaWina83cb6ee3308
diff --git a/src/renderer/src/components/editor/markdown-tokenizer-start.test.ts b/src/renderer/src/components/editor/markdown-tokenizer-start.test.ts
index c190b3b7e5a..f6f4dc3f5a8 100644
--- a/src/renderer/src/components/editor/markdown-tokenizer-start.test.ts
+++ b/src/renderer/src/components/editor/markdown-tokenizer-start.test.ts
@@ -1,7 +1,9 @@
import { expect, it, vi } from 'vitest'
import { createTiptapMarkedFacade } from './tiptap-marked-facade'
+import { createOrcaDetailsExtensions } from './rich-markdown-details-extension'
import {
createMarkdownTokenizerStart,
+ createMarkdownTokenizerStartCursor,
registerMarkdownTokenizerStart
} from './markdown-tokenizer-start'
@@ -94,3 +96,40 @@ it('skips custom searches when their required marker is absent', () => {
marked.lexer('Prose\n<DETAILS>')
expect(search).toHaveReturnedWith(5)
})
+
+it('recognizes a suffix-start directive skipped by an earlier line-start search', () => {
+ const marked = createTiptapMarkedFacade()
+ marked.use({
+ extensions: [
+ {
+ name: 'directive',
+ level: 'block',
+ start: createMarkdownTokenizerStart(':::x', true),
+ tokenizer: (source) =>
+ source.startsWith(':::x') ? { type: 'directive', raw: ':::x', text: 'found' } : undefined
+ }
+ ]
+ })
+ expect(marked.lexer('Prose\n\nx:::x\n\n:::x').map(({ type, raw }) => ({ type, raw }))).toEqual([
+ { type: 'paragraph', raw: 'Prose' },
+ { type: 'space', raw: '\n\n' },
+ { type: 'paragraph', raw: 'x' },
+ { type: 'directive', raw: ':::x' },
+ { type: 'space', raw: '\n\n' },
+ { type: 'directive', raw: ':::x' }
+ ])
+})
+
+function getDetailsTokenizerStart() {
+ const tokenizer = createOrcaDetailsExtensions()[0].config.markdownTokenizer
+ if (!tokenizer || typeof tokenizer === 'function' || typeof tokenizer.start !== 'function') {
+ throw new Error('Details tokenizer must expose its start hint')
+ }
+ return tokenizer.start
+}
+
+it('rechecks details start hints when a suffix discards their lookbehind', () => {
+ const start = createMarkdownTokenizerStartCursor(getDetailsTokenizerStart())
+ const source = 'before\n <details>\n\nnext\n<details>'
+ expect([0, 7, 10, 26].map((offset) => start(source.slice(offset)))).toEqual([10, 19, 16, -1])
+})
diff --git a/src/renderer/src/components/editor/markdown-tokenizer-start.ts b/src/renderer/src/components/editor/markdown-tokenizer-start.ts
index b32a4539fac..77b7c227578 100644
--- a/src/renderer/src/components/editor/markdown-tokenizer-start.ts
+++ b/src/renderer/src/components/editor/markdown-tokenizer-start.ts
@@ -1,4 +1,7 @@
-const startMarkers = new WeakMap<(source: string) => number | void, string>()
+type MarkdownTokenizerStart = (source: string) => number | void
+
+const startMarkers = new WeakMap<MarkdownTokenizerStart, string>()
+const startCursorFactories = new WeakMap<MarkdownTokenizerStart, () => MarkdownTokenizerStart>()
export function registerMarkdownTokenizerStart(marker: string, start: (source: string) => number) {
startMarkers.set(start, marker)
@@ -14,9 +17,60 @@ export function createMarkdownTokenizerStart(marker: string, lineStart = false)
}
return position
}
+ if (lineStart && /[\n\r\u2028\u2029]/.test(marker)) {
+ return registerMarkdownTokenizerStart(marker, start)
+ }
+ startCursorFactories.set(start, () =>
+ createMarkdownTokenizerStartSearchCursor(start, 0, lineStart ? marker : undefined)
+ )
return registerMarkdownTokenizerStart(marker, start)
}
export function getMarkdownTokenizerStartMarker(start: (source: string) => number | void) {
return startMarkers.get(start)
}
+
+/** Bind to one blockTokens call, where Marked passes monotonically shorter suffixes. */
+export function createMarkdownTokenizerStartCursor(
+ start: MarkdownTokenizerStart
+): MarkdownTokenizerStart {
+ return startCursorFactories.get(start)?.() ?? start
+}
+
+/** Removing a prefix must only remove matches; each match needs at most lookbehindLength preceding units. */
+export function registerMarkdownLookbehindTokenizerStart(
+ marker: string,
+ start: (source: string) => number,
+ lookbehindLength: number
+) {
+ startCursorFactories.set(start, () =>
+ createMarkdownTokenizerStartSearchCursor(start, lookbehindLength)
+ )
+ return registerMarkdownTokenizerStart(marker, start)
+}
+
+function createMarkdownTokenizerStartSearchCursor(
+ start: (source: string) => number,
+ lookbehindLength: number,
+ suffixStartMarker?: string
+): MarkdownTokenizerStart {
+ let firstLength: number | undefined
+ let nextPosition: number | undefined
+ return (source) => {
+ firstLength ??= source.length
+ const position = firstLength - source.length
+ // A suffix can begin at a marker an earlier line-start search skipped.
+ if (suffixStartMarker !== undefined && source.startsWith(suffixStartMarker)) {
+ return 0
+ }
+ if (nextPosition === -1) {
+ return -1
+ }
+ if (nextPosition !== undefined && nextPosition >= position + lookbehindLength) {
+ return nextPosition - position
+ }
+ const match = start(source)
+ nextPosition = match === -1 ? -1 : position + match
+ return match
+ }
+}
diff --git a/src/renderer/src/components/editor/rich-markdown-details-extension.ts b/src/renderer/src/components/editor/rich-markdown-details-extension.ts
index 5162c72cd6f..58e6359d3ab 100644
--- a/src/renderer/src/components/editor/rich-markdown-details-extension.ts
+++ b/src/renderer/src/components/editor/rich-markdown-details-extension.ts
@@ -1,4 +1,4 @@
-import { registerMarkdownTokenizerStart } from './markdown-tokenizer-start'
+import { registerMarkdownLookbehindTokenizerStart } from './markdown-tokenizer-start'
import { guardMarkdownDirectiveTokenizer } from './markdown-directive-tokenizer'
import { decodeHtmlEntities, type AnyExtension, type Editor } from '@tiptap/core'
import { Details, DetailsContent, DetailsSummary } from '@tiptap/extension-details'
@@ -233,8 +233,10 @@ const OrcaDetails = Details.extend({
name: 'details',
level: 'block',
// HTML blocks open only at a line start; marked passes source.slice(1), so offset 0 never qualifies.
- start: registerMarkdownTokenizerStart('<', (source) =>
- source.search(/(?<=[\n\r] {0,3})<details\b/i)
+ start: registerMarkdownLookbehindTokenizerStart(
+ '<',
+ (source) => source.search(/(?<=[\n\r] {0,3})<details\b/i),
+ 4
),
tokenize(src, _tokens, lexer) {
const detailsBlock = matchDetailsHtmlBlock(src, 0)
diff --git a/src/renderer/src/components/editor/tiptap-marked-facade.ts b/src/renderer/src/components/editor/tiptap-marked-facade.ts
index 87950efdeb9..f06b6a20231 100644
--- a/src/renderer/src/components/editor/tiptap-marked-facade.ts
+++ b/src/renderer/src/components/editor/tiptap-marked-facade.ts
@@ -1,4 +1,7 @@
-import { getMarkdownTokenizerStartMarker } from './markdown-tokenizer-start'
+import {
+ createMarkdownTokenizerStartCursor,
+ getMarkdownTokenizerStartMarker
+} from './markdown-tokenizer-start'
import {
Hooks,
Lexer,
@@ -88,10 +91,12 @@ export function createTiptapMarkedFacade(): typeof marked {
if (!extensions?.startBlock) {
return super.blockTokens(source, tokens, lastParagraphClipped)
}
- const startBlock = extensions.startBlock.filter((start) => {
- const marker = getMarkdownTokenizerStartMarker(start)
- return !marker || source.includes(marker)
- })
+ const startBlock = extensions.startBlock
+ .filter((start) => {
+ const marker = getMarkdownTokenizerStartMarker(start)
+ return !marker || source.includes(marker)
+ })
+ .map(createMarkdownTokenizerStartCursor)
this.options.extensions = { ...extensions, startBlock }
try {
return super.blockTokens(source, tokens, lastParagraphClipped)