Orca is the place for working with a fleet of agents (e.g. Grok Bot, Claude Code). Run any coding agent with your own subscription. Available on desktop, mobile and remote runtime.
TypeScript adeagent-ideai-agentsautomationbotclaude-codecodexghosttygrok-bothermes-agentmobile-appmuseopenclawopencodeorchestrationpiruntimeterminalworktreesyc-backed
Reuse Markdown tokenizer and details start searches across paragraph suffixes (#35374)
OrcaWina83cb6ee3308
diff --git a/src/renderer/src/components/editor/markdown-tokenizer-start.test.ts b/src/renderer/src/components/editor/markdown-tokenizer-start.test.tsindex c190b3b7e5a..f6f4dc3f5a8 100644--- a/src/renderer/src/components/editor/markdown-tokenizer-start.test.ts+++ b/src/renderer/src/components/editor/markdown-tokenizer-start.test.ts@@ -1,7 +1,9 @@import { expect, it, vi } from 'vitest'import { createTiptapMarkedFacade } from './tiptap-marked-facade'+import { createOrcaDetailsExtensions } from './rich-markdown-details-extension'import {createMarkdownTokenizerStart,+ createMarkdownTokenizerStartCursor,registerMarkdownTokenizerStart} from './markdown-tokenizer-start'@@ -94,3 +96,40 @@ it('skips custom searches when their required marker is absent', () => {marked.lexer('Prose\n<DETAILS>')expect(search).toHaveReturnedWith(5)})++it('recognizes a suffix-start directive skipped by an earlier line-start search', () => {+ const marked = createTiptapMarkedFacade()+ marked.use({+ extensions: [+ {+ name: 'directive',+ level: 'block',+ start: createMarkdownTokenizerStart(':::x', true),+ tokenizer: (source) =>+ source.startsWith(':::x') ? { type: 'directive', raw: ':::x', text: 'found' } : undefined+ }+ ]+ })+ expect(marked.lexer('Prose\n\nx:::x\n\n:::x').map(({ type, raw }) => ({ type, raw }))).toEqual([+ { type: 'paragraph', raw: 'Prose' },+ { type: 'space', raw: '\n\n' },+ { type: 'paragraph', raw: 'x' },+ { type: 'directive', raw: ':::x' },+ { type: 'space', raw: '\n\n' },+ { type: 'directive', raw: ':::x' }+ ])+})++function getDetailsTokenizerStart() {+ const tokenizer = createOrcaDetailsExtensions()[0].config.markdownTokenizer+ if (!tokenizer || typeof tokenizer === 'function' || typeof tokenizer.start !== 'function') {+ throw new Error('Details tokenizer must expose its start hint')+ }+ return tokenizer.start+}++it('rechecks details start hints when a suffix discards their lookbehind', () => {+ const start = createMarkdownTokenizerStartCursor(getDetailsTokenizerStart())+ const source = 'before\n <details>\n\nnext\n<details>'+ expect([0, 7, 10, 26].map((offset) => start(source.slice(offset)))).toEqual([10, 19, 16, -1])+})diff --git a/src/renderer/src/components/editor/markdown-tokenizer-start.ts b/src/renderer/src/components/editor/markdown-tokenizer-start.tsindex b32a4539fac..77b7c227578 100644--- a/src/renderer/src/components/editor/markdown-tokenizer-start.ts+++ b/src/renderer/src/components/editor/markdown-tokenizer-start.ts@@ -1,4 +1,7 @@-const startMarkers = new WeakMap<(source: string) => number | void, string>()+type MarkdownTokenizerStart = (source: string) => number | void++const startMarkers = new WeakMap<MarkdownTokenizerStart, string>()+const startCursorFactories = new WeakMap<MarkdownTokenizerStart, () => MarkdownTokenizerStart>()export function registerMarkdownTokenizerStart(marker: string, start: (source: string) => number) {startMarkers.set(start, marker)@@ -14,9 +17,60 @@ export function createMarkdownTokenizerStart(marker: string, lineStart = false)}return position}+ if (lineStart && /[\n\r\u2028\u2029]/.test(marker)) {+ return registerMarkdownTokenizerStart(marker, start)+ }+ startCursorFactories.set(start, () =>+ createMarkdownTokenizerStartSearchCursor(start, 0, lineStart ? marker : undefined)+ )return registerMarkdownTokenizerStart(marker, start)}export function getMarkdownTokenizerStartMarker(start: (source: string) => number | void) {return startMarkers.get(start)}++/** Bind to one blockTokens call, where Marked passes monotonically shorter suffixes. */+export function createMarkdownTokenizerStartCursor(+ start: MarkdownTokenizerStart+): MarkdownTokenizerStart {+ return startCursorFactories.get(start)?.() ?? start+}++/** Removing a prefix must only remove matches; each match needs at most lookbehindLength preceding units. */+export function registerMarkdownLookbehindTokenizerStart(+ marker: string,+ start: (source: string) => number,+ lookbehindLength: number+) {+ startCursorFactories.set(start, () =>+ createMarkdownTokenizerStartSearchCursor(start, lookbehindLength)+ )+ return registerMarkdownTokenizerStart(marker, start)+}++function createMarkdownTokenizerStartSearchCursor(+ start: (source: string) => number,+ lookbehindLength: number,+ suffixStartMarker?: string+): MarkdownTokenizerStart {+ let firstLength: number | undefined+ let nextPosition: number | undefined+ return (source) => {+ firstLength ??= source.length+ const position = firstLength - source.length+ // A suffix can begin at a marker an earlier line-start search skipped.+ if (suffixStartMarker !== undefined && source.startsWith(suffixStartMarker)) {+ return 0+ }+ if (nextPosition === -1) {+ return -1+ }+ if (nextPosition !== undefined && nextPosition >= position + lookbehindLength) {+ return nextPosition - position+ }+ const match = start(source)+ nextPosition = match === -1 ? -1 : position + match+ return match+ }+}diff --git a/src/renderer/src/components/editor/rich-markdown-details-extension.ts b/src/renderer/src/components/editor/rich-markdown-details-extension.tsindex 5162c72cd6f..58e6359d3ab 100644--- a/src/renderer/src/components/editor/rich-markdown-details-extension.ts+++ b/src/renderer/src/components/editor/rich-markdown-details-extension.ts@@ -1,4 +1,4 @@-import { registerMarkdownTokenizerStart } from './markdown-tokenizer-start'+import { registerMarkdownLookbehindTokenizerStart } from './markdown-tokenizer-start'import { guardMarkdownDirectiveTokenizer } from './markdown-directive-tokenizer'import { decodeHtmlEntities, type AnyExtension, type Editor } from '@tiptap/core'import { Details, DetailsContent, DetailsSummary } from '@tiptap/extension-details'@@ -233,8 +233,10 @@ const OrcaDetails = Details.extend({name: 'details',level: 'block',// HTML blocks open only at a line start; marked passes source.slice(1), so offset 0 never qualifies.- start: registerMarkdownTokenizerStart('<', (source) =>- source.search(/(?<=[\n\r] {0,3})<details\b/i)+ start: registerMarkdownLookbehindTokenizerStart(+ '<',+ (source) => source.search(/(?<=[\n\r] {0,3})<details\b/i),+ 4),tokenize(src, _tokens, lexer) {const detailsBlock = matchDetailsHtmlBlock(src, 0)diff --git a/src/renderer/src/components/editor/tiptap-marked-facade.ts b/src/renderer/src/components/editor/tiptap-marked-facade.tsindex 87950efdeb9..f06b6a20231 100644--- a/src/renderer/src/components/editor/tiptap-marked-facade.ts+++ b/src/renderer/src/components/editor/tiptap-marked-facade.ts@@ -1,4 +1,7 @@-import { getMarkdownTokenizerStartMarker } from './markdown-tokenizer-start'+import {+ createMarkdownTokenizerStartCursor,+ getMarkdownTokenizerStartMarker+} from './markdown-tokenizer-start'import {Hooks,Lexer,@@ -88,10 +91,12 @@ export function createTiptapMarkedFacade(): typeof marked {if (!extensions?.startBlock) {return super.blockTokens(source, tokens, lastParagraphClipped)}- const startBlock = extensions.startBlock.filter((start) => {- const marker = getMarkdownTokenizerStartMarker(start)- return !marker || source.includes(marker)- })+ const startBlock = extensions.startBlock+ .filter((start) => {+ const marker = getMarkdownTokenizerStartMarker(start)+ return !marker || source.includes(marker)+ })+ .map(createMarkdownTokenizerStartCursor)this.options.extensions = { ...extensions, startBlock }try {return super.blockTokens(source, tokens, lastParagraphClipped)