The agent that grows with you
Python aiai-agentai-agentsanthropicchatgptclaudeclaude-codecodexhermeshermes-agentllmnous-researchopenai
import { normalizeMathDelimiters } from '@assistant-ui/react-streamdown'
import { isLikelyProseFence, sanitizeLanguageTag } from '@/lib/markdown-code'
import { clampHtmlNestingDepth } from '@/lib/markdown-html-depth'
import { mediaKind, mediaMarkdownHref } from '@/lib/media'
import { previewMarkdownHref } from '@/lib/preview-targets'
import { stripPreviewTargets } from '@/lib/preview-targets'
import { linkifySessionRefs } from '@/lib/session-refs'
import { DIRECTIVE_WITH_BODY_RE } from '@/lib/transcript-directives'
// Same tag set as agent/think_scrubber.py THINK_TAG_NAMES, plus desktop-only
// `scratchpad`/`analysis`.
const REASONING_TAGS = 'think|thinking|reasoning|thought|reasoning_scratchpad|scratchpad|analysis'
// A run of adjacent closed blocks is one match, so the seam check below sees the
// prose on either side of the whole run rather than the previous block's `>`.
const REASONING_BLOCK_RE = new RegExp(`(?:<(${REASONING_TAGS})>[\\s\\S]*?<\\/\\1>\\s*)+`, 'gi')
// An open tag that starts its own block with no close tag yet. The block-boundary
// requirement is what lets a real reasoning preamble (always its own block) vanish
// while prose that merely mentions `<thinking>` mid-sentence survives — the same
// line agent/think_scrubber.py draws.
const OPEN_REASONING_BLOCK_RE = new RegExp(`(^|\\n)[ \\t]*<(${REASONING_TAGS})>[\\s\\S]*$`, 'i')
// A half-arrived open tag (`<thin`) at a block boundary is not a tag yet, so the
// pass above lets it paint as prose for one frame and then erase it — the same
// paint/un-paint class as #62774, one frame long. Hold it back the way
// agent/think_scrubber.py `_hold_partial`/`_max_partial_suffix` does, but only
// for prefixes of the known tag names so `<div` at a line start still renders.
const REASONING_TAG_PREFIXES = Array.from(
new Set(REASONING_TAGS.split('|').flatMap(tag => Array.from({ length: tag.length }, (_, i) => tag.slice(0, i + 1))))
).join('|')
const PARTIAL_OPEN_REASONING_TAG_RE = new RegExp(`(^|\\n)[ \\t]*<(?:${REASONING_TAG_PREFIXES})?$`, 'i')
const PREVIEW_MARKER_RE = /\[Preview:[^\]]+\]\(#preview[:/][^)]+\)/gi
const FENCE_LINE_RE = /^([ \t]*)(`{3,}|~{3,})([^\n]*)$/
const FENCE_OPEN_LINE_RE = /^([ \t]*(?:>[ \t]*)*(?:(?:[-+*]|\d+[.)])[ \t]+)?[ \t]*)(`{3,}|~{3,})([^\n]*)$/
const FENCE_CLOSE_LINE_RE = /^[ \t]*(?:>[ \t]*)*(`{3,}|~{3,})[ \t]*\r?$/
const INLINE_CODE_SPLIT_RE = /(`[^`\n]+`)/g
// Math spans as remark-math will see them: a `$$…$$` block, which may span
// lines, or a same-line `$…$`. A delimiter escaped as `\$` is prose — that is
// exactly how escapeCurrencyDollarsPreservingMath marks a price, so the
// lookbehinds keep `\$5 and \$10` out of the math branch.
//
// The inline body steps over escape pairs (`\\[^\n]`) rather than excluding
// `$` outright, because `\$` is a literal dollar INSIDE math (`$x + \$5$`) and
// must not end the span — the same distinction findClosingSingleDollar draws
// via isEscapedAt. The two alternatives are disjoint on their first character,
// so the body cannot backtrack ambiguously.
//
// The inline branch deliberately has NO lookbehind on its CLOSING `$`: the
// body alternation consumes escape pairs atomically, so any `$` the engine
// reaches after the body is by construction unescaped — a trailing `\\$`
// (escaped backslash, valid TeX) would otherwise defeat a one-character
// lookbehind and unshield the span ($a\\$ mis-split, #92371). The display
// branch keeps its guard: its `[\s\S]*?` body does not step over escape
// pairs, so it genuinely needs it.
const MATH_SPAN_SPLIT_RE = /((?<!\\)\$\$[\s\S]*?(?<!\\)\$\$|(?<!\\)\$(?:[^\n$\\]|\\[^\n])+?\$)/g
const LATEX_DISPLAY_OPEN_LINE_RE = /^([ \t]*(?:>[ \t]*)*(?:(?:[-+*]|\d+[.)])[ \t]+)?[ \t]*)\\{1,2}\[[ \t]*\r?$/
const LATEX_DISPLAY_CLOSE_LINE_RE = /^([ \t]*(?:>[ \t]*)*(?:(?:[-+*]|\d+[.)])[ \t]+)?[ \t]*)\\{1,2}\][ \t]*\r?$/
const CUSTOM_DISPLAY_MATH_LINE_RE = /^([ \t]*(?:>[ \t]*)*(?:(?:[-+*]|\d+[.)])[ \t]+)?[ \t]*)\[\/math\][ \t]*\r?$/
// A `$$` that opens a block and is immediately followed by the first line of the
// equation, e.g. `$$\begin{aligned}`. Both patterns anchor the `$$` to the start
// of the line (modulo blockquote/list prefix), which is also what keeps them from
// firing inside an inline code span — `` `$$a `` leads with a backtick.
const HUGGING_DISPLAY_MATH_OPEN_RE =
/^([ \t]*(?:>[ \t]*)*(?:(?:[-+*]|\d+[.)])[ \t]+)?[ \t]*)\$\$[ \t]*(\S[^\n]*?)[ \t]*\r?$/
const HUGGING_DISPLAY_MATH_CLOSE_RE = /^([ \t]*(?:>[ \t]*)*[ \t]*)(\S[^\n]*?)\$\$[ \t]*\r?$/
// Bare-URL autolink matcher. The character classes EXCLUDE `*` so a URL that
// abuts markdown emphasis with no separating space (e.g. `**label: https://x**`,
// a very common LLM pattern) doesn't swallow the trailing `**` into the href.
// `*` is never meaningful in a real URL path, and GFM's own autolink extension
// likewise strips trailing emphasis/punctuation — so dropping it here is safe
// and keeps the emphasis run intact. Other trailing punctuation is still peeled
// off by the final `[^\s<>"'`*.,;:!?]` class.
const RAW_URL_RE = /https?:\/\/[^\s<>"'`*]+[^\s<>"'`*.,;:!?]/g
const URL_TRAILING_PUNCTUATION = '.,;:!?'
// Markdown that already owns the URLs inside it, which the bare-URL autolinker
// steps over whole (#49822): inline links and images `[label](dest "title")`,
// full and collapsed references `[label][ref]`, and the label + destination of
// a definition `[ref]: url`. Labels may nest one bracket pair (an IPv6 host,
// `[see [docs]]`) and destinations one paren pair (`…/wiki/Foo_(bar)`). A label
// or inline target still streaming in (`[https://exa`, `[x](https://exa`) owns
// the rest of its line, so the tail repair sees the link it is building instead
// of a wrapped fragment. Only the outer group captures, so split() leaves the
// owned spans at odd indices.
const LINK_LABEL_BODY = String.raw`(?:[^[\]\\\n]|\\.|\[(?:[^[\]\\\n]|\\.)*\])*`
const LINK_LABEL = String.raw`\[${LINK_LABEL_BODY}\]`
const MARKDOWN_LINK_SPLIT_RE = new RegExp(
`(${[
String.raw`!?${LINK_LABEL}\((?:[^()\n]|\([^()\n]*\))*(?:\)|$)`,
String.raw`${LINK_LABEL}\[(?:[^[\]\\\n]|\\.)*\]`,
String.raw`^[ \t]{0,3}${LINK_LABEL}:[ \t]*\S*`,
String.raw`\[${LINK_LABEL_BODY}$`
].join('|')})`,
'gm'
)
// A fenced block whose entire body is a loopback URL is the preview-pane
// hand-off: the widget already paints that address, so the whole block is
// dropped instead of painting the raw text twice. Bare loopback URLs in
// PROSE are user-facing content and must survive (#121683) — "dev server at
// http://localhost:3000" is the address the reader needs, wherever it sits
// in the sentence.
const LOCAL_PREVIEW_ONLY_RE = /^https?:\/\/(?:localhost|127\.0\.0\.1|0\.0\.0\.0|\[::1\])(?::\d+)?\/?$/i
const URL_ONLY_LINE_RE = /^\s*https?:\/\/\S+\s*$/i
// Autolink-shaped spans (bare or angle-bracketed http(s) URLs) that must be
// skipped by lone-tilde escaping: `~` is legal in URL paths and must survive.
const URL_LIKE_SPLIT_RE = /(<https?:\/\/[^>\s]+>|https?:\/\/[^\s<>"'`*]+[^\s<>"'`*.,;:!?])/g
// A `~` that is neither part of `~~~` fence, part of `~~` strikethrough, nor
// an already-escaped `\~`. Escaping it with `\~` makes `marked` render a
// literal tilde, so CJK ranges (`1~10`) and approximation prefixes (`~¥0.089`)
// no longer pair up into a GFM strikethrough span.
const LONE_TILDE_RE = /(?<![\\~])~(?!~)/g
// HTML-shaped prose tokens (`<tool_call>`, `<observation>`...) that are NOT
// real inline elements get swallowed by the HTML-aware renderer (parse5 sees
// an unclosed tag and consumes the rest of the message). Match unknown tag-like
// runs and escape them to entities; known safe tags pass through untouched.
const HTML_TAG_RE = /<\/?([A-Za-z][A-Za-z0-9:_-]*)(?:\s+[^<>]*?)?\/?>/g
const SAFE_HTML_TAG_NAMES = new Set([
'a',
'abbr',
'b',
'blockquote',
'br',
'cite',
'code',
'data',
'del',
'details',
'div',
'em',
'figcaption',
'figure',
'h1',
'h2',
'h3',
'h4',
'h5',
'h6',
'hr',
'i',
'img',
'ins',
'kbd',
'li',
'mark',
'ol',
'p',
'pre',
'q',
'rp',
'rt',
'ruby',
's',
'samp',
'small',
'span',
'strong',
'sub',
'summary',
'sup',
'table',
'tbody',
'td',
'tfoot',
'th',
'thead',
'tr',
'u',
'ul',
'var',
'wbr'
])
const CITATION_MARKER_RE = /(?<=[\p{L}\p{N})\].,!?:;"'”’])\[(?:\d+(?:\s*,\s*\d+)*)\](?!\()/gu
// Web-citation transport markers (Gemini-style grounding): private-use
// delimiters U+E200/U+E201 wrap a `citeturn<n>search<m>` id list, with U+E202
// separating ids — `\uE200citeturn0search11\uE202turn2search0\uE201`, or the
// single-id `\uE200citeturn0search0\uE201`. The delimiters paint as
// replacement glyphs (the reported "triple bars") and the ids are protocol
// noise a reader cannot follow to a source (#120587). Strip the whole marker;
// a marker that cannot be resolved is dropped, never invented into a link.
// The bare no-delimiter alternative only fires with the `cite` head, so plain
// prose can't trip it. Scoped to this shape: stray private-use characters
// (icon fonts, user content) are left alone.
const CITATION_TRANSPORT_MARKER_RE =
/\uE200(?:cite)?(?:\uE202?turn\d+search\d+)+\uE201?|citeturn\d+search\d+(?:turn\d+search\d+)*/gu
// The `Sources` section header the bundled grounded-citations skill renders —
// `## Sources` (ATX, 1-6 hashes) or plain `Sources:`, case-insensitive,
// optionally bolded, colon optional. Mirrors the skill's own header pattern
// (`_SOURCES_HEADER_RE` in scripts/sources.py) so only the section shape it
// actually emits is recognized.
const SOURCES_HEADER_RE = /^[ \t]*(?:#{1,6}[ \t]*)?(?:\*\*)?sources:?(?:\*\*)?[ \t]*\r?$/i
// Numbered source-list entries inside a `Sources` section — a `[3] https://…`
// line (optionally behind a list marker or a `-`/`–`/`:` separator), as the
// bundled grounded-citations skill renders its entries. Such an entry anchors
// the bare `[3]` marker in prose: the marker cites a listed source, so it
// must survive the orphan-marker strip (#110370). Entries only count after
// the LAST `Sources` header and never inside a fenced block, so ordinary
// `[7] todo` lines or fenced examples can't anchor anything.
const SOURCE_LIST_ENTRY_RE = /^[ \t]*(?:[-*+][ \t]+)?\[((?:\d+(?:\s*,\s*\d+)*))\][ \t]*(?:[-–:][ \t]*)?https?:\/\/\S/
// Any fence line toggles code-block state while scanning for the `Sources`
// section — the same toggle the bundled skill uses to drop fenced code from
// a draft's prose.
const FENCE_TOGGLE_RE = /^[ \t]*(?:```|~~~)/
// Markdown links whose target is a filesystem path on the agent's machine:
// `[report](/home/user/report.md)`, `[notes](file:///srv/notes.txt)`,
// `[todo](~/todo.md)`, `[log](C:\logs\run.txt)`. Negative lookbehind keeps
// image syntax (``) on its existing inline pipeline. Plain
// targets exclude `)`/whitespace, matching how LLMs actually emit these;
// CommonMark angle-bracket destinations (`[notes](<~/My Notes/todo.md>`) are
// matched separately so paths with spaces route to the preview pipeline too
// (#102782) — `routeFileLinksToPreview` strips the surrounding `<>`.
const FILE_LINK_RE =
/(?<!!)\[(?<label>[^\]\n]+)\]\((?<target>(?:<(?:file:\/\/|\/|~\/|[a-z]:[\\/])[^>]*>)|(?:file:\/\/|\/|~\/|[a-z]:[\\/])[^)\s]*)\)/gi
// Transcript directives (`::name{...}`) carry DATA in their attributes: a task
// brief, a signed URL with `_`/`~`/`&` in its query. Every prose rewrite here
// corrupts them (the URL autolinker wraps `url="https://…"` in `<…>`, currency
// escaping leaves `\$`), and the parser reads what survives as markdown
// (`*by week*` → <em>, `~/x` → strikethrough), splitting the paragraph so the
// card is never claimed. So directives are lifted out before the rewrites and
// put back with every inline metachar backslash-escaped; the parser consumes
// the escapes and the renderer sees the directive byte-for-byte.
const MARKDOWN_INLINE_META_RE = /[\\`*_~[\]<>$&]/g
const DIRECTIVE_SLOT_RE = /\uE000(\d+)\uE001/g
const DIRECTIVE_OR_CODE_RE = new RegExp(`(${DIRECTIVE_WITH_BODY_RE.source})|${INLINE_CODE_SPLIT_RE.source}`, 'g')
/**
* Returns true when `body` contains a line that's exactly `marker` (modulo
* leading/trailing horizontal whitespace) — i.e. an unambiguous close fence
* for an opening fence with the same marker.
*
* Implemented with string comparisons (not RegExp) so that input-derived
* `marker` values can never bleed into a regex pattern. This matters for
* CodeQL's `js/incomplete-hostname-regexp` dataflow, which would otherwise
* trace test-fixture URLs from the input through `marker` into the regex
* source, even though `marker` is captured by `(`{3,}|~{3,})` and can only
* ever be backticks or tildes.
*/
function hasCloseFenceLine(body: string, marker: string): boolean {
const lines = body.split('\n')
// Original regex required `\n` immediately before the close fence, so the
// first line of `body` (which has no preceding newline within `body`)
// cannot itself be the close fence.
for (let i = 1; i < lines.length; i += 1) {
const line = lines[i]
let lo = 0
let hi = line.length
while (lo < hi && (line[lo] === ' ' || line[lo] === '\t')) {
lo += 1
}
while (hi > lo && (line[hi - 1] === ' ' || line[hi - 1] === '\t')) {
hi -= 1
}
if (line.slice(lo, hi) === marker) {
return true
}
}
return false
}
function scrubBacktickNoise(text: string): string {
const balancedFenceRe = /(^|\n)([ \t]*)(`{3,}|~{3,})([^\n]*)\n([\s\S]*?)\n[ \t]*\3[ \t]*(?=\n|$)/g
const protectedRanges: { end: number; start: number }[] = []
let match: RegExpExecArray | null
while ((match = balancedFenceRe.exec(text)) !== null) {
const start = match.index + match[1].length
protectedRanges.push({ end: balancedFenceRe.lastIndex, start })
}
const danglingCodeFenceRe = /(^|\n)[ \t]*(`{3,}|~{3,})([a-z0-9][a-z0-9+#-]{0,15})[ \t]*\n([\s\S]*)$/gi
while ((match = danglingCodeFenceRe.exec(text)) !== null) {
const start = match.index + match[1].length
const marker = match[2] || '```'
const info = match[3] || ''
const body = match[4] || ''
if (!hasCloseFenceLine(body, marker) && sanitizeLanguageTag(info) && !isLikelyProseFence(info, body)) {
protectedRanges.push({ end: text.length, start })
break
}
}
protectedRanges.sort((a, b) => a.start - b.start)
const fenceNoiseRe = /`{3,}/g
let out = ''
let cursor = 0
for (const range of protectedRanges) {
out += text.slice(cursor, range.start).replace(fenceNoiseRe, '')
out += text.slice(range.start, range.end)
cursor = range.end
}
out += text.slice(cursor).replace(fenceNoiseRe, '')
for (let pass = 0; pass < 2; pass += 1) {
// Match EXACTLY 2 backticks (not part of a longer run) on each side.
// Without the lookbehind/lookahead, two adjacent triple-backtick
// fences with only whitespace between them get spliced together —
// e.g. ```bash\n...\n```\n\n```latex matches the regex's
// last-2-of-bash-close + \n\n + first-2-of-latex-open and the
// surrounding fence markers collapse into a single longer block,
// which the markdown parser then treats as ONE giant code block.
out = out.replace(/(?<!`)``(?!`)\s*(?<!`)``(?!`)/g, '')
out = out.replace(/(^|[^`])``(?=\s|[.,;:!?)\]'"\u2014\u2013-]|$)/g, '$1')
}
return out
}
// Runs on the ACCUMULATED text every streaming flush, so an unterminated block
// must already be hidden here: otherwise the chain of thought paints as prose
// until the close tag lands and then the whole span vanishes in one frame
// (#62774). Removing a closed block between two words keeps one space so `no` +
// `Hermes` does not fuse into `noHermes`. The seam check reads the two chars at
// the match edges rather than slicing the accumulated text, which would copy
// O(n) per closed block on every flush.
function stripReasoningBlocks(text: string): string {
const closed = text.replace(REASONING_BLOCK_RE, (match: string, _tag: string, offset: number, whole: string) => {
const prev = whole[offset - 1]
const next = whole[offset + match.length]
return prev && next && !/\s/.test(prev) && !/\s/.test(next) ? ' ' : ''
})
return closed.replace(OPEN_REASONING_BLOCK_RE, '$1').replace(PARTIAL_OPEN_REASONING_TAG_RE, '$1')
}
interface MarkdownSegment {
code: boolean
// A closed fence whose closer is the very next line.
empty: boolean
text: string
}
// Prose and fenced-code segments in order; joined, they are `text` exactly. A
// fence opens only on its own line (after any blockquote / list-item prefix)
// and closes only on a line holding the same character at least as long — the
// rule normalizeFenceBlocks applies. A ``` inside a code line, or the inner
// fences of a ````-fenced markdown example, is code, not a boundary: treating it
// as one ran the prose rewrites over the rest of the listing (`arr[0]` lost its
// index, `$5` gained a backslash) and dropped the closing fences.
function splitFencedCode(text: string): MarkdownSegment[] {
const segments: MarkdownSegment[] = []
let cursor = 0
let fence: null | { marker: string; openerEnd: number; start: number } = null
for (let lineStart = 0; ;) {
const newline = text.indexOf('\n', lineStart)
const lineEnd = newline === -1 ? text.length : newline
const line = text.slice(lineStart, lineEnd)
if (!fence) {
const open = line.match(FENCE_OPEN_LINE_RE)
// A backtick run with a backtick in its info string is inline code.
if (open && !(open[2][0] === '`' && open[3].includes('`'))) {
fence = { marker: open[2], openerEnd: lineEnd, start: lineStart + open[1].length }
}
} else {
const close = line.match(FENCE_CLOSE_LINE_RE)
if (close && close[1][0] === fence.marker[0] && close[1].length >= fence.marker.length) {
segments.push(
{ code: false, empty: false, text: text.slice(cursor, fence.start) },
{ code: true, empty: lineStart === fence.openerEnd + 1, text: text.slice(fence.start, lineEnd) }
)
cursor = lineEnd
fence = null
}
}
if (newline === -1) {
break
}
lineStart = newline + 1
}
if (fence) {
segments.push(
{ code: false, empty: false, text: text.slice(cursor, fence.start) },
{ code: true, empty: false, text: text.slice(fence.start) }
)
} else {
segments.push({ code: false, empty: false, text: text.slice(cursor) })
}
return segments
}
function stripEmptyFenceBlocks(text: string): string {
return splitFencedCode(text)
.filter(segment => !segment.empty)
.map(segment => segment.text)
.join('')
}
function isUrlOnlyBlock(lines: string[]): boolean {
const nonEmpty = lines.filter(line => line.trim())
return nonEmpty.length > 0 && nonEmpty.every(line => URL_ONLY_LINE_RE.test(line))
}
// Where a bare URL match really ends. Peels what the prose wrapped around it:
// the `)` of `(see https://x)`, the `]` of `[https://x]`, and punctuation that
// peel exposes (`(https://x.)`). A closer stays when an opener inside the URL
// pairs with it, so `…/wiki/Foo_(bar)` and `http://[::1]/` keep theirs.
function isPairedCloser(url: string, index: number): boolean {
const close = url[index]
const open = close === ')' ? '(' : '['
let depth = 0
for (let cursor = 0; cursor < index; cursor += 1) {
if (url[cursor] === open) {
depth += 1
} else if (url[cursor] === close && depth > 0) {
depth -= 1
}
}
return depth > 0
}
function bareUrlEnd(url: string): number {
let end = url.length
while (end > 0) {
const last = url[end - 1]
const strayCloser = (last === ')' || last === ']') && !isPairedCloser(url, end - 1)
if (!strayCloser && !URL_TRAILING_PUNCTUATION.includes(last)) {
break
}
end -= 1
}
return end
}
function linkBareUrls(text: string): string {
return text.replace(RAW_URL_RE, (match: string, index: number) => {
if (text[index - 1] === '<') {
return match
}
const end = bareUrlEnd(match)
return `<${match.slice(0, end)}>${match.slice(end)}`
})
}
function autoLinkRawUrls(text: string): string {
return text
.split(MARKDOWN_LINK_SPLIT_RE)
.map((part, index) => (index % 2 === 1 ? part : linkBareUrls(part)))
.join('')
}
function escapeLoneTildes(text: string): string {
return text
.split(URL_LIKE_SPLIT_RE)
.map(part => {
if (/^<?https?:\/\//i.test(part)) {
return part
}
return part.replace(LONE_TILDE_RE, '\\~')
})
.join('')
}
function escapeUnknownHtmlLikeTags(text: string): string {
return text.replace(HTML_TAG_RE, (tag: string, name: string) => {
if (SAFE_HTML_TAG_NAMES.has(name.toLowerCase())) {
return tag
}
return tag.replace(/</g, '<').replace(/>/g, '>')
})
}
// Rewrite filesystem-path links to the renderer's hash-href door (#82140).
// A plain path/file: href names a file on the AGENT's machine: Streamdown's
// URL hardening blocks `file:`/`~/` outright, and an absolute path renders
// as a dead anchor (file:// is blocked in the renderer; on a remote gateway
// the file isn't on this disk at all). `#preview/…` / `#media:…` hrefs pass
// hardening by design and route through PreviewAttachment/MediaAttachment,
// which resolve the path at VIEW time against the session's backend — local
// reads the file directly, remote fetches it over the authenticated /api/fs
// bridge — so the same transcript works from every machine that opens it.
function routeFileLinksToPreview(text: string): string {
return text.replace(FILE_LINK_RE, (match: string, ...args: unknown[]) => {
const groups = args.at(-1) as { label: string; target: string }
const target = groups.target.replace(/^<|>$/g, '')
const href = mediaKind(target) === 'file' ? previewMarkdownHref(target) : mediaMarkdownHref(target)
return `[${groups.label}](${href})`
})
}
// Mirror of the bundled grounded-citations skill's `_split_draft()`: find the
// LAST `Sources` header, then collect entry ids only from the section that
// header opens (fenced code excluded). A marker survives only when this
// response actually lists a corresponding `[n] url` entry.
function collectSourceListIds(text: string): Set<string> {
const ids = new Set<string>()
const lines = text.split('\n')
let inFence = false
let headerIndex = -1
for (let index = 0; index < lines.length; index += 1) {
const line = lines[index]
if (FENCE_TOGGLE_RE.test(line)) {
inFence = !inFence
continue
}
if (!inFence && SOURCES_HEADER_RE.test(line)) {
headerIndex = index
}
}
if (headerIndex === -1) {
return ids
}
inFence = false
for (let index = headerIndex + 1; index < lines.length; index += 1) {
const line = lines[index]
if (FENCE_TOGGLE_RE.test(line)) {
inFence = !inFence
continue
}
if (inFence) {
continue
}
const match = line.match(SOURCE_LIST_ENTRY_RE)
if (match) {
for (const id of match[1].split(',')) {
ids.add(id.trim())
}
}
}
return ids
}
function rewriteProseSegment(segment: string, sourceListIds: Set<string>): string {
return linkifySessionRefs(
escapeLoneTildes(
autoLinkRawUrls(
routeFileLinksToPreview(
escapeUnknownHtmlLikeTags(
segment
.replace(/`{3,}/g, '')
.replace(CITATION_TRANSPORT_MARKER_RE, '')
.replace(CITATION_MARKER_RE, marker => {
const ids = marker
.slice(1, -1)
.split(',')
.map(id => id.trim())
return ids.every(id => sourceListIds.has(id)) ? marker : ''
})
)
)
)
)
)
}
/**
* Lift every directive outside inline code out of `text`, leaving a slot the
* prose rewrites cannot match. `restore` puts each back shielded: inline
* markdown backslash-escaped so the parser yields it as plain text.
*/
function liftDirectives(text: string): { text: string; restore: (rewritten: string) => string } {
const lifted: string[] = []
if (text.includes('::') && !text.includes('\uE000')) {
// One leftmost-wins scan: a code span that opens first keeps a directive
// inside it as code; a directive that opens first keeps the backticks in
// its attribute values (`brief="wrap the `run` command"`) as data.
text = text.replace(DIRECTIVE_OR_CODE_RE, (match, directive?: string) =>
directive ? `\uE000${lifted.push(directive) - 1}\uE001` : match
)
}
return {
text,
restore: rewritten =>
lifted.length === 0
? rewritten
: rewritten.replace(DIRECTIVE_SLOT_RE, (_slot, index: string) =>
lifted[Number(index)].replace(MARKDOWN_INLINE_META_RE, '\\$&')
)
}
}
/**
* Apply the prose rewrites to visible prose only.
*
* Inline code has always been shielded here. Math has to be shielded for the
* same reason: these rewrites read TeX as prose. `CITATION_MARKER_RE` is the
* one that bites — its lookbehind accepts any letter, so the `t` of `\sqrt`
* qualifies and `$\sqrt[3]{8}$` loses its index to what looks like a citation
* marker, long before KaTeX sees it.
*
* Split on math spans is odd-index-is-a-delimiter (capturing split), not a
* `startsWith('$')` test, so a prose segment that merely opens with a stray
* dollar can't be mistaken for math.
*/
function normalizeVisibleProse(text: string, sourceListIds: Set<string>): string {
return text
.split(INLINE_CODE_SPLIT_RE)
.map(part =>
part.startsWith('`')
? part
: part
.split(MATH_SPAN_SPLIT_RE)
.map((segment, index) => (index % 2 === 1 ? segment : rewriteProseSegment(segment, sourceListIds)))
.join('')
)
.join('')
}
function isEscapedAt(text: string, index: number): boolean {
let slashCount = 0
for (let cursor = index - 1; cursor >= 0 && text[cursor] === '\\'; cursor -= 1) {
slashCount += 1
}
return slashCount % 2 === 1
}
/**
* True when the `$` at `index` opens a currency amount rather than math.
*
* Two shapes, and the second is why this helper exists. The US shape `$5`
* hugs its digits, so "followed by a digit" identifies it. Most of the rest
* of the world writes a currency PREFIX plus a space — `R$ 12.345` (BRL),
* `US$ 1,200`, `AU$ 40`. Treating only the hugging shape as currency left
* `R$ 12.345 … R$ 98.765` with two bare dollars on one line, so remark-math
* (`singleDollarTextMath: true`) paired them and painted the whole sentence
* between two prices as an equation.
*
* The spaced shape additionally requires a letter immediately before the `$`
* — that prefix is what makes it a currency symbol. A bare `$ 5` keeps its
* old behavior, so spaced inline math like `$ x^2 $` is untouched.
*/
function isCurrencyOpenerAt(text: string, index: number): boolean {
if (text[index] !== '$' || isEscapedAt(text, index) || text[index - 1] === '$' || text[index + 1] === '$') {
return false
}
if (/\d/u.test(text[index + 1] || '')) {
return true
}
return /^[ \u00a0]\d/u.test(text.slice(index + 1, index + 3)) && /^[A-Za-z]$/u.test(text[index - 1] || '')
}
function findClosingSingleDollar(text: string, openingIndex: number): number {
for (let cursor = openingIndex + 1; cursor < text.length && text[cursor] !== '\n'; cursor += 1) {
if (text[cursor] !== '$' || isEscapedAt(text, cursor)) {
continue
}
// A `$$` run belongs to display math, not to this inline candidate.
if (text[cursor - 1] === '$' || text[cursor + 1] === '$') {
continue
}
return cursor
}
return -1
}
function isLikelyNumericInlineMath(body: string, followingCharacter: string): boolean {
const value = body.trim()
if (!/^\d/u.test(value)) {
return false
}
// Currency ranges and prose fragments can sit between two price openers,
// e.g. `$5-$10` or `$5, then $10`. They are not balanced math spans.
if (/[+\-*/=<>^_,;:(]$/u.test(value)) {
return false
}
if (/https?:\/\//iu.test(value)) {
return false
}
// A dollar immediately followed by a letter/number is more likely the next
// opener in prose such as `$5 and $10` or `$5 and $x$`. Preserve it only
// when the candidate body itself carries an unambiguous math signal.
if (/^\p{N}/u.test(followingCharacter)) {
return false
}
if (/^[\p{L}\\]/u.test(followingCharacter)) {
return /\\[A-Za-z]+|[+*/=<>^_{}]/u.test(value)
}
return true
}
function opensCompleteInlineMath(text: string, openingIndex: number): boolean {
const closingIndex = findClosingSingleDollar(text, openingIndex)
if (closingIndex === -1) {
return false
}
const body = text.slice(openingIndex + 1, closingIndex)
return /^[\p{L}\p{N}\\{([|+\-=_^]/u.test(body)
}
/**
* Escape price openers without corrupting balanced numeric inline math.
*
* The upstream helper deliberately treats every `$` followed by a digit as
* currency. That turns `$4\in A$` into `\$4\in A$`; remark-math then pairs
* the orphan closing dollar with a later formula and renders the intervening
* prose as math. We retain the price behavior for `$5 and $10` and `$5-$10`,
* but preserve balanced, same-line numeric math spans.
*/
function escapeCurrencyDollarsPreservingMath(text: string): string {
let out = ''
let copiedThrough = 0
for (let cursor = 0; cursor < text.length; cursor += 1) {
if (!isCurrencyOpenerAt(text, cursor)) {
continue
}
const closingIndex = findClosingSingleDollar(text, cursor)
if (
closingIndex !== -1 &&
// A second amount on the same line is the NEXT opener, never this
// span's closer: `R$ 12.345 … R$ 98.765` is two prices, not one
// equation wrapping the prose between them.
!isCurrencyOpenerAt(text, closingIndex) &&
!opensCompleteInlineMath(text, closingIndex) &&
isLikelyNumericInlineMath(text.slice(cursor + 1, closingIndex), text[closingIndex + 1] || '')
) {
cursor = closingIndex
continue
}
out += `${text.slice(copiedThrough, cursor)}\\$`
copiedThrough = cursor + 1
}
return out + text.slice(copiedThrough)
}
// East Asian script and punctuation ranges: CJK Symbols/Punctuation, Hiragana,
// Katakana, Han (incl. Extension A and compatibility ideographs), Hangul, and
// the fullwidth/halfwidth forms. Fullwidth punctuation (() , :) is
// near-universal in CJK prose and is included so a `$foo(bar)$`-shaped span
// with no Han glyph in its body still classifies as prose.
const CJK_RE = /[\u3000-\u303f\u3040-\u30ff\u3400-\u4dbf\u4e00-\u9fff\uf900-\ufaff\uac00-\ud7af\uff00-\uffef]/u
/**
* Escape the opening `$` of any same-line single-dollar span whose body
* contains East Asian script or punctuation, so remark-math reads it as a
* literal dollar instead of pairing it with the later `$` and typesetting the
* intervening prose as one KaTeX inline formula (#103546).
*
* A bare `$identifier` in CJK prose (written twice in one sentence) is the
* classic false positive of `singleDollarTextMath: true`: the whole sentence
* renders through KaTeX — CJK glyphs in a serif fallback face at 1.21em, and
* copy-out yields per-character math-italic codepoints, not the source text.
* CJK prose sets no inter-word spaces and rarely writes `$…$` math around
* non-Latin text, so a CJK body is prose with near certainty.
*
* Escaping only the OPENING `$` is enough: the closing `$` loses its partner
* and renders literally. Real math is untouched — its body carries no CJK —
* and `$$` display runs are skipped by the same `$$`-run guard the currency
* escape uses. A non-CJK span is consumed through its closer: otherwise that
* closer is scanned as the next opener, and CJK prose between two formulas
* makes the escape land on the first equation's real closing `$`. The one
* accepted tradeoff: genuine inline math whose body names a CJK variable
* (`$x = 变量$`) renders as literal prose. Losing one equation is far cheaper
* than corrupting a sentence's copy-out.
*/
function escapeCjkProseDollars(text: string): string {
let out = ''
let copiedThrough = 0
for (let cursor = 0; cursor < text.length; cursor += 1) {
if (text[cursor] !== '$' || text[cursor - 1] === '$' || isEscapedAt(text, cursor)) {
continue
}
const closingIndex = findClosingSingleDollar(text, cursor)
if (closingIndex === -1) {
continue
}
const body = text.slice(cursor + 1, closingIndex)
if (!CJK_RE.test(body)) {
cursor = closingIndex
continue
}
out += `${text.slice(copiedThrough, cursor)}\\$`
copiedThrough = cursor + 1
}
return out + text.slice(copiedThrough)
}
/**
* Moves the `$$` delimiters of a MULTI-LINE display-math block onto their own
* lines: `$$\begin{aligned}` … `\end{aligned}$$` becomes a `$$`-only line, the
* body, then a `$$`-only line.
*
* remark-math's flow-math construct is fence-shaped: whatever follows the
* opening `$$` on the same line is read as an info string and DISCARDED, and the
* closing `$$` is only recognized alone on its own line. So the hugging form
* loses its first line, never closes, and KaTeX paints the remains as raw source
* text. Models emit this form constantly.
*
* Single-line `$$…$$` is left alone — it routes through the inline math-text
* construct and already renders.
*/
function splitHuggingDisplayMath(text: string): string {
const lines = text.split('\n')
const out: string[] = []
for (let index = 0; index < lines.length; index += 1) {
const openingMatch = lines[index].match(HUGGING_DISPLAY_MATH_OPEN_RE)
// `$$x^2$$` closes on the same line — not our case.
if (!openingMatch || openingMatch[2].endsWith('$$')) {
out.push(lines[index])
continue
}
let closingIndex = -1
for (let candidate = index + 1; candidate < lines.length; candidate += 1) {
if (HUGGING_DISPLAY_MATH_CLOSE_RE.test(lines[candidate])) {
closingIndex = candidate
break
}
}
if (closingIndex === -1) {
out.push(lines[index])
continue
}
const closingMatch = lines[closingIndex].match(HUGGING_DISPLAY_MATH_CLOSE_RE) as RegExpMatchArray
const openingCarriageReturn = lines[index].endsWith('\r') ? '\r' : ''
const closingCarriageReturn = lines[closingIndex].endsWith('\r') ? '\r' : ''
// The container prefix (blockquote marker, list bullet) has to be replayed
// onto the delimiter lines, or the block falls out of its container.
out.push(
`${openingMatch[1]}$$${openingCarriageReturn}`,
`${openingMatch[1]}${openingMatch[2]}${openingCarriageReturn}`
)
out.push(...lines.slice(index + 1, closingIndex))
// The break this split INTRODUCES takes the block's own line ending, not the
// closing line's — a final `\end{aligned}$$` with no trailing CRLF (it's the
// last line of the message) would otherwise emit a bare LF into an
// otherwise-CRLF block.
out.push(
`${closingMatch[1]}${closingMatch[2]}${openingCarriageReturn}`,
`${closingMatch[1]}$$${closingCarriageReturn}`
)
index = closingIndex
}
return out.join('\n')
}
function normalizeDisplayMathForMarkdown(text: string): string {
const lines = text.split('\n')
for (let index = 0; index < lines.length; index += 1) {
const latexMatch = lines[index].match(LATEX_DISPLAY_OPEN_LINE_RE)
const customMatch = lines[index].match(CUSTOM_DISPLAY_MATH_LINE_RE)
const openingMatch = latexMatch || customMatch
if (!openingMatch) {
continue
}
const prefix = openingMatch[1] || ''
const closingPattern = latexMatch ? LATEX_DISPLAY_CLOSE_LINE_RE : CUSTOM_DISPLAY_MATH_LINE_RE
for (let closingIndex = index + 1; closingIndex < lines.length; closingIndex += 1) {
const closingMatch = lines[closingIndex].match(closingPattern)
if (!closingMatch) {
continue
}
const openingCarriageReturn = lines[index].endsWith('\r') ? '\r' : ''
const closingCarriageReturn = lines[closingIndex].endsWith('\r') ? '\r' : ''
const closingPrefix = closingMatch[1] || ''
lines[index] = `${prefix}$$${openingCarriageReturn}`
lines[closingIndex] = `${closingPrefix}$$${closingCarriageReturn}`
index = closingIndex
break
}
}
return lines.join('\n')
}
function normalizeProseMath(text: string): string {
// remark-math requires multiline display delimiters on their own lines.
// Normalize those locally before the dependency handles inline forms;
// its compact `$$body$$` rewrite makes the first equation line metadata
// and leaks the trailing `$$` into KaTeX's error fallback.
//
// splitHuggingDisplayMath runs LAST because normalizeMathDelimiters is itself
// a source of the hugging form: a multi-line `\[…\]` comes out of it as
// `$$\begin{aligned}…\end{aligned}$$`. Running afterwards catches both the
// hugging math the model emitted and the hugging math the rewrite produced.
const normalized = splitHuggingDisplayMath(normalizeMathDelimiters(normalizeDisplayMathForMarkdown(text)))
const cjkEscaped = escapeCjkProseDollars(normalized)
return escapeCurrencyDollarsPreservingMath(cjkEscaped)
}
function extend(out: string[], lines: string[]) {
for (const line of lines) {
out.push(line)
}
}
function pushProseFence(out: string[], indent: string, info: string, lines: string[]) {
if (info) {
out.push(`${indent}${info}`.trimEnd())
}
extend(out, lines)
}
function findClosingFence(lines: string[], start: number, marker: string): number {
for (let cursor = start + 1; cursor < lines.length; cursor += 1) {
const closeMatch = (lines[cursor] || '').match(FENCE_LINE_RE)
if (!closeMatch) {
continue
}
const closeMarker = closeMatch[2] || ''
const closeInfo = (closeMatch[3] || '').trim()
if (!closeInfo && closeMarker[0] === marker[0] && closeMarker.length >= marker.length) {
return cursor
}
}
return -1
}
// Languages that should be routed to the math (KaTeX) renderer instead of
// being shown as a syntax-highlighted code block.
//
// We deliberately recognize ONLY `math` here, not `latex` or `tex`.
// Reasoning: GitHub-style markdown uses ` ```math ` to mean "render as
// math" and ` ```latex `/` ```tex ` to mean "show LaTeX/TeX source code"
// (syntax highlighted). Conflating the two breaks code blocks where a
// user is *discussing* LaTeX rather than embedding it (e.g.,
// ```latex\n\begin{equation}\n E = mc^2\n\end{equation}``` shown as a
// teaching example). Anyone who wants math rendered should use ```math.
const MATH_FENCE_LANGUAGES = new Set(['math'])
function isMathFence(language: string): boolean {
return MATH_FENCE_LANGUAGES.has(language.toLowerCase())
}
function normalizeFenceBlocks(text: string): string {
const sourceLines = text.split('\n')
const out: string[] = []
let index = 0
while (index < sourceLines.length) {
const line = sourceLines[index] || ''
const match = line.match(FENCE_LINE_RE)
if (!match) {
out.push(line)
index += 1
continue
}
const indent = match[1] || ''
const marker = match[2] || '```'
const infoRaw = (match[3] || '').trim()
const languageToken = infoRaw.split(/\s+/, 1)[0] || ''
const language = sanitizeLanguageTag(languageToken)
const openerValid = !infoRaw || Boolean(language)
if (!openerValid) {
out.push(`${indent}${infoRaw}`.trimEnd())
index += 1
continue
}
const closeIndex = findClosingFence(sourceLines, index, marker)
const bodyLines = sourceLines.slice(index + 1, closeIndex === -1 ? sourceLines.length : closeIndex)
const body = bodyLines.join('\n')
if (closeIndex !== -1 && !body.trim()) {
index = closeIndex + 1
continue
}
if (closeIndex !== -1 && LOCAL_PREVIEW_ONLY_RE.test(body.trim())) {
index = closeIndex + 1
continue
}
if (closeIndex !== -1 && isUrlOnlyBlock(bodyLines)) {
extend(out, bodyLines)
index = closeIndex + 1
continue
}
if (closeIndex === -1) {
if (!body.trim()) {
index += 1
continue
}
if (isLikelyProseFence(infoRaw, body)) {
pushProseFence(out, indent, infoRaw, bodyLines)
} else if (isMathFence(language)) {
// Streaming math fence — rewrite the language tag to "math".
// remark-math + rehype-katex pick up ```math fenced blocks via
// the language-math class on the resulting <code> element. We
// keep the fence intact (instead of converting to $$..$$) so
// any literal `$$` characters in the body don't collide with
// an outer math wrapper. No close emitted yet — streaming.
out.push(`${indent}${marker}math`)
extend(out, bodyLines)
} else {
out.push(`${indent}${marker}${language}`)
extend(out, bodyLines)
}
break
}
if (isLikelyProseFence(infoRaw, body)) {
pushProseFence(out, indent, infoRaw, bodyLines)
index = closeIndex + 1
continue
}
if (isMathFence(language)) {
// Closed math fence — rewrite the language tag to "math" so
// rehype-katex's language-math class detection picks it up.
// Body stays untouched (no $$..$$ rewrite) so authors can write
// arbitrary LaTeX including `$$display$$` markers without them
// colliding with our wrapper. Without this rewrite the block
// would render as a syntax-highlighted "latex" code listing.
out.push(`${indent}${marker}math`)
extend(out, bodyLines)
out.push(`${indent}${marker}`)
index = closeIndex + 1
continue
}
out.push(`${indent}${marker}${language}`)
extend(out, bodyLines)
out.push(`${indent}${marker}`)
index = closeIndex + 1
}
return out.join('\n')
}
export function preprocessMarkdown(text: string): string {
const cleaned = stripReasoningBlocks(text).replace(PREVIEW_MARKER_RE, '')
const scrubbed = scrubBacktickNoise(cleaned)
const normalizedFences = normalizeFenceBlocks(scrubbed)
const strippedEmptyFences = stripEmptyFenceBlocks(normalizedFences)
// Anchors come from the response's `Sources` section only: a marker
// survives the orphan strip exactly when this response lists a matching
// `[n] url` entry there (see collectSourceListIds).
const sourceListIds = collectSourceListIds(strippedEmptyFences)
return splitFencedCode(strippedEmptyFences)
.map(({ code, text: part }) => {
// Fence blocks pass through untouched.
if (code) {
return part
}
// Run only on prose segments so `$5` literals and `\(` inside code
// blocks stay intact. The HTML-depth clamp belongs here for the same
// reason: a fenced block renders as code and never reaches rehype-raw,
// so escaping tags inside one would corrupt the listing for nothing.
// Directives sit out the rewrites entirely (see liftDirectives).
const { text: prose, restore } = liftDirectives(part)
return restore(
clampHtmlNestingDepth(normalizeVisibleProse(stripPreviewTargets(normalizeProseMath(prose)), sourceListIds))
)
})
.join('')
}
/**
* Math-only normalization for static file previews. Mirrors the math half of
* `preprocessMarkdown` — delimiter normalization (`\(…\)`, `\[…\]`), display
* math on its own lines, and currency-dollar escaping — but deliberately skips
* the chat-only transforms (reasoning-block stripping, `@session:` ref
* linking, preview-target stripping, raw-URL autolinking, citation-marker
* stripping). A file's prose is author content, not model output, so those
* rewrites must not touch it. Code fences and inline code spans pass through
* untouched so `$`, `\(` and `\begin` inside listings are never mangled.
*
* ` ```math ` fences need no preprocessing: remark/rehype emit them as
* `<code class="language-math">` and the memoized rehype-katex wrapper renders
* them regardless of this function.
*/
export function normalizeFilePreviewMath(text: string): string {
return splitFencedCode(text)
.map(({ code, text: part }) => {
if (code) {
return part
}
return part
.split(INLINE_CODE_SPLIT_RE)
.map(segment => (segment.startsWith('`') ? segment : normalizeProseMath(segment)))
.join('')
})
.join('')
}