Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.
Go agentagenticaichatbotdsh-pluginembeddingsevaluationgenerative-aigolangknowledge-basellmmulti-tenantollamaopenaiquestion-answeringragrerankingsemantic-searchvector-searchwiki
import hljs from 'highlight.js'
import { Marked, Renderer } from 'marked'
import markedKatex from 'marked-katex-extension'
import { domPurifyAllowedUriRegexp } from './markdownDomPurify.ts'
import { literalSingleTildeExtension } from './markedLiteralTilde.ts'
import { stripDuplicateLeadingH1 } from './markdownTitleDedupe.ts'
import { escapeHTML, safeMarkdownToHTML, sanitizeDocumentPreviewHTML } from './security.ts'
const PREVIEW_IMAGE_ATTRIBUTES = 'loading="lazy" decoding="async" fetchpriority="low"'
const HIGHLIGHT_AUTO_MAX_CHARS = 32_768
export function isSafePreviewImageHref(href: unknown): href is string {
if (typeof href !== 'string') return false
const trimmed = href.trim()
if (!trimmed || trimmed.startsWith('#')) return false
return domPurifyAllowedUriRegexp.test(trimmed)
}
function highlightPreviewCode(source: string, lang?: string): string {
if (lang && hljs.getLanguage(lang)) {
try {
return hljs.highlight(source, { language: lang }).value
} catch {
// Fall through to auto-detect or plain text.
}
}
if (source.length > HIGHLIGHT_AUTO_MAX_CHARS) {
return escapeHTML(source)
}
return hljs.highlightAuto(source).value
}
function createPreviewMarkdownRenderer(): Renderer {
const renderer = new Renderer()
// GFM task lists: marked emits <input type="checkbox">, which the DOMPurify
// tag allowlist strips (leaving a plain bullet) — render a styled span.
renderer.checkbox = ({ checked }) =>
`<span class="task-checkbox${checked ? ' checked' : ''}"></span>`
renderer.image = ({ href, title, text }) => {
if (!isSafePreviewImageHref(href)) return ''
const safeHref = href.trim().replace(/"/g, '"')
const safeTitle = title ? ` title="${escapeHTML(title)}"` : ''
return `<img src="${safeHref}" alt="${escapeHTML(text || '')}"${safeTitle} class="markdown-image" ${PREVIEW_IMAGE_ATTRIBUTES}>`
}
renderer.code = ({ text, lang }) => {
const source = typeof text === 'string' ? text : ''
return `<pre><code class="hljs">${highlightPreviewCode(source, lang)}</code></pre>`
}
return renderer
}
const previewMarked = new Marked({
breaks: true,
gfm: true,
renderer: createPreviewMarkdownRenderer(),
})
previewMarked.use(markedKatex({ throwOnError: false, nonStandard: true }))
previewMarked.use(literalSingleTildeExtension)
function preprocessMathDelimiters(rawText: string): string {
if (!rawText || typeof rawText !== 'string') return ''
return rawText
.replace(/\\\[([\s\S]*?)\\\]/g, '$$$$$1$$$$')
.replace(/\\\(([\s\S]*?)\\\)/g, '$$$1$$')
}
export interface DocumentPreviewMarkdownOptions {
/**
* Names already shown by the surrounding UI (drawer header, preview
* toolbar). A leading level-1 heading that normalizes to one of them is
* dropped so the document title is not rendered twice.
*/
titles?: Array<string | null | undefined>
}
export function renderDocumentPreviewMarkdown(
markdown: string,
sanitize: (html: string) => string = sanitizeDocumentPreviewHTML,
options: DocumentPreviewMarkdownOptions = {},
): string {
// Strip the duplicated heading before delimiter preprocessing so the
// comparison sees the original text (math delimiters stay untouched).
const deduped = options.titles?.length
? stripDuplicateLeadingH1(markdown, options.titles)
: markdown
const mathSafe = preprocessMathDelimiters(deduped)
return sanitize(previewMarked.parse(safeMarkdownToHTML(mathSafe)) as string)
}