Open-source Grok Bot alternative for named AI teammates. Local-first desktop, web, and mobile clients with per-bot tools, instructions, and memory. Independent fork of T3 Code.
import { fromMarkdown } from "mdast-util-from-markdown";
import { gfmFromMarkdown } from "mdast-util-gfm";
import { gfm } from "micromark-extension-gfm";
import {
generativeBlockSummary,
generativeKindForLanguage,
} from "../markdown/generative/generativeSchemas";
export interface RosterLastMessage {
text: string;
at: string;
/** The chat the message went to, when known. */
threadId?: string;
}
type MarkdownNode = ReturnType<typeof fromMarkdown> | MarkdownNodeChild;
type MarkdownNodeChild = ReturnType<typeof fromMarkdown>["children"][number];
const markdownPreviewOptions = {
extensions: [gfm()],
mdastExtensions: [gfmFromMarkdown()],
};
const markdownPreviewCache = new Map<string, string>();
const MARKDOWN_PREVIEW_CACHE_LIMIT = 500;
/**
* One-line plain text for a roster preview. Chat messages are markdown, and a
* preview row shows only the words: emphasis, code ticks and fences, link and
* image syntax, headings, and list or quote markers go, and whitespace
* collapses to single spaces. Links keep their label; images drop out; code
* and URLs stay literal. The text is parsed with the same micromark/GFM stack
* the chat renders with, and results are cached by text so a roster render
* never reparses an unchanged message.
*/
export function flattenMarkdownPreview(markdown: string): string {
const cached = markdownPreviewCache.get(markdown);
if (cached !== undefined) return cached;
// A row shows one line, so parse whole blocks from the top only until the
// flattened text is long enough. Parsing costs several milliseconds per long
// answer. Cuts fall on blank lines outside code fences, so they never land
// inside a link, image, or code span and leak raw syntax into the preview.
// A message longer than the parse limit only parses blocks that end inside
// it; past the last such block, a cheap strip of the next few thousand
// characters stands in, so one enormous line cannot stall a roster render.
const complete = markdown.length <= MARKDOWN_PREVIEW_PARSE_LIMIT;
const source = complete ? markdown : markdown.slice(0, MARKDOWN_PREVIEW_PARSE_LIMIT);
// Chunks parse apart, so a reference image or link in one chunk still needs
// the definitions another chunk holds; they flatten to nothing themselves.
const definitions = (source.match(MARKDOWN_REFERENCE_DEFINITION) ?? []).join("\n");
const withDefinitions = definitions ? `\n\n${definitions}` : "";
let flattened = "";
let offset = 0;
while (offset < source.length && flattened.length < MARKDOWN_PREVIEW_TEXT_TARGET) {
const end = markdownPreviewChunkEnd(source, offset, complete);
if (end === null) break;
const chunk = flattenMarkdownText(source.slice(offset, end) + withDefinitions);
if (chunk.length > 0) flattened = flattened.length > 0 ? `${flattened} ${chunk}` : chunk;
offset = end;
}
if (flattened.length === 0 && !complete) {
flattened = stripMarkdownRoughly(
markdown.slice(offset, offset + MARKDOWN_PREVIEW_ROUGH_LIMIT).trimStart(),
);
}
if (markdownPreviewCache.size >= MARKDOWN_PREVIEW_CACHE_LIMIT) markdownPreviewCache.clear();
markdownPreviewCache.set(markdown, flattened);
return flattened;
}
const MARKDOWN_PREVIEW_TEXT_TARGET = 280;
const MARKDOWN_PREVIEW_CHUNK_TARGET = 600;
const MARKDOWN_PREVIEW_PARSE_LIMIT = 20_000;
const MARKDOWN_PREVIEW_ROUGH_LIMIT = 2_000;
const MARKDOWN_FENCE = /^ {0,3}(`{3,}|~{3,})/;
const MARKDOWN_REFERENCE_DEFINITION = /^ {0,3}\[(?!\^)(?:[^\]\\\n]|\\.)+\]:[ \t]*\S.*$/gm;
/**
* End of the next chunk starting at `offset`: a blank line outside a fenced
* code block, preferably at least the chunk target long. When `complete` is
* false the source is a cut prefix, so its end is not a block boundary and the
* result is the last blank line seen, or null when there is none.
*/
function markdownPreviewChunkEnd(
markdown: string,
offset: number,
complete: boolean,
): number | null {
let fence: string | null = null;
let lastBoundary: number | null = null;
let lineStart = offset;
while (lineStart < markdown.length) {
const newline = markdown.indexOf("\n", lineStart);
const lineEnd = newline === -1 ? markdown.length : newline + 1;
const line = markdown.slice(lineStart, lineEnd);
const marker = MARKDOWN_FENCE.exec(line)?.[1];
if (fence === null) {
if (marker !== undefined) fence = marker;
else if (newline !== -1 && line.trim().length === 0) {
if (lineEnd - offset >= MARKDOWN_PREVIEW_CHUNK_TARGET) return lineEnd;
lastBoundary = lineEnd;
}
} else if (
marker !== undefined &&
marker[0] === fence[0] &&
marker.length >= fence.length &&
line.trim() === marker
) {
fence = null;
}
lineStart = lineEnd;
}
return complete ? markdown.length : lastBoundary;
}
/**
* Rough plain text for the tail of a message too long to parse: drops image
* syntax and alt text, keeps link labels, and removes code ticks, emphasis,
* and line markers. It is only a fallback, so literal brackets or underscores
* next to words may survive or go.
*/
function stripMarkdownRoughly(markdown: string): string {
const stripProse = (text: string) =>
withoutImagesRoughly(text)
.replace(/\[([^\]]*)\](?:\([^)]*\)?|\[[^\]]*\]?)/g, "$1")
.replace(/^[ \t]{0,3}(?:#{1,6}|>|[-*+]|\d{1,9}[.)])[ \t]+/gm, "")
.replace(/~~|\*+|(?<![A-Za-z0-9])_+|_+(?![A-Za-z0-9])/g, "");
let result = "";
let offset = 0;
while (offset < markdown.length) {
const opening = unescapedIndexOf(markdown, "`", offset);
// An image that starts before the next code span is dropped whole, so a
// backtick inside its label cannot split it and leak the description.
const image = unescapedIndexOf(markdown, "![", offset);
if (image >= 0 && (opening < 0 || image < opening)) {
result += `${stripProse(markdown.slice(offset, image))} `;
const end = roughImageEnd(markdown, image);
if (end === null) break;
offset = end;
continue;
}
if (opening < 0) {
result += stripProse(markdown.slice(offset));
break;
}
const delimiter = /^`+/.exec(markdown.slice(opening))![0];
const closing = markdown.indexOf(delimiter, opening + delimiter.length);
if (closing < 0) {
result += stripProse(markdown.slice(offset).replace(/`+/g, ""));
break;
}
result += stripProse(markdown.slice(offset, opening));
result += markdown.slice(opening + delimiter.length, closing);
offset = closing + delimiter.length;
}
return result.replace(/\s+/g, " ").trim();
}
/**
* Text with every `![` image dropped through the end of its balanced bracket
* group and any balanced parenthesis or reference group after it, honoring
* backslash escapes. An image with no balanced close drops the rest of the
* text, so alt text never leaks even when some ordinary words go with it.
*/
function withoutImagesRoughly(markdown: string): string {
let result = "";
let offset = 0;
for (;;) {
const start = markdown.indexOf("![", offset);
if (start === -1) return result + markdown.slice(offset);
result += `${markdown.slice(offset, start)} `;
const end = roughImageEnd(markdown, start);
if (end === null) return result;
offset = end;
}
}
/**
* Index just past the `![` image at `start`: its balanced label and any
* balanced parenthesis or reference group after it. Null when the label or
* target never closes.
*/
function roughImageEnd(markdown: string, start: number): number | null {
const labelEnd = balancedGroupEnd(markdown, start + 1, "[", "]");
if (labelEnd === null) return null;
const next = markdown[labelEnd];
if (next !== "(" && next !== "[") return labelEnd;
return balancedGroupEnd(markdown, labelEnd, next, next === "(" ? ")" : "]");
}
/** Index of `needle` at or after `from` that no backslash escapes, or -1. */
function unescapedIndexOf(text: string, needle: string, from: number): number {
for (
let index = text.indexOf(needle, from);
index >= 0;
index = text.indexOf(needle, index + 1)
) {
let backslashes = 0;
while (text[index - 1 - backslashes] === "\\") backslashes += 1;
if (backslashes % 2 === 0) return index;
}
return -1;
}
/** Index just past the close matching the `open` at `start`, or null. */
function balancedGroupEnd(text: string, start: number, open: string, close: string): number | null {
let depth = 0;
for (let index = start; index < text.length; index += 1) {
const char = text[index];
if (char === "\\") index += 1;
else if (char === open) depth += 1;
else if (char === close && --depth === 0) return index + 1;
}
return null;
}
function flattenMarkdownText(markdown: string): string {
const parts: string[] = [];
collectPreviewText(fromMarkdown(markdown, markdownPreviewOptions), parts);
return parts.join("").replace(/\s+/g, " ").trim();
}
function collectPreviewText(node: MarkdownNode, parts: string[]): void {
switch (node.type) {
case "text":
case "inlineCode":
parts.push(node.value);
break;
case "code": {
// Generative blocks carry JSON; preview them by name instead.
const kind = generativeKindForLanguage(node.lang ?? "");
parts.push(kind ? ` ${generativeBlockSummary(kind, node.value)} ` : node.value);
break;
}
case "html":
parts.push(visibleHtmlText(node.value));
break;
case "image":
case "imageReference":
case "definition":
case "break":
case "thematicBreak":
parts.push(" ");
break;
default:
if ("children" in node) {
for (const child of node.children) collectPreviewText(child, parts);
}
}
// Block boundaries become spaces so adjacent paragraphs or list items never
// glue their words together.
if (node.type !== "text" && !isPhrasingNode(node)) parts.push(" ");
}
/**
* The words raw HTML shows once the chat renders it: tags and comments go,
* and line-breaking elements leave a space so their words do not glue.
*/
function visibleHtmlText(html: string): string {
const text = html
.replace(/<!--[\s\S]*?(?:-->|$)/g, "")
.replace(/<\/?(?:br|p|div|li|ul|ol|h[1-6]|tr|td|th|table|blockquote|pre|hr)\b[^>]*>/gi, " ")
.replace(/<[^>]*>/g, "")
.replace(/\s+/g, " ");
return text.includes("&") ? decodeHtmlEntities(text) : text;
}
/**
* Decodes character references the way the rendered chat does. Markdown parsing
* knows every named HTML entity, so the text is parsed with all other ASCII
* punctuation escaped and its words read back; unknown names stay as written.
*/
function decodeHtmlEntities(text: string): string {
const escaped = text.trim().replace(/[!-"$-%'-/:<-@[-`{-~]/g, "\\$&");
const parts: string[] = [];
collectPreviewText(fromMarkdown(escaped), parts);
// Keep the edge spaces that separate this HTML from neighbouring text.
const lead = text.startsWith(" ") ? " " : "";
const trail = text.endsWith(" ") ? " " : "";
return `${lead}${parts.join("").trim()}${trail}`;
}
function isPhrasingNode(node: MarkdownNode): boolean {
switch (node.type) {
case "text":
case "inlineCode":
case "emphasis":
case "strong":
case "delete":
case "link":
case "linkReference":
case "footnoteReference":
case "html":
return true;
default:
return false;
}
}
export function resolveLatestRosterMessage(
fallback: RosterLastMessage | null,
messages: ReadonlyArray<{
role: "user" | "assistant" | "system";
text: string;
createdAt: string;
parentThreadId?: string | null | undefined;
}>,
threadId?: string | null,
): RosterLastMessage | null {
let latest: RosterLastMessage | null = null;
for (let index = messages.length - 1; index >= 0; index -= 1) {
const message = messages[index];
if (!message || message.parentThreadId != null || message.role === "system") continue;
const text = flattenMarkdownPreview(message.text);
if (text.length > 0) {
latest = { text, at: message.createdAt };
break;
}
}
// A fallback that flattens to nothing (an image-only attachment, say) must
// not beat an older visible answer on timestamp alone.
// The fallback is kept per bot, so one sent to another chat does not
// describe the chat that is open. With no open chat (its last chat was
// archived or deleted) no fallback describes anything current.
const sameChatFallback =
fallback && threadId !== null && (fallback.threadId ?? threadId) === threadId ? fallback : null;
const flatFallback = sameChatFallback
? { ...sameChatFallback, text: flattenMarkdownPreview(sameChatFallback.text) }
: null;
const usableFallback = flatFallback && flatFallback.text.length > 0 ? flatFallback : null;
if (!latest) return usableFallback;
if (!usableFallback) return latest;
return latest.at >= usableFallback.at ? latest : usableFallback;
}