The Open Standard for Generative UI
import type { ThemeTokens } from "../theme";
export type TokenKind =
| "string"
| "state"
| "number"
| "atom"
| "type"
| "ident"
| "keyword"
| "operator"
| "punct"
| "text";
export interface Token {
kind: TokenKind;
value: string;
}
/** Shared by the Debug editor and the Inspect stream log. */
export function tokenColor(t: ThemeTokens): Record<TokenKind, string> {
return {
string: t.syntaxString,
state: t.syntaxState,
number: t.syntaxNumber,
atom: t.syntaxAtom,
type: t.syntaxType,
ident: t.syntaxIdent,
keyword: t.syntaxKeyword,
operator: t.syntaxOperator,
punct: t.syntaxPunct,
text: t.syntaxText,
};
}
const RULES: { kind: Exclude<TokenKind, "text">; re: RegExp }[] = [
{ kind: "string", re: /^"(?:[^"\\]|\\.)*"?/ },
{ kind: "state", re: /^\$[A-Za-z_][\w]*/ },
{ kind: "number", re: /^-?\d+(?:\.\d+)?/ },
{ kind: "atom", re: /^(?:true|false|null)\b/ },
{ kind: "type", re: /^[A-Z][\w]*/ },
{ kind: "ident", re: /^[a-z_][\w]*/ },
{ kind: "keyword", re: /^@[A-Za-z_][\w]*/ },
{ kind: "operator", re: /^=/ },
{ kind: "punct", re: /^[[\]{}(),:]/ },
];
/**
* Regroup tokens into one array per source line, dropping the newlines.
* The editor renders a row per line so the gutter number stays with its line
* even when the line soft-wraps. Always returns at least one (empty) line.
*/
export function toTokenLines(tokens: Token[]): Token[][] {
const lines: Token[][] = [[]];
for (const token of tokens) {
const parts = token.value.split("\n");
for (const [index, part] of parts.entries()) {
if (index > 0) lines.push([]);
if (part) lines[lines.length - 1]!.push({ kind: token.kind, value: part });
}
}
return lines;
}
/** Same token rules as the docs highlighter, without CodeMirror. */
export function tokenizeLang(source: string): Token[] {
const tokens: Token[] = [];
// Runs of one kind collapse into a single token — the fallback below matches a
// character at a time, so whitespace would otherwise dominate the output.
const push = (kind: TokenKind, value: string) => {
const last = tokens[tokens.length - 1];
if (last?.kind === kind) last.value += value;
else tokens.push({ kind, value });
};
let i = 0;
while (i < source.length) {
const slice = source.slice(i);
let matched = false;
for (const rule of RULES) {
const match = rule.re.exec(slice);
if (!match) continue;
const value = match[0];
const kind: TokenKind = rule.kind === "ident" && value === "root" ? "keyword" : rule.kind;
push(kind, value);
i += value.length;
matched = true;
break;
}
if (!matched) {
push("text", source[i]!);
i += 1;
}
}
return tokens;
}