The Open Standard for Generative UI
// ─────────────────────────────────────────────────────────────────────────────
// Lexer for openui-lang
// ─────────────────────────────────────────────────────────────────────────────
import { T, type Token } from "./tokens";
/**
* Tokenize an openui-lang source string into a flat token array.
*
* Handles all token types: identifiers, literals, operators,
* state variables ($name), dot access, ternary.
*/
export function tokenize(src: string): Token[] {
const tokens: Token[] = [];
let i = 0;
const n = src.length;
while (i < n) {
// Skip horizontal whitespace (not newlines — they're significant)
while (i < n && (src[i] === " " || src[i] === "\t" || src[i] === "\r")) i++;
if (i >= n) break;
const c = src[i];
// ── Newline ────────────────────────────────────────────────────────
if (c === "\n") {
tokens.push({ t: T.Newline });
i++;
continue;
}
// ── Single-character punctuation (brackets, comma, colon) ─────────
if (c === "(") {
tokens.push({ t: T.LParen });
i++;
continue;
}
if (c === ")") {
tokens.push({ t: T.RParen });
i++;
continue;
}
if (c === "[") {
tokens.push({ t: T.LBrack });
i++;
continue;
}
if (c === "]") {
tokens.push({ t: T.RBrack });
i++;
continue;
}
if (c === "{") {
tokens.push({ t: T.LBrace });
i++;
continue;
}
if (c === "}") {
tokens.push({ t: T.RBrace });
i++;
continue;
}
if (c === ",") {
tokens.push({ t: T.Comma });
i++;
continue;
}
if (c === ":") {
tokens.push({ t: T.Colon });
i++;
continue;
}
// ── Equals / EqEq ─────────────────────────────────────────────────
if (c === "=") {
if (i + 1 < n && src[i + 1] === "=") {
tokens.push({ t: T.EqEq });
i += 2;
} else {
tokens.push({ t: T.Equals });
i++;
}
continue;
}
// ── Not / NotEq ───────────────────────────────────────────────────
if (c === "!") {
if (i + 1 < n && src[i + 1] === "=") {
tokens.push({ t: T.NotEq });
i += 2;
} else {
tokens.push({ t: T.Not });
i++;
}
continue;
}
// ── Greater / GreaterEq ───────────────────────────────────────────
if (c === ">") {
if (i + 1 < n && src[i + 1] === "=") {
tokens.push({ t: T.GreaterEq });
i += 2;
} else {
tokens.push({ t: T.Greater });
i++;
}
continue;
}
// ── Less / LessEq ────────────────────────────────────────────────
if (c === "<") {
if (i + 1 < n && src[i + 1] === "=") {
tokens.push({ t: T.LessEq });
i += 2;
} else {
tokens.push({ t: T.Less });
i++;
}
continue;
}
// ── And (&&) ──────────────────────────────────────────────────────
if (c === "&") {
if (i + 1 < n && src[i + 1] === "&") {
tokens.push({ t: T.And });
i += 2;
} else {
tokens.push({ t: T.And });
i++;
}
continue;
}
// ── Or (||) ───────────────────────────────────────────────────────
if (c === "|") {
if (i + 1 < n && src[i + 1] === "|") {
tokens.push({ t: T.Or });
i += 2;
} else {
tokens.push({ t: T.Or });
i++;
}
continue;
}
// ── Dot ───────────────────────────────────────────────────────────
if (c === ".") {
tokens.push({ t: T.Dot });
i++;
continue;
}
// ── Question ──────────────────────────────────────────────────────
if (c === "?") {
tokens.push({ t: T.Question });
i++;
continue;
}
// ── Plus ──────────────────────────────────────────────────────────
if (c === "+") {
tokens.push({ t: T.Plus });
i++;
continue;
}
// ── Star ──────────────────────────────────────────────────────────
if (c === "*") {
tokens.push({ t: T.Star });
i++;
continue;
}
// ── Slash ─────────────────────────────────────────────────────────
if (c === "/") {
tokens.push({ t: T.Slash });
i++;
continue;
}
// ── Percent ───────────────────────────────────────────────────────
if (c === "%") {
tokens.push({ t: T.Percent });
i++;
continue;
}
// ── String literal: "..." ─────────────────────────────────────────
if (c === '"') {
const start = i;
i++; // skip opening quote
let isClosed = false;
// Fast-forward to the closing quote, respecting escapes
while (i < n) {
if (src[i] === "\\") {
i += 2; // skip backslash and the escaped character
} else if (src[i] === '"') {
i++; // include the closing quote
isClosed = true;
break;
} else {
i++;
}
}
const rawString = src.slice(start, i);
try {
// Let JavaScript's native JSON parser handle all unescaping (\n, \t, \uXXXX, etc.)
// If the string is incomplete (streaming), we add a closing quote to parse what we have so far.
const validJsonString = isClosed ? rawString : rawString + '"';
tokens.push({ t: T.Str, v: JSON.parse(validJsonString) });
} catch {
// Fallback if JSON.parse fails (e.g., malformed unicode escape during streaming)
// Strip the quotes and return the raw text so the UI doesn't crash
const stripped = rawString.replace(/^"|"$/g, "");
tokens.push({ t: T.Str, v: stripped });
}
continue;
}
// ── String literal: '...' (single quotes) ────────────────────────
if (c === "'") {
i++; // skip opening quote
let result = "";
let isClosed = false;
while (i < n) {
if (src[i] === "\\") {
i++; // skip backslash
if (i < n) {
const esc = src[i];
if (esc === "'") result += "'";
else if (esc === "\\") result += "\\";
else if (esc === "n") result += "\n";
else if (esc === "t") result += "\t";
else result += esc; // pass through other escaped chars
i++;
}
} else if (src[i] === "'") {
i++; // skip closing quote
isClosed = true;
break;
} else {
result += src[i];
i++;
}
}
void isClosed; // consumed for streaming parity with double-quote path
tokens.push({ t: T.Str, v: result });
continue;
}
// ── Minus: negative number literal or subtraction operator ────────
if (c === "-") {
const prev = tokens.length > 0 ? tokens[tokens.length - 1] : null;
const afterValue =
prev != null &&
(prev.t === T.Num ||
prev.t === T.Str ||
prev.t === T.Ident ||
prev.t === T.Type ||
prev.t === T.RParen ||
prev.t === T.RBrack ||
prev.t === T.True ||
prev.t === T.False ||
prev.t === T.Null ||
prev.t === T.StateVar ||
prev.t === T.BuiltinCall);
if (!afterValue && i + 1 < n && src[i + 1] >= "0" && src[i + 1] <= "9") {
// Negative number literal — fall through to number parsing below
} else {
// Binary subtraction operator (or unary minus handled by expression parser)
tokens.push({ t: T.Minus });
i++;
continue;
}
}
// ── Number literal: 42, -3, 1.5, 1e10 ────────────────────────────
const isDigit = c >= "0" && c <= "9";
const isNegDigit = c === "-" && i + 1 < n && src[i + 1] >= "0" && src[i + 1] <= "9";
if (isDigit || isNegDigit) {
const start = i;
if (src[i] === "-") i++; // optional minus
while (i < n && src[i] >= "0" && src[i] <= "9") i++; // integer part
if (i < n && src[i] === "." && i + 1 < n && src[i + 1] >= "0" && src[i + 1] <= "9") {
// optional decimal — only if a digit follows the dot
i++;
while (i < n && src[i] >= "0" && src[i] <= "9") i++;
}
if (i < n && (src[i] === "e" || src[i] === "E")) {
// optional exponent
i++;
if (i < n && (src[i] === "+" || src[i] === "-")) i++;
while (i < n && src[i] >= "0" && src[i] <= "9") i++;
}
tokens.push({ t: T.Num, v: +src.slice(start, i) });
continue;
}
// ── State variable: $identifier ───────────────────────────────────
if (
c === "$" &&
i + 1 < n &&
((src[i + 1] >= "a" && src[i + 1] <= "z") ||
(src[i + 1] >= "A" && src[i + 1] <= "Z") ||
src[i + 1] === "_")
) {
const start = i;
i++; // skip $
while (
i < n &&
((src[i] >= "a" && src[i] <= "z") ||
(src[i] >= "A" && src[i] <= "Z") ||
(src[i] >= "0" && src[i] <= "9") ||
src[i] === "_")
)
i++;
tokens.push({ t: T.StateVar, v: src.slice(start, i) });
continue;
}
// ── Keyword or identifier ─────────────────────────────────────────
const isAlpha = (c >= "a" && c <= "z") || (c >= "A" && c <= "Z") || c === "_";
if (isAlpha) {
const start = i;
while (
i < n &&
((src[i] >= "a" && src[i] <= "z") ||
(src[i] >= "A" && src[i] <= "Z") ||
(src[i] >= "0" && src[i] <= "9") ||
src[i] === "_")
)
i++;
const word = src.slice(start, i);
if (word === "true") {
tokens.push({ t: T.True });
continue;
}
if (word === "false") {
tokens.push({ t: T.False });
continue;
}
if (word === "null") {
tokens.push({ t: T.Null });
continue;
}
// PascalCase → component type name; lowercase → variable reference
const kind = c >= "A" && c <= "Z" ? T.Type : T.Ident;
tokens.push({ t: kind, v: word });
continue;
}
// ── Builtin call: @identifier ───────────────────────────────────
if (
c === "@" &&
i + 1 < n &&
((src[i + 1] >= "a" && src[i + 1] <= "z") ||
(src[i + 1] >= "A" && src[i + 1] <= "Z") ||
src[i + 1] === "_")
) {
i++; // skip @
const start = i;
while (
i < n &&
((src[i] >= "a" && src[i] <= "z") ||
(src[i] >= "A" && src[i] <= "Z") ||
(src[i] >= "0" && src[i] <= "9") ||
src[i] === "_")
)
i++;
tokens.push({ t: T.BuiltinCall, v: src.slice(start, i) });
continue;
}
i++; // skip any other character (e.g. #, emojis)
}
tokens.push({ t: T.EOF });
return tokens;
}