Skip to content

The Open Standard for Generative UI

12.6 KB
Raw Blame
// ─────────────────────────────────────────────────────────────────────────────
// Lexer for openui-lang
// ─────────────────────────────────────────────────────────────────────────────

import { T, type Token } from "./tokens";

/**
 * Tokenize an openui-lang source string into a flat token array.
 *
 * Handles all token types: identifiers, literals, operators,
 * state variables ($name), dot access, ternary.
 */
export function tokenize(src: string): Token[] {
  const tokens: Token[] = [];
  let i = 0;
  const n = src.length;

  while (i < n) {
    // Skip horizontal whitespace (not newlines — they're significant)
    while (i < n && (src[i] === " " || src[i] === "\t" || src[i] === "\r")) i++;
    if (i >= n) break;

    const c = src[i];

    // ── Newline ────────────────────────────────────────────────────────
    if (c === "\n") {
      tokens.push({ t: T.Newline });
      i++;
      continue;
    }

    // ── Single-character punctuation (brackets, comma, colon) ─────────
    if (c === "(") {
      tokens.push({ t: T.LParen });
      i++;
      continue;
    }
    if (c === ")") {
      tokens.push({ t: T.RParen });
      i++;
      continue;
    }
    if (c === "[") {
      tokens.push({ t: T.LBrack });
      i++;
      continue;
    }
    if (c === "]") {
      tokens.push({ t: T.RBrack });
      i++;
      continue;
    }
    if (c === "{") {
      tokens.push({ t: T.LBrace });
      i++;
      continue;
    }
    if (c === "}") {
      tokens.push({ t: T.RBrace });
      i++;
      continue;
    }
    if (c === ",") {
      tokens.push({ t: T.Comma });
      i++;
      continue;
    }
    if (c === ":") {
      tokens.push({ t: T.Colon });
      i++;
      continue;
    }

    // ── Equals / EqEq ─────────────────────────────────────────────────
    if (c === "=") {
      if (i + 1 < n && src[i + 1] === "=") {
        tokens.push({ t: T.EqEq });
        i += 2;
      } else {
        tokens.push({ t: T.Equals });
        i++;
      }
      continue;
    }

    // ── Not / NotEq ───────────────────────────────────────────────────
    if (c === "!") {
      if (i + 1 < n && src[i + 1] === "=") {
        tokens.push({ t: T.NotEq });
        i += 2;
      } else {
        tokens.push({ t: T.Not });
        i++;
      }
      continue;
    }

    // ── Greater / GreaterEq ───────────────────────────────────────────
    if (c === ">") {
      if (i + 1 < n && src[i + 1] === "=") {
        tokens.push({ t: T.GreaterEq });
        i += 2;
      } else {
        tokens.push({ t: T.Greater });
        i++;
      }
      continue;
    }

    // ── Less / LessEq ────────────────────────────────────────────────
    if (c === "<") {
      if (i + 1 < n && src[i + 1] === "=") {
        tokens.push({ t: T.LessEq });
        i += 2;
      } else {
        tokens.push({ t: T.Less });
        i++;
      }
      continue;
    }

    // ── And (&&) ──────────────────────────────────────────────────────
    if (c === "&") {
      if (i + 1 < n && src[i + 1] === "&") {
        tokens.push({ t: T.And });
        i += 2;
      } else {
        tokens.push({ t: T.And });
        i++;
      }
      continue;
    }

    // ── Or (||) ───────────────────────────────────────────────────────
    if (c === "|") {
      if (i + 1 < n && src[i + 1] === "|") {
        tokens.push({ t: T.Or });
        i += 2;
      } else {
        tokens.push({ t: T.Or });
        i++;
      }
      continue;
    }

    // ── Dot ───────────────────────────────────────────────────────────
    if (c === ".") {
      tokens.push({ t: T.Dot });
      i++;
      continue;
    }

    // ── Question ──────────────────────────────────────────────────────
    if (c === "?") {
      tokens.push({ t: T.Question });
      i++;
      continue;
    }

    // ── Plus ──────────────────────────────────────────────────────────
    if (c === "+") {
      tokens.push({ t: T.Plus });
      i++;
      continue;
    }

    // ── Star ──────────────────────────────────────────────────────────
    if (c === "*") {
      tokens.push({ t: T.Star });
      i++;
      continue;
    }

    // ── Slash ─────────────────────────────────────────────────────────
    if (c === "/") {
      tokens.push({ t: T.Slash });
      i++;
      continue;
    }

    // ── Percent ───────────────────────────────────────────────────────
    if (c === "%") {
      tokens.push({ t: T.Percent });
      i++;
      continue;
    }

    // ── String literal: "..." ─────────────────────────────────────────
    if (c === '"') {
      const start = i;
      i++; // skip opening quote
      let isClosed = false;
      // Fast-forward to the closing quote, respecting escapes
      while (i < n) {
        if (src[i] === "\\") {
          i += 2; // skip backslash and the escaped character
        } else if (src[i] === '"') {
          i++; // include the closing quote
          isClosed = true;
          break;
        } else {
          i++;
        }
      }
      const rawString = src.slice(start, i);
      try {
        // Let JavaScript's native JSON parser handle all unescaping (\n, \t, \uXXXX, etc.)
        // If the string is incomplete (streaming), we add a closing quote to parse what we have so far.
        const validJsonString = isClosed ? rawString : rawString + '"';
        tokens.push({ t: T.Str, v: JSON.parse(validJsonString) });
      } catch {
        // Fallback if JSON.parse fails (e.g., malformed unicode escape during streaming)
        // Strip the quotes and return the raw text so the UI doesn't crash
        const stripped = rawString.replace(/^"|"$/g, "");
        tokens.push({ t: T.Str, v: stripped });
      }
      continue;
    }

    // ── String literal: '...' (single quotes) ────────────────────────
    if (c === "'") {
      i++; // skip opening quote
      let result = "";
      let isClosed = false;
      while (i < n) {
        if (src[i] === "\\") {
          i++; // skip backslash
          if (i < n) {
            const esc = src[i];
            if (esc === "'") result += "'";
            else if (esc === "\\") result += "\\";
            else if (esc === "n") result += "\n";
            else if (esc === "t") result += "\t";
            else result += esc; // pass through other escaped chars
            i++;
          }
        } else if (src[i] === "'") {
          i++; // skip closing quote
          isClosed = true;
          break;
        } else {
          result += src[i];
          i++;
        }
      }
      void isClosed; // consumed for streaming parity with double-quote path
      tokens.push({ t: T.Str, v: result });
      continue;
    }

    // ── Minus: negative number literal or subtraction operator ────────
    if (c === "-") {
      const prev = tokens.length > 0 ? tokens[tokens.length - 1] : null;
      const afterValue =
        prev != null &&
        (prev.t === T.Num ||
          prev.t === T.Str ||
          prev.t === T.Ident ||
          prev.t === T.Type ||
          prev.t === T.RParen ||
          prev.t === T.RBrack ||
          prev.t === T.True ||
          prev.t === T.False ||
          prev.t === T.Null ||
          prev.t === T.StateVar ||
          prev.t === T.BuiltinCall);

      if (!afterValue && i + 1 < n && src[i + 1] >= "0" && src[i + 1] <= "9") {
        // Negative number literal — fall through to number parsing below
      } else {
        // Binary subtraction operator (or unary minus handled by expression parser)
        tokens.push({ t: T.Minus });
        i++;
        continue;
      }
    }

    // ── Number literal: 42, -3, 1.5, 1e10 ────────────────────────────
    const isDigit = c >= "0" && c <= "9";
    const isNegDigit = c === "-" && i + 1 < n && src[i + 1] >= "0" && src[i + 1] <= "9";
    if (isDigit || isNegDigit) {
      const start = i;
      if (src[i] === "-") i++; // optional minus
      while (i < n && src[i] >= "0" && src[i] <= "9") i++; // integer part
      if (i < n && src[i] === "." && i + 1 < n && src[i + 1] >= "0" && src[i + 1] <= "9") {
        // optional decimal — only if a digit follows the dot
        i++;
        while (i < n && src[i] >= "0" && src[i] <= "9") i++;
      }
      if (i < n && (src[i] === "e" || src[i] === "E")) {
        // optional exponent
        i++;
        if (i < n && (src[i] === "+" || src[i] === "-")) i++;
        while (i < n && src[i] >= "0" && src[i] <= "9") i++;
      }
      tokens.push({ t: T.Num, v: +src.slice(start, i) });
      continue;
    }

    // ── State variable: $identifier ───────────────────────────────────
    if (
      c === "$" &&
      i + 1 < n &&
      ((src[i + 1] >= "a" && src[i + 1] <= "z") ||
        (src[i + 1] >= "A" && src[i + 1] <= "Z") ||
        src[i + 1] === "_")
    ) {
      const start = i;
      i++; // skip $
      while (
        i < n &&
        ((src[i] >= "a" && src[i] <= "z") ||
          (src[i] >= "A" && src[i] <= "Z") ||
          (src[i] >= "0" && src[i] <= "9") ||
          src[i] === "_")
      )
        i++;
      tokens.push({ t: T.StateVar, v: src.slice(start, i) });
      continue;
    }

    // ── Keyword or identifier ─────────────────────────────────────────
    const isAlpha = (c >= "a" && c <= "z") || (c >= "A" && c <= "Z") || c === "_";
    if (isAlpha) {
      const start = i;
      while (
        i < n &&
        ((src[i] >= "a" && src[i] <= "z") ||
          (src[i] >= "A" && src[i] <= "Z") ||
          (src[i] >= "0" && src[i] <= "9") ||
          src[i] === "_")
      )
        i++;
      const word = src.slice(start, i);
      if (word === "true") {
        tokens.push({ t: T.True });
        continue;
      }
      if (word === "false") {
        tokens.push({ t: T.False });
        continue;
      }
      if (word === "null") {
        tokens.push({ t: T.Null });
        continue;
      }
      // PascalCase → component type name; lowercase → variable reference
      const kind = c >= "A" && c <= "Z" ? T.Type : T.Ident;
      tokens.push({ t: kind, v: word });
      continue;
    }

    // ── Builtin call: @identifier ───────────────────────────────────
    if (
      c === "@" &&
      i + 1 < n &&
      ((src[i + 1] >= "a" && src[i + 1] <= "z") ||
        (src[i + 1] >= "A" && src[i + 1] <= "Z") ||
        src[i + 1] === "_")
    ) {
      i++; // skip @
      const start = i;
      while (
        i < n &&
        ((src[i] >= "a" && src[i] <= "z") ||
          (src[i] >= "A" && src[i] <= "Z") ||
          (src[i] >= "0" && src[i] <= "9") ||
          src[i] === "_")
      )
        i++;
      tokens.push({ t: T.BuiltinCall, v: src.slice(start, i) });
      continue;
    }

    i++; // skip any other character (e.g. #, emojis)
  }

  tokens.push({ t: T.EOF });
  return tokens;
}