Skip to content

The Open Standard for Generative UI

25.2 KB
Raw Blame
import {
  captureParserParseException,
  captureParserParseResult,
  prepareParserParseTelemetry,
} from "../telemetry/runtime";
import type { ASTNode, Statement } from "./ast";
import { isASTNode, walkAST } from "./ast";
import { ACTION_DEFS, BUILTIN_CALLS, isBuiltin, isReservedCall, RESERVED_CALLS } from "./builtins";
import { parseExpression } from "./expressions";
import { tokenize } from "./lexer";
import { materializeValue, nameRawCallArgs } from "./materialize";
import { autoClose, split, type RawStmt } from "./statements";
import { T } from "./tokens";
import {
  isElementNode,
  type ElementNode,
  type JSONSchemaDef,
  type LibraryJSONSchema,
  type MaterializeCtx,
  type MutationStatementInfo,
  type ParamDef,
  type ParamMap,
  type ParseResult,
  type QueryStatementInfo,
  type ValidationError,
} from "./types";
import { getSchemaDefaultValue, pushValidationIssue } from "./validation";

// ─────────────────────────────────────────────────────────────────────────────
// Result building
// ─────────────────────────────────────────────────────────────────────────────

function emptyResult(incomplete = true): ParseResult {
  return {
    root: null,
    meta: {
      incomplete,
      unresolved: [],
      orphaned: [],
      statementCount: 0,
      errors: [],
    },
    stateDeclarations: {},
    queryStatements: [],
    mutationStatements: [],
  };
}

/**
 * Walk an AST node to collect all StateRef ($variable) names referenced
 * within. Used at parse time to pre-compute per-query state dependencies.
 */
export function collectQueryDeps(node: unknown): string[] {
  if (!isASTNode(node)) return [];
  const refs = new Set<string>();
  walkAST(node, (current) => {
    if (current.k === "StateRef") refs.add(current.n);
  });
  return [...refs];
}

/**
 * Classify a raw statement + parsed expression into a typed Statement.
 * Determined at parse time from token type + expression shape.
 */
function classifyStatement(raw: RawStmt, expr: ASTNode): Statement {
  // Query(...) → query declaration — check BEFORE $var to handle `$foo = Query(...)` correctly
  if (expr.k === "Comp" && expr.name === RESERVED_CALLS.Query) {
    const deps = collectQueryDeps(expr.args[1]);
    return {
      kind: "query",
      id: raw.id,
      call: { callee: RESERVED_CALLS.Query, args: expr.args },
      expr,
      deps: deps.length > 0 ? deps : undefined,
    };
  }
  // Mutation(...) → mutation declaration
  if (expr.k === "Comp" && expr.name === RESERVED_CALLS.Mutation) {
    return {
      kind: "mutation",
      id: raw.id,
      call: { callee: RESERVED_CALLS.Mutation, args: expr.args },
      expr,
    };
  }
  // $variables → state declaration
  if (raw.idTokenType === T.StateVar) {
    return { kind: "state", id: raw.id, init: expr };
  }
  // Everything else → value declaration
  return { kind: "value", id: raw.id, expr };
}

/**
 * Extract typed statements from the symbol table.
 * State defaults are materialized to plain values (no raw AST in output).
 */
/** A `$name` at the very end of streamed text, which may still be growing (`$d` of `$dishes`). */
interface PartialRef {
  name: string;
  stmts: ReadonlySet<Statement>;
}

function extractStatements(
  stmts: Statement[],
  ctx: MaterializeCtx,
  partialRef?: PartialRef,
): {
  stateDeclarations: Record<string, unknown>;
  queryStatements: QueryStatementInfo[];
  mutationStatements: MutationStatementInfo[];
} {
  const stateDeclarations: Record<string, unknown> = {};
  const queryStatements: QueryStatementInfo[] = [];
  const mutationStatements: MutationStatementInfo[] = [];

  for (const stmt of stmts) {
    switch (stmt.kind) {
      case "state":
        stateDeclarations[stmt.id] = materializeValue(stmt.init, ctx);
        break;
      case "query":
        nameRawCallArgs(stmt.call.args, ctx, stmt.id);
        queryStatements.push({
          statementId: stmt.id,
          toolAST: stmt.call.args[0] ?? null,
          argsAST: stmt.call.args[1] ?? null,
          defaultsAST: stmt.call.args[2] ?? null,
          refreshAST: stmt.call.args[3] ?? null,
          deps: stmt.deps,
          complete: true,
        });
        break;
      case "mutation":
        nameRawCallArgs(stmt.call.args, ctx, stmt.id);
        mutationStatements.push({
          statementId: stmt.id,
          toolAST: stmt.call.args[0] ?? null,
          argsAST: stmt.call.args[1] ?? null,
        });
        break;
    }
  }

  // Auto-declare: any $var referenced in code but not explicitly declared → null default.
  // collectQueryDeps already walks AST for StateRef nodes — reuse it here.
  for (const stmt of stmts) {
    const nodes =
      stmt.kind === "state"
        ? [stmt.init]
        : stmt.kind === "value"
          ? [stmt.expr]
          : stmt.kind === "query" || stmt.kind === "mutation"
            ? stmt.call.args
            : [];
    for (const node of nodes) {
      for (const dep of collectQueryDeps(node)) {
        if (dep === partialRef?.name && partialRef.stmts.has(stmt)) continue;
        if (!(dep in stateDeclarations)) {
          stateDeclarations[dep] = null;
        }
      }
    }
  }

  return { stateDeclarations, queryStatements, mutationStatements };
}

const DEFAULT_ROOT_STATEMENT_ID = "root";

/** A call to a library component: not an `@` function, built-in, Query or Mutation. */
function isComponentCall(node: ASTNode | undefined): node is Extract<ASTNode, { k: "Comp" }> {
  return node?.k === "Comp" && !node.fn && !isBuiltin(node.name) && !isReservedCall(node.name);
}

/** Entry: `root`, else the first statement that calls the root component. */
function pickEntry(
  stmtMap: Map<string, Statement>,
  rootName?: string,
): { id: string; fallback: boolean } | null {
  if (stmtMap.has(DEFAULT_ROOT_STATEMENT_ID))
    return { id: DEFAULT_ROOT_STATEMENT_ID, fallback: false };
  for (const stmt of stmtMap.values()) {
    if (stmt.kind === "value" && isComponentCall(stmt.expr) && stmt.expr.name === rootName)
      return { id: stmt.id, fallback: true };
  }
  return null;
}

/** Follow refs to the entry's value; a component call that did not render was dropped by validation. */
function resolvesToComponentCall(node: ASTNode | undefined, syms: Map<string, ASTNode>): boolean {
  const seen = new Set<string>();
  while (node?.k === "Ref" && !seen.has(node.n)) {
    seen.add(node.n);
    node = syms.get(node.n);
  }
  return isComponentCall(node);
}

/** True when a runtime expression can produce a component, e.g. `$t ? A : B`. */
function containsComponent(node: ASTNode): boolean {
  let found = false;
  walkAST(node, (current) => {
    if (isComponentCall(current)) found = true;
  });
  return found;
}

function buildResult(
  stmtMap: Map<string, Statement>,
  typedStmts: Statement[],
  wasIncomplete: boolean,
  stmtCount: number,
  cat: ParamMap | undefined,
  rootName?: string,
  partialRef?: PartialRef,
): ParseResult {
  const entry = pickEntry(stmtMap, rootName);

  const syms = new Map<string, ASTNode>();
  for (const [id, stmt] of stmtMap) {
    syms.set(id, stmt.kind === "state" ? stmt.init : stmt.expr);
  }
  const unres: string[] = [];
  const errors: ValidationError[] = [];
  // Track orphaned: start with value statement IDs, delete as they're reached.
  // Exclude root, state ($var), query, and mutation declarations — they're consumed separately.
  const unreached = new Set<string>();
  for (const [id, stmt] of stmtMap) {
    if (id === entry?.id) continue;
    if (stmt.kind === "state" || stmt.kind === "query" || stmt.kind === "mutation") continue;
    unreached.add(id);
  }
  const ctx: MaterializeCtx = {
    syms,
    cat,
    errors,
    unres,
    visited: new Set(),
    partial: wasIncomplete,
    currentStatementId: entry?.id,
    unreached,
  };

  let root: ElementNode | null = null;
  let rootExpression: ASTNode | undefined;
  if (entry) {
    const materialized = materializeValue(syms.get(entry.id)!, ctx);
    if (isElementNode(materialized)) {
      root = materialized;
      root.statementId = entry.id;
    } else if (isASTNode(materialized) && containsComponent(materialized)) {
      rootExpression = materialized;
    }
  }

  const { stateDeclarations, queryStatements, mutationStatements } = extractStatements(
    typedStmts,
    ctx,
    partialRef,
  );

  // Streaming consumers drop parser errors until the stream ends, so this reads as a stream-end check.
  ctx.currentStatementId = entry?.id;
  const component = rootName ?? "";
  if (!entry) {
    pushValidationIssue(ctx, component, "", { code: "no-root", reason: "missing" });
  } else if (!root && !rootExpression && !resolvesToComponentCall(syms.get(entry.id), syms)) {
    pushValidationIssue(ctx, component, "", { code: "no-root", reason: "not-component" });
  } else if (entry.fallback) {
    pushValidationIssue(ctx, component, "", {
      code: "no-root",
      reason: root ? "fallback" : "missing",
    });
  }

  // With no entry nothing is reachable; `no-root` alone describes the problem.
  const orphaned = entry ? [...unreached] : [];

  return {
    root,
    ...(rootExpression ? { rootExpression } : {}),
    meta: {
      incomplete: wasIncomplete,
      unresolved: unres,
      orphaned,
      statementCount: stmtCount,
      errors: errors,
    },
    stateDeclarations,
    queryStatements,
    mutationStatements,
  };
}

// ─────────────────────────────────────────────────────────────────────────────
// Public API
// ─────────────────────────────────────────────────────────────────────────────

function skipString(input: string, start: number): number {
  if (input[start] !== '"') return start;
  let i = start + 1;
  while (i < input.length) {
    const c = input[i];
    if (c === "\\") {
      i += 2; // skip escape character and the escaped character
    } else if (c === '"') {
      return i + 1; // return index after the closing quote
    } else {
      i++;
    }
  }
  return i; // return length if string was unclosed
}

/** Extract code from markdown fences, or return as-is if no fences found.
 *  String-context-aware: skips ``` inside double-quoted strings. */
export function stripFences(input: string): string {
  const blocks: string[] = [];
  let i = 0;

  while (i < input.length) {
    // Scan for opening ``` while tracking string context
    let fenceStart = -1;
    while (i < input.length) {
      const nextI = skipString(input, i);
      if (nextI > i) {
        i = nextI;
        continue;
      }

      const c = input[i];
      if (
        c === "`" &&
        i + 1 < input.length &&
        input[i + 1] === "`" &&
        i + 2 < input.length &&
        input[i + 2] === "`"
      ) {
        fenceStart = i;
        break;
      }
      i++;
    }

    if (fenceStart === -1) break;

    // Skip language tag until newline
    let j = fenceStart + 3;
    while (j < input.length && input[j] !== "\n") j++;
    if (j >= input.length) {
      // No newline after opening fence (streaming) — take everything after fence marker + lang tag
      blocks.push(input.slice(fenceStart + 3).replace(/^[^\n]*\n?/, ""));
      i = input.length;
      break;
    }
    j++; // skip the newline

    // Scan for closing ``` while tracking string context
    let closePos = -1;
    let k = j;
    while (k < input.length) {
      const nextK = skipString(input, k);
      if (nextK > k) {
        k = nextK;
        continue;
      }
      const c = input[k];
      if (
        c === "`" &&
        k + 1 < input.length &&
        input[k + 1] === "`" &&
        k + 2 < input.length &&
        input[k + 2] === "`"
      ) {
        closePos = k;
        break;
      }
      k++;
    }

    if (closePos !== -1) {
      blocks.push(input.slice(j, closePos));
      i = closePos + 3;
    } else {
      // No closing fence found (streaming) — take everything after opening fence
      blocks.push(input.slice(j));
      i = input.length;
    }
  }

  if (blocks.length > 0) return blocks.join("\n");

  // Fallback: if input starts with ``` but wasn't matched (e.g. no newline after fence)
  if (input.startsWith("```")) {
    let j = 3;
    while (j < input.length && input[j] !== "\n") j++;
    const start = j < input.length ? j + 1 : 3;
    // Try to strip trailing ```
    const body = input.slice(start);
    const trailingFence = body.lastIndexOf("```");
    if (trailingFence !== -1) {
      return body.slice(0, trailingFence);
    }
    return body;
  }

  return input;
}

/** Strip // and # line comments outside of strings (handles both " and ' delimiters). */
function stripComments(input: string): string {
  let inStr: false | '"' | "'" = false;
  return input
    .split("\n")
    .map((line) => {
      for (let i = 0; i < line.length; i++) {
        const c = line[i];
        if (inStr) {
          if (c === "\\" && i + 1 < line.length) {
            i++; // skip escaped char
            continue;
          }
          if (c === inStr) inStr = false;
          continue;
        }
        if (c === '"' || c === "'") {
          inStr = c;
          continue;
        }
        // // style comments
        if (c === "/" && line[i + 1] === "/") {
          return line.substring(0, i).trimEnd();
        }
        // # style comments (Python/YAML style — LLMs sometimes use these)
        if (c === "#") {
          return line.substring(0, i).trimEnd();
        }
      }
      return line;
    })
    .join("\n");
}

/** Clean LLM response: strip fences, comments, whitespace. */
function preprocess(input: string): string {
  return stripComments(stripFences(input.trim())).trim();
}

/**
 * Parse a complete openui-lang string in one pass.
 *
 * @param input  - Full openui-lang source text (may be partial/streaming)
 * @param cat    - Param map for positional-arg → named-prop mapping
 * @returns      ParseResult with root ElementNode (or null) and metadata
 */
export function parse(input: string, cat: ParamMap, rootName?: string): ParseResult {
  const trimmed = preprocess(input);
  if (!trimmed) return emptyResult();

  const { text, wasIncomplete } = autoClose(trimmed);
  const stmts = split(tokenize(text));
  if (!stmts.length) return emptyResult(wasIncomplete);

  const stmtMap = new Map<string, Statement>();
  for (const s of stmts) {
    const expr = parseExpression(s.tokens);
    const stmt = classifyStatement(s, expr);
    stmtMap.set(s.id, stmt);
  }
  // Derive from map to deduplicate — Map.set overwrites duplicates
  const typedStmts = [...stmtMap.values()];

  return buildResult(stmtMap, typedStmts, wasIncomplete, stmtMap.size, cat, rootName);
}

export interface StreamParser {
  /** Feed the next SSE/stream chunk and get the latest ParseResult.
   *  Pass `complete` once the stream has ended. */
  push(chunk: string, complete?: boolean): ParseResult;
  /** Set the full text — diffs against internal buffer, pushes only the delta.
   *  Resets automatically if the text was replaced (not appended).
   *  Pass `complete` once the stream has ended. */
  set(fullText: string, complete?: boolean): ParseResult;
  /** Get the latest ParseResult without consuming new data. */
  getResult(complete?: boolean): ParseResult;
}

export function createStreamParser(cat: ParamMap, rootName?: string): StreamParser {
  let buf = ""; // raw accumulated input (kept for set() diffing)
  // Preprocessed view of `buf` (fences + comments stripped, same as parse()'s
  // preprocess). The completed-statement scan runs over THIS, never the raw
  // buffer — so leading markdown prose / ```fences``` (e.g. an apostrophe in
  // "You're …" before the program) can't corrupt statement-boundary detection.
  let cleaned = "";
  let completedEnd = 0; // watermark: how far into `cleaned` is already completed
  const completedStmtMap = new Map<string, Statement>();

  let completedCount = 0;

  function addStmt(text: string) {
    // `text` is sliced from `cleaned`, so it's already fence/comment-free.
    const t = text.trim();
    if (!t) return;
    for (const s of split(tokenize(t))) {
      const expr = parseExpression(s.tokens);
      const stmt = classifyStatement(s, expr);
      completedStmtMap.set(s.id, stmt);
      completedCount++;
    }
  }

  // Recompute `cleaned` from `buf`. If the already-completed prefix is no longer
  // a prefix of the new cleaned text (e.g. an opening ```fence``` just appeared
  // and shifted the stripped output), the watermark + cache are stale, so reset
  // and re-scan. When the prefix is stable (the common streaming case) the cache
  // is kept; pending text is overlaid on a copy when building the current result.
  function refreshCleaned() {
    const next = preprocess(buf);
    if (!next.startsWith(cleaned.slice(0, completedEnd))) {
      completedEnd = 0;
      completedStmtMap.clear();
      completedCount = 0;
    }
    cleaned = next;
  }

  function scanNewCompleted(): number {
    let depth = 0,
      ternaryDepth = 0,
      inStr: false | '"' | "'" = false,
      esc = false;
    let stmtStart = completedEnd;

    for (let i = completedEnd; i < cleaned.length; i++) {
      const c = cleaned[i];
      if (esc) {
        esc = false;
        continue;
      }
      if (c === "\\" && inStr) {
        esc = true;
        continue;
      }
      if (inStr) {
        if (c === inStr) inStr = false;
        continue;
      }
      if (c === '"' || c === "'") {
        inStr = c;
        continue;
      }

      if (c === "(" || c === "[" || c === "{") depth++;
      else if (c === ")" || c === "]" || c === "}") depth = Math.max(0, depth - 1);
      // Track ternary ? and : at bracket depth 0 (colons inside {} are object key separators)
      else if (c === "?" && depth === 0) ternaryDepth++;
      else if (c === ":" && depth === 0 && ternaryDepth > 0) ternaryDepth--;
      else if (c === "\n" && depth <= 0 && ternaryDepth <= 0) {
        // Before splitting, look ahead past whitespace to see if the next
        // meaningful character is `?` or `:` — ternary continuation.
        let peek = i + 1;
        while (
          peek < cleaned.length &&
          (cleaned[peek] === " " ||
            cleaned[peek] === "\t" ||
            cleaned[peek] === "\r" ||
            cleaned[peek] === "\n")
        )
          peek++;
        if (
          peek < cleaned.length &&
          (cleaned[peek] === "?" || (cleaned[peek] === ":" && ternaryDepth > 0))
        ) {
          continue; // ternary continuation — don't split
        }
        // Depth-0 newline = end of a statement
        const t = cleaned.slice(stmtStart, i).trim();
        if (t) addStmt(t);
        stmtStart = i + 1; // next statement begins after this newline
        completedEnd = i + 1; // advance the "already processed" watermark
      }
    }

    return stmtStart; // start of the current pending (incomplete) statement
  }

  function currentResult(complete = false): ParseResult {
    refreshCleaned();
    const pendingStart = scanNewCompleted();
    const pendingText = cleaned.slice(pendingStart).trim();

    // No pending text — all statements are complete
    if (!pendingText) {
      if (completedCount === 0) return emptyResult();
      return buildResult(
        completedStmtMap,
        [...completedStmtMap.values()],
        false,
        completedCount,
        cat,
        rootName,
      );
    }

    // `cleaned` is already preprocessed (fences + comments stripped); just
    // autoclose the incomplete trailing statement so it's syntactically valid.
    const { text: closed, wasIncomplete } = autoClose(pendingText);
    const stmts = split(tokenize(closed));

    if (!stmts.length) {
      if (completedCount === 0) return emptyResult(wasIncomplete);
      return buildResult(
        completedStmtMap,
        [...completedStmtMap.values()],
        wasIncomplete,
        completedCount,
        cat,
        rootName,
      );
    }

    // New IDs can render progressively; existing IDs are replaced only when
    // the pending expression no longer needs automatic closing.
    const allStmtMap = new Map(completedStmtMap);
    const pendingStmts = new Set<Statement>();
    for (const s of stmts) {
      if (completedStmtMap.has(s.id) && wasIncomplete) continue;
      const expr = parseExpression(s.tokens);
      const stmt = classifyStatement(s, expr);
      allStmtMap.set(s.id, stmt);
      pendingStmts.add(stmt);
    }
    // A `$name` ending the raw text may still be growing (`$d` of `$dishes`): skip it unless
    // the stream is complete or the same name already appears in full in the pending text.
    const tail = complete ? undefined : /\$\w+$/.exec(buf)?.[0];
    const tailName =
      tail &&
      pendingText.endsWith(tail) &&
      !new RegExp(`\\${tail}(?!\\w)`).test(pendingText.slice(0, -tail.length))
        ? tail
        : undefined;
    // Derive from map to deduplicate
    const allTypedStmts = [...allStmtMap.values()];

    return buildResult(
      allStmtMap,
      allTypedStmts,
      wasIncomplete,
      completedCount + stmts.length,
      cat,
      rootName,
      tailName ? { name: tailName, stmts: pendingStmts } : undefined,
    );
  }

  function reset() {
    buf = "";
    cleaned = "";
    completedEnd = 0;
    completedStmtMap.clear();
    completedCount = 0;
  }

  return {
    push(chunk, complete) {
      buf += chunk;
      return currentResult(complete);
    },
    set(fullText, complete) {
      if (fullText.length < buf.length || !fullText.startsWith(buf)) {
        reset();
      }
      const delta = fullText.slice(buf.length);
      if (delta) buf += delta;
      return currentResult(complete);
    },
    getResult: currentResult,
  };
}

export interface Parser {
  parse(input: string): ParseResult;
}

type ActionDefs = Map<string, unknown>;

// {$ref: "#/$defs/ActionExpression"} -> its body, so action slots validate as inline unions
function inlineActionRefs(node: unknown, defs: ActionDefs): unknown {
  if (!node || typeof node !== "object") return node;
  const ref = (node as { $ref?: unknown }).$ref;
  if (typeof ref === "string") return defs.has(ref) ? inlineActionRefs(defs.get(ref), defs) : node;
  const copy = (Array.isArray(node) ? [...node] : { ...node }) as Record<string, unknown>;
  let changed = false;
  for (const [key, value] of Object.entries(copy)) {
    copy[key] = inlineActionRefs(value, defs);
    changed ||= copy[key] !== value;
  }
  return changed ? copy : node;
}

function compileParams(def: JSONSchemaDef, defs: ActionDefs): ParamDef[] {
  const properties = def.properties ?? {};
  const required = def.required ?? [];
  return Object.keys(properties).map((key) => ({
    name: key,
    required: required.includes(key),
    defaultValue: getSchemaDefaultValue(properties[key]),
    schema: defs.size ? inlineActionRefs(properties[key], defs) : properties[key],
  }));
}

export function compileSchema(schema: LibraryJSONSchema): ParamMap {
  const map: ParamMap = new Map();
  const components = schema.properties && new Set(Object.keys(schema.properties));
  // Action $defs (ActionExpression, OpenUrl, custom actions) are data shapes, never components
  const defs: ActionDefs = new Map();
  for (const name of [...Object.keys(ACTION_DEFS), ...Object.keys(schema.actions ?? {})]) {
    const body = schema.$defs?.[name];
    if (body && !components?.has(name)) defs.set(`#/$defs/${name}`, body);
  }
  for (const [name, def] of Object.entries(schema.$defs ?? {})) {
    // Skip non-component defs, e.g. zod's hoisted recursive schemas (__schema0).
    if (components ? !components.has(name) : defs.has(`#/$defs/${name}`)) continue;
    map.set(name, { params: compileParams(def, defs) });
  }
  // The call registry: the built-ins plus the library's functions and actions
  map.callDefs = new Map(BUILTIN_CALLS);
  for (const [name, fn] of Object.entries(schema.functions ?? {})) {
    map.callDefs.set(name, { kind: "function", params: compileParams(fn.params, defs) });
  }
  for (const [name, action] of Object.entries(schema.actions ?? {})) {
    map.callDefs.set(name, { kind: "action", params: compileParams(action.params, defs) });
  }
  return map;
}

/**
 * Create a parser from a library JSON Schema document.
 * Pass `library.toJSONSchema()` to get the schema.
 *
 * @example
 * ```ts
 * const parser = createParser(library.toJSONSchema());
 * const result = parser.parse(openuiLangString);
 * ```
 */
export function createParser(schema: LibraryJSONSchema, rootName?: string): Parser {
  const paramMap = compileSchema(schema);
  return {
    parse(input: string): ParseResult {
      const telemetry = prepareParserParseTelemetry();
      try {
        const result = parse(input, paramMap, rootName);
        captureParserParseResult(telemetry, result);
        return result;
      } catch (error) {
        captureParserParseException(telemetry);
        throw error;
      }
    },
  };
}

/**
 * Create a streaming parser from a library JSON Schema document.
 * Pass `library.toJSONSchema()` to get the schema.
 */
export function createStreamingParser(schema: LibraryJSONSchema, rootName?: string): StreamParser {
  return createStreamParser(compileSchema(schema), rootName);
}