Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

13.8 KB
Raw Blame
/**
 * Shared helpers for the reference generators.
 *
 * docs.cua.ai compiles the committed .mdx with @mdx-js and shims only four
 * fumadocs-ui components (Callout, Card, Tabs, Steps), and it builds the nav
 * from explicit `meta.json` page lists. Generated pages therefore use plain
 * Markdown plus those components, carry the AUTO-GENERATED banner (the docs
 * block linter and link checker recognise it), and are written byte-for-byte
 * deterministically so `--check` is a plain string comparison.
 */

import * as fs from 'fs';
import * as path from 'path';
import { normalizeEmDashes } from '../prose-style';

export const REPO_ROOT = path.resolve(__dirname, '../../..');
export const DOCS_CONTENT = path.join(REPO_ROOT, 'docs', 'content', 'docs');

export const GENERATED_MARKER = 'AUTO-GENERATED FILE - DO NOT EDIT DIRECTLY';

/** Escapes MDX-significant characters outside inline code spans. */
export function escapeMdxText(value: string): string {
  return value
    .split(/(`[^`]*`)/g)
    .map((segment) => {
      if (segment.startsWith('`') && segment.endsWith('`') && segment.length > 1) {
        return segment;
      }
      return segment
        .replace(/\{/g, '{')
        .replace(/\}/g, '}')
        .replace(/</g, '&lt;')
        .replace(/>/g, '&gt;');
    })
    .join('');
}

/** A single-line, pipe-safe table cell. */
export function escapeTableCell(value: string): string {
  return escapeMdxText(value.replace(/\r?\n+/g, ' ').trim()).replace(/\|/g, '\\|');
}

/** Inline code that survives any backticks inside `value`. */
export function inlineCode(value: string): string {
  const runs = value.match(/`+/g) ?? [];
  const longest = runs.reduce((max, run) => Math.max(max, run.length), 0);
  const fence = '`'.repeat(longest + 1);
  const pad = value.startsWith('`') || value.endsWith('`') ? ' ' : '';
  return `${fence}${pad}${value}${pad}${fence}`;
}

/** A table-safe inline code span (pipes escaped). */
export function codeCell(value: string): string {
  return inlineCode(value.replace(/\r?\n+/g, ' ')).replace(/\|/g, '\\|');
}

/** A fenced code block whose fence is longer than any backtick run inside. */
export function codeFence(lang: string, code: string, meta = ''): string {
  const runs = code.match(/`{3,}/g) ?? [];
  const longest = runs.reduce((max, run) => Math.max(max, run.length), 2);
  const fence = '`'.repeat(longest + 1);
  const info = [lang, meta].filter(Boolean).join(' ');
  return `${fence}${info}\n${code.replace(/\n+$/, '')}\n${fence}`;
}

/**
 * The heading anchor rehype-slug (github-slugger) derives from plain text,
 * for ASCII headings: lower-case, punctuation dropped, spaces to hyphens.
 */
export function slug(text: string): string {
  return text
    .toLowerCase()
    .replace(/[^a-z0-9 _-]/g, '')
    .replace(/ /g, '-');
}

/** YAML frontmatter with JSON-quoted scalars (always valid YAML). */
export function frontmatter(title: string, description: string): string {
  return ['---', `title: ${JSON.stringify(title)}`, `description: ${JSON.stringify(description)}`, '---'].join(
    '\n'
  );
}

export interface GeneratedPage {
  title: string;
  description: string;
  /** The command that regenerates the page, e.g. `pnpm --dir docs docs:generate:cua-cli`. */
  generator: string;
  /** What the page is generated from, e.g. `cua dump-docs --type cli`. */
  source: string;
  /** Product version stamped into the banner. */
  version?: string;
  /** fumadocs-ui imports the page needs (Callout, Card, Tabs, Steps only). */
  components?: Array<'Callout' | 'Cards' | 'Tabs' | 'Steps'>;
  body: string;
}

const COMPONENT_IMPORTS: Record<string, string> = {
  Callout: "import { Callout } from 'fumadocs-ui/components/callout';",
  Cards: "import { Card, Cards } from 'fumadocs-ui/components/card';",
  Tabs: "import { Tab, Tabs } from 'fumadocs-ui/components/tabs';",
  Steps: "import { Step, Steps } from 'fumadocs-ui/components/steps';",
};

/** Renders a complete generated page: frontmatter, banner, imports, body. */
export function renderPage(page: GeneratedPage): string {
  const banner = [
    '{/*',
    `  ${GENERATED_MARKER}`,
    `  Generated by: ${page.generator}`,
    `  Source: ${page.source}`,
    ...(page.version ? [`  Version: ${page.version}`] : []),
    '*/}',
  ].join('\n');
  const imports = [...new Set(page.components ?? [])].sort().map((c) => COMPONENT_IMPORTS[c]);
  const parts = [frontmatter(page.title, page.description), banner];
  if (imports.length) parts.push(imports.join('\n'));
  parts.push(page.body.replace(/\n{3,}/g, '\n\n').trim());
  return normalizeEmDashes(parts.join('\n\n')) + '\n';
}

/** Language tabs; every panel is Markdown. `groupId` syncs tabs across a page. */
export function tabs(panels: Array<[label: string, markdown: string]>, groupId?: string): string {
  const items = panels.map(([label]) => `'${label.replace(/'/g, "\\'")}'`).join(', ');
  const group = groupId ? ` groupId="${groupId}" persist` : '';
  const out = [`<Tabs items={[${items}]}${group}>`];
  for (const [label, markdown] of panels) {
    out.push(`<Tab value="${label}">`, '', markdown.trim(), '', '</Tab>');
  }
  out.push('</Tabs>');
  return out.join('\n');
}

/** A `meta.json` with an explicit page list (the prod nav has no `...` rest). */
export function metaJson(title: string, pages: string[], extra: Record<string, unknown> = {}): string {
  return JSON.stringify({ title, ...extra, pages }, null, 2) + '\n';
}

/**
 * JSON with object keys sorted at every level: a stable snapshot even when the
 * producer's key order is not (Swift's JSONEncoder, for one).
 */
export function stableJson(value: unknown): string {
  const sort = (v: unknown): unknown => {
    if (Array.isArray(v)) return v.map(sort);
    if (v && typeof v === 'object') {
      return Object.fromEntries(
        Object.keys(v as Record<string, unknown>)
          .sort()
          .map((k) => [k, sort((v as Record<string, unknown>)[k])])
      );
    }
    return v;
  };
  return JSON.stringify(sort(value), null, 1) + '\n';
}

/** Whether `content` is a generated page (banner near the top). */
export function isGenerated(content: string): boolean {
  return content.slice(0, 600).includes(GENERATED_MARKER);
}

/**
 * Writes (or, with `checkOnly`, compares) every file in `files`, keyed by
 * absolute path. Generated pages found in `ownedDirs` but absent from `files`
 * are stale: removed on generate, reported on check. With `generator`, only
 * pages whose banner names that command count, so generators can share a
 * directory. Returns the drifted repo-relative paths.
 */
export function syncFiles(
  files: Map<string, string>,
  checkOnly: boolean,
  ownedDirs: string[] = [],
  generator?: string
): string[] {
  const drift: string[] = [];
  const rel = (p: string) => path.relative(REPO_ROOT, p).split(path.sep).join('/');
  for (const [file, content] of [...files].sort(([a], [b]) => a.localeCompare(b))) {
    const current = fs.existsSync(file) ? fs.readFileSync(file, 'utf-8') : null;
    if (current !== content) {
      drift.push(rel(file));
      if (!checkOnly) {
        fs.mkdirSync(path.dirname(file), { recursive: true });
        fs.writeFileSync(file, content);
      }
    }
  }
  for (const dir of ownedDirs) {
    if (!fs.existsSync(dir)) continue;
    for (const name of fs.readdirSync(dir).sort()) {
      const file = path.join(dir, name);
      if (!name.endsWith('.mdx') || files.has(file)) continue;
      const content = fs.readFileSync(file, 'utf-8');
      if (!isGenerated(content)) continue;
      if (generator && !content.slice(0, 600).includes(`Generated by: ${generator}`)) continue;
      drift.push(`${rel(file)} (stale)`);
      if (!checkOnly) fs.rmSync(file);
    }
  }
  return drift;
}

/** Shared `main` tail: report drift and exit non-zero in check mode. */
export function finish(name: string, drift: string[], checkOnly: boolean, regenerate: string): void {
  if (checkOnly && drift.length) {
    for (const file of drift) console.error(`${file} is missing or out of sync with its source`);
    console.error(`Run '${regenerate}' to update the ${name} reference.`);
    process.exit(1);
  }
  console.log(
    checkOnly
      ? `${name} reference is up to date.`
      : drift.length
        ? `${name} reference: wrote ${drift.length} file(s).`
        : `${name} reference: no changes.`
  );
}

/** Parses `--check` / `--check-only` from argv. */
export function isCheckMode(argv: string[] = process.argv.slice(2)): boolean {
  return argv.includes('--check') || argv.includes('--check-only');
}

// ============================================================================
// Curated headers and tested examples
// ============================================================================

/** Generator-owned curated prose (`headers/<product>/<page>.md`). */
export const HEADERS_DIR = path.join(REPO_ROOT, 'scripts', 'docs-generators', 'headers');
/** Example programs the generators embed as tested blocks (`examples/<product>/...`). */
export const EXAMPLES_DIR = path.join(REPO_ROOT, 'scripts', 'docs-generators', 'examples');

/**
 * A curated header, `{{name}}` placeholders filled from `vars`; '' when the
 * file does not exist. An unknown placeholder fails, so a renamed variable
 * cannot leak into a page.
 */
export function readHeader(rel: string, vars: Record<string, string> = {}, dir = HEADERS_DIR): string {
  const file = path.join(dir, rel);
  if (!fs.existsSync(file)) return '';
  return fs
    .readFileSync(file, 'utf-8')
    .replace(/\{\{(\w+)\}\}/g, (_, name: string) => {
      if (!(name in vars)) throw new Error(`${rel}: unknown placeholder {{${name}}}`);
      return vars[name];
    })
    .trim();
}

export interface Example {
  /** What the example shows: an item or `Owner.method` (the file name without its extension). */
  target: string;
  /** Tab label: Python, TypeScript, Swift, Kotlin or Rust. */
  lang: string;
  fence: string;
  /** Disposition attributes from the file's first line (`test`, `prelude`, `env`). */
  attrs: Record<string, string>;
  code: string;
  /** Repo-relative source path. */
  file: string;
}

const EXAMPLE_LANGS: Record<string, [lang: string, fence: string, comment: string]> = {
  '.py': ['Python', 'python', '#'],
  '.ts': ['TypeScript', 'ts', '//'],
  '.swift': ['Swift', 'swift', '//'],
  '.kt': ['Kotlin', 'kotlin', '//'],
  '.rs': ['Rust', 'rust', '//'],
};
const EXAMPLE_ATTRS = new Set(['test', 'prelude', 'env']);

/**
 * Parses one example file. Its first line declares how CI runs it, in the
 * docs code-block policy's terms: `# docs: test="docs" prelude="spacesd"`.
 */
export function parseExample(file: string, text: string): Example {
  const ext = path.extname(file);
  const spec = EXAMPLE_LANGS[ext];
  if (!spec) throw new Error(`${file}: unsupported example extension ${ext}`);
  const [lang, fence, comment] = spec;
  const [first, ...rest] = text.replace(/\r\n/g, '\n').split('\n');
  const head = first.match(new RegExp(`^${comment.replace(/\//g, '\\/')}\\s*docs:\\s*(.*)$`));
  if (!head) throw new Error(`${file}: the first line must be \`${comment} docs: test="<lane>" ...\``);
  const attrs = Object.fromEntries([...head[1].matchAll(/([a-z]+)="([^"]*)"/g)].map((m) => [m[1], m[2]]));
  for (const key of Object.keys(attrs)) {
    if (!EXAMPLE_ATTRS.has(key)) throw new Error(`${file}: unknown attribute ${key}`);
  }
  if (!attrs.test) throw new Error(`${file}: examples must be tested (test="<lane>")`);
  // Pages collapse runs of blank lines, so the page shows exactly what runs.
  const code = rest.join('\n').replace(/^\n+/, '').replace(/\s+$/, '').replace(/\n{3,}/g, '\n\n') + '\n';
  return {
    target: path.basename(file, ext),
    lang,
    fence,
    attrs,
    code,
    file: path.relative(REPO_ROOT, file).split(path.sep).join('/'),
  };
}

/** Every example under `dir`, keyed by target, in tab order. */
export function loadExamples(dir: string): Map<string, Example[]> {
  const out = new Map<string, Example[]>();
  if (!fs.existsSync(dir)) return out;
  const order = Object.values(EXAMPLE_LANGS).map(([lang]) => lang);
  for (const name of fs.readdirSync(dir).sort()) {
    const file = path.join(dir, name);
    if (!fs.statSync(file).isFile() || name.startsWith('.') || name.endsWith('.md')) continue;
    const ex = parseExample(file, fs.readFileSync(file, 'utf-8'));
    out.set(ex.target, [...(out.get(ex.target) ?? []), ex]);
  }
  for (const list of out.values()) list.sort((a, b) => order.indexOf(a.lang) - order.indexOf(b.lang));
  return out;
}

/** A stable block id for an example: `sandboxes-create-py`. */
export function exampleId(ex: Example): string {
  const suffix = { Python: 'py', TypeScript: 'ts', Swift: 'swift', Kotlin: 'kt', Rust: 'rs' }[ex.lang] ?? 'x';
  const base = ex.target
    .replace(/([a-z0-9])([A-Z])/g, '$1-$2')
    .toLowerCase()
    .replace(/[^a-z0-9]+/g, '-')
    .replace(/^-|-$/g, '');
  return `${base}-${suffix}`;
}

/**
 * Examples as tested fences (one tab per language when there are several),
 * preceded by an MDX comment naming the source files. Python blocks get their
 * own session so each runs alone, not after the page's earlier blocks.
 */
export function renderExamples(examples: Example[], groupId?: string): string {
  if (!examples.length) return '';
  const fenceOf = (ex: Example) => {
    const id = exampleId(ex);
    const attrs: Array<[string, string]> = [['test', ex.attrs.test], ['id', id]];
    if (ex.lang === 'Python') attrs.push(['session', id]);
    if (ex.attrs.prelude) attrs.push(['prelude', ex.attrs.prelude]);
    if (ex.attrs.env) attrs.push(['env', ex.attrs.env]);
    return codeFence(ex.fence, ex.code, attrs.map(([k, v]) => `${k}="${v}"`).join(' '));
  };
  const source = `{/* Examples: ${examples.map((e) => e.file).join(', ')} */}`;
  const body =
    examples.length === 1 ? fenceOf(examples[0]) : tabs(examples.map((ex) => [ex.lang, fenceOf(ex)]), groupId);
  return `${source}\n\n${body}`;
}

/** Word count of a page (the checklist caps reference pages at 5,000 words). */
export function wordCount(content: string): number {
  return content.split(/\s+/).filter(Boolean).length;
}