Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

20.1 KB
Raw Blame
#!/usr/bin/env npx tsx

/**
 * Rust crate reference generator.
 *
 * Builds rustdoc JSON for the public cua crates with a pinned nightly (the
 * `scripts/docs-generators/rustdoc-json` tool owns the pin and the matching
 * `rustdoc-types`), then renders `reference/cua-sdk/rust/*.mdx`: a few pages
 * per crate, grouped by source module, with an anchor per item.
 *
 * Usage:
 *   pnpm --dir docs docs:generate:cua-rust            # build JSON + write pages
 *   pnpm --dir docs docs:check:cua-rust               # drift check (CI)
 *   npx tsx scripts/docs-generators/rust-crates.ts --dump FILE [--check]
 *                                                     # render an existing dump
 *
 * `rust/index.mdx` (crates, dependency and stability) is rendered from
 * headers/cua-sdk/rust/index.md.
 */

import { execFileSync } from 'child_process';
import * as fs from 'fs';
import * as path from 'path';
import {
  DOCS_CONTENT,
  REPO_ROOT,
  codeFence,
  escapeMdxText,
  finish,
  inlineCode,
  isCheckMode,
  metaJson,
  readHeader,
  renderPage,
  slug,
  syncFiles,
} from './lib/mdx';

// ============================================================================
// Dump schema (scripts/docs-generators/rustdoc-json/src/main.rs)
// ============================================================================

export interface Member {
  name: string;
  signature: string;
  docs: string;
  deprecated?: string;
}

export interface ItemDoc {
  path: string;
  module: string;
  name: string;
  kind: string;
  source: string;
  signature: string;
  docs: string;
  deprecated?: string;
  fields?: Member[];
  variants?: Member[];
  methods?: Member[];
  traits?: string[];
  /** Other public paths of the same item (see `dedupe`). */
  aliases?: string[];
}

export interface CrateDoc {
  name: string;
  lib: string;
  docs: string;
  items: ItemDoc[];
}

export interface Dump {
  tool: string;
  toolchain: string;
  format_version: number;
  crates: CrateDoc[];
}

// ============================================================================
// Pages
// ============================================================================

export interface PageSpec {
  slug: string;
  crate: string;
  title: string;
  description: string;
  /** Source files (`src/...`, prefix match) whose items belong here. */
  sources: string[];
  /** Catch-all page for the crate's items no other page claims. */
  fallback?: boolean;
}

/**
 * The stable crates: `cua-sdk` (the Rust host API, the same objects the
 * UniFFI bindings expose) and `cua-spacesd-client`. The unstable crates
 * (`cua-sandbox-core`, `cua-spaces`) have no pages; rust/index.mdx points at
 * `cargo doc` for them (see its stability tiers).
 */
export const PAGES: PageSpec[] = [
  {
    slug: 'cua-sdk',
    crate: 'cua-sdk',
    title: 'cua-sdk',
    description: 'The Rust host API: the Cua object and everything the language bindings export, plus the Rust-only items.',
    sources: [],
    fallback: true,
  },
  {
    slug: 'cua-spacesd-client',
    crate: 'cua-spacesd-client',
    title: 'cua-spacesd-client',
    description: 'The async cua-spacesd client crate (native gRPC or gRPC-Web).',
    sources: [],
    fallback: true,
  },
];

export const CRATES = [...new Set(PAGES.map((p) => p.crate))];

const OUTPUT_DIR = path.join(DOCS_CONTENT, 'cua-sdk', 'reference', 'rust');
const UNIFFI_DIR = path.join(DOCS_CONTENT, 'cua-sdk', 'reference');
const UNIFFI_ROUTE = '/cua-sdk/reference';
const UNIFFI_GENERATOR = 'Generated by: pnpm --dir docs docs:generate:cua-sdk';

/**
 * Where the Cua SDK object pages (the cua-sdk generator) document each
 * exported name: `## \`Name\`` objects and functions, `## \`Name\` record`
 * and `## \`Name\` enum`. cua-sdk's `native` items are that same API, with a
 * Rust tab, so the crate page links there instead of repeating it.
 */
export function uniffiAnchors(dir = UNIFFI_DIR, route = UNIFFI_ROUTE): Map<string, string> {
  const out = new Map<string, string>();
  const walk = (rel: string) => {
    const abs = path.join(dir, rel);
    if (!fs.existsSync(abs)) return;
    for (const name of fs.readdirSync(abs).sort()) {
      const file = path.join(abs, name);
      if (fs.statSync(file).isDirectory()) {
        walk(path.join(rel, name));
        continue;
      }
      if (!name.endsWith('.mdx')) continue;
      const text = fs.readFileSync(file, 'utf-8');
      if (!text.slice(0, 600).includes(UNIFFI_GENERATOR)) continue;
      const page = path.join(rel, name.replace(/\.mdx$/, '')).split(path.sep).join('/').replace(/(^|\/)index$/, '');
      for (const line of text.split('\n')) {
        const m = line.match(/^##\s+`([A-Za-z_]\w*)`( record| enum)?\s*$/);
        if (m && !out.has(m[1])) out.set(m[1], `${route}/${page}#${slug(`${m[1]}${m[2] ?? ''}`)}`);
      }
    }
  };
  walk('');
  return out;
}
const TOOL_DIR = path.join(REPO_ROOT, 'scripts', 'docs-generators', 'rustdoc-json');
const GENERATOR = 'pnpm --dir docs docs:generate:cua-rust';

/**
 * One item per definition: a type re-exported at the crate root and
 * reachable through its module is the same item. Keeps the shortest path
 * and records the others as `aliases`.
 */
export function dedupe(items: ItemDoc[]): ItemDoc[] {
  const byKey = new Map<string, ItemDoc>();
  const key = (i: ItemDoc) => [i.kind, i.name, i.source, i.signature].join('\u0000');
  for (const item of [...items].sort((a, b) => a.path.length - b.path.length || a.path.localeCompare(b.path))) {
    const k = key(item);
    const first = byKey.get(k);
    if (first) first.aliases = [...(first.aliases ?? []), item.path];
    else byKey.set(k, { ...item, aliases: [] });
  }
  return [...byKey.values()];
}

/** Assigns every item of every crate to exactly one page. */
export function assign(dump: Dump, pages: PageSpec[] = PAGES): Map<string, ItemDoc[]> {
  const out = new Map<string, ItemDoc[]>(pages.map((p) => [p.slug, []]));
  for (const krate of dump.crates) {
    const own = pages.filter((p) => p.crate === krate.name);
    if (!own.length) throw new Error(`no reference page for crate ${krate.name}`);
    const fallback = own.find((p) => p.fallback) ?? own[0];
    for (const item of dedupe(krate.items)) {
      const page = own.find((p) => p.sources.some((s) => item.source.startsWith(s))) ?? fallback;
      out.get(page.slug)!.push(item);
    }
  }
  return out;
}

/** Rustdoc Markdown to MDX-safe Markdown. */
export function docMarkdown(docs: string): string {
  const out: string[] = [];
  let fence: string | null = null;
  for (const raw of docs.split('\n')) {
    const line = raw.replace(/\s+$/, '');
    const open = line.match(/^(\s*)(```+|~~~+)\s*(.*)$/);
    if (fence === null && open) {
      fence = open[2];
      // Rustdoc code blocks default to Rust; drop rustdoc-only attributes.
      const attrs = open[3].split(',').map((a) => a.trim()).filter(Boolean);
      const lang = attrs.find((a) => !/^(ignore|no_run|should_panic|compile_fail|edition\d+|test_harness)$/.test(a)) ?? 'rust';
      out.push(`${open[1]}${fence}${lang}`);
      continue;
    }
    if (fence !== null) {
      if (line.trim().startsWith(fence)) fence = null;
      // Hidden doctest lines (`# use ...`) are not shown by rustdoc either.
      else if (/^\s*#( |$)/.test(line)) continue;
      out.push(line);
      continue;
    }
    // Reference-style link definitions ([`Foo`]: crate::Foo).
    if (/^\s*\[[^\]]+\]:\s/.test(line)) continue;
    let text = line
      // Headings would nest under the item heading (and H1 is renderer-owned).
      .replace(/^\s{0,3}#{1,6}\s+(.*)$/, '**$1**')
      // Intra-doc links: [text][`Foo`] and [`Foo`] -> the text.
      .replace(/\[([^\]]+)\]\[[^\]]*\]/g, '$1')
      .replace(/\[(`[^`\]]+`)\](?!\()/g, '$1')
      // [text](`Foo`) / [text](crate::Foo): intra-doc targets, not URLs.
      .replace(/\[([^\]]+)\]\((?!https?:|\/|#)[^)]*\)/g, '$1')
      // Autolinks would be escaped into visible angle brackets.
      .replace(/<(https?:\/\/[^>\s]+)>/g, '$1');
    text = escapeMdxText(text);
    out.push(text);
  }
  if (fence !== null) out.push(fence);
  return out.join('\n').trim();
}

function relPath(item: ItemDoc, krate: CrateDoc): string {
  return item.path.startsWith(`${krate.lib}::`) ? item.path.slice(krate.lib.length + 2) : item.path;
}

/**
 * The summary line of a rustdoc comment: its first paragraph's first
 * sentence, intra-doc links reduced to their text, capped at `max`
 * characters. The full comment is one click away in the source; the page
 * stays small because docs.cua.ai eager-imports every page (twice: compiled
 * and raw) into one SSR chunk.
 */
export function summary(docs: string, max = 200): string {
  const para = docs.trim().split(/\n\s*\n/)[0] ?? '';
  if (/^\s*(```|~~~)/.test(para)) return '';
  let text = para
    .replace(/\s*\n\s*/g, ' ')
    .replace(/\[([^\]]+)\]\[[^\]]*\]/g, '$1')
    .replace(/\[(`[^`\]]+`)\](?!\()/g, '$1')
    .replace(/\[([^\]]+)\]\((?!https?:)[^)]*\)/g, '$1')
    .replace(/<(https?:\/\/[^>\s]+)>/g, '$1')
    .trim();
  const sentence = text.match(/^(.+?[.!?])(?=\s+[A-Z(`*])/);
  if (sentence) text = sentence[1];
  if (text.length > max) {
    const cut = text.slice(0, max);
    text = cut.slice(0, Math.max(cut.lastIndexOf(' '), max - 20)).replace(/[,;:\s]+$/, '') + ' ...';
  }
  // An unbalanced code span would swallow the rest of the line.
  if ((text.match(/`/g) ?? []).length % 2) text = text.replace(/`/g, '');
  return text;
}

/**
 * `async_trait` desugars `async fn` into lifetimes and a boxed future;
 * show the `async fn` a reader writes.
 */
export function cleanSignature(sig: string): string {
  const m = sig.match(
    /^(.*?fn \w+)<([^>]*'async_trait[^>]*)>\((.*)\) -> Pin<Box<dyn Future<Output = (.*)> \+ Send \+ 'async_trait>>(?: where .*)?$/
  );
  if (!m) return sig;
  const generics = m[2]
    .split(',')
    .map((g) => g.trim())
    .filter((g) => g && !g.startsWith("'"));
  const args = m[3].replace(/&'life\d+ /g, '&');
  const head = m[1].replace(/fn /, 'async fn ');
  const out = m[4] === '()' ? '' : ` -> ${m[4]}`;
  return `${head}${generics.length ? `<${generics.join(', ')}>` : ''}(${args})${out}`;
}

/** A long constant value (an error message, a URL list) is elided. */
export function shortValue(sig: string): string {
  const m = sig.match(/^(pub (?:const|static) [^=]+= )(.*);$/);
  return m && m[2].length > 48 ? `${m[1]}/* ... */;` : sig;
}

/** A trailing `// summary` comment, for members inside a code block. */
function comment(docs: string, deprecated?: string, max = 40): string {
  const parts = [deprecated ? 'Deprecated.' : '', max ? summary(docs, max) : ''].filter(Boolean);
  return parts.length ? ` // ${parts.join(' ')}` : '';
}

/** One Rust block: the item's declaration with its members, one line each. */
export function declaration(item: ItemDoc): string {
  const lines: string[] = [];
  const sig = cleanSignature(item.signature);
  const fields = item.fields ?? [];
  const variants = item.variants ?? [];
  if (fields.length || variants.length) {
    lines.push(sig.replace(/\s*\{ \/\* fields below \*\/ \}$/, '').replace(/\s*\{.*\}$/, '') + ' {');
    for (const f of fields) lines.push(`  pub ${cleanSignature(f.signature)},${comment(f.docs, f.deprecated, 0)}`);
    for (const v of variants) lines.push(`  ${v.signature},${comment(v.docs, v.deprecated, 0)}`);
    lines.push('}');
  } else {
    lines.push(sig);
  }
  const methods = item.methods ?? [];
  if (methods.length) {
    if (item.kind !== 'trait') lines.push(`impl ${item.name} {`);
    else if (lines[0].endsWith(sig)) lines[lines.length - 1] = `${sig} {`;
    for (const m of methods) lines.push(`  ${cleanSignature(m.signature).replace(/^pub /, '')};${comment(m.docs, m.deprecated, 0)}`);
    lines.push('}');
  }
  return codeFence('rust', lines.join('\n'));
}

function summaryLine(item: ItemDoc): string[] {
  const out: string[] = [];
  if (item.deprecated) {
    out.push(`**Deprecated:** ${escapeMdxText(item.deprecated.replace(/^deprecated:? ?/, ''))}`, '');
  }
  const text = summary(item.docs, 56);
  if (text) out.push(docMarkdown(text), '');
  return out;
}

export function renderItem(item: ItemDoc, krate: CrateDoc): string {
  return [`### ${inlineCode(relPath(item, krate))}`, '', ...summaryLine(item), declaration(item)].join('\n');
}

/** Free items (functions, constants, aliases, statics, macros): one block. */
function freeItems(items: ItemDoc[], krate: CrateDoc): string {
  const lines: string[] = [];
  let module = '';
  for (const item of items) {
    const mod = relPath({ ...item, path: item.module } as ItemDoc, krate);
    if (mod !== module) {
      if (lines.length) lines.push('');
      if (mod !== item.module || item.module !== krate.lib) lines.push(`// mod ${mod === krate.lib ? krate.lib : mod}`);
      module = mod;
    }
    const doc = summary(item.docs, 40);
    if (item.deprecated) {
      const note = item.deprecated.replace(/^deprecated:? ?/, '');
      lines.push(note ? `#[deprecated = ${JSON.stringify(note)}]` : '#[deprecated]');
    }
    if (doc) lines.push(`/// ${doc}`);
    lines.push(shortValue(cleanSignature(item.signature)));
  }
  return codeFence('rust', lines.join('\n'));
}

const TYPE_KINDS = new Set(['struct', 'enum', 'union', 'trait']);
const FREE_SECTIONS: Array<[kinds: string[], heading: string]> = [
  [['function'], 'Functions'],
  [['constant', 'static'], 'Constants'],
  [['type'], 'Type aliases'],
  [['macro'], 'Macros'],
];

export function renderCratePage(
  spec: PageSpec,
  krate: CrateDoc,
  items: ItemDoc[],
  dump: Dump,
  version: string,
  uniffi: Map<string, string> = new Map()
): string {
  const body: string[] = [];
  const primary = spec.fallback === true;
  const part = spec.title.split(': ')[1];
  body.push(
    (primary ? 'Public API' : 'Part of the public API') +
      ` of the ${inlineCode(krate.name)} crate (${inlineCode(krate.lib)})` +
      (primary || !part ? '.' : `: ${part}.`) +
      ' Summaries and declarations only (`cargo doc -p ' +
      krate.name +
      ' --open` in `libs/cua` has the full comments); see [Rust crates](/cua-sdk/reference/rust) for stability.'
  );
  if (primary && krate.docs) body.push(docMarkdown(krate.docs.trim().split(/\n\s*\n/)[0]));
  const siblings = PAGES.filter((p) => p.crate === spec.crate && p.slug !== spec.slug);
  if (siblings.length) {
    body.push(
      `More of this crate: ${siblings.map((p) => `[${escapeMdxText(p.title.split(': ')[1] ?? p.title)}](/cua-sdk/reference/rust/${p.slug})`).join(', ')}.`
    );
  }
  const sorted = [...items].sort((a, b) => a.path.localeCompare(b.path));
  const modules = sorted.filter((i) => i.kind === 'module');
  if (modules.length) {
    body.push('## Modules');
    body.push(
      modules
        .map((m) => {
          const text = summary(m.docs, 56);
          return `- ${inlineCode(relPath(m, krate))}${text ? `: ${docMarkdown(text)}` : ''}`;
        })
        .join('\n')
    );
  }
  // cua-sdk's exported (UniFFI) API: one line each, linking the full entry.
  const linked = (i: ItemDoc) =>
    krate.name === 'cua-sdk' && i.source.startsWith('src/native/') && uniffi.has(i.name);
  const exported = sorted.filter(linked);
  if (exported.length) {
    body.push('## Exported API');
    body.push(
      'These are the objects, records, enums and functions the language bindings export, documented once in the [Cua SDK reference](/cua-sdk/reference), whose Rust tabs show each signature. Objects are `Arc<T>` handles whose methods are `async fn` returning `Result<_, CuaError>`, records are structs with public fields, and names are snake_case.'
    );
    body.push(
      exported
        .map((i) => {
          return `[${inlineCode(relPath(i, krate))}](${uniffi.get(i.name)})`;
        })
        .join(', ') + '.'
    );
  }
  const rest = sorted.filter((i) => !linked(i));
  const types = rest.filter((i) => TYPE_KINDS.has(i.kind));
  if (types.length) {
    body.push('## Types');
    for (const item of types) body.push(renderItem(item, krate));
  }
  for (const [kinds, heading] of FREE_SECTIONS) {
    const group = rest
      .filter((i) => kinds.includes(i.kind))
      .sort((a, b) => a.module.localeCompare(b.module) || a.name.localeCompare(b.name));
    if (!group.length) continue;
    body.push(`## ${heading}`, freeItems(group, krate));
  }
  // Re-exports of crates outside the public set (the
  // protos): one line, since their own crates document them.
  const reexports = rest.filter((i) => i.kind === 'reexport');
  if (reexports.length) {
    body.push(
      '## Re-exports',
      reexports.map((r) => inlineCode(r.signature.replace(/^pub use /, '').replace(/;$/, ''))).join(', ') + '.'
    );
  }
  return renderPage({
    title: spec.title,
    description: spec.description,
    generator: GENERATOR,
    source: `rustdoc JSON (${dump.toolchain}, format ${dump.format_version}) of ${krate.name}`,
    version,
    body: body.join('\n\n'),
  });
}

/** One row per crate: its pages and the summary line of its crate docs. */
function crateTable(dump: Dump): string {
  const rows = CRATES.map((name) => {
    const krate = dump.crates.find((c) => c.name === name);
    const pages = PAGES.filter((p) => p.crate === name)
      .map((p) => `[${escapeMdxText(p.title)}](/cua-sdk/reference/rust/${p.slug})`)
      .join(', ');
    const text = krate ? summary(krate.docs, 160) : '';
    return `| ${inlineCode(name)} | ${text ? docMarkdown(text).replace(/\|/g, '\\|') : ''} | ${pages} |`;
  });
  return ['| Crate | What it is | Reference |', '| --- | --- | --- |', ...rows].join('\n');
}

/** Every generated file, keyed by absolute path. */
export function renderAll(
  dump: Dump,
  version: string,
  outDir = OUTPUT_DIR,
  uniffi: Map<string, string> = uniffiAnchors()
): Map<string, string> {
  const files = new Map<string, string>();
  const assigned = assign(dump);
  for (const spec of PAGES) {
    const krate = dump.crates.find((c) => c.name === spec.crate);
    if (!krate) throw new Error(`dump has no crate ${spec.crate}`);
    files.set(path.join(outDir, `${spec.slug}.mdx`), renderCratePage(spec, krate, assigned.get(spec.slug)!, dump, version, uniffi));
  }
  files.set(
    path.join(outDir, 'index.mdx'),
    renderPage({
      title: 'Rust crates',
      description: 'The public Rust crates of the Cua SDK, how to depend on them, and what is stable.',
      generator: GENERATOR,
      source: `rustdoc JSON (${dump.toolchain}, format ${dump.format_version})`,
      version,
      body: [
        'Rust hosts (CLIs, Tauri apps, services) use the crates directly instead of a binding. The crates are not on crates.io, so there is no docs.rs page: each crate has one page here with every public item.',
        crateTable(dump),
        readHeader('cua-sdk/rust/index.md', { version }),
      ]
        .filter(Boolean)
        .join('\n\n'),
    })
  );
  files.set(path.join(outDir, 'meta.json'), metaJson('Rust crates', PAGES.map((p) => p.slug)));
  return files;
}

// ============================================================================
// Main
// ============================================================================

/** A caller's RUSTUP_TOOLCHAIN would override the tool's pinned nightly. */
function withoutToolchainOverride(env: NodeJS.ProcessEnv): NodeJS.ProcessEnv {
  const { RUSTUP_TOOLCHAIN: _ignored, ...rest } = env;
  return rest;
}

function buildDump(): Dump {
  const target = path.join(TOOL_DIR, 'target');
  const out = path.join(target, 'dump.json');
  // cwd = the tool dir, so rustup picks its pinned nightly (rust-toolchain.toml).
  execFileSync(
    'cargo',
    [
      'run',
      '--quiet',
      '--release',
      '--',
      '--workspace',
      path.join(REPO_ROOT, 'libs', 'cua'),
      '--target-dir',
      path.join(target, 'crates'),
      '--out',
      out,
      ...CRATES,
    ],
    { cwd: TOOL_DIR, stdio: 'inherit', env: withoutToolchainOverride(process.env) }
  );
  return JSON.parse(fs.readFileSync(out, 'utf-8'));
}

function main(): void {
  const args = process.argv.slice(2);
  const checkOnly = isCheckMode(args);
  const dumpIndex = args.indexOf('--dump');
  const dump: Dump =
    dumpIndex >= 0 ? JSON.parse(fs.readFileSync(args[dumpIndex + 1], 'utf-8')) : buildDump();
  const version = fs.readFileSync(path.join(REPO_ROOT, 'libs', 'cua', 'VERSION'), 'utf-8').trim();
  const drift = syncFiles(renderAll(dump, version), checkOnly, [OUTPUT_DIR], GENERATOR);
  finish('Rust crate', drift, checkOnly, GENERATOR);
}

if (require.main === module) main();