Skip to content

Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.

14.0 KB
Raw Blame
// Build data/examples.json: the machine-readable index of every runnable
// example in the repo, derived from what is actually on disk.
//
// The /examples gallery page renders from this file, so the page can never
// advertise an example that was deleted or miss one that was added. Run it
// after adding or removing an examples directory:
//
//   npm run build:examples
//
// Discovery rules, in order:
//   1. Single-file HTML demos at the root of examples/.
//   2. Project directories under examples/ (each has its own README).
//   3. Per-package examples/ directories anywhere in the repo, excluding
//      node_modules and the robinhood/ suite (a separate vendored set).
//
// Every entry carries a title, a description, its runnable files, and the
// command or URL that runs it. Descriptions come from the example's own README
// or an HTML <title>/comment, never from a hand-kept list here.

import { readFileSync, writeFileSync, readdirSync, existsSync, statSync, renameSync } from 'node:fs';
import { join, relative, resolve, basename, dirname } from 'node:path';
import { fileURLToPath } from 'node:url';

// Write through a temp file in the same directory, then rename. rename(2) is
// atomic on the same filesystem, so a reader (the page, a test, another agent
// in this shared worktree) always sees either the old file or the new one,
// never a half-written one.
function writeAtomic(path, contents) {
	const tmp = `${path}.tmp-${process.pid}`;
	writeFileSync(tmp, contents);
	renameSync(tmp, path);
}

const root = join(dirname(fileURLToPath(import.meta.url)), '..');
const OUT = join(root, 'data/examples.json');

const SKIP_DIRS = new Set(['node_modules', '.git', 'dist', 'dist-lib', 'build', 'robinhood']);

function read(path) {
	try {
		return readFileSync(path, 'utf8');
	} catch {
		return '';
	}
}

// First real prose line of a markdown file, stripped of markup.
function firstProse(md) {
	const lines = md.split('\n');
	let seenHeading = false;
	const para = [];
	for (const raw of lines) {
		const line = raw.trim();
		if (line.startsWith('# ')) {
			seenHeading = true;
			continue;
		}
		if (!seenHeading) continue;
		const structural = !line || /^([#>|]|```|---|!\[)/.test(line);
		if (!para.length && structural) continue;
		if (para.length && structural) break;
		para.push(line);
	}
	return para
		.join(' ')
		.replace(/\[([^\]]+)\]\([^)]+\)/g, '$1')
		.replace(/`([^`]*)`/g, '$1')
		.replace(/\*\*([^*]+)\*\*/g, '$1')
		.replace(/[*_]/g, '')
		.replace(/\s+/g, ' ')
		.trim();
}

function firstHeading(md) {
	const m = md.match(/^#\s+(.+)$/m);
	return m ? m[1].replace(/`/g, '').trim() : null;
}

function clamp(text, max = 260) {
	if (!text || text.length <= max) return text;
	const window = text.slice(0, max);
	const ends = [...window.matchAll(/[.!?](?=\s|$)/g)].map((m) => m.index + 1);
	const cut = ends.length ? ends[ends.length - 1] : window.lastIndexOf(' ');
	return window.slice(0, cut > 0 ? cut : max).trim();
}

function decodeEntities(s) {
	return s
		.replace(/&lt;/g, '<')
		.replace(/&gt;/g, '>')
		.replace(/&amp;/g, '&')
		.replace(/&quot;/g, '"')
		.replace(/&#39;/g, "'")
		.replace(/&nbsp;/g, ' ');
}

// An HTML demo describes itself, in priority order, with a meta description,
// its own visible intro paragraph, or the leading source comment. Anything
// else would be copy invented here, so the field is left empty instead.
function describeHtml(path) {
	const html = read(path);
	const title = html.match(/<title>([^<]+)<\/title>/i)?.[1]?.trim();
	const meta = html.match(/<meta\s+name=["']description["']\s+content=["']([^"']+)/i)?.[1];
	// Take the first paragraph that is actually prose. A demo's first <p> is
	// often a live status line ("Finding a public agent to show…"), which reads
	// as a description but is really runtime state that changes on load.
	const body = html.split(/<body[^>]*/i)[1] || '';
	// An element whose id or class names a runtime slot holds state, not prose.
	const isStatusSlot = (attrs) =>
		/\b(?:id|class)="[^"]*\b(?:status|state|loading|error|log|output|result)\b/i.test(attrs);
	let bodyText = '';
	for (const match of body.matchAll(/<p([^>]*)>([\s\S]{25,400}?)<\/p>/gi)) {
		const [, attrs, inner] = match;
		const text = inner.replace(/<[^>]+>/g, ' ').replace(/\s+/g, ' ').trim();
		// A trailing ellipsis is the giveaway for in-progress copy ("Loading…").
		if (!text || text.endsWith('…') || isStatusSlot(attrs)) continue;
		bodyText = text;
		break;
	}
	const lead = html.match(/^\s*<!--\s*([\s\S]{20,400}?)-->/m)?.[1];
	const raw = meta || bodyText || lead || '';
	const h1 = html.match(/<h1[^>]*>([\s\S]{2,90}?)<\/h1>/i)?.[1]?.replace(/<[^>]+>/g, '');
	return {
		title: decodeEntities(title || h1 || basename(path, '.html')).trim(),
		description: clamp(decodeEntities(raw).replace(/\s+/g, ' ').trim()),
	};
}

function listFiles(dir) {
	if (!existsSync(dir)) return [];
	return readdirSync(dir)
		.filter((f) => !f.startsWith('.'))
		.filter((f) => statSync(join(dir, f)).isFile())
		.sort();
}

function runnableFiles(dir) {
	return listFiles(dir).filter((f) => /\.(mjs|js|ts|html|sh|json)$/.test(f) && f !== 'package.json');
}

// How a reader actually runs this example, stated concretely.
// The concrete way a reader runs this, or null when the directory is reference
// material (schema samples, agent-as-files definitions) with nothing to execute.
function runHint(dirRel, files) {
	const script = files.find((f) => /\.(mjs|js)$/.test(f));
	const shell = files.find((f) => f.endsWith('.sh'));
	const html = files.find((f) => f.endsWith('.html'));
	if (script) return `node ${dirRel}/${script}`;
	if (shell) return `bash ${dirRel}/${shell}`;
	if (html) return `npm run dev, then open /${dirRel}/${html}`;
	// A project with its own package.json runs through its declared script.
	const pkgPath = join(root, dirRel, 'package.json');
	if (existsSync(pkgPath)) {
		try {
			const scripts = JSON.parse(read(pkgPath)).scripts || {};
			for (const name of ['start', 'dev', 'demo', 'run']) {
				if (scripts[name]) return `cd ${dirRel} && npm install && npm run ${name}`;
			}
		} catch {
			/* a malformed package.json just means no hint */
		}
	}
	return null;
}

const entries = [];

// 1. Single-file HTML demos at the root of examples/.
const examplesDir = join(root, 'examples');
for (const file of listFiles(examplesDir)) {
	if (!file.endsWith('.html')) continue;
	const abs = join(examplesDir, file);
	const { title, description } = describeHtml(abs);
	entries.push({
		id: `demo:${file}`,
		kind: 'html-demo',
		group: 'Web component demos',
		title,
		description,
		path: `examples/${file}`,
		files: [file],
		run: `http://localhost:3000/examples/${file}`,
		runKind: 'url',
	});
}

// 2. Project directories under examples/.
for (const name of readdirSync(examplesDir)) {
	const abs = join(examplesDir, name);
	if (!existsSync(abs) || !statSync(abs).isDirectory()) continue;
	if (SKIP_DIRS.has(name)) continue;
	const readme = read(join(abs, 'README.md'));
	const files = runnableFiles(abs);
	let projectPkgDescription = '';
	try {
		projectPkgDescription = JSON.parse(read(join(abs, 'package.json'))).description || '';
	} catch {
		/* not every example project is a package */
	}
	entries.push({
		id: `project:${name}`,
		kind: 'project',
		group: 'Example projects',
		title: firstHeading(readme) || name,
		description: clamp(firstProse(readme)) || clamp(projectPkgDescription),
		path: `examples/${name}`,
		files,
		run: runHint(`examples/${name}`, files),
		runKind: 'command',
		readme: existsSync(join(abs, 'README.md')) ? `examples/${name}/README.md` : null,
	});
}

// 3. Per-package examples/ directories.
function findExampleDirs(dir, depth = 0) {
	if (depth > 3) return [];
	const found = [];
	let items;
	try {
		items = readdirSync(dir);
	} catch {
		return found;
	}
	for (const name of items) {
		if (name.startsWith('.') || SKIP_DIRS.has(name)) continue;
		const abs = join(dir, name);
		let st;
		try {
			st = statSync(abs);
		} catch {
			continue;
		}
		if (!st.isDirectory()) continue;
		if (name === 'examples' && dir !== root) {
			found.push(abs);
			continue;
		}
		if (name === 'examples' || name === 'example') found.push(abs);
		found.push(...findExampleDirs(abs, depth + 1));
	}
	return found;
}

const seen = new Set(entries.map((e) => e.path));
for (const abs of findExampleDirs(root)) {
	const rel = relative(root, abs);
	if (rel.startsWith('examples/') || seen.has(rel)) continue;
	const owner = dirname(rel);
	const files = runnableFiles(abs);
	if (!files.length) continue;
	const readme = read(join(abs, 'README.md'));
	const ownerReadme = read(join(root, owner, 'README.md'));
	let pkgName = owner;
	let pkgDescription = '';
	try {
		const pkg = JSON.parse(read(join(root, owner, 'package.json')));
		pkgName = pkg.name || owner;
		pkgDescription = pkg.description || '';
	} catch {
		/* not every examples owner is an npm package */
	}
	entries.push({
		id: `package:${owner}`,
		kind: 'package',
		group: 'Package examples',
		title: pkgName,
		description: clamp(firstProse(readme) || firstProse(ownerReadme) || pkgDescription),
		path: rel,
		files,
		run: runHint(rel, files),
		runKind: 'command',
		readme: existsSync(join(abs, 'README.md')) ? `${rel}/README.md` : null,
		owner,
	});
}

// Source titles and descriptions may carry em/en dashes; the repo bans both
// glyphs in committed files, so normalize them to a plain hyphen on the way out.
const plainDashes = (s) => (typeof s === 'string' ? s.replace(/[\u2013\u2014]/g, '-') : s);
for (const e of entries) {
	e.title = plainDashes(e.title);
	e.description = plainDashes(e.description);
}

entries.sort((a, b) => a.group.localeCompare(b.group) || a.title.localeCompare(b.title));

const payload = {
	$comment:
		'Generated by scripts/build-examples-index.mjs from what is on disk. Do not edit by hand; run npm run build:examples.',
	generated_by: 'scripts/build-examples-index.mjs',
	counts: {
		total: entries.length,
		html_demos: entries.filter((e) => e.kind === 'html-demo').length,
		projects: entries.filter((e) => e.kind === 'project').length,
		packages: entries.filter((e) => e.kind === 'package').length,
	},
	examples: entries,
};

writeAtomic(OUT, `${JSON.stringify(payload, null, '\t')}\n`);

// Mirror the same index into docs/examples.md between markers, so the doc's
// inventory is regenerated rather than hand-kept. Everything outside the
// markers is hand-written and left untouched.
//
// EXAMPLES_DOC retargets that mirror at another file. The marker-guard test
// needs to feed this generator a deliberately broken doc, and doing that to the
// real docs/examples.md leaves the repo holding the broken fixture whenever the
// run dies before its restore (a SIGKILLed vitest fork, a cancelled CI job),
// which then fails every later run against a doc nobody edited. Pointing the
// test at a scratch file keeps the hazard out of the worktree entirely.
const DOC = process.env.EXAMPLES_DOC ? resolve(process.env.EXAMPLES_DOC) : join(root, 'docs/examples.md');
// Repo-relative reads best for the real doc; anything outside the repo (a test's
// scratch file) is named in full rather than as a pile of `../`.
const _docRel = relative(root, DOC);
const DOC_LABEL = !_docRel || _docRel.startsWith('..') ? DOC : _docRel;
const START = '<!-- BEGIN GENERATED EXAMPLES INDEX (npm run build:examples) -->';
const END = '<!-- END GENERATED EXAMPLES INDEX -->';

function mdEscape(text) {
	return (text || '').replace(/\|/g, '\\|');
}

const byGroup = new Map();
for (const entry of entries) {
	if (!byGroup.has(entry.group)) byGroup.set(entry.group, []);
	byGroup.get(entry.group).push(entry);
}

const lines = [
	START,
	'',
	'<!-- Generated by scripts/build-examples-index.mjs. Do not edit inside these markers. -->',
	'',
	'## Every runnable example in the repo',
	'',
	`The repo ships ${payload.counts.total} examples: ${payload.counts.html_demos} web component demos, ` +
		`${payload.counts.projects} end-to-end projects, and ${payload.counts.packages} package example sets. ` +
		'This table is generated from what is on disk, so it cannot list one that was deleted or miss one that was added. ' +
		'The machine-readable version is [data/examples.json](../data/examples.json).',
	'',
];

for (const [group, items] of byGroup) {
	lines.push(`### ${group}`, '');
	lines.push('| Example | What it shows | Run it |', '|---|---|---|');
	for (const item of items) {
		const link = `[\`${item.path}\`](../${item.path})`;
		const what = mdEscape(item.description) || 'See the directory for details.';
		const run = item.run ? `\`${mdEscape(item.run)}\`` : 'Reference files, nothing to execute';
		lines.push(`| ${link} | ${what} | ${run} |`);
	}
	lines.push('');
}

lines.push(
	'Run any command from the repo root after `npm install`. The web component demos need `npm run dev` first, since they import from `src/` so edits hot-reload into them. Anything that can spend money is gated behind an environment variable and off by default.',
	'',
	END,
);

const block = lines.join('\n');
const doc = readFileSync(DOC, 'utf8');
const startAt = doc.indexOf(START);
const endAt = doc.indexOf(END);

let next;
if (startAt !== -1 && endAt !== -1 && endAt > startAt) {
	next = `${doc.slice(0, startAt)}${block}${doc.slice(endAt + END.length)}`;
} else if (startAt === -1 && endAt === -1) {
	next = `${doc.trimEnd()}\n\n---\n\n${block}\n`;
} else {
	throw new Error(
		`${DOC_LABEL} has a malformed generated block (START at ${startAt}, END at ${endAt}). ` +
			'Fix or remove the markers by hand; refusing to rewrite the file.',
	);
}

// A generator that can shrink a hand-written doc is a generator that will
// eventually eat one. The prose outside the markers must survive every run.
const handWritten = (text) => text.split(START)[0].trim().length;
if (handWritten(next) < handWritten(doc)) {
	throw new Error(
		`Refusing to write ${DOC_LABEL}: the hand-written section above the generated block ` +
			`would shrink from ${handWritten(doc)} to ${handWritten(next)} characters.`,
	);
}
writeAtomic(DOC, next.endsWith('\n') ? next : `${next}\n`);

console.log(
	`wrote data/examples.json + docs/examples.md index: ${payload.counts.total} examples ` +
		`(${payload.counts.html_demos} demos, ${payload.counts.projects} projects, ${payload.counts.packages} packages)`,
);