Skip to content

🎥 Make videos programmatically with React

7.0 KB
Raw Blame
import {join} from 'node:path';
import {scenarios} from '../scenarios';
import {runSkillEvalComparison} from './compare';
import {
	createSkillEvalId,
	createSkillEvalName,
	getSkillEvalDir,
	getSkillEvalPath,
	toEvalUrl,
	writeSkillEval,
} from './eval';
import {exportStaticSite} from './export-static-site';
import {maxParallelSkillEvalRuns, validateSkillEvalRunCount} from './run-count';
import {runSkillEval} from './run-skill-eval';
import {runWithConcurrency} from './run-with-concurrency';

const serverUrl = 'http://localhost:4321';

const getScenario = (id: string | undefined) => {
	if (!id) {
		if (scenarios.length === 1) {
			return scenarios[0];
		}

		throw new Error('Pass a scenario id.');
	}

	const scenario = scenarios.find((candidate) => candidate.id === id);

	if (!scenario) {
		throw new Error(`Unknown scenario "${id}". Run "bun run eval list".`);
	}

	return scenario;
};

const parseRunCount = (args: string[]) => {
	let runCount = 1;

	for (let index = 0; index < args.length; index++) {
		const arg = args[index];

		if (arg === '--runs') {
			const value = args[index + 1];

			if (!value) {
				throw new Error('Pass a value after --runs.');
			}

			runCount = Number(value);
			index++;
		} else if (arg.startsWith('--runs=')) {
			runCount = Number(arg.slice('--runs='.length));
		} else {
			throw new Error(`Unknown option "${arg}".`);
		}
	}

	return validateSkillEvalRunCount(runCount, '--runs');
};

const parseCompareOptions = (args: string[]) => {
	let beforeGitRef: string | undefined;
	const runCountArgs: string[] = [];

	for (let index = 0; index < args.length; index++) {
		const arg = args[index];

		if (arg === '--runs') {
			const value = args[index + 1];

			if (!value) {
				throw new Error('Pass a value after --runs.');
			}

			runCountArgs.push(arg, value);
			index++;
		} else if (arg.startsWith('--runs=')) {
			runCountArgs.push(arg);
		} else if (!beforeGitRef) {
			beforeGitRef = arg;
		} else {
			throw new Error(`Unknown option "${arg}".`);
		}
	}

	return {
		beforeGitRef,
		runCount: parseRunCount(runCountArgs),
	};
};

const parseExportOptions = (args: string[]) => {
	let outDir: string | undefined;
	const targets: string[] = [];

	for (let index = 0; index < args.length; index++) {
		const arg = args[index];

		if (arg === '--out') {
			const value = args[index + 1];

			if (!value) {
				throw new Error('Pass a value after --out.');
			}

			outDir = value;
			index++;
		} else if (arg.startsWith('--out=')) {
			outDir = arg.slice('--out='.length);
		} else {
			targets.push(arg);
		}
	}

	if (targets.length === 0) {
		throw new Error('Pass at least one eval path.');
	}

	return {outDir, targets};
};

const main = async () => {
	const command = process.argv[2];

	if (command === 'dev') {
		await import('./server');
	} else if (command === 'compare') {
		const scenario = getScenario(process.argv[3]);
		const {beforeGitRef, runCount} = parseCompareOptions(process.argv.slice(4));
		const evalId = createSkillEvalId(scenario.id);
		process.stdout.write(
			`Comparing ${scenario.id} with ${scenario.model}${
				beforeGitRef ? ` against ${beforeGitRef}` : ''
			} (${runCount} ${runCount === 1 ? 'run' : 'runs'})\n`,
		);
		const result = await runSkillEvalComparison(scenario, {
			beforeGitRef,
			evalId,
			onLog: (chunk) => process.stdout.write(chunk),
			runCount,
		});

		if (result.skipped) {
			process.stdout.write(`${result.reason}\n`);
			return;
		}

		const evalName = createSkillEvalName({id: evalId, type: 'comparison'});
		await writeSkillEval({
			comparisonPath: join(result.comparison.comparisonDir, 'comparison.json'),
			completedAt: result.comparison.completedAt,
			createdAt: result.comparison.createdAt,
			evalDir: getSkillEvalDir(scenario.id, evalId),
			id: evalId,
			name: evalName,
			runCount,
			scenarioId: scenario.id,
			type: 'comparison',
		});
		process.stdout.write(
			result.comparison.runs
				?.map(
					(run) =>
						`${scenario.id} #${run.index}: ${run.before.hash} -> ${run.after.hash}\n`,
				)
				.join('') ??
				`${scenario.id}: ${result.comparison.before.hash} -> ${result.comparison.after.hash}\n`,
		);
		process.stdout.write(
			`Eval: ${evalName} - ${serverUrl}${toEvalUrl({id: evalId, scenarioId: scenario.id})}\n`,
		);
		process.stdout.write(
			`Eval metadata: ${getSkillEvalPath(scenario.id, evalId)}\n`,
		);
		process.stdout.write(`Preview: ${serverUrl}\n`);
	} else if (command === 'export') {
		const {outDir, targets} = parseExportOptions(process.argv.slice(3));
		const result = await exportStaticSite({outDir, targets});

		process.stdout.write(`Exported ${result.targetCount} result(s).\n`);
		process.stdout.write(`Index: ${result.indexHtmlPath}\n`);
		process.stdout.write(`Deploy: ${result.deployCommand}\n`);
	} else if (command === 'list') {
		for (const scenario of scenarios) {
			process.stdout.write(`${scenario.id}\t${scenario.model}\n`);
		}
	} else if (command === 'run') {
		const scenarioId = process.argv[3];
		if (!scenarioId) {
			throw new Error('Pass a scenario id or --all.');
		}

		const scenariosToRun =
			scenarioId === '--all' ? scenarios : [getScenario(scenarioId)];
		const runCount = parseRunCount(process.argv.slice(4));

		for (const scenario of scenariosToRun) {
			const createdAt = new Date().toISOString();
			const evalId = createSkillEvalId(scenario.id);
			process.stdout.write(
				`Running ${scenario.id} with ${scenario.model} (${runCount} ${
					runCount === 1 ? 'run' : 'runs'
				})\n`,
			);
			const runIndexes = Array.from(
				{length: runCount},
				(_, index) => index + 1,
			);
			const results = await runWithConcurrency({
				inputs: runIndexes,
				limit: maxParallelSkillEvalRuns,
				worker: async (runIndex) => {
					const result = await runSkillEval({
						...scenario,
						evalId,
						evalRunIndex: runIndex,
						runLabel:
							runCount === 1
								? undefined
								: `run-${String(runIndex).padStart(2, '0')}`,
						skillSnapshot: {
							isWorkingTree: true,
							label: 'after',
						},
					});
					process.stdout.write(
						`Wrote run #${runIndex}: ${result.manifestPath}\n`,
					);
					return result;
				},
			});

			const completedAt = new Date().toISOString();
			const evalName = createSkillEvalName({id: evalId, type: 'run'});
			await writeSkillEval({
				completedAt,
				createdAt,
				evalDir: getSkillEvalDir(scenario.id, evalId),
				id: evalId,
				name: evalName,
				runCount,
				runs: results.map((result, index) => ({
					index: index + 1,
					manifestPath: result.manifestPath,
				})),
				scenarioId: scenario.id,
				type: 'run',
			});
			process.stdout.write(
				`Completed ${results.length} ${results.length === 1 ? 'run' : 'runs'} for ${scenario.id}\n`,
			);
			process.stdout.write(
				`Eval: ${evalName} - ${serverUrl}${toEvalUrl({id: evalId, scenarioId: scenario.id})}\n`,
			);
			process.stdout.write(
				`Eval metadata: ${getSkillEvalPath(scenario.id, evalId)}\n`,
			);
		}
	} else {
		process.stdout.write(
			'Usage: bun run eval <list|run|compare|export|dev> [scenario-id|--all] [base-ref] [--runs 1-4]\n',
		);
		process.exit(command ? 1 : 0);
	}
};

main();