Skip to content

Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.

40.6 KB
Raw Blame
#!/usr/bin/env node
// Operator CLI for the @trythreews content queue (data/x-content/queue.json).
// The same engine runs in production as /api/cron/x-content.
//
//   npm run x:content -- check                     validate every item
//   npm run x:content -- plan                      when each item lands, and what is blocking it
//   npm run x:content -- run --dry-run [--id slug]  print the exact X API calls for the next due item
//   npm run x:content -- run --id slug              publish one item now (owner-approved posts only)
//   npm run x:content -- import <blog-slug|url> --as post|article --id slug [--lane l] [--pattern p]
//   npm run x:content -- prepare-video <input> --out public/x-media/<id>/clip.mp4 [--captions file.srt] [--speed 3] [--item slug]
//   npm run x:content -- scout <url> [--format square]      what a page offers a scenario: controls, fields, numbers
//   npm run x:content -- prove <slug> [--no-film]            run the item's scenario against the live product and film it
//   npm run x:content -- prove --status draft               film every item in a status that carries a scenario
//   npm run x:content -- prove --file data/x-content/stories/<slug>.json   film a story that is not in the queue yet
//   npm run x:content -- adopt [<slug>...]                   move finished stories from data/x-content/stories/ into the queue
//   npm run x:content -- advance [<slug>] [--dry-run]        take every draft and review item as far as it can go:
//                                                            film, review, and release by policy where the queue allows it
//   npm run x:content -- advance --ship                      and publish what is approved as bundles, so it goes out
//                                                            without waiting for a deploy (needs X_CONTENT_BUNDLE_SECRET)
//   npm run x:content -- remix <slug> --as card|gif|art|reel [--headline "..."] [--body "..."] [--at 6]
//                                  [--from 2 --to 8] [--width 720] [--prompt "..."]
//                                                            lead a filmed post with a black card, a GIF cut from
//                                                            the reel, or AI key art; the reel moves to the first reply
//   npm run x:content -- pause <slug>                        take a post back before it goes out
//   npm run x:content -- review <slug> [--no-editor]      the editorial bar: lint, live fact checks, AI editor
//   npm run x:content -- review --status review            review every item awaiting review
//   npm run x:content -- approve <slug>                    owner gate: release a reviewed item
//   npm run x:content -- approve --status review           release every item whose review passed
//
// Env: reads .env.local then .env. DATABASE_URL gives plan/run the shared
// publish ledger; X_API_KEY, X_API_SECRET, X_ACCESS_TOKEN, X_ACCESS_SECRET are
// needed only to publish.

import { existsSync, mkdirSync, readdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
import { dirname, extname, relative, resolve } from 'node:path';
import { spawnSync } from 'node:child_process';
import { fileURLToPath } from 'node:url';

import { QUEUE_PATH, loadQueue, validateItem, validateQueue } from '../api/_lib/x-content/queue.js';
import { DEFAULT_CADENCE, TIERS, currentSlot, pickDue, reservedKinds, reservedSeries, seriesOf, slotLabel, slotOpenings, tierOf } from '../api/_lib/x-content/schedule.js';
import { loadLifts, rankItems } from '../api/_lib/x-content/priority.js';
import { activeHolds, inventory, stockLabel } from '../api/_lib/x-content/runner.js';
import { runTick } from '../api/_lib/x-content/runner.js';
import { dbStore, memoryStore } from '../api/_lib/x-content/state.js';
import { MAX_SPEED, VIDEO_LIMITS, mediaType, parseFfmpegProbe, videoFilterChain } from '../api/_lib/x-content/media.js';
import { weightedLength } from '../api/_lib/x-content/quality.js';
import { approvalProblems, loadReview, reviewItem } from '../api/_lib/x-content/review.js';
import { FORMATS, proofPath, proofProblems, proveItem, scoutPage } from '../api/_lib/x-content/reel.js';
import { approvalPolicy, policyBlockers, releaseDigest, vetoUntil } from '../api/_lib/x-content/approval.js';
import { learnLifts, outcomesStore } from '../api/_lib/x-content/outcomes.js';
import { browserProblem } from '../api/_lib/x-content/verify.js';
import { FORMATS as FEED_FORMATS, HEAD_KINDS as HEAD_AS, formatMix, formatOf, headSpecProblems, withFormats } from '../api/_lib/x-content/formats.js';
import { placeHead, rebuildHead } from './lib/x-heads.mjs';

const root = resolve(fileURLToPath(new URL('..', import.meta.url)));
const DAY = 24 * 60 * 60_000;

function loadEnvFile(path) {
	if (!existsSync(path)) return;
	for (const line of readFileSync(path, 'utf8').split('\n')) {
		const match = line.match(/^\s*([A-Z0-9_]+)\s*=\s*(.*?)\s*$/i);
		if (match && !process.env[match[1]]) process.env[match[1]] = match[2].replace(/^["']|["']$/g, '');
	}
}
loadEnvFile(resolve(root, '.env.local'));
loadEnvFile(resolve(root, '.env'));

const args = process.argv.slice(2);
const VALUE_FLAGS = new Set(['--id', '--as', '--lane', '--pattern', '--out', '--captions', '--item', '--now', '--status', '--speed', '--format', '--shot', '--file', '--headline', '--body', '--at', '--from', '--to', '--width', '--prompt', '--alt']);
const positional = args.filter((arg, index) => !arg.startsWith('--') && !VALUE_FLAGS.has(args[index - 1]));
const has = (flag) => args.includes(`--${flag}`);
const option = (name, fallback = null) => {
	const inline = args.find((arg) => arg.startsWith(`--${name}=`));
	if (inline) return inline.slice(name.length + 3);
	const index = args.indexOf(`--${name}`);
	return index >= 0 && args[index + 1] && !args[index + 1].startsWith('--') ? args[index + 1] : fallback;
};
const command = positional[0] || 'check';

function fail(message) {
	console.error(message);
	process.exit(1);
}

// Reviews, proofs and the line all drive a real browser. When this machine
// cannot start one, nothing they record would be about the post.
async function requireBrowser() {
	const problem = await browserProblem();
	if (problem) fail(`x-content: ${problem}. Nothing was reviewed or recorded.`);
}

const store = () => (process.env.DATABASE_URL ? dbStore() : memoryStore());

async function check() {
	const queue = loadQueue(root);
	const state = await store().load();
	const { problems, notes } = validateQueue(queue, root, { state });
	for (const note of notes) console.log(`note  ${note}`);
	let blocking = 0;
	for (const item of queue.items) {
		for (const problem of problems[item.id]) {
			// Drafts are allowed to be unfinished; anything headed for X is not.
			const severe = ['review', 'approved'].includes(item.status);
			if (severe) blocking++;
			console[severe ? 'error' : 'log'](`${severe ? 'error' : 'draft'} ${item.id}: ${problem}`);
		}
	}
	if (blocking) fail(`x-content: ${blocking} problem(s) in items under review or approved`);
	console.log(`x-content: ${queue.items.length} item(s) checked`);
}

async function plan() {
	const queue = loadQueue(root);
	const s = store();
	const state = await s.load();
	const { problems } = validateQueue(queue, root, { state });
	const now = Date.now();
	const cadence = { ...DEFAULT_CADENCE, ...queue.cadence };
	const seed = process.env.X_CONTENT_SCHEDULE_SEED || null;
	const published = new Map((state.published || []).map((row) => [row.id, row]));
	// Rank the way a tick does: with what the account's own posts measured.
	const learned = process.env.DATABASE_URL ? learnLifts((await outcomesStore().load().catch(() => ({ posts: [] }))).posts, { now }) : null;
	const lifts = learned ? Object.assign(loadLifts(root) || new Map(), { learned }) : loadLifts(root);
	const reviews = new Map(queue.items.map((item) => [item.id, loadReview(root, item.id)]));
	const holds = activeHolds(state, queue.items, root, now);
	console.log(`ledger: ${s.label}${seed ? '' : '   (slot minutes are placeholders here: X_CONTENT_SCHEDULE_SEED only exists in production)'}\n`);

	console.log('Slots (T1 flagship, T2 features, T3 proof of work, article every few days):');
	const openings = slotOpenings(now, cadence, seed).filter((slot) => slot.opensAt > now - DAY && slot.opensAt < now + DAY);
	const openKey = currentSlot(now, cadence, seed).slot?.key;
	for (const slot of openings) {
		const used = [...published.values()].find((row) => row.slot === slot.key);
		const state_ = used ? `used by ${used.id}` : slot.key === openKey ? 'OPEN NOW' : slot.opensAt <= now ? 'passed' : 'upcoming';
		console.log(`  ${new Date(slot.opensAt).toISOString().slice(0, 16)}Z  ${slotLabel(slot)}  ${state_}`);
	}

	console.log('\nStock (approved, ready, not held), one slot per tier per day:');
	for (const row of inventory(queue.items.filter((item) => !problems[item.id].length), state, root, now, cadence)) console.log(`  ${stockLabel(row)}${row.low ? '   LOW' : ''}`);
	// The feed format each approved post would lead with, and the last few sent.
	const approved = queue.items.filter((item) => item.status === 'approved' && !published.has(item.id));
	const mix = formatMix(approved);
	console.log(`  by format: ${FEED_FORMATS.map((format) => `${format} ${mix[format]}`).join(', ')}`);
	const recent = withFormats(state.published, queue.items).slice(-6).map((row) => row.format || '?');
	if (recent.length) console.log(`  last sent: ${recent.join(' > ')}   (at most ${queue.quality?.maximumSameFormatInARow ?? 'any'} of one format in a row)`);

	// Items a kind or series slot owns are ranked for that slot, not for their tier.
	const reserved = reservedKinds(cadence);
	const series = reservedSeries(cadence);
	const ownSlot = (item) => reserved.has(item.kind) || series.has(seriesOf(item));
	const groups = [
		...TIERS.map((tier) => ({ label: `T${tier}`, match: (item) => !ownSlot(item) && tierOf(item) === tier })),
		...[...series].map((name) => ({ label: `${name} series`, match: (item) => !reserved.has(item.kind) && seriesOf(item) === name })),
		...[...reserved].map((kind) => ({ label: `${kind} slot`, match: (item) => item.kind === kind })),
	];
	for (const group of groups) {
		const waiting = queue.items.filter((item) => group.match(item) && !published.has(item.id));
		if (!waiting.length) continue;
		console.log(`\n${group.label} by priority:`);
		const ranked = rankItems(waiting, { lifts, published: withFormats(state.published, queue.items), quality: queue.quality, reviews, now });
		for (const row of ranked) {
			const item = row.item;
			const flags = [
				item.status !== 'approved' ? item.status : null,
				problems[item.id].length ? `${problems[item.id].length} problem(s)` : null,
				holds.has(item.id) ? `held until ${state.holds[item.id].until}: ${state.holds[item.id].reason}` : null,
				Date.parse(item.notBefore) > now ? `embargoed until ${item.notBefore}` : null,
			].filter(Boolean);
			const parts = Object.entries(row.parts).map(([key, value]) => `${key} ${value > 0 ? '+' : ''}${value}`).join(', ');
			console.log(`  ${String(row.score).padStart(6)}  ${item.id.padEnd(26)} ${String(formatOf(item)).padEnd(7)} ${flags.length ? `[${flags.join('; ')}]` : 'ready'}`);
			const outlook = row.volumeChance == null
				? `predicted ${row.predictedLift}x the account median`
				: `${Math.round(row.volumeChance * 100)}% chance of a volume response${row.volumeSignals.length ? `: ${row.volumeSignals.join(', ')}` : ''}`;
			console.log(`          ${parts}  (${outlook})`);
		}
		for (const item of waiting.filter((item) => !ranked.some((row) => row.item.id === item.id))) console.log(`    gone  ${item.id.padEnd(26)} [expired ${item.expiresAt}]`);
	}
	if (published.size) {
		console.log('\nSent:');
		for (const row of published.values()) console.log(`  ${row.publishedAt}  ${row.id.padEnd(26)} ${row.slot ? `slot ${row.slot} ` : ''}${row.url}`);
	}

	const publishable = queue.items.filter((item) => !problems[item.id].length);
	const next = pickDue({ items: publishable, allItems: queue.items, state, now, cadence: queue.cadence, quality: queue.quality, seed, lifts, reviews, exclude: holds });
	console.log(`\nnow: ${next.item ? `${next.item.id} would fill slot ${next.slot.key} (${next.slot.kind ? `${next.slot.kind}, T${next.tier}` : `T${next.tier}`}${next.filledDown ? `, filling a T${next.slot.tier} slot` : ''}, score ${next.score})` : next.reason}`);
}

async function run() {
	const dryRun = has('dry-run');
	if (!dryRun && !process.env.DATABASE_URL) fail('Publishing needs DATABASE_URL so the shared ledger prevents a double post. Use --dry-run to preview.');
	const s = store();
	if (!dryRun && !(await s.acquireLock())) fail('Another publish (the cron or another operator) holds the lock. Try again in a few minutes.');
	try {
		const now = option('now') ? Date.parse(option('now')) : Date.now();
		const result = await runTick({ root, store: s, now, dryRun, requestedId: option('id') });
		for (const row of result.blocked) console.error(`blocked ${row.id}:\n  ${row.problems.join('\n  ')}`);
		if (result.preview) {
			console.log(`Preview of ${result.preview.id} (${result.preview.kind}), score ${result.preview.score ?? 'n/a'}:\n`);
			for (const call of result.preview.calls) console.log(JSON.stringify(call, null, 2));
		} else if (result.published) {
			for (const row of result.held || []) console.log(`held  ${row.id} until ${row.hold.until}: ${row.reason}`);
			console.log(`Published ${result.published.id}: ${result.published.url}`);
		} else {
			for (const row of result.held || []) console.log(`held  ${row.id} until ${row.hold.until}: ${row.reason}`);
			console.log(result.reason);
			if (result.skipped) process.exit(1);
		}
	} finally {
		if (!dryRun) await s.releaseLock();
	}
}

// --- import --------------------------------------------------------------

function slugOk(value) {
	return /^[a-z0-9][a-z0-9-]{1,80}$/.test(String(value || ''));
}

async function fetchBytes(url) {
	const response = await fetch(url, { headers: { 'user-agent': 'three.ws content importer (+https://three.ws)' }, redirect: 'follow' });
	if (!response.ok) throw new Error(`${url} answered HTTP ${response.status}`);
	return { buffer: Buffer.from(await response.arrayBuffer()), type: response.headers.get('content-type') || '' };
}

// A three.ws URL maps onto this checkout, so importing our own blog never
// depends on the live site being deployed.
function localPathFor(url) {
	const parsed = new URL(url);
	if (parsed.hostname !== 'three.ws' && parsed.hostname !== 'www.three.ws') return null;
	const path = decodeURIComponent(parsed.pathname);
	for (const candidate of [path.slice(1), `public${path}`, `${path.slice(1)}.html`]) {
		if (candidate && existsSync(resolve(root, candidate)) && extname(candidate)) return resolve(root, candidate);
	}
	return null;
}

async function loadSource(ref) {
	if (/^https?:\/\//.test(ref)) {
		const local = localPathFor(ref);
		const html = local ? readFileSync(local, 'utf8') : (await fetchBytes(ref)).buffer.toString('utf8');
		return { html, baseUrl: ref };
	}
	if (slugOk(ref) && existsSync(resolve(root, `blog/${ref}.html`))) {
		return { html: readFileSync(resolve(root, `blog/${ref}.html`), 'utf8'), baseUrl: `https://three.ws/blog/${ref}` };
	}
	fail(`${ref} is neither a URL nor a post in blog/`);
}

// Store an image as an upload-ready file under public/x-media/<id>/. SVG and
// AVIF are rasterized because X accepts neither.
async function saveImage(src, baseUrl, dir, name) {
	const absolute = new URL(src, baseUrl).toString();
	const local = localPathFor(absolute);
	const buffer = local ? readFileSync(local) : (await fetchBytes(absolute)).buffer;
	const { default: sharp } = await import('sharp');
	const meta = await sharp(buffer, { animated: true }).metadata();
	let ext = { jpeg: '.jpg', png: '.png', webp: '.webp', gif: '.gif' }[meta.format];
	let bytes = buffer;
	if (!ext) {
		bytes = await sharp(buffer, { density: 192 }).png().toBuffer();
		ext = '.png';
	}
	if (ext !== '.gif' && bytes.length > 5 * 1024 * 1024) {
		bytes = await sharp(bytes).resize({ width: 2400, withoutEnlargement: true }).jpeg({ quality: 88, mozjpeg: true }).toBuffer();
		ext = '.jpg';
	}
	const path = `${dir}/${name}${ext}`;
	mkdirSync(resolve(root, dir), { recursive: true });
	writeFileSync(resolve(root, path), bytes);
	return path;
}

// The house style bans en and em dashes; imported prose gets a comma instead.
const noDashes = (text) => String(text).replace(/\s*[\u2013\u2014]\s*/g, ', ');

const meta = (doc, selector) => doc.querySelector(selector)?.getAttribute('content')?.trim() || '';

async function importSource() {
	const ref = positional[1];
	const as = option('as', 'post');
	const id = option('id');
	if (!ref) fail('Usage: import <blog-slug|url> --as post|article --id slug');
	if (!['post', 'article'].includes(as)) fail('--as must be post or article');
	if (!slugOk(id)) fail('--id must be a lowercase slug');
	const queue = loadQueue(root);
	if (queue.items.some((item) => item.id === id)) fail(`${id} is already in the queue`);

	const { parse } = await import('node-html-parser');
	const { html, baseUrl } = await loadSource(ref);
	const doc = parse(html);
	const canonical = doc.querySelector('link[rel="canonical"]')?.getAttribute('href') || meta(doc, 'meta[property="og:url"]') || baseUrl;
	const title = (meta(doc, 'meta[property="og:title"]') || doc.querySelector('title')?.textContent || id).trim();
	const description = meta(doc, 'meta[property="og:description"]') || meta(doc, 'meta[name="description"]');
	const ogImage = meta(doc, 'meta[property="og:image"]') || meta(doc, 'meta[name="twitter:image"]');
	const mediaDir = `public/x-media/${id}`;
	const cover = ogImage ? await saveImage(ogImage, baseUrl, mediaDir, '00-cover') : null;

	const item = {
		id,
		status: 'draft',
		kind: as,
		lane: option('lane', as === 'article' ? 'article' : 'community'),
		pattern: option('pattern', as === 'article' ? 'longform' : 'mechanism'),
		notBefore: new Date(Math.ceil((Date.now() + 86400000) / 3600000) * 3600000).toISOString(),
		source: { url: canonical, importedAt: new Date().toISOString() },
	};

	if (as === 'post') {
		const link = ` ${canonical}`;
		let lead = description || title;
		while (weightedLength(`${lead}${link}`) > 280) lead = lead.replace(/\s+\S+$/, '');
		item.posts = [{ text: `${lead}${link}`, media: cover ? [{ path: cover, alt: description || title }] : [] }];
	} else {
		const { default: TurndownService } = await import('turndown');
		const { gfm } = await import('@joplin/turndown-plugin-gfm');
		const content =
			doc.querySelector('article') || doc.querySelector('main .post-wrap') || doc.querySelector('main') || doc.querySelector('body');
		for (const selector of ['script', 'style', 'nav', 'header', 'footer', 'aside', 'form', 'button', 'svg', 'canvas', 'iframe', 'video', 'noscript', '.post-meta', '#nav-container', '#footer-container']) {
			for (const node of content.querySelectorAll(selector)) node.remove();
		}
		// Site chrome inside the post body: "← Blog" style back links.
		for (const anchor of content.querySelectorAll('a')) {
			if (/^\s*[\u2190<]/.test(anchor.textContent)) (anchor.parentNode?.textContent.trim() === anchor.textContent.trim() ? anchor.parentNode : anchor).remove();
		}
		// The title becomes the Article title, not a heading inside the body.
		content.querySelector('h1')?.remove();
		let index = 0;
		for (const img of content.querySelectorAll('img')) {
			const src = img.getAttribute('src');
			if (!src || src.startsWith('data:')) {
				img.remove();
				continue;
			}
			try {
				img.setAttribute('src', await saveImage(src, baseUrl, mediaDir, String(++index).padStart(2, '0')));
			} catch (err) {
				console.warn(`skipped image ${src}: ${err.message}`);
				img.remove();
			}
		}
		for (const anchor of content.querySelectorAll('a[href]')) {
			anchor.setAttribute('href', new URL(anchor.getAttribute('href'), baseUrl).toString());
		}
		const turndown = new TurndownService({ headingStyle: 'atx', codeBlockStyle: 'fenced', bulletListMarker: '-' });
		turndown.use(gfm);
		turndown.remove(['style', 'script']);
		const markdown = noDashes(turndown.turndown(content.innerHTML))
			.replace(/\n{3,}/g, '\n\n')
			.replace(/(\n\s*(?:\* \* \*|-{3,})\s*)+$/, '')
			.trim();
		const body = `data/x-content/articles/${id}.md`;
		mkdirSync(resolve(root, dirname(body)), { recursive: true });
		writeFileSync(resolve(root, body), `${markdown}\n\n---\n\nOriginally published at [${new URL(canonical).host}${new URL(canonical).pathname}](${canonical}).\n`);
		item.article = { title: noDashes(title.replace(/\s*[|\u2013\u2014-]\s*three\.ws\s*$/i, '')), body };
		if (cover) item.article.cover = { path: cover };
		if (description) item.posts = [{ text: noDashes(description) }];
	}

	queue.items.push(item);
	writeFileSync(resolve(root, QUEUE_PATH), `${JSON.stringify(queue, null, '\t')}\n`);
	console.log(`Added ${id} as a ${as} draft from ${canonical}`);
	if (item.article) console.log(`Article body: ${item.article.body}`);
	console.log(`Media: ${mediaDir}/`);
	console.log('Edit the copy in your own voice, run `npm run x:content -- check`, then set status to review.');
}

// --- video -----------------------------------------------------------------

function ffmpegPath() {
	const bundled = resolve(root, 'node_modules/ffmpeg-static/ffmpeg');
	return existsSync(bundled) ? bundled : 'ffmpeg';
}

function prepareVideo() {
	const input = positional[1];
	const out = option('out');
	if (!input || !existsSync(input)) fail('Usage: prepare-video <input> --out public/x-media/<id>/clip.mp4 [--captions file.srt] [--speed 3] [--item slug]');
	if (!out || !out.startsWith('public/x-media/') || mediaType(out)?.kind !== 'video' || extname(out) !== '.mp4') fail('--out must be an .mp4 under public/x-media/');
	const captions = option('captions');
	if (captions && !existsSync(captions)) fail(`captions file ${captions} is missing`);

	const L = VIDEO_LIMITS;
	// Most of the feed autoplays muted, so burned-in captions carry the story;
	// --speed compensates for a browser recording that could only render a few
	// frames a second. See videoFilterChain for why speeding up beats padding.
	const speed = Number(option('speed', '1'));
	if (!(speed > 0 && speed <= MAX_SPEED)) fail(`--speed must be greater than 0 and at most ${MAX_SPEED}`);
	const filters = videoFilterChain({ speed, subtitlesPath: captions ? resolve(captions) : null, limits: L });
	mkdirSync(resolve(root, dirname(out)), { recursive: true });
	const ffmpeg = ffmpegPath();
	const encode = spawnSync(ffmpeg, [
		'-y', '-hide_banner', '-i', input,
		'-t', String(L.maxDurationSec),
		'-vf', filters.join(','),
		// Audio rides the same speed change, or it drifts out of the picture.
		...(speed === 1 ? [] : ['-af', `atempo=${speed}`]),
		'-fpsmax', String(L.maxFps),
		'-c:v', 'libx264', '-profile:v', 'high', '-preset', 'slow', '-crf', '20', '-pix_fmt', 'yuv420p',
		'-c:a', 'aac', '-b:a', '128k', '-ar', '48000', '-ac', '2',
		'-movflags', '+faststart',
		resolve(root, out),
	], { encoding: 'utf8', stdio: ['ignore', 'inherit', 'pipe'] });
	if (encode.status !== 0) fail(`ffmpeg failed:\n${encode.stderr.split('\n').slice(-15).join('\n')}`);

	const probeRun = spawnSync(ffmpeg, ['-hide_banner', '-i', resolve(root, out)], { encoding: 'utf8' });
	const probe = parseFfmpegProbe(probeRun.stderr);
	if (!probe) fail(`could not read the encoded file:\n${probeRun.stderr}`);
	console.log(`Encoded ${relative(root, resolve(root, out))}`);
	console.log(JSON.stringify(probe, null, '\t'));

	const itemId = option('item');
	if (!itemId) {
		console.log('\nAdd this as the media entry\'s "probe", or rerun with --item <slug> to write it.');
		return;
	}
	const queue = loadQueue(root);
	const item = queue.items.find((row) => row.id === itemId);
	if (!item) fail(`no queue item named ${itemId}`);
	const post = (item.posts ||= [{ text: '' }])[0];
	post.media = [{ path: out, probe }];
	writeFileSync(resolve(root, QUEUE_PATH), `${JSON.stringify(queue, null, '\t')}\n`);
	console.log(`\nAttached to ${itemId} as the head post's only media.`);
}

// --- proof reels -----------------------------------------------------------

async function scout() {
	await requireBrowser();
	const url = positional[1];
	if (!/^https:\/\//.test(String(url))) fail('Usage: scout <https url> [--format landscape|square|portrait] [--shot file.png]');
	const format = option('format', 'landscape');
	if (!FORMATS[format]) fail(`--format must be one of ${Object.keys(FORMATS).join(', ')}`);
	const page = await scoutPage(url, { format, shot: option('shot') });
	console.log(`${page.url}  HTTP ${page.status}  "${page.title}"  (${format}, ${FORMATS[format].width}x${FORMATS[format].height - FORMATS[format].bar} logical)`);
	const section = (title, rows) => rows.length && console.log(`\n${title}\n${rows.map((row) => `  ${row}`).join('\n')}`);
	section('Headings', page.headings);
	section('Controls (click by this text)', page.buttons);
	section('Fields (type "into" the placeholder or label)', page.fields);
	section('Links', page.links);
	section('Canvases', page.canvases);
	section('Lines that carry a number (candidates for read)', page.numbers);
	section('First lines of the page', page.text.slice(0, 25));
}

function printProof(id, proof) {
	console.log(`\n=== ${id}: ${proof.passed ? 'PROVED' : 'FAILED'} ===`);
	for (const step of proof.steps) console.log(`  ${step.ok ? 'pass' : 'FAIL'}  ${String(step.index).padStart(2)}  ${step.kind.padEnd(7)} ${String(step.target).slice(0, 44).padEnd(44)} ${String(step.frames).padStart(4)} frames  ${step.detail}`);
	if (Object.keys(proof.facts).length) console.log(`  read: ${Object.entries(proof.facts).map(([name, value]) => `${name} = "${value}"`).join(', ')}`);
	for (const hit of proof.responses) console.log(`  answered: ${hit.method} ${hit.path} ${hit.status} in ${hit.seconds} s`);
	for (const cut of proof.cuts) console.log(`  cut: ${cut.seconds} s after step ${cut.afterStep}, labelled in the reel`);
	if (proof.video) console.log(`  reel: ${proof.video.path}  ${proof.video.durationSec} s, ${proof.video.width}x${proof.video.height}, ${proof.video.frames} frames, ${Math.round(proof.video.motion * 100)}% changing`);
}

// Films one item. Returns the proof. The reel leads the post unless the item
// declares another head, which is then rebuilt from this reel (x-heads.mjs)
// and the reel moves to the reply the head names.
async function film(item, { filming = true } = {}) {
	const failureShot = resolve(root, `node_modules/.cache/x-content/${item.id}-failure.png`);
	mkdirSync(dirname(failureShot), { recursive: true });
	const { proof, media } = await proveItem(item, { root, film: filming, failureShot });
	printProof(item.id, proof);
	if (!proof.passed) console.log(`  the page as it was when the step failed: ${relative(root, failureShot)}`);
	// The reel replaces whatever carried it before. Anything else on its post
	// goes with it, because X allows a video no company on its post.
	if (media) {
		for (const post of item.posts) post.media = (post.media || []).filter((row) => row.path !== media.path);
		const { head } = await placeHead(item, media, { root, log: console.log });
		for (const post of item.posts) if (!post.media?.length) delete post.media;
		console.log(`  recorded in ${proofPath(item.id)}${head ? `; head rebuilt as ${relative(root, resolve(root, head.path))}` : ''}`);
	}
	return proof;
}

const saveJson = (path, value) => writeFileSync(path, `${JSON.stringify(value, null, '\t')}\n`);
const saveQueue = (queue) => saveJson(resolve(root, QUEUE_PATH), queue);

// Stories are queue items that live one to a file until they are finished, so
// several can be written and filmed at once without every author rewriting
// the same queue file.
const STORY_DIR = 'data/x-content/stories';

async function prove() {
	await requireBrowser();
	const filming = !has('no-film');
	const file = option('file');
	if (file) {
		const path = resolve(root, file);
		if (!existsSync(path)) fail(`${file} does not exist`);
		const item = JSON.parse(readFileSync(path, 'utf8'));
		if (!item.scenario) fail(`${file} carries no scenario`);
		const proof = await film(item, { filming });
		if (filming && proof.passed) saveJson(path, item);
		if (!proof.passed) process.exit(1);
		return;
	}
	const queue = loadQueue(root);
	const status = option('status');
	const id = positional[1];
	const items = queue.items.filter((item) => item.scenario && (id ? item.id === id : status ? item.status === status : false));
	if (!items.length) fail(id ? `${id} is not in the queue, or carries no scenario` : 'Usage: prove <slug> | prove --status draft | prove --file <story.json> [--no-film]');
	let failed = 0;
	for (const item of items) {
		const proof = await film(item, { filming });
		if (!proof.passed) failed++;
		else if (filming) saveQueue(queue);
	}
	if (failed) process.exit(1);
}

async function adopt() {
	const dir = resolve(root, STORY_DIR);
	const wanted = positional.slice(1);
	const files = existsSync(dir) ? readdirSync(dir).filter((name) => name.endsWith('.json') && (!wanted.length || wanted.includes(name.replace(/\.json$/, '')))) : [];
	if (!files.length) fail(`no stories to adopt in ${STORY_DIR}${wanted.length ? ` named ${wanted.join(', ')}` : ''}`);
	const queue = loadQueue(root);
	let refused = 0;
	const adopted = [];
	for (const name of files) {
		const path = resolve(dir, name);
		const story = JSON.parse(readFileSync(path, 'utf8'));
		const existing = queue.items.findIndex((item) => item.id === story.id);
		if (existing >= 0 && ['approved', 'posted'].includes(queue.items[existing].status)) {
			refused++;
			console.log(`skip  ${story.id}: the queue already holds it as ${queue.items[existing].status}`);
			continue;
		}
		// A story arrives as a draft whatever it called itself: review and
		// approval are steps the queue takes, not claims a file makes.
		const item = { ...story, status: 'draft' };
		const problems = validateItem(item, root, { quality: queue.quality || {} });
		if (existing >= 0) queue.items[existing] = item;
		else queue.items.push(item);
		adopted.push(path);
		console.log(`ok    ${story.id}: in the queue as a draft${problems.length ? `, with ${problems.length} problem(s) to fix: ${problems[0]}` : ''}`);
	}
	// The queue is written before any story is removed, so a failure part way
	// through leaves every story where it was.
	saveQueue(queue);
	for (const path of adopted) rmSync(path);
	if (refused) process.exit(1);
}

// Changes what leads a filmed post, without filming it again. The head is built
// from the reel on record, the reel moves to the reply the head names, and the
// item goes back to draft, because a review only covers the bytes it saw:
// `advance` reviews it again and releases it by policy.
async function remix() {
	const id = positional[1];
	const usage = `Usage: remix <slug> [--as ${HEAD_AS.join('|')}] [--headline "Line one\\nLine two"] [--body "..."] [--at 6] [--from 2 --to 8] [--width 720] [--prompt "..."] [--alt "..."]\n       Without --as, rebuilds the head the item already declares.`;
	if (!id) fail(usage);
	const queue = loadQueue(root);
	const item = queue.items.find((row) => row.id === id);
	if (!item) fail(`${id} is not in the queue`);
	const as = option('as') ?? item.head?.as;
	if (!HEAD_AS.includes(as)) fail(usage);
	await requireBrowser();
	if (item.status === 'posted') fail(`${id} has already gone out`);
	if (!item.scenario) fail(`${id} carries no scenario, so it has no reel to build a head from`);

	const previous = item.head?.as === as ? item.head : {};
	const text = (name) => option(name)?.replace(/\\n/g, '\n');
	const number = (name) => (option(name) == null ? undefined : Number(option(name)));
	const head = { ...previous, as };
	for (const [key, value] of Object.entries({ headline: text('headline'), body: text('body'), prompt: text('prompt'), alt: text('alt'), at: number('at'), from: number('from'), to: number('to'), width: number('width') })) {
		if (value !== undefined) head[key] = value;
	}
	if (as === 'reel') delete item.head;
	else item.head = head;
	const before = headSpecProblems(item);
	if (before.length) fail(`${id}: ${before.join('; ')}`);

	// The art lanes call NVIDIA NIM, whose key lives on the Cloud Run service.
	if (as === 'art') hydrateReviewEnv();
	const { head: media } = await rebuildHead(item, { root, log: console.log });
	const wasStatus = item.status;
	if (['review', 'approved', 'paused'].includes(item.status)) item.status = 'draft';
	saveQueue(queue);
	const problems = validateItem({ ...item, status: 'review' }, root, { quality: queue.quality || {} });
	console.log(`ok    ${id}: ${media ? `leads with ${media.path}${media.generated ? ` (AI-generated by ${media.generated.model})` : ''}` : 'leads with its reel again'}; ${wasStatus === item.status ? item.status : `${wasStatus} -> ${item.status}`}`);
	if (media?.alt) console.log(`      alt: ${media.alt}`);
	for (const problem of problems) console.log(`      fix: ${problem}`);
	console.log(`      next: npm run x:content -- advance ${id}`);
}

async function pause() {
	const id = positional[1];
	const queue = loadQueue(root);
	const item = queue.items.find((row) => row.id === id);
	if (!item) fail('Usage: pause <slug>');
	if (item.status === 'posted') fail(`${id} has already gone out`);
	item.status = 'paused';
	saveQueue(queue);
	console.log(`ok    ${id}: paused. It will not go out until it is approved again.`);
}

// One command for the whole line. Each item is taken as far as it can go and
// stops at the first thing that needs a person, with the reason.
async function advance() {
	await requireBrowser();
	const queue = loadQueue(root);
	const id = positional[1];
	const dryRun = has('dry-run');
	const policy = approvalPolicy(queue);
	const quality = queue.quality || {};
	const items = queue.items.filter((item) => (id ? item.id === id : ['draft', 'review'].includes(item.status)));
	if (!items.length) fail(id ? `${id} is not in the queue` : 'nothing to advance: no draft or review items');
	console.log(`approval: ${policy.mode === 'auto' ? `by policy for tier ${policy.tiers.join(', ')}${policy.articles ? ' and for articles' : ''}, embargoed ${policy.vetoHours} h` : 'by the owner'}`);
	if (!dryRun) hydrateReviewEnv();

	const rows = [];
	const released = [];
	for (const item of items) {
		const row = { id: item.id, from: item.status, reached: item.status, stopped: null };
		rows.push(row);
		try {
			if (item.scenario && proofProblems(item, root).length) {
				if (dryRun) {
					row.stopped = 'would film its scenario';
					continue;
				}
				const proof = await film(item);
				if (!proof.passed) {
					const failed = proof.steps.find((step) => !step.ok);
					row.stopped = `the feature did not pass its scenario at step ${failed.index} (${failed.kind} ${failed.target}): ${failed.detail}`;
					continue;
				}
				saveQueue(queue);
			}
			const problems = validateItem({ ...item, status: 'review' }, root, { quality });
			if (problems.length) {
				row.stopped = problems[0];
				continue;
			}
			if (dryRun) {
				row.stopped = 'would review it';
				continue;
			}
			item.status = 'review';
			row.reached = 'review';
			saveQueue(queue);
			const record = await reviewItem(item, { root, glossary: quality.glossary || [], quality });
			printReview(record);
			if (!record.passed) {
				row.stopped = record.blockers[0];
				continue;
			}
			const blockers = policyBlockers(item, record, policy, root);
			if (blockers.length) {
				row.stopped = `passed review; waits for the owner because ${blockers[0]}`;
				continue;
			}
			item.status = 'approved';
			item.notBefore = vetoUntil(item, policy);
			item.approvedBy = 'policy';
			row.reached = 'approved';
			saveQueue(queue);
			released.push({ id: item.id, tier: tierOf(item), notBefore: item.notBefore, head: item.posts?.[0]?.text || item.article?.title || '' });
		} catch (err) {
			row.stopped = String(err.message || err).split('\n')[0];
		}
	}

	console.log('\nitem                         from     reached   stopped at');
	for (const row of rows) console.log(`${row.id.padEnd(28)} ${row.from.padEnd(8)} ${row.reached.padEnd(9)} ${row.stopped || ''}`);
	if (released.length) {
		const digest = releaseDigest(released);
		console.log(`\n${digest.title}\n${digest.detail}`);
		const { sendOpsAlert } = await import('../api/_lib/alerts.js');
		await sendOpsAlert(digest.title, digest.detail, { severity: 'info' }).catch((err) => console.log(`(the ops alert was not recorded: ${err.message})`));
	}
	if (has('ship') && !dryRun) {
		// Everything approved and not yet sent, whoever approved it: the bundles
		// are what production reads, so they should always say what the queue says.
		const ship = spawnSync(process.execPath, [resolve(root, 'scripts/x-bundle.mjs'), 'publish', '--approved'], { cwd: root, stdio: 'inherit' });
		if (ship.status !== 0) process.exitCode = 1;
	}
}

// --- review ----------------------------------------------------------------

// Credentials the review needs, taken from the Cloud Run service when they are
// not in the local env files.
const REVIEW_ENV = ['X_API_KEY', 'X_API_SECRET', 'OPENROUTER_API_KEY', 'OPENAI_API_KEY', 'NVIDIA_API_KEY'];

function hydrateReviewEnv() {
	if (!process.env.GOOGLE_CLOUD_PROJECT) process.env.GOOGLE_CLOUD_PROJECT = 'aerial-vehicle-466722-p5';
	for (const name of REVIEW_ENV.filter((key) => !process.env[key])) {
		const read = spawnSync(process.execPath, [resolve(root, 'scripts/read-service-env.mjs'), `^${name}$`, '--raw'], { cwd: root, encoding: 'utf8' });
		const value = read.status === 0 ? read.stdout.trim() : '';
		if (value) process.env[name] = value;
	}
	if (!process.env.GITHUB_TOKEN && !process.env.GH_TOKEN) {
		const token = spawnSync('gh', ['auth', 'token'], { encoding: 'utf8' });
		if (token.status === 0 && token.stdout.trim()) process.env.GITHUB_TOKEN = token.stdout.trim();
	}
}

function printReview(record) {
	const mark = (ok) => (ok ? 'pass' : 'FAIL');
	console.log(`\n=== ${record.id}: ${record.passed ? 'PASSED' : 'NOT READY'} ===`);
	for (const text of record.texts) console.log(`  > ${text.replace(/\n/g, '\n    ')}`);
	console.log('\nFacts');
	for (const check of record.verification.checks) console.log(`  ${mark(check.ok)}  ${check.kind} ${check.target}${check.claim ? ` [${check.claim}]` : ''}: ${check.detail}`);
	if (record.lint.length) {
		console.log('\nEditorial lint');
		for (const row of record.lint) console.log(`  ${row.severity.padEnd(8)} ${row.where}: ${row.message}`);
	}
	if (record.editor) {
		const e = record.editor;
		console.log(`\nEditor (${e.model}): ${e.verdict}`);
		console.log(`  ${Object.entries(e.scores).map(([key, value]) => `${key} ${value}`).join(' | ')}`);
		console.log(`  ${e.summary || ''}`);
		for (const issue of e.issues) console.log(`  ${issue.severity.padEnd(8)} ${issue.area}: "${issue.quote}" ${issue.problem}\n           fix: ${issue.fix}`);
		if (e.rewrite?.posts?.length) {
			console.log('  suggested rewrite:');
			for (const text of e.rewrite.posts) console.log(`    > ${text}`);
			for (const finding of e.rewriteFindings || []) console.log(`    rewrite not usable as is: ${finding}`);
		}
		if (e.fallbacks?.length) console.log(`  (fell back past: ${e.fallbacks.map((f) => f.split(':')[0]).join(', ')})`);
	}
	if (record.blockers.length) {
		console.log('\nBlocking approval');
		for (const blocker of record.blockers) console.log(`  - ${blocker}`);
	}
	console.log(`\nRecorded in data/x-content/reviews/${record.id}.json`);
}

async function review() {
	await requireBrowser();
	const queue = loadQueue(root);
	const status = option('status');
	const id = positional[1];
	const items = queue.items.filter((item) => (id ? item.id === id : status ? item.status === status : false));
	if (!items.length) fail('Usage: review <slug> | review --status review [--no-editor]');
	hydrateReviewEnv();
	let failed = 0;
	for (const item of items) {
		const record = await reviewItem(item, { root, glossary: queue.quality?.glossary || [], quality: queue.quality || {}, skipEditor: has('no-editor') });
		printReview(record);
		if (!record.passed) failed++;
	}
	if (failed) process.exit(1);
}

// The owner's gate, as a command instead of a hand edit. It only ever moves an
// item to `approved`, and only while a passing review record covers the exact
// bytes that are in the queue right now, which is the same rule the publisher
// enforces at send time.
async function approve() {
	const queue = loadQueue(root);
	const status = option('status');
	const id = positional[1];
	const items = queue.items.filter((item) => (id ? item.id === id : status ? item.status === status : false));
	if (!items.length) fail('Usage: approve <slug> | approve --status review');

	let blocked = 0;
	for (const item of items) {
		const problems = approvalProblems({ ...item, status: 'approved' }, root);
		if (problems.length) {
			blocked++;
			console.log(`hold  ${item.id}: ${problems.join('; ')}`);
			continue;
		}
		item.status = 'approved';
		console.log(`ok    ${item.id}: approved; it goes out in priority order once ready (embargo ${item.notBefore.slice(0, 10)})`);
	}
	writeFileSync(resolve(root, QUEUE_PATH), `${JSON.stringify(queue, null, '\t')}\n`);
	if (blocked) process.exit(1);
}

const commands = { check, plan, run, review, approve, scout, prove, adopt, advance, remix, pause, import: importSource, 'prepare-video': prepareVideo };
if (!commands[command]) fail(`Unknown command ${command}. Commands: ${Object.keys(commands).join(', ')}`);
await commands[command]();