Skip to content

Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.

feat(x-content): publish an approved post without a production deploy

nirholasd3c4ded20823
diff --git a/api/_lib/x-content/overlay.js b/api/_lib/x-content/overlay.js
new file mode 100644
index 000000000..5d42cfc28
--- /dev/null
+++ b/api/_lib/x-content/overlay.js
@@ -0,0 +1,505 @@
+// The overlay root: how an approved post reaches production without a deploy.
+//
+// The engine reads everything from files under one root (the queue, the review
+// records, the media), and in production that root is the Cloud Run image, so a
+// post approved today could not go out until someone shipped a new image. This
+// module removes that ceiling without touching a single editorial rule.
+//
+// `npm run x:bundle -- publish <id>` packs an approved item, its full review
+// record, and every file it needs into a bundle, uploads the files to object
+// storage under content-addressed keys, and lists the bundle in one index that
+// is signed with X_CONTENT_BUNDLE_SECRET. At the start of a tick production
+// verifies that signature, then materializes a directory that mirrors the repo
+// layout: the image's own data/ and public/ reached through symlinks, the
+// bundle files written on top after their bytes hash to what the index
+// recorded. The unchanged engine then runs with `root` pointing at it. The
+// review record is still bound to the content hash, still expires, and the
+// media limits, link checks, and probes still run, because the engine cannot
+// tell an overlay from an image.
+//
+// Nothing here can stop a tick. Any failure (no secret, no storage, a bad
+// signature, a file that does not match its hash) falls back to the image root,
+// which is exactly what production did before the overlay existed.
+
+import { createHash, createHmac, timingSafeEqual } from 'node:crypto';
+import { existsSync, lstatSync, mkdirSync, readFileSync, readdirSync, readlinkSync, renameSync, rmSync, statSync, symlinkSync, unlinkSync, writeFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { isAbsolute, join, resolve, sep } from 'node:path';
+import { MEDIA_ROOTS, mediaType } from './media.js';
+import { QUEUE_PATH, loadArticle, loadQueue, validateItem } from './queue.js';
+import { loadReview, reviewPath } from './review.js';
+
+export const INDEX_KEY = 'x-content/bundles/index.json';
+export const INDEX_VERSION = 1;
+export const ARTICLE_ROOT = 'data/x-content/articles/';
+export const REVIEW_ROOT = 'data/x-content/reviews/';
+export const proofPath = (id) => `data/x-content/proofs/${id}.json`;
+export const fileKey = (id, sha256) => `x-content/bundles/${id}/${sha256}`;
+export const DEFAULT_OVERLAY_DIR = join(tmpdir(), 'x-content-overlay');
+
+// A short secret is a guessable one, and this secret is the only thing standing
+// between write access to the bucket and a post on @trythreews.
+export const MIN_SECRET_LENGTH = 32;
+
+// Marks a directory as one this module built, so materialize never prunes a
+// directory that holds someone's files.
+const MARKER = '.x-content-overlay';
+// Top-level entries of the image that the overlay exposes. data/ and public/
+// are what the engine reads; the rest is what a command probe or a textFrom
+// check may look for under root.
+const EXPOSED = ['data', 'public', 'scripts', 'docs', 'package.json'];
+
+const SLUG = /^[a-z0-9][a-z0-9-]{1,80}$/;
+const SHA256 = /^[0-9a-f]{64}$/;
+
+export class OverlayError extends Error {
+ constructor(code, message) {
+ super(message);
+ this.name = 'OverlayError';
+ this.code = code;
+ }
+}
+
+const sha256 = (bytes) => createHash('sha256').update(bytes).digest('hex');
+
+// ── Canonical JSON and the signature ────────────────────────────────────────
+// Keys sorted at every depth, so the bytes that are signed do not depend on the
+// order a writer happened to build an object in, or on a store that reorders
+// keys. Values follow JSON.stringify: undefined members are dropped, undefined
+// array entries become null.
+export function canonical(value) {
+ if (value && typeof value.toJSON === 'function') return canonical(value.toJSON());
+ if (Array.isArray(value)) return `[${value.map((row) => canonical(row === undefined ? null : row)).join(',')}]`;
+ if (value && typeof value === 'object') {
+ const keys = Object.keys(value).filter((key) => value[key] !== undefined && typeof value[key] !== 'function').sort();
+ return `{${keys.map((key) => `${JSON.stringify(key)}:${canonical(value[key])}`).join(',')}}`;
+ }
+ return JSON.stringify(value) ?? 'null';
+}
+
+export const bundleHash = ({ item, review, files }) => sha256(canonical({ item, review, files }));
+
+function secretFrom(env) {
+ const secret = String(env?.X_CONTENT_BUNDLE_SECRET ?? '').trim();
+ if (!secret) throw new OverlayError('unconfigured', 'X_CONTENT_BUNDLE_SECRET is not set');
+ if (secret.length < MIN_SECRET_LENGTH) throw new OverlayError('unconfigured', `X_CONTENT_BUNDLE_SECRET is shorter than ${MIN_SECRET_LENGTH} characters; generate one with \`openssl rand -hex 32\``);
+ return secret;
+}
+
+export function signIndex({ version, updatedAt, bundles }, env = process.env) {
+ return createHmac('sha256', secretFrom(env)).update(canonical({ version, updatedAt, bundles })).digest('hex');
+}
+
+export function verifyIndex(index, env = process.env) {
+ const expected = Buffer.from(signIndex(index, env), 'hex');
+ const given = SHA256.test(String(index?.signature || '')) ? Buffer.from(index.signature, 'hex') : null;
+ if (!given || given.length !== expected.length || !timingSafeEqual(given, expected)) {
+ throw new OverlayError('bad_signature', 'the bundle index signature does not verify; it was not written with this X_CONTENT_BUNDLE_SECRET, or it was changed after it was signed');
+ }
+ return index;
+}
+
+// ── Paths ───────────────────────────────────────────────────────────────────
+// Why a path may not be written into the overlay, or null when it may. The
+// queue and the review records are the overlay's own: a bundle file is never
+// allowed to stand in for either.
+export function bundlePathProblem(path) {
+ if (typeof path !== 'string' || !path) return 'is empty';
+ if (path.includes('..')) return 'contains ..';
+ if (isAbsolute(path) || path.startsWith('/') || /^[a-zA-Z]:/.test(path)) return 'is absolute';
+ if (path.includes('\\') || path.includes('\0')) return 'contains a backslash or a null byte';
+ if (path.split('/').some((part) => !part || part === '.')) return 'has an empty or dot segment';
+ if (![...MEDIA_ROOTS, ARTICLE_ROOT].some((prefix) => path.startsWith(prefix))) return `is outside ${[...MEDIA_ROOTS, ARTICLE_ROOT].join(', ')}`;
+ if (path === QUEUE_PATH || path.startsWith(REVIEW_ROOT)) return 'is reserved for the queue and the review records';
+ return null;
+}
+
+function checkBundle(bundle) {
+ const id = String(bundle?.id || '');
+ if (!SLUG.test(id)) throw new OverlayError('bad_bundle', `bundle id ${JSON.stringify(bundle?.id)} is not a lowercase slug`);
+ if (bundle.item?.id !== id) throw new OverlayError('bad_bundle', `${id}: the bundled item is ${JSON.stringify(bundle.item?.id)}`);
+ if (!bundle.review || bundle.review.id !== id) throw new OverlayError('bad_bundle', `${id}: the bundled review record is for ${JSON.stringify(bundle.review?.id)}`);
+ if (!Array.isArray(bundle.files)) throw new OverlayError('bad_bundle', `${id}: files is not a list`);
+ for (const file of bundle.files) {
+ const problem = bundlePathProblem(file?.path);
+ if (problem) throw new OverlayError('bad_path', `${id}: file path ${JSON.stringify(file?.path)} ${problem}`);
+ if (!SHA256.test(String(file.sha256 || ''))) throw new OverlayError('bad_bundle', `${id}: ${file.path} has no sha256`);
+ if (!Number.isInteger(file.bytes) || file.bytes < 0) throw new OverlayError('bad_bundle', `${id}: ${file.path} has no byte count`);
+ }
+ if (bundle.hash !== bundleHash(bundle)) throw new OverlayError('bad_bundle', `${id}: the bundle hash does not cover its item, review record, and files`);
+ return bundle;
+}
+
+// ── Storage ─────────────────────────────────────────────────────────────────
+// The real store. Loaded on first use, so the cron module stays importable and
+// cheap while the overlay is switched off.
+export function r2Storage() {
+ const r2 = () => import('../r2.js');
+ return {
+ label: 'object storage',
+ async put(key, buffer, contentType) {
+ const { putObject } = await r2();
+ await putObject({ key, body: buffer, contentType });
+ },
+ async get(key) {
+ const { getObjectBuffer } = await r2();
+ try {
+ return await getObjectBuffer(key);
+ } catch (err) {
+ if (err?.name === 'NoSuchKey' || err?.$metadata?.httpStatusCode === 404) return null;
+ throw err;
+ }
+ },
+ async has(key) {
+ const { headObject } = await r2();
+ return Boolean(await headObject(key));
+ },
+ async remove(key) {
+ const { deleteObject } = await r2();
+ await deleteObject(key);
+ },
+ };
+}
+
+// `has` is optional on a store: one that only has put, get, and remove answers
+// by reading the object.
+const stored = async (storage, key) => (storage.has ? storage.has(key) : (await storage.get(key)) !== null);
+
+const contentTypeOf = (path) => mediaType(path)?.mime || (path.endsWith('.md') ? 'text/markdown; charset=utf-8' : path.endsWith('.json') ? 'application/json' : 'application/octet-stream');
+
+// ── Building and publishing ─────────────────────────────────────────────────
+// Every file the item needs at publish time: each post's media, and for an
+// Article its body, cover, and inline images. An item with a scenario also
+// carries its proof record.
+function itemPaths(item, root) {
+ const paths = [];
+ for (const post of item.posts || []) for (const media of post.media || []) paths.push(media.path);
+ if (item.kind === 'article') {
+ paths.push(item.article.body, item.article.cover.path);
+ for (const image of loadArticle(root, item.article).images) paths.push(image.path);
+ }
+ if (existsSync(resolve(root, proofPath(item.id)))) paths.push(proofPath(item.id));
+ return [...new Set(paths)];
+}
+
+export function buildBundle(root, id, { now = Date.now() } = {}) {
+ const item = (loadQueue(root).items || []).find((row) => row.id === id);
+ if (!item) throw new Error(`${id} cannot be bundled: no queue item has that id`);
+ const reasons = [];
+ if (item.status !== 'approved') reasons.push(`status is ${item.status}; only an approved item is bundled`);
+ reasons.push(...validateItem(item, root));
+ if (!reasons.length) {
+ for (const path of itemPaths(item, root)) {
+ const problem = bundlePathProblem(path);
+ if (problem) reasons.push(`file path ${path} ${problem}`);
+ }
+ }
+ if (reasons.length) throw new Error(`${id} cannot be bundled: ${reasons.join('; ')}`);
+
+ const review = loadReview(root, id);
+ const files = itemPaths(item, root).map((path) => {
+ const bytes = readFileSync(resolve(root, path));
+ return { path, sha256: sha256(bytes), bytes: bytes.length };
+ });
+ return { id, item, review, files, bundledAt: new Date(now).toISOString(), hash: bundleHash({ item, review, files }) };
+}
+
+// Two bundles may share a file (one hero image on two posts), but never
+// disagree about its bytes: each review record is bound to one exact file.
+function fileConflicts(bundles) {
+ const seen = new Map();
+ const conflicts = [];
+ for (const bundle of bundles) {
+ for (const file of bundle.files) {
+ const prior = seen.get(file.path);
+ if (prior && prior.sha256 !== file.sha256) conflicts.push(`${file.path} differs between ${prior.id} and ${bundle.id}`);
+ else seen.set(file.path, { id: bundle.id, sha256: file.sha256 });
+ }
+ }
+ return conflicts;
+}
+
+async function writeIndex({ bundles, storage, env, now }) {
+ const index = { version: INDEX_VERSION, updatedAt: new Date(now).toISOString(), bundles };
+ index.signature = signIndex(index, env);
+ await storage.put(INDEX_KEY, Buffer.from(`${JSON.stringify(index, null, '\t')}\n`), 'application/json');
+ return index;
+}
+
+async function removeFiles(storage, id, files, keep = new Set()) {
+ for (const file of files) if (!keep.has(file.sha256)) await storage.remove(fileKey(id, file.sha256));
+}
+
+// `replaceIndex` starts a new index when the stored one cannot be verified,
+// which is what an operator needs after rotating the secret.
+export async function publishBundles({ root, ids, storage = r2Storage(), env = process.env, now = Date.now(), replaceIndex = false }) {
+ secretFrom(env);
+ const built = [];
+ const failures = [];
+ for (const id of [...new Set(ids)]) {
+ try {
+ built.push(buildBundle(root, id, { now }));
+ } catch (err) {
+ failures.push(err.message);
+ }
+ }
+ if (failures.length) throw new Error(failures.join('\n'));
+
+ let existing = null;
+ try {
+ existing = await readIndex({ storage, env });
+ } catch (err) {
+ if (!(replaceIndex && err instanceof OverlayError && err.code !== 'unconfigured')) throw err;
+ }
+ const replaced = (existing?.bundles || []).filter((bundle) => built.some((row) => row.id === bundle.id));
+ const bundles = [...(existing?.bundles || []).filter((bundle) => !replaced.includes(bundle)), ...built];
+ const conflicts = fileConflicts(bundles);
+ if (conflicts.length) throw new Error(`bundles disagree about a file, so one of them would fail its review: ${conflicts.join('; ')}`);
+
+ // Files first, then the index, so the index never names a file that is not
+ // there yet.
+ let uploaded = 0;
+ let skipped = 0;
+ for (const bundle of built) {
+ for (const file of bundle.files) {
+ const key = fileKey(bundle.id, file.sha256);
+ if (await stored(storage, key)) {
+ skipped++;
+ continue;
+ }
+ await storage.put(key, readFileSync(resolve(root, file.path)), contentTypeOf(file.path));
+ uploaded++;
+ }
+ }
+ await writeIndex({ bundles, storage, env, now });
+ for (const old of replaced) {
+ const current = built.find((row) => row.id === old.id);
+ await removeFiles(storage, old.id, old.files, new Set(current.files.map((file) => file.sha256)));
+ }
+ return { published: built.map((bundle) => bundle.id), uploaded, skipped };
+}
+
+// Drops a bundle and its files. The last bundle takes the index with it, so an
+// empty bucket reads as "no index" rather than as an index of nothing.
+export async function removeBundle({ id, storage = r2Storage(), env = process.env, now = Date.now() }) {
+ const index = await readIndex({ storage, env });
+ const bundle = index?.bundles.find((row) => row.id === id);
+ if (!bundle) return { removed: false, remaining: index?.bundles.length || 0 };
+ const bundles = index.bundles.filter((row) => row !== bundle);
+ if (bundles.length) await writeIndex({ bundles, storage, env, now });
+ else await storage.remove(INDEX_KEY);
+ await removeFiles(storage, id, bundle.files);
+ return { removed: true, remaining: bundles.length };
+}
+
+export async function readIndex({ storage = r2Storage(), env = process.env }) {
+ secretFrom(env);
+ const bytes = await storage.get(INDEX_KEY);
+ if (!bytes) return null;
+ let index;
+ try {
+ index = JSON.parse(bytes.toString('utf8'));
+ } catch {
+ throw new OverlayError('bad_signature', 'the bundle index is not JSON, so there is nothing to verify');
+ }
+ verifyIndex(index, env);
+ if (index.version !== INDEX_VERSION) throw new OverlayError('bad_bundle', `bundle index version ${index.version} is not ${INDEX_VERSION}`);
+ if (!Array.isArray(index.bundles)) throw new OverlayError('bad_bundle', 'the bundle index has no bundle list');
+ index.bundles.forEach(checkBundle);
+ return index;
+}
+
+// Downloads every file the index names and checks it against its recorded hash.
+export async function verifyBundles({ storage = r2Storage(), env = process.env }) {
+ const index = await readIndex({ storage, env });
+ if (!index) return { index: null, bundles: 0, files: 0, problems: [] };
+ const problems = fileConflicts(index.bundles);
+ let files = 0;
+ for (const bundle of index.bundles) {
+ for (const file of bundle.files) {
+ files++;
+ const bytes = await storage.get(fileKey(bundle.id, file.sha256));
+ if (!bytes) problems.push(`${bundle.id}: ${file.path} is missing from storage`);
+ else if (sha256(bytes) !== file.sha256) problems.push(`${bundle.id}: ${file.path} does not hash to ${file.sha256}`);
+ }
+ }
+ return { index, bundles: index.bundles.length, files, problems };
+}
+
+// ── The overlay directory ───────────────────────────────────────────────────
+function kindOf(path) {
+ try {
+ const stat = lstatSync(path);
+ return stat.isSymbolicLink() ? 'link' : stat.isDirectory() ? 'directory' : 'file';
+ } catch (err) {
+ if (err.code === 'ENOENT') return null;
+ throw err;
+ }
+}
+
+const isDirectory = (path) => existsSync(path) && statSync(path).isDirectory();
+
+// Removes what is at `path` and never what a symlink there points to.
+function discard(path) {
+ const kind = kindOf(path);
+ if (kind === 'directory') rmSync(path, { recursive: true, force: true });
+ else if (kind) unlinkSync(path);
+}
+
+function link(target, path) {
+ if (kindOf(path) === 'link' && readlinkSync(path) === target) return;
+ discard(path);
+ symlinkSync(target, path);
+}
+
+// Written beside the destination and renamed over it. A rename replaces a
+// symlink instead of following it, so a write can never land in the image.
+function writeAtomic(path, bytes) {
+ const partial = `${path}.${process.pid}.partial`;
+ discard(partial);
+ writeFileSync(partial, bytes);
+ if (kindOf(path) === 'directory') discard(path);
+ renameSync(partial, path);
+}
+
+const parentOf = (path) => (path.includes('/') ? path.slice(0, path.lastIndexOf('/')) : '');
+const nameOf = (path) => path.slice(path.lastIndexOf('/') + 1);
+
+// Every directory on the way to each path, nearest the root included.
+function directoriesOf(paths) {
+ const directories = new Set();
+ for (const path of paths) for (let parent = parentOf(path); parent; parent = parentOf(parent)) directories.add(parent);
+ return directories;
+}
+
+// Lays `dir` out as a view of the image with room for `writes`, the relative
+// paths of the files the overlay is about to write. A directory on the way to
+// any of those paths becomes a real directory whose image entries are linked
+// one by one; every other directory is a single link to the image's own. Works
+// for a write at any depth, and leaves nothing behind from an earlier tick
+// that this one does not want.
+function layout({ imageRoot, dir, writes }) {
+ const owned = new Set(writes);
+ const parents = directoriesOf(writes);
+
+ const visit = (relative) => {
+ const here = join(dir, relative);
+ if (kindOf(here) !== 'directory') {
+ discard(here);
+ mkdirSync(here, { recursive: true });
+ }
+ const wanted = new Set(relative ? [] : [MARKER]);
+ const source = join(imageRoot, relative);
+ const names = isDirectory(source) ? readdirSync(source) : [];
+ for (const name of relative ? names : names.filter((entry) => EXPOSED.includes(entry))) {
+ const child = relative ? `${relative}/${name}` : name;
+ wanted.add(name);
+ if (!owned.has(child) && !parents.has(child)) link(join(source, name), join(here, name));
+ }
+ for (const path of [...owned, ...parents]) if (parentOf(path) === relative) wanted.add(nameOf(path));
+ for (const path of parents) if (parentOf(path) === relative) visit(path);
+ for (const name of readdirSync(here)) if (!wanted.has(name)) discard(join(here, name));
+ };
+ visit('');
+}
+
+const inside = (child, parent) => child === parent || child.startsWith(`${parent}${sep}`);
+
+// The overlay prunes what it does not recognize, so it only ever takes a
+// directory that is new, empty, or already its own.
+function claimDirectory(dir, imageRoot) {
+ if (inside(imageRoot, dir)) throw new OverlayError('bad_dir', `${dir} contains the image root`);
+ if (EXPOSED.some((name) => inside(dir, join(imageRoot, name)))) throw new OverlayError('bad_dir', `${dir} is inside a directory the overlay mirrors`);
+ const kind = kindOf(dir);
+ if (kind && kind !== 'directory') throw new OverlayError('bad_dir', `${dir} is not a directory`);
+ if (kind && readdirSync(dir).length && !existsSync(join(dir, MARKER))) throw new OverlayError('bad_dir', `${dir} holds files that are not an overlay; pick an empty directory`);
+ mkdirSync(dir, { recursive: true, mode: 0o700 });
+}
+
+const hasBytes = (path, hash) => kindOf(path) === 'file' && sha256(readFileSync(path)) === hash;
+
+// The image queue with the bundled items laid over it. A post the image records
+// as posted is history and is never replaced; cadence and quality settings are
+// always the image's.
+export function mergeQueue(imageQueue, bundles) {
+ const items = [...(imageQueue.items || [])];
+ const applied = [];
+ const superseded = [];
+ for (const bundle of bundles) {
+ const at = items.findIndex((item) => item.id === bundle.id);
+ if (at >= 0 && items[at].status === 'posted') {
+ superseded.push(bundle.id);
+ continue;
+ }
+ if (at >= 0) items[at] = bundle.item;
+ else items.push(bundle.item);
+ applied.push(bundle);
+ }
+ return { queue: { ...imageQueue, items }, applied, superseded };
+}
+
+// Builds the overlay directory and returns its root. `files` counts the bundle
+// files in the overlay, `reused` how many of them an earlier tick had already
+// downloaded.
+export async function materialize({ imageRoot, storage = r2Storage(), env = process.env, dir = DEFAULT_OVERLAY_DIR, now = Date.now(), index = null }) {
+ const image = resolve(imageRoot);
+ const root = resolve(dir);
+ const verified = index ? verifyIndex(index, env) : await readIndex({ storage, env });
+ if (!verified) throw new OverlayError('no_index', 'there is no bundle index in storage');
+ verified.bundles.forEach(checkBundle);
+
+ const { queue, applied, superseded } = mergeQueue(loadQueue(image), verified.bundles);
+ const conflicts = fileConflicts(applied);
+ if (conflicts.length) throw new OverlayError('bad_bundle', `bundles disagree about a file: ${conflicts.join('; ')}`);
+ const files = new Map();
+ for (const bundle of applied) for (const file of bundle.files) files.set(file.path, { ...file, id: bundle.id });
+
+ const writes = [QUEUE_PATH, ...applied.map((bundle) => reviewPath(bundle.id)), ...files.keys()];
+ const directories = directoriesOf(writes);
+ for (const path of files.keys()) if (directories.has(path)) throw new OverlayError('bad_path', `${path} is both a file and a directory on the way to another file`);
+
+ claimDirectory(root, image);
+ writeAtomic(join(root, MARKER), `${JSON.stringify({ imageRoot: image, materializedAt: new Date(now).toISOString() })}\n`);
+ layout({ imageRoot: image, dir: root, writes });
+
+ let reused = 0;
+ for (const file of files.values()) {
+ const path = join(root, file.path);
+ if (hasBytes(path, file.sha256)) {
+ reused++;
+ continue;
+ }
+ const bytes = await storage.get(fileKey(file.id, file.sha256));
+ if (!bytes) throw new OverlayError('bad_file', `${file.id}: ${file.path} is missing from storage`);
+ if (sha256(bytes) !== file.sha256) throw new OverlayError('bad_file', `${file.id}: ${file.path} does not hash to the ${file.sha256.slice(0, 12)} the index recorded`);
+ writeAtomic(path, bytes);
+ }
+ for (const bundle of applied) writeAtomic(join(root, reviewPath(bundle.id)), `${JSON.stringify(bundle.review, null, '\t')}\n`);
+ writeAtomic(join(root, QUEUE_PATH), `${JSON.stringify(queue, null, '\t')}\n`);
+
+ return { root, bundles: applied.map((bundle) => bundle.id), files: files.size, reused, superseded };
+}
+
+// The one call the cron makes: the root this tick should run against, and why.
+// It answers with the image root whenever the overlay is switched off or cannot
+// be trusted, and it never throws.
+export async function resolveRoot({ imageRoot, env = process.env, storage = null, dir = DEFAULT_OVERLAY_DIR, now = Date.now() }) {
+ const image = (reason) => ({ root: imageRoot, overlay: null, reason });
+ const unavailable = (reason) => {
+ console.warn('[x-content] overlay unavailable, using the image queue', reason);
+ return image(reason);
+ };
+ if (env.X_CONTENT_REMOTE_QUEUE !== 'true') return image('X_CONTENT_REMOTE_QUEUE is not true');
+ try {
+ secretFrom(env);
+ if (!storage && !(await import('../r2.js')).objectStorageConfigured()) return unavailable('object storage is not configured');
+ const store = storage || r2Storage();
+ const index = await readIndex({ storage: store, env });
+ if (!index) return image('there is no bundle index in storage');
+ const { root, ...overlay } = await materialize({ imageRoot, storage: store, env, dir, now, index });
+ if (!overlay.bundles.length) return image('every bundle in the index is already posted in the image queue');
+ return { root, overlay, reason: `${overlay.bundles.length} bundle(s) laid over the image queue` };
+ } catch (err) {
+ return unavailable(String(err?.message || err));
+ }
+}
diff --git a/api/cron/x-content.js b/api/cron/x-content.js
index 31237106b..678377ee3 100644
--- a/api/cron/x-content.js
+++ b/api/cron/x-content.js
@@ -7,6 +7,12 @@
// or an X Article followed by posts that quote it. Pacing, jitter, rotation,
// and the publish ledger live in api/_lib/x-content/.
//
+// With X_CONTENT_REMOTE_QUEUE=true the tick first lays the signed bundles from
+// object storage over the image (api/_lib/x-content/overlay.js), so a post
+// approved after the last deploy goes out without another one. The engine and
+// every editorial rule are the same either way, and a tick whose overlay cannot
+// be verified runs against the image queue instead of failing.
+//
// Posting is off until the owner turns it on. With X_CONTENT_AUTO_PUBLISH unset
// the tick runs in preview mode and returns the exact X API calls it would make,
// so the queue can be verified in production before a single post goes out.
@@ -19,6 +25,7 @@ import { json, method, wrapCron } from '../_lib/http.js';
import { requireCron } from '../_lib/cron-auth.js';
import { runTick } from '../_lib/x-content/runner.js';
import { dbStore } from '../_lib/x-content/state.js';
+import { resolveRoot } from '../_lib/x-content/overlay.js';
export default wrapCron(async (req, res) => {
if (!method(req, res, ['GET'])) return;
@@ -31,9 +38,10 @@ export default wrapCron(async (req, res) => {
}
const live = process.env.X_CONTENT_AUTO_PUBLISH === 'true';
try {
- const result = await runTick({ root: process.cwd(), store, dryRun: !live });
+ const { root, overlay, reason: queueReason } = await resolveRoot({ imageRoot: process.cwd() });
+ const result = await runTick({ root, store, dryRun: !live });
if (result.blocked.length) console.warn('[cron] x-content approved items failing validation', result.blocked);
- json(res, 200, { ok: true, mode: live ? 'publish' : 'preview', ...result });
+ json(res, 200, { ok: true, mode: live ? 'publish' : 'preview', ...result, queue: overlay ? 'overlay' : 'image', queueReason, overlay });
} catch (err) {
console.error('[cron] x-content publish failed; progress is saved and the next tick resumes', err);
json(res, 502, { ok: false, mode: live ? 'publish' : 'preview', error: String(/** @type {any} */ (err)?.message || err) });
diff --git a/scripts/x-bundle.mjs b/scripts/x-bundle.mjs
new file mode 100644
index 000000000..f4c6bf1e2
--- /dev/null
+++ b/scripts/x-bundle.mjs
@@ -0,0 +1,176 @@
+#!/usr/bin/env node
+// Operator CLI for the content queue's bundles: how an approved post reaches
+// production without a deploy (api/_lib/x-content/overlay.js).
+//
+// node scripts/x-bundle.mjs publish <id> [<id>...] bundle approved items and upload them
+// node scripts/x-bundle.mjs publish --approved every approved item the ledger has not published
+// node scripts/x-bundle.mjs list the verified index, one bundle per line
+// node scripts/x-bundle.mjs pull [--dir <path>] build the root production will see, locally
+// node scripts/x-bundle.mjs remove <id> drop a bundle and its files
+// node scripts/x-bundle.mjs verify check the signature and every file hash
+//
+// `publish --replace-index` starts a new index when the stored one no longer
+// verifies, which is the step that follows rotating the secret.
+//
+// Env: reads .env.local then .env. The storage credentials (S3_ENDPOINT,
+// S3_BUCKET, S3_ACCESS_KEY_ID, S3_SECRET_ACCESS_KEY) and the signing secret
+// (X_CONTENT_BUNDLE_SECRET) are taken from the Cloud Run service when they are
+// not set locally, so a bundle is signed with the secret production verifies
+// against. DATABASE_URL gives `publish --approved` the shared publish ledger.
+
+import { existsSync, readFileSync } from 'node:fs';
+import { resolve } from 'node:path';
+import { spawnSync } from 'node:child_process';
+import { fileURLToPath } from 'node:url';
+
+import { loadQueue } from '../api/_lib/x-content/queue.js';
+import { REVIEW_MAX_AGE_DAYS } from '../api/_lib/x-content/review.js';
+import { dbStore, memoryStore } from '../api/_lib/x-content/state.js';
+import { DEFAULT_OVERLAY_DIR, buildBundle, materialize, publishBundles, r2Storage, readIndex, removeBundle, verifyBundles } from '../api/_lib/x-content/overlay.js';
+
+const root = resolve(fileURLToPath(new URL('..', import.meta.url)));
+const DAY = 24 * 60 * 60_000;
+
+function loadEnvFile(path) {
+ if (!existsSync(path)) return;
+ for (const line of readFileSync(path, 'utf8').split('\n')) {
+ const match = line.match(/^\s*([A-Z0-9_]+)\s*=\s*(.*?)\s*$/i);
+ if (match && !process.env[match[1]]) process.env[match[1]] = match[2].replace(/^["']|["']$/g, '');
+ }
+}
+loadEnvFile(resolve(root, '.env.local'));
+loadEnvFile(resolve(root, '.env'));
+
+const args = process.argv.slice(2);
+const VALUE_FLAGS = new Set(['--dir']);
+const positional = args.filter((arg, index) => !arg.startsWith('--') && !VALUE_FLAGS.has(args[index - 1]));
+const has = (flag) => args.includes(`--${flag}`);
+const option = (name, fallback = null) => {
+ const inline = args.find((arg) => arg.startsWith(`--${name}=`));
+ if (inline) return inline.slice(name.length + 3);
+ const index = args.indexOf(`--${name}`);
+ return index >= 0 && args[index + 1] && !args[index + 1].startsWith('--') ? args[index + 1] : fallback;
+};
+const command = positional[0] || 'list';
+
+function fail(message) {
+ console.error(message);
+ process.exit(1);
+}
+
+// What api/_lib/env.js reads for the storage client, plus the signing secret.
+const BUNDLE_ENV = ['S3_ENDPOINT', 'S3_BUCKET', 'S3_ACCESS_KEY_ID', 'S3_SECRET_ACCESS_KEY', 'X_CONTENT_BUNDLE_SECRET'];
+
+function hydrateBundleEnv() {
+ if (!process.env.GOOGLE_CLOUD_PROJECT) process.env.GOOGLE_CLOUD_PROJECT = 'aerial-vehicle-466722-p5';
+ for (const name of BUNDLE_ENV.filter((key) => !String(process.env[key] ?? '').trim())) {
+ const read = spawnSync(process.execPath, [resolve(root, 'scripts/read-service-env.mjs'), `^${name}$`, '--raw'], { cwd: root, encoding: 'utf8' });
+ const value = read.status === 0 ? read.stdout.trim() : '';
+ if (value) process.env[name] = value;
+ }
+ const missing = BUNDLE_ENV.filter((key) => !String(process.env[key] ?? '').trim());
+ if (missing.includes('X_CONTENT_BUNDLE_SECRET')) {
+ fail('X_CONTENT_BUNDLE_SECRET is not set here or on the Cloud Run service. Generate one with `openssl rand -hex 32`, store it on the service as a Secret Manager reference, and run this again.');
+ }
+ if (missing.length) fail(`Object storage is not configured: ${missing.join(', ')} missing here and on the Cloud Run service.`);
+}
+
+const kb = (bytes) => `${(bytes / 1024).toFixed(1)} KB`;
+const totalBytes = (bundle) => bundle.files.reduce((sum, file) => sum + file.bytes, 0);
+
+async function approvedIds() {
+ const store = process.env.DATABASE_URL ? dbStore() : memoryStore();
+ const state = await store.load();
+ const published = new Set((state.published || []).map((row) => row.id));
+ console.log(`ledger: ${store.label}`);
+ return loadQueue(root).items.filter((item) => item.status === 'approved' && !published.has(item.id)).map((item) => item.id);
+}
+
+async function publish() {
+ const named = positional.slice(1);
+ if (!named.length && !has('approved')) fail('Usage: publish <id> [<id>...] | publish --approved [--replace-index]');
+ const ids = named.length ? named : await approvedIds();
+ if (!ids.length) {
+ console.log('Nothing to bundle: no approved item is waiting to be published.');
+ return;
+ }
+
+ // Named items are all or nothing. A sweep of everything approved skips what
+ // cannot be bundled, says why, and still ships the rest.
+ const ready = [];
+ const refused = [];
+ for (const id of ids) {
+ try {
+ const bundle = buildBundle(root, id);
+ ready.push(id);
+ console.log(`ready ${id}: ${bundle.files.length} file(s), ${kb(totalBytes(bundle))}`);
+ } catch (err) {
+ refused.push(id);
+ console.error(`refuse ${err.message}`);
+ }
+ }
+ if (refused.length && named.length) fail(`Nothing was uploaded: ${refused.join(', ')} cannot be bundled.`);
+ if (!ready.length) fail('Nothing was uploaded: no item could be bundled.');
+
+ const result = await publishBundles({ root, ids: ready, storage: r2Storage(), replaceIndex: has('replace-index') });
+ console.log(`\nPublished ${result.published.join(', ')}: ${result.uploaded} file(s) uploaded, ${result.skipped} already stored.`);
+ console.log('Production reads the index at the start of its next tick when X_CONTENT_REMOTE_QUEUE is true.');
+ if (refused.length) process.exit(1);
+}
+
+function reviewWindow(bundle, now) {
+ const age = (now - Date.parse(bundle.review.reviewedAt)) / DAY;
+ if (!(age <= REVIEW_MAX_AGE_DAYS)) return `review EXPIRED (${Math.floor(age)} days old, limit ${REVIEW_MAX_AGE_DAYS})`;
+ return `review ${Math.floor(age)} day(s) old, ${Math.ceil(REVIEW_MAX_AGE_DAYS - age)} left`;
+}
+
+async function list() {
+ const index = await readIndex({ storage: r2Storage() });
+ if (!index) {
+ console.log('No bundle index in storage.');
+ return;
+ }
+ const now = Date.now();
+ console.log(`index signed and verified, updated ${index.updatedAt}, ${index.bundles.length} bundle(s)\n`);
+ for (const bundle of index.bundles) {
+ console.log(` ${bundle.id.padEnd(30)} T${bundle.item.tier} ${String(bundle.item.kind).padEnd(7)} ${bundle.files.length} file(s), ${kb(totalBytes(bundle))} bundled ${bundle.bundledAt} ${reviewWindow(bundle, now)}`);
+ }
+}
+
+async function pull() {
+ const dir = resolve(option('dir', DEFAULT_OVERLAY_DIR));
+ const result = await materialize({ imageRoot: root, storage: r2Storage(), dir });
+ console.log(`bundles: ${result.bundles.join(', ') || 'none'}`);
+ if (result.superseded.length) console.log(`already posted in this checkout's queue, so left alone: ${result.superseded.join(', ')}`);
+ console.log(`files: ${result.files} (${result.reused} reused from an earlier pull)`);
+ console.log(`root: ${result.root}`);
+}
+
+async function remove() {
+ const id = positional[1];
+ if (!id) fail('Usage: remove <id>');
+ const result = await removeBundle({ id, storage: r2Storage() });
+ if (!result.removed) fail(`${id} is not in the bundle index.`);
+ console.log(`Removed ${id} and its files. ${result.remaining ? `${result.remaining} bundle(s) remain.` : 'The index was the last object, so it is gone too.'}`);
+}
+
+async function verify() {
+ const result = await verifyBundles({ storage: r2Storage() });
+ if (!result.index) {
+ console.log('No bundle index in storage; nothing to verify.');
+ return;
+ }
+ console.log(`signature: ok (index updated ${result.index.updatedAt})`);
+ for (const problem of result.problems) console.error(`error ${problem}`);
+ if (result.problems.length) fail(`x-bundle: ${result.problems.length} problem(s) across ${result.bundles} bundle(s)`);
+ console.log(`files: ${result.files} downloaded across ${result.bundles} bundle(s), every hash matches`);
+}
+
+const commands = { publish, list, pull, remove, verify };
+if (!commands[command]) fail(`Unknown command ${command}. Commands: ${Object.keys(commands).join(', ')}`);
+hydrateBundleEnv();
+try {
+ await commands[command]();
+} catch (err) {
+ fail(`x-bundle: ${err.code ? `${err.code}: ` : ''}${err.message}`);
+}
diff --git a/tests/x-content-overlay.test.js b/tests/x-content-overlay.test.js
new file mode 100644
index 000000000..5b833516e
--- /dev/null
+++ b/tests/x-content-overlay.test.js
@@ -0,0 +1,591 @@
+import { afterEach, describe, it, expect, vi } from 'vitest';
+import { existsSync, lstatSync, mkdtempSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { dirname, join } from 'node:path';
+
+import {
+ INDEX_KEY, OverlayError, buildBundle, bundleHash, bundlePathProblem, canonical, fileKey, materialize, mergeQueue, proofPath, publishBundles,
+ readIndex, removeBundle, resolveRoot, signIndex, verifyBundles, verifyIndex,
+} from '../api/_lib/x-content/overlay.js';
+import { loadQueue, validateQueue } from '../api/_lib/x-content/queue.js';
+import { contentHash, loadReview, reviewPath } from '../api/_lib/x-content/review.js';
+import { readMedia } from '../api/_lib/x-content/media.js';
+import { runTick } from '../api/_lib/x-content/runner.js';
+import { memoryStore } from '../api/_lib/x-content/state.js';
+
+const HOUR = 3_600_000;
+const SECRET = 'a'.repeat(64);
+const env = { X_CONTENT_BUNDLE_SECRET: SECRET, X_CONTENT_REMOTE_QUEUE: 'true' };
+const CADENCE = { windowMinutes: 45, minimumMinutesApart: 240, dailyCap: 3, slots: [{ tier: 3, at: '08:00' }, { tier: 1, at: '16:00' }, { tier: 2, at: '22:00' }] };
+const QUALITY = { queueSimilarityLimit: 0.34, archiveSimilarityLimit: 0.42 };
+
+const FRESH = 'Rig Doctor names the skeleton convention of any humanoid GLB, then lists which bones animate and which stay put: three.ws/rig-doctor';
+const OLDER = 'Materialize measures the true solid volume of a repaired mesh and prices a physical print from that measurement: three.ws/materialize';
+
+function write(root, path, content) {
+ mkdirSync(dirname(join(root, path)), { recursive: true });
+ writeFileSync(join(root, path), content);
+}
+
+function writeQueue(root, items, over = {}) {
+ write(root, 'data/x-content/queue.json', JSON.stringify({ account: 'trythreews', cadence: CADENCE, quality: QUALITY, items, ...over }));
+}
+
+function writeReview(root, item, over = {}) {
+ const record = { id: item.id, contentHash: contentHash(item, root), reviewedAt: new Date(Date.now() - HOUR).toISOString(), passed: true, blockers: [], ...over };
+ write(root, reviewPath(item.id), JSON.stringify(record));
+ return record;
+}
+
+const post = (id, text, over = {}) => ({
+ id, status: 'approved', kind: 'post', tier: 1, lane: id, pattern: id, notBefore: '2026-09-01T00:00:00Z',
+ posts: [{ text, media: [{ path: `public/x-media/${id}/card.png`, alt: `The ${id} page` }] }],
+ ...over,
+});
+
+// The operator's checkout: one approved, reviewed post the image has never seen.
+function checkout(over = {}) {
+ const root = mkdtempSync(join(tmpdir(), 'x-overlay-checkout-'));
+ const item = post('fresh', FRESH, over);
+ write(root, 'public/x-media/fresh/card.png', Buffer.from('the fresh card, byte for byte'));
+ writeQueue(root, [item]);
+ writeReview(root, item);
+ return { root, item };
+}
+
+// The production image: an older deploy with its own queue, history, and media.
+function image(items = null) {
+ const root = mkdtempSync(join(tmpdir(), 'x-overlay-image-'));
+ const older = post('older', OLDER, { status: 'posted' });
+ write(root, 'public/x-media/older/card.png', Buffer.from('the older card'));
+ write(root, 'public/announce/img/hero.webp', Buffer.from('a hero that only the image has'));
+ write(root, 'data/x-content/volume-model.json', JSON.stringify({ intercept: -2, baseRate: 0.1, features: [] }));
+ write(root, 'data/archives/trythreews_tweets_1.json', JSON.stringify([{ text: 'An unrelated earlier post about something else entirely.' }]));
+ write(root, 'scripts/probe.mjs', 'process.exit(0);\n');
+ write(root, 'package.json', '{"name":"image"}\n');
+ writeQueue(root, items || [older]);
+ writeReview(root, older);
+ return root;
+}
+
+function memoryStorage() {
+ const objects = new Map();
+ const puts = [];
+ return {
+ objects,
+ puts,
+ async put(key, buffer) {
+ puts.push(key);
+ objects.set(key, Buffer.from(buffer));
+ },
+ async get(key) {
+ return objects.has(key) ? Buffer.from(objects.get(key)) : null;
+ },
+ async remove(key) {
+ objects.delete(key);
+ },
+ };
+}
+
+const scratch = () => join(mkdtempSync(join(tmpdir(), 'x-overlay-out-')), 'overlay');
+
+async function published(over = {}) {
+ const source = checkout(over);
+ const storage = memoryStorage();
+ await publishBundles({ root: source.root, ids: ['fresh'], storage, env });
+ return { ...source, storage };
+}
+
+// A signed index around hand-built bundles, for the shapes buildBundle refuses
+// to produce.
+async function forge(storage, bundles) {
+ const index = { version: 1, updatedAt: new Date().toISOString(), bundles: bundles.map((bundle) => ({ ...bundle, hash: bundleHash(bundle) })) };
+ index.signature = signIndex(index, env);
+ await storage.put(INDEX_KEY, Buffer.from(JSON.stringify(index)));
+ return index;
+}
+
+const rejection = async (promise) => {
+ try {
+ await promise;
+ } catch (err) {
+ return err;
+ }
+ return null;
+};
+
+afterEach(() => {
+ vi.restoreAllMocks();
+ vi.unstubAllEnvs();
+});
+
+describe('canonical JSON and the signature', () => {
+ it('does not depend on key order at any depth', () => {
+ const left = { b: 1, a: { d: [1, { z: 1, y: 2 }], c: 'x' }, gone: undefined };
+ const right = { a: { c: 'x', d: [1, { y: 2, z: 1 }] }, b: 1 };
+ expect(canonical(left)).toBe(canonical(right));
+ expect(canonical(left)).toBe('{"a":{"c":"x","d":[1,{"y":2,"z":1}]},"b":1}');
+ expect(canonical([1, 2])).not.toBe(canonical([2, 1]));
+ expect(canonical(JSON.parse(JSON.stringify(left)))).toBe(canonical(left));
+ });
+
+ it('signs an index and verifies it again', () => {
+ const index = { version: 1, updatedAt: '2026-09-29T00:00:00.000Z', bundles: [{ id: 'fresh', item: { id: 'fresh', tier: 1 } }] };
+ index.signature = signIndex(index, env);
+ expect(index.signature).toMatch(/^[0-9a-f]{64}$/);
+ expect(verifyIndex(index, env)).toBe(index);
+ expect(verifyIndex({ signature: index.signature, bundles: [{ item: { tier: 1, id: 'fresh' }, id: 'fresh' }], updatedAt: index.updatedAt, version: 1 }, env)).toBeTruthy();
+ });
+
+ it('rejects a tampered index, a tampered item, and the wrong secret', async () => {
+ const { storage } = await published();
+ const stored = JSON.parse(storage.objects.get(INDEX_KEY).toString('utf8'));
+ expect((await readIndex({ storage, env })).bundles.map((bundle) => bundle.id)).toEqual(['fresh']);
+
+ const tampered = [
+ { ...stored, updatedAt: '2030-01-01T00:00:00.000Z' },
+ { ...stored, bundles: [{ ...stored.bundles[0], item: { ...stored.bundles[0].item, posts: [{ text: 'Buy this other coin now' }] } }] },
+ { ...stored, bundles: [] },
+ { ...stored, signature: 'not a signature' },
+ { ...stored, signature: undefined },
+ ];
+ for (const index of tampered) {
+ storage.objects.set(INDEX_KEY, Buffer.from(JSON.stringify(index)));
+ const err = await rejection(readIndex({ storage, env }));
+ expect(err).toBeInstanceOf(OverlayError);
+ expect(err.code).toBe('bad_signature');
+ }
+
+ storage.objects.set(INDEX_KEY, Buffer.from(JSON.stringify(stored)));
+ expect((await rejection(readIndex({ storage, env: { X_CONTENT_BUNDLE_SECRET: 'b'.repeat(64) } }))).code).toBe('bad_signature');
+ expect((await rejection(readIndex({ storage, env: {} }))).code).toBe('unconfigured');
+ expect((await rejection(readIndex({ storage, env: { X_CONTENT_BUNDLE_SECRET: 'short' } }))).code).toBe('unconfigured');
+ expect(await readIndex({ storage: memoryStorage(), env })).toBeNull();
+ });
+});
+
+describe('buildBundle', () => {
+ it('packs the item, its review record, and the hash of every file', () => {
+ const { root, item } = checkout();
+ const bundle = buildBundle(root, 'fresh', { now: Date.parse('2026-09-29T12:00:00Z') });
+ expect(bundle.item).toEqual(item);
+ expect(bundle.review).toEqual(loadReview(root, 'fresh'));
+ expect(bundle.files).toEqual([{ path: 'public/x-media/fresh/card.png', sha256: expect.stringMatching(/^[0-9a-f]{64}$/), bytes: 29 }]);
+ expect(bundle.bundledAt).toBe('2026-09-29T12:00:00.000Z');
+ expect(bundle.hash).toBe(bundleHash(bundle));
+ });
+
+ it('refuses an item that is not approved', () => {
+ const { root } = checkout({ status: 'review' });
+ expect(() => buildBundle(root, 'fresh')).toThrow(/fresh cannot be bundled: status is review/);
+ expect(() => buildBundle(root, 'absent')).toThrow(/no queue item/);
+ });
+
+ it('refuses an item whose review record does not match its content', () => {
+ const { root, item } = checkout();
+ writeQueue(root, [{ ...item, posts: [{ ...item.posts[0], text: FRESH.replace('Rig Doctor', 'The Rig Doctor') }] }]);
+ expect(() => buildBundle(root, 'fresh')).toThrow(/changed after the last review/);
+
+ writeQueue(root, [item]);
+ writeFileSync(join(root, 'public/x-media/fresh/card.png'), Buffer.from('a different card'));
+ expect(() => buildBundle(root, 'fresh')).toThrow(/changed after the last review/);
+ });
+
+ it('refuses a review that is too old or did not pass', () => {
+ const { root, item } = checkout();
+ writeReview(root, item, { reviewedAt: new Date(Date.now() - 20 * 24 * HOUR).toISOString() });
+ expect(() => buildBundle(root, 'fresh')).toThrow(/days old/);
+ writeReview(root, item, { passed: false, blockers: ['link three.ws/rig-doctor: HTTP 404'] });
+ expect(() => buildBundle(root, 'fresh')).toThrow(/did not pass/);
+ });
+
+ it('bundles the body, cover, and inline images of an Article', () => {
+ const root = mkdtempSync(join(tmpdir(), 'x-overlay-checkout-'));
+ write(root, 'public/x-media/long/cover.png', Buffer.from('cover'));
+ write(root, 'public/x-media/long/inline.png', Buffer.from('inline'));
+ write(root, 'data/x-content/articles/long.md', '## Why\n\nBecause.\n\n![Hero](../../../public/x-media/long/inline.png)\n\nThe end.');
+ const item = {
+ id: 'long', status: 'approved', kind: 'article', tier: 1, lane: 'article', pattern: 'longform', notBefore: '2026-09-01T00:00:00Z',
+ article: { title: 'How rigs get named', body: 'data/x-content/articles/long.md', cover: { path: 'public/x-media/long/cover.png' } },
+ posts: [{ text: 'The long version of how Rig Doctor reads a skeleton.' }],
+ };
+ writeQueue(root, [item]);
+ writeReview(root, item);
+ expect(buildBundle(root, 'long').files.map((file) => file.path)).toEqual(['data/x-content/articles/long.md', 'public/x-media/long/cover.png', 'public/x-media/long/inline.png']);
+ });
+});
+
+describe('publishBundles', () => {
+ it('uploads each file once and is idempotent', async () => {
+ const { root } = checkout();
+ const storage = memoryStorage();
+ const first = await publishBundles({ root, ids: ['fresh', 'fresh'], storage, env });
+ expect(first).toEqual({ published: ['fresh'], uploaded: 1, skipped: 0 });
+ const [file] = buildBundle(root, 'fresh').files;
+ expect(storage.puts).toEqual([fileKey('fresh', file.sha256), INDEX_KEY]);
+ expect(storage.objects.get(fileKey('fresh', file.sha256)).equals(readFileSync(join(root, file.path)))).toBe(true);
+
+ const second = await publishBundles({ root, ids: ['fresh'], storage, env });
+ expect(second).toEqual({ published: ['fresh'], uploaded: 0, skipped: 1 });
+ expect(storage.puts.filter((key) => key !== INDEX_KEY)).toHaveLength(1);
+ const index = await readIndex({ storage, env });
+ expect(index.bundles.map((bundle) => bundle.id)).toEqual(['fresh']);
+ expect([...storage.objects.keys()].sort()).toEqual([fileKey('fresh', file.sha256), INDEX_KEY].sort());
+ });
+
+ it('uploads nothing when any named item cannot be bundled', async () => {
+ const { root, item } = checkout();
+ writeQueue(root, [item, post('draft', OLDER, { status: 'draft' })]);
+ const storage = memoryStorage();
+ await expect(publishBundles({ root, ids: ['fresh', 'draft'], storage, env })).rejects.toThrow(/draft cannot be bundled/);
+ expect(storage.objects.size).toBe(0);
+ await expect(publishBundles({ root, ids: ['fresh'], storage, env: {} })).rejects.toThrow(/X_CONTENT_BUNDLE_SECRET/);
+ });
+
+ it('replaces a bundle with the same id and drops the file it no longer uses', async () => {
+ const { root, item, storage } = await published();
+ const before = buildBundle(root, 'fresh').files[0];
+ writeFileSync(join(root, 'public/x-media/fresh/card.png'), Buffer.from('a reshot card'));
+ writeReview(root, item);
+ await publishBundles({ root, ids: ['fresh'], storage, env });
+ const after = buildBundle(root, 'fresh').files[0];
+ expect(after.sha256).not.toBe(before.sha256);
+ expect((await readIndex({ storage, env })).bundles).toHaveLength(1);
+ expect(storage.objects.has(fileKey('fresh', before.sha256))).toBe(false);
+ expect(storage.objects.has(fileKey('fresh', after.sha256))).toBe(true);
+ });
+
+ it('refuses to build on an index it cannot verify unless told to replace it', async () => {
+ const { root, storage } = await published();
+ const rotated = { X_CONTENT_BUNDLE_SECRET: 'c'.repeat(64) };
+ expect((await rejection(publishBundles({ root, ids: ['fresh'], storage, env: rotated }))).code).toBe('bad_signature');
+ await publishBundles({ root, ids: ['fresh'], storage, env: rotated, replaceIndex: true });
+ expect((await readIndex({ storage, env: rotated })).bundles.map((bundle) => bundle.id)).toEqual(['fresh']);
+ });
+
+ it('removes a bundle, its files, and the index once it is empty', async () => {
+ const { storage } = await published();
+ expect(await removeBundle({ id: 'absent', storage, env })).toEqual({ removed: false, remaining: 1 });
+ expect(await removeBundle({ id: 'fresh', storage, env })).toEqual({ removed: true, remaining: 0 });
+ expect(storage.objects.size).toBe(0);
+ expect(await readIndex({ storage, env })).toBeNull();
+ });
+
+ it('verifies every stored file against the index', async () => {
+ const { root, storage } = await published();
+ expect(await verifyBundles({ storage, env })).toMatchObject({ bundles: 1, files: 1, problems: [] });
+ const [file] = buildBundle(root, 'fresh').files;
+ storage.objects.set(fileKey('fresh', file.sha256), Buffer.from('swapped'));
+ expect((await verifyBundles({ storage, env })).problems.join('\n')).toMatch(/does not hash/);
+ storage.objects.delete(fileKey('fresh', file.sha256));
+ expect((await verifyBundles({ storage, env })).problems.join('\n')).toMatch(/missing from storage/);
+ });
+});
+
+describe('materialize', () => {
+ it('builds a root the unchanged engine reads as an approved, reviewed item', async () => {
+ const { root: source, item, storage } = await published();
+ const imageRoot = image();
+ const result = await materialize({ imageRoot, storage, env, dir: scratch() });
+ expect(result).toMatchObject({ bundles: ['fresh'], files: 1, reused: 0, superseded: [] });
+
+ const queue = loadQueue(result.root);
+ expect(queue.items.map((row) => `${row.id}:${row.status}`)).toEqual(['older:posted', 'fresh:approved']);
+ expect(queue.items[1]).toEqual(item);
+ expect(queue.cadence).toEqual(CADENCE);
+ expect(queue.quality).toEqual(QUALITY);
+
+ const { problems } = validateQueue(queue, result.root);
+ expect(problems.fresh).toEqual([]);
+ expect(problems.older).toEqual([]);
+ expect(loadReview(result.root, 'fresh')).toEqual(loadReview(source, 'fresh'));
+ expect(loadReview(result.root, 'fresh').contentHash).toBe(contentHash(item, result.root));
+
+ const media = readMedia(item.posts[0].media[0], result.root);
+ expect(media.buffer.equals(readFileSync(join(source, 'public/x-media/fresh/card.png')))).toBe(true);
+ expect(media).toMatchObject({ mime: 'image/png', kind: 'image', category: 'tweet_image' });
+ expect(lstatSync(join(result.root, 'public/x-media/fresh/card.png')).isFile()).toBe(true);
+ });
+
+ it('reaches everything the image holds through the overlay', async () => {
+ const { storage } = await published();
+ const imageRoot = image();
+ const { root } = await materialize({ imageRoot, storage, env, dir: scratch() });
+
+ expect(JSON.parse(readFileSync(join(root, 'data/x-content/volume-model.json'), 'utf8')).baseRate).toBe(0.1);
+ expect(readFileSync(join(root, 'public/announce/img/hero.webp'), 'utf8')).toBe('a hero that only the image has');
+ expect(readFileSync(join(root, 'public/x-media/older/card.png'), 'utf8')).toBe('the older card');
+ expect(loadReview(root, 'older')).toEqual(loadReview(imageRoot, 'older'));
+ expect(existsSync(join(root, 'scripts/probe.mjs'))).toBe(true);
+ expect(existsSync(join(root, 'package.json'))).toBe(true);
+
+ // A directory the overlay never writes into is one link; a directory on the
+ // way to a bundle file is real, with the image's entries linked inside it.
+ expect(lstatSync(join(root, 'data/archives')).isSymbolicLink()).toBe(true);
+ expect(lstatSync(join(root, 'public/announce')).isSymbolicLink()).toBe(true);
+ expect(lstatSync(join(root, 'public/x-media/older')).isSymbolicLink()).toBe(true);
+ for (const path of ['data', 'data/x-content', 'data/x-content/reviews', 'public', 'public/x-media', 'public/x-media/fresh']) {
+ expect(lstatSync(join(root, path)).isDirectory()).toBe(true);
+ }
+ expect(lstatSync(join(root, 'data/x-content/reviews/older.json')).isSymbolicLink()).toBe(true);
+ });
+
+ it('writes a bundle file at any depth, and never into the image', async () => {
+ const source = checkout();
+ const deep = 'public/announce/img/launch/2026/card.png';
+ const item = post('fresh', FRESH, { posts: [{ text: FRESH, media: [{ path: deep, alt: 'The launch card' }] }] });
+ write(source.root, deep, Buffer.from('the reviewed card'));
+ writeQueue(source.root, [item]);
+ writeReview(source.root, item);
+ const storage = memoryStorage();
+ await publishBundles({ root: source.root, ids: ['fresh'], storage, env });
+
+ const imageRoot = image([post('fresh', OLDER), post('older', OLDER, { status: 'posted' })]);
+ write(imageRoot, deep, Buffer.from('the card the last deploy shipped'));
+ writeFileSync(join(imageRoot, reviewPath('fresh')), '{"id":"fresh","stale":true}');
+ const before = readFileSync(join(imageRoot, 'data/x-content/queue.json'), 'utf8');
+
+ const { root } = await materialize({ imageRoot, storage, env, dir: scratch() });
+ expect(readFileSync(join(root, deep), 'utf8')).toBe('the reviewed card');
+ expect(readFileSync(join(root, 'public/announce/img/hero.webp'), 'utf8')).toBe('a hero that only the image has');
+ expect(loadQueue(root).items.map((row) => row.id)).toEqual(['fresh', 'older']);
+ expect(loadQueue(root).items[0]).toEqual(item);
+ expect(validateQueue(loadQueue(root), root).problems.fresh).toEqual([]);
+
+ expect(readFileSync(join(imageRoot, deep), 'utf8')).toBe('the card the last deploy shipped');
+ expect(readFileSync(join(imageRoot, reviewPath('fresh')), 'utf8')).toBe('{"id":"fresh","stale":true}');
+ expect(readFileSync(join(imageRoot, 'data/x-content/queue.json'), 'utf8')).toBe(before);
+ });
+
+ it('never replaces an item the image records as posted', async () => {
+ const { storage } = await published();
+ const posted = post('fresh', OLDER, { status: 'posted' });
+ const imageRoot = image([posted]);
+ write(imageRoot, 'public/x-media/fresh/card.png', Buffer.from('what actually went out'));
+ const result = await materialize({ imageRoot, storage, env, dir: scratch() });
+
+ expect(result).toMatchObject({ bundles: [], files: 0, superseded: ['fresh'] });
+ expect(loadQueue(result.root).items).toEqual([posted]);
+ expect(readFileSync(join(result.root, 'public/x-media/fresh/card.png'), 'utf8')).toBe('what actually went out');
+ expect(mergeQueue({ cadence: CADENCE, items: [posted] }, [{ id: 'fresh', item: post('fresh', FRESH) }]).queue).toEqual({ cadence: CADENCE, items: [posted] });
+ });
+
+ it('reuses what an earlier tick downloaded and drops what is no longer bundled', async () => {
+ const { storage } = await published();
+ const imageRoot = image();
+ const dir = scratch();
+ await materialize({ imageRoot, storage, env, dir });
+ const gets = [];
+ const get = storage.get;
+ storage.get = async (key) => {
+ gets.push(key);
+ return get(key);
+ };
+ expect(await materialize({ imageRoot, storage, env, dir })).toMatchObject({ bundles: ['fresh'], files: 1, reused: 1 });
+ expect(gets).toEqual([INDEX_KEY]);
+
+ // A file that rotted on disk is fetched again instead of trusted.
+ writeFileSync(join(dir, 'public/x-media/fresh/card.png'), 'rot');
+ expect(await materialize({ imageRoot, storage, env, dir })).toMatchObject({ files: 1, reused: 0 });
+ expect(readFileSync(join(dir, 'public/x-media/fresh/card.png'), 'utf8')).toBe('the fresh card, byte for byte');
+
+ const other = checkout();
+ const next = post('next', OLDER.replace('Materialize', 'The volume tool'));
+ write(other.root, 'public/x-media/next/card.png', Buffer.from('the next card'));
+ writeQueue(other.root, [next]);
+ writeReview(other.root, next);
+ await publishBundles({ root: other.root, ids: ['next'], storage, env });
+ await removeBundle({ id: 'fresh', storage, env });
+
+ const result = await materialize({ imageRoot, storage, env, dir });
+ expect(result.bundles).toEqual(['next']);
+ expect(existsSync(join(dir, 'public/x-media/fresh'))).toBe(false);
+ expect(existsSync(join(dir, reviewPath('fresh')))).toBe(false);
+ expect(loadQueue(dir).items.map((row) => row.id)).toEqual(['older', 'next']);
+ expect(readFileSync(join(imageRoot, 'public/x-media/older/card.png'), 'utf8')).toBe('the older card');
+ expect(readFileSync(join(imageRoot, 'data/archives/trythreews_tweets_1.json'), 'utf8')).toMatch(/unrelated earlier post/);
+ });
+
+ it('throws bad_file when the stored bytes do not match the recorded hash', async () => {
+ const { root, storage } = await published();
+ const [file] = buildBundle(root, 'fresh').files;
+ storage.objects.set(fileKey('fresh', file.sha256), Buffer.from('swapped in the bucket'));
+ const dir = scratch();
+ const err = await rejection(materialize({ imageRoot: image(), storage, env, dir }));
+ expect(err).toBeInstanceOf(OverlayError);
+ expect(err.code).toBe('bad_file');
+ expect(existsSync(join(dir, 'public/x-media/fresh/card.png'))).toBe(false);
+
+ storage.objects.delete(fileKey('fresh', file.sha256));
+ expect((await rejection(materialize({ imageRoot: image(), storage, env, dir: scratch() }))).code).toBe('bad_file');
+ });
+
+ it('rejects path traversal and every path outside the media roots', async () => {
+ expect(bundlePathProblem('public/x-media/fresh/card.png')).toBeNull();
+ expect(bundlePathProblem('data/x-content/articles/fresh.md')).toBeNull();
+ expect(bundlePathProblem(proofPath('fresh'))).toBeNull();
+
+ const { item, root } = checkout();
+ const review = loadReview(root, 'fresh');
+ const bytes = Buffer.from('payload');
+ const sha256 = buildBundle(root, 'fresh').files[0].sha256;
+ const paths = [
+ '../outside.png', 'public/../../outside.png', 'public/x-media/..', '/etc/passwd', 'C:/windows/card.png', 'scripts/x-content.mjs', 'api/cron/x-content.js',
+ 'public\\x-media\\card.png', 'public//card.png', 'public/./card.png', 'data/x-content/queue.json', 'data/x-content/reviews/fresh.json', '',
+ ];
+ for (const path of paths) {
+ expect(bundlePathProblem(path)).toBeTruthy();
+ const storage = memoryStorage();
+ await storage.put(fileKey('fresh', sha256), bytes);
+ await forge(storage, [{ id: 'fresh', item, review, files: [{ path, sha256, bytes: bytes.length }], bundledAt: new Date().toISOString() }]);
+ const imageRoot = image();
+ const dir = scratch();
+ const err = await rejection(materialize({ imageRoot, storage, env, dir }));
+ expect(err).toBeInstanceOf(OverlayError);
+ expect(err.code).toBe('bad_path');
+ expect(existsSync(dir)).toBe(false);
+ expect(existsSync(join(dirname(dir), 'outside.png'))).toBe(false);
+ }
+ });
+
+ it('rejects a bundle whose id, item, or review record do not agree', async () => {
+ const { item, root } = checkout();
+ const review = loadReview(root, 'fresh');
+ const bundledAt = new Date().toISOString();
+ for (const bundle of [
+ { id: '../fresh', item: { ...item, id: '../fresh' }, review: { ...review, id: '../fresh' }, files: [], bundledAt },
+ { id: 'fresh', item: { ...item, id: 'other' }, review, files: [], bundledAt },
+ { id: 'fresh', item, review: { ...review, id: 'other' }, files: [], bundledAt },
+ ]) {
+ const storage = memoryStorage();
+ await forge(storage, [bundle]);
+ expect((await rejection(materialize({ imageRoot: image(), storage, env, dir: scratch() }))).code).toBe('bad_bundle');
+ }
+ });
+
+ it('only takes a directory that is empty or already an overlay', async () => {
+ const { storage } = await published();
+ const imageRoot = image();
+ const occupied = mkdtempSync(join(tmpdir(), 'x-overlay-occupied-'));
+ writeFileSync(join(occupied, 'notes.txt'), 'someone else keeps files here');
+ expect((await rejection(materialize({ imageRoot, storage, env, dir: occupied }))).code).toBe('bad_dir');
+ expect(readFileSync(join(occupied, 'notes.txt'), 'utf8')).toBe('someone else keeps files here');
+ expect((await rejection(materialize({ imageRoot, storage, env, dir: imageRoot }))).code).toBe('bad_dir');
+ expect((await rejection(materialize({ imageRoot, storage, env, dir: dirname(imageRoot) }))).code).toBe('bad_dir');
+ expect((await rejection(materialize({ imageRoot, storage, env, dir: join(imageRoot, 'public/overlay') }))).code).toBe('bad_dir');
+ expect(existsSync(join(imageRoot, 'public/x-media/older/card.png'))).toBe(true);
+
+ const empty = mkdtempSync(join(tmpdir(), 'x-overlay-empty-'));
+ expect((await materialize({ imageRoot, storage, env, dir: empty })).bundles).toEqual(['fresh']);
+ });
+
+ it('carries the proof record of an item with a scenario', async () => {
+ const source = checkout();
+ const proof = Buffer.from(`${JSON.stringify({ id: 'fresh', scenario: 'rig-doctor', steps: [{ name: 'diagnose', ok: true }] }, null, '\t')}\n`);
+ write(source.root, proofPath('fresh'), proof);
+ const bundle = buildBundle(source.root, 'fresh');
+ expect(bundle.files.map((file) => file.path)).toEqual(['public/x-media/fresh/card.png', 'data/x-content/proofs/fresh.json']);
+
+ const storage = memoryStorage();
+ expect(await publishBundles({ root: source.root, ids: ['fresh'], storage, env })).toMatchObject({ uploaded: 2 });
+ const imageRoot = image();
+ const { root, files } = await materialize({ imageRoot, storage, env, dir: scratch() });
+ expect(files).toBe(2);
+ expect(readFileSync(join(root, 'data/x-content/proofs/fresh.json')).equals(proof)).toBe(true);
+ expect(lstatSync(join(root, 'data/x-content/proofs/fresh.json')).isFile()).toBe(true);
+ expect(existsSync(join(imageRoot, 'data/x-content/proofs'))).toBe(false);
+ });
+});
+
+describe('resolveRoot', () => {
+ const quiet = () => vi.spyOn(console, 'warn').mockImplementation(() => {});
+
+ it('uses the overlay when the index verifies', async () => {
+ const { storage } = await published();
+ const imageRoot = image();
+ const dir = scratch();
+ const result = await resolveRoot({ imageRoot, env, storage, dir });
+ expect(result.root).toBe(dir);
+ expect(result.overlay).toMatchObject({ bundles: ['fresh'], files: 1, reused: 0 });
+ expect(result.overlay.root).toBeUndefined();
+ });
+
+ it('falls back to the image root for every failure and never throws', async () => {
+ const warn = quiet();
+ const { root, storage } = await published();
+ const imageRoot = image();
+ const fallback = (result) => {
+ expect(result.root).toBe(imageRoot);
+ expect(result.overlay).toBeNull();
+ expect(result.reason).toBeTruthy();
+ return result.reason;
+ };
+
+ expect(fallback(await resolveRoot({ imageRoot, env: { X_CONTENT_BUNDLE_SECRET: SECRET }, storage, dir: scratch() }))).toMatch(/X_CONTENT_REMOTE_QUEUE/);
+ expect(fallback(await resolveRoot({ imageRoot, env: { ...env, X_CONTENT_REMOTE_QUEUE: 'TRUE' }, storage, dir: scratch() }))).toMatch(/X_CONTENT_REMOTE_QUEUE/);
+ expect(fallback(await resolveRoot({ imageRoot, env: { ...env, X_CONTENT_REMOTE_QUEUE: '1' }, storage, dir: scratch() }))).toMatch(/X_CONTENT_REMOTE_QUEUE/);
+ expect(warn).not.toHaveBeenCalled();
+
+ expect(fallback(await resolveRoot({ imageRoot, env: { X_CONTENT_REMOTE_QUEUE: 'true' }, storage, dir: scratch() }))).toMatch(/X_CONTENT_BUNDLE_SECRET is not set/);
+ expect(fallback(await resolveRoot({ imageRoot, env, storage: memoryStorage(), dir: scratch() }))).toMatch(/no bundle index/);
+ expect(fallback(await resolveRoot({ imageRoot, env: { ...env, X_CONTENT_BUNDLE_SECRET: 'd'.repeat(64) }, storage, dir: scratch() }))).toMatch(/signature does not verify/);
+
+ for (const name of ['S3_ENDPOINT', 'S3_BUCKET', 'S3_PUBLIC_DOMAIN', 'S3_ACCESS_KEY_ID', 'S3_SECRET_ACCESS_KEY']) vi.stubEnv(name, '');
+ expect(fallback(await resolveRoot({ imageRoot, env, dir: scratch() }))).toMatch(/object storage is not configured/);
+
+ const down = { ...memoryStorage(), get: async () => { throw new Error('socket hang up'); } };
+ expect(fallback(await resolveRoot({ imageRoot, env, storage: down, dir: scratch() }))).toMatch(/socket hang up/);
+
+ const posted = image([post('fresh', OLDER, { status: 'posted' })]);
+ expect((await resolveRoot({ imageRoot: posted, env, storage, dir: scratch() })).root).toBe(posted);
+
+ const [file] = buildBundle(root, 'fresh').files;
+ storage.objects.set(fileKey('fresh', file.sha256), Buffer.from('swapped in the bucket'));
+ expect(fallback(await resolveRoot({ imageRoot, env, storage, dir: scratch() }))).toMatch(/does not hash/);
+
+ const missing = mkdtempSync(join(tmpdir(), 'x-overlay-image-'));
+ const result = await resolveRoot({ imageRoot: missing, env, storage, dir: scratch() });
+ expect(result).toMatchObject({ root: missing, overlay: null });
+
+ expect(warn).toHaveBeenCalledWith('[x-content] overlay unavailable, using the image queue', expect.stringMatching(/does not hash/));
+ });
+});
+
+describe('a tick against the overlay', () => {
+ it('previews the bundled post with the bytes that were reviewed', async () => {
+ const { root: source, storage } = await published();
+ const imageRoot = image();
+ const { root, overlay } = await resolveRoot({ imageRoot, env, storage, dir: scratch() });
+ expect(overlay.bundles).toEqual(['fresh']);
+
+ // 16:50 UTC today: the 16:00 flagship slot is open whatever its jitter.
+ const now = Date.parse(`${new Date().toISOString().slice(0, 10)}T16:50:00Z`);
+ const result = await runTick({ root, store: memoryStore(), dryRun: true, now, env: {} });
+ expect(result.blocked).toEqual([]);
+ expect(result.preview.id).toBe('fresh');
+ expect(result.preview.calls.map((call) => call.call)).toEqual(['media.upload', 'media.metadata', 'tweets.create']);
+ expect(result.preview.calls[0]).toMatchObject({ bytes: readFileSync(join(source, 'public/x-media/fresh/card.png')).length, media_type: 'image/png', media_category: 'tweet_image' });
+ expect(result.preview.calls[1].alt).toBe('The fresh page');
+ expect(result.preview.calls[2].text).toBe(FRESH);
+
+ // The same tick against the image alone has nothing to send.
+ const without = await runTick({ root: imageRoot, store: memoryStore(), dryRun: true, now, env: {} });
+ expect(without.preview).toBeUndefined();
+ });
+
+ it('blocks a bundled post whose review has expired, exactly as the image queue would', async () => {
+ const source = checkout();
+ const storage = memoryStorage();
+ await publishBundles({ root: source.root, ids: ['fresh'], storage, env });
+ const index = await readIndex({ storage, env });
+ const aged = { ...index.bundles[0], review: { ...index.bundles[0].review, reviewedAt: new Date(Date.now() - 15 * 24 * HOUR).toISOString() } };
+ await forge(storage, [aged]);
+
+ const { root } = await materialize({ imageRoot: image(), storage, env, dir: scratch() });
+ const now = Date.parse(`${new Date().toISOString().slice(0, 10)}T16:50:00Z`);
+ const result = await runTick({ root, store: memoryStore(), dryRun: true, now, env: {} });
+ expect(result.preview).toBeUndefined();
+ expect(result.blocked.map((row) => row.id)).toEqual(['fresh']);
+ expect(result.blocked[0].problems.join('\n')).toMatch(/days old/);
+ });
+});