Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.
JavaScript 3d-generationai-agentsanimationavatardigital-humangenerative-aiglbgltfimage-to-3dllmmcpmcp-servermodel-context-protocolsolanatext-to-3dthreejsweb-componentswebglwebxrx402
feat(x-content): publish an approved post without a production deploy
nirholasd3c4ded20823
diff --git a/api/_lib/x-content/overlay.js b/api/_lib/x-content/overlay.jsnew file mode 100644index 000000000..5d42cfc28--- /dev/null+++ b/api/_lib/x-content/overlay.js@@ -0,0 +1,505 @@+// The overlay root: how an approved post reaches production without a deploy.+//+// The engine reads everything from files under one root (the queue, the review+// records, the media), and in production that root is the Cloud Run image, so a+// post approved today could not go out until someone shipped a new image. This+// module removes that ceiling without touching a single editorial rule.+//+// `npm run x:bundle -- publish <id>` packs an approved item, its full review+// record, and every file it needs into a bundle, uploads the files to object+// storage under content-addressed keys, and lists the bundle in one index that+// is signed with X_CONTENT_BUNDLE_SECRET. At the start of a tick production+// verifies that signature, then materializes a directory that mirrors the repo+// layout: the image's own data/ and public/ reached through symlinks, the+// bundle files written on top after their bytes hash to what the index+// recorded. The unchanged engine then runs with `root` pointing at it. The+// review record is still bound to the content hash, still expires, and the+// media limits, link checks, and probes still run, because the engine cannot+// tell an overlay from an image.+//+// Nothing here can stop a tick. Any failure (no secret, no storage, a bad+// signature, a file that does not match its hash) falls back to the image root,+// which is exactly what production did before the overlay existed.++import { createHash, createHmac, timingSafeEqual } from 'node:crypto';+import { existsSync, lstatSync, mkdirSync, readFileSync, readdirSync, readlinkSync, renameSync, rmSync, statSync, symlinkSync, unlinkSync, writeFileSync } from 'node:fs';+import { tmpdir } from 'node:os';+import { isAbsolute, join, resolve, sep } from 'node:path';+import { MEDIA_ROOTS, mediaType } from './media.js';+import { QUEUE_PATH, loadArticle, loadQueue, validateItem } from './queue.js';+import { loadReview, reviewPath } from './review.js';++export const INDEX_KEY = 'x-content/bundles/index.json';+export const INDEX_VERSION = 1;+export const ARTICLE_ROOT = 'data/x-content/articles/';+export const REVIEW_ROOT = 'data/x-content/reviews/';+export const proofPath = (id) => `data/x-content/proofs/${id}.json`;+export const fileKey = (id, sha256) => `x-content/bundles/${id}/${sha256}`;+export const DEFAULT_OVERLAY_DIR = join(tmpdir(), 'x-content-overlay');++// A short secret is a guessable one, and this secret is the only thing standing+// between write access to the bucket and a post on @trythreews.+export const MIN_SECRET_LENGTH = 32;++// Marks a directory as one this module built, so materialize never prunes a+// directory that holds someone's files.+const MARKER = '.x-content-overlay';+// Top-level entries of the image that the overlay exposes. data/ and public/+// are what the engine reads; the rest is what a command probe or a textFrom+// check may look for under root.+const EXPOSED = ['data', 'public', 'scripts', 'docs', 'package.json'];++const SLUG = /^[a-z0-9][a-z0-9-]{1,80}$/;+const SHA256 = /^[0-9a-f]{64}$/;++export class OverlayError extends Error {+ constructor(code, message) {+ super(message);+ this.name = 'OverlayError';+ this.code = code;+ }+}++const sha256 = (bytes) => createHash('sha256').update(bytes).digest('hex');++// ── Canonical JSON and the signature ────────────────────────────────────────+// Keys sorted at every depth, so the bytes that are signed do not depend on the+// order a writer happened to build an object in, or on a store that reorders+// keys. Values follow JSON.stringify: undefined members are dropped, undefined+// array entries become null.+export function canonical(value) {+ if (value && typeof value.toJSON === 'function') return canonical(value.toJSON());+ if (Array.isArray(value)) return `[${value.map((row) => canonical(row === undefined ? null : row)).join(',')}]`;+ if (value && typeof value === 'object') {+ const keys = Object.keys(value).filter((key) => value[key] !== undefined && typeof value[key] !== 'function').sort();+ return `{${keys.map((key) => `${JSON.stringify(key)}:${canonical(value[key])}`).join(',')}}`;+ }+ return JSON.stringify(value) ?? 'null';+}++export const bundleHash = ({ item, review, files }) => sha256(canonical({ item, review, files }));++function secretFrom(env) {+ const secret = String(env?.X_CONTENT_BUNDLE_SECRET ?? '').trim();+ if (!secret) throw new OverlayError('unconfigured', 'X_CONTENT_BUNDLE_SECRET is not set');+ if (secret.length < MIN_SECRET_LENGTH) throw new OverlayError('unconfigured', `X_CONTENT_BUNDLE_SECRET is shorter than ${MIN_SECRET_LENGTH} characters; generate one with \`openssl rand -hex 32\``);+ return secret;+}++export function signIndex({ version, updatedAt, bundles }, env = process.env) {+ return createHmac('sha256', secretFrom(env)).update(canonical({ version, updatedAt, bundles })).digest('hex');+}++export function verifyIndex(index, env = process.env) {+ const expected = Buffer.from(signIndex(index, env), 'hex');+ const given = SHA256.test(String(index?.signature || '')) ? Buffer.from(index.signature, 'hex') : null;+ if (!given || given.length !== expected.length || !timingSafeEqual(given, expected)) {+ throw new OverlayError('bad_signature', 'the bundle index signature does not verify; it was not written with this X_CONTENT_BUNDLE_SECRET, or it was changed after it was signed');+ }+ return index;+}++// ── Paths ───────────────────────────────────────────────────────────────────+// Why a path may not be written into the overlay, or null when it may. The+// queue and the review records are the overlay's own: a bundle file is never+// allowed to stand in for either.+export function bundlePathProblem(path) {+ if (typeof path !== 'string' || !path) return 'is empty';+ if (path.includes('..')) return 'contains ..';+ if (isAbsolute(path) || path.startsWith('/') || /^[a-zA-Z]:/.test(path)) return 'is absolute';+ if (path.includes('\\') || path.includes('\0')) return 'contains a backslash or a null byte';+ if (path.split('/').some((part) => !part || part === '.')) return 'has an empty or dot segment';+ if (![...MEDIA_ROOTS, ARTICLE_ROOT].some((prefix) => path.startsWith(prefix))) return `is outside ${[...MEDIA_ROOTS, ARTICLE_ROOT].join(', ')}`;+ if (path === QUEUE_PATH || path.startsWith(REVIEW_ROOT)) return 'is reserved for the queue and the review records';+ return null;+}++function checkBundle(bundle) {+ const id = String(bundle?.id || '');+ if (!SLUG.test(id)) throw new OverlayError('bad_bundle', `bundle id ${JSON.stringify(bundle?.id)} is not a lowercase slug`);+ if (bundle.item?.id !== id) throw new OverlayError('bad_bundle', `${id}: the bundled item is ${JSON.stringify(bundle.item?.id)}`);+ if (!bundle.review || bundle.review.id !== id) throw new OverlayError('bad_bundle', `${id}: the bundled review record is for ${JSON.stringify(bundle.review?.id)}`);+ if (!Array.isArray(bundle.files)) throw new OverlayError('bad_bundle', `${id}: files is not a list`);+ for (const file of bundle.files) {+ const problem = bundlePathProblem(file?.path);+ if (problem) throw new OverlayError('bad_path', `${id}: file path ${JSON.stringify(file?.path)} ${problem}`);+ if (!SHA256.test(String(file.sha256 || ''))) throw new OverlayError('bad_bundle', `${id}: ${file.path} has no sha256`);+ if (!Number.isInteger(file.bytes) || file.bytes < 0) throw new OverlayError('bad_bundle', `${id}: ${file.path} has no byte count`);+ }+ if (bundle.hash !== bundleHash(bundle)) throw new OverlayError('bad_bundle', `${id}: the bundle hash does not cover its item, review record, and files`);+ return bundle;+}++// ── Storage ─────────────────────────────────────────────────────────────────+// The real store. Loaded on first use, so the cron module stays importable and+// cheap while the overlay is switched off.+export function r2Storage() {+ const r2 = () => import('../r2.js');+ return {+ label: 'object storage',+ async put(key, buffer, contentType) {+ const { putObject } = await r2();+ await putObject({ key, body: buffer, contentType });+ },+ async get(key) {+ const { getObjectBuffer } = await r2();+ try {+ return await getObjectBuffer(key);+ } catch (err) {+ if (err?.name === 'NoSuchKey' || err?.$metadata?.httpStatusCode === 404) return null;+ throw err;+ }+ },+ async has(key) {+ const { headObject } = await r2();+ return Boolean(await headObject(key));+ },+ async remove(key) {+ const { deleteObject } = await r2();+ await deleteObject(key);+ },+ };+}++// `has` is optional on a store: one that only has put, get, and remove answers+// by reading the object.+const stored = async (storage, key) => (storage.has ? storage.has(key) : (await storage.get(key)) !== null);++const contentTypeOf = (path) => mediaType(path)?.mime || (path.endsWith('.md') ? 'text/markdown; charset=utf-8' : path.endsWith('.json') ? 'application/json' : 'application/octet-stream');++// ── Building and publishing ─────────────────────────────────────────────────+// Every file the item needs at publish time: each post's media, and for an+// Article its body, cover, and inline images. An item with a scenario also+// carries its proof record.+function itemPaths(item, root) {+ const paths = [];+ for (const post of item.posts || []) for (const media of post.media || []) paths.push(media.path);+ if (item.kind === 'article') {+ paths.push(item.article.body, item.article.cover.path);+ for (const image of loadArticle(root, item.article).images) paths.push(image.path);+ }+ if (existsSync(resolve(root, proofPath(item.id)))) paths.push(proofPath(item.id));+ return [...new Set(paths)];+}++export function buildBundle(root, id, { now = Date.now() } = {}) {+ const item = (loadQueue(root).items || []).find((row) => row.id === id);+ if (!item) throw new Error(`${id} cannot be bundled: no queue item has that id`);+ const reasons = [];+ if (item.status !== 'approved') reasons.push(`status is ${item.status}; only an approved item is bundled`);+ reasons.push(...validateItem(item, root));+ if (!reasons.length) {+ for (const path of itemPaths(item, root)) {+ const problem = bundlePathProblem(path);+ if (problem) reasons.push(`file path ${path} ${problem}`);+ }+ }+ if (reasons.length) throw new Error(`${id} cannot be bundled: ${reasons.join('; ')}`);++ const review = loadReview(root, id);+ const files = itemPaths(item, root).map((path) => {+ const bytes = readFileSync(resolve(root, path));+ return { path, sha256: sha256(bytes), bytes: bytes.length };+ });+ return { id, item, review, files, bundledAt: new Date(now).toISOString(), hash: bundleHash({ item, review, files }) };+}++// Two bundles may share a file (one hero image on two posts), but never+// disagree about its bytes: each review record is bound to one exact file.+function fileConflicts(bundles) {+ const seen = new Map();+ const conflicts = [];+ for (const bundle of bundles) {+ for (const file of bundle.files) {+ const prior = seen.get(file.path);+ if (prior && prior.sha256 !== file.sha256) conflicts.push(`${file.path} differs between ${prior.id} and ${bundle.id}`);+ else seen.set(file.path, { id: bundle.id, sha256: file.sha256 });+ }+ }+ return conflicts;+}++async function writeIndex({ bundles, storage, env, now }) {+ const index = { version: INDEX_VERSION, updatedAt: new Date(now).toISOString(), bundles };+ index.signature = signIndex(index, env);+ await storage.put(INDEX_KEY, Buffer.from(`${JSON.stringify(index, null, '\t')}\n`), 'application/json');+ return index;+}++async function removeFiles(storage, id, files, keep = new Set()) {+ for (const file of files) if (!keep.has(file.sha256)) await storage.remove(fileKey(id, file.sha256));+}++// `replaceIndex` starts a new index when the stored one cannot be verified,+// which is what an operator needs after rotating the secret.+export async function publishBundles({ root, ids, storage = r2Storage(), env = process.env, now = Date.now(), replaceIndex = false }) {+ secretFrom(env);+ const built = [];+ const failures = [];+ for (const id of [...new Set(ids)]) {+ try {+ built.push(buildBundle(root, id, { now }));+ } catch (err) {+ failures.push(err.message);+ }+ }+ if (failures.length) throw new Error(failures.join('\n'));++ let existing = null;+ try {+ existing = await readIndex({ storage, env });+ } catch (err) {+ if (!(replaceIndex && err instanceof OverlayError && err.code !== 'unconfigured')) throw err;+ }+ const replaced = (existing?.bundles || []).filter((bundle) => built.some((row) => row.id === bundle.id));+ const bundles = [...(existing?.bundles || []).filter((bundle) => !replaced.includes(bundle)), ...built];+ const conflicts = fileConflicts(bundles);+ if (conflicts.length) throw new Error(`bundles disagree about a file, so one of them would fail its review: ${conflicts.join('; ')}`);++ // Files first, then the index, so the index never names a file that is not+ // there yet.+ let uploaded = 0;+ let skipped = 0;+ for (const bundle of built) {+ for (const file of bundle.files) {+ const key = fileKey(bundle.id, file.sha256);+ if (await stored(storage, key)) {+ skipped++;+ continue;+ }+ await storage.put(key, readFileSync(resolve(root, file.path)), contentTypeOf(file.path));+ uploaded++;+ }+ }+ await writeIndex({ bundles, storage, env, now });+ for (const old of replaced) {+ const current = built.find((row) => row.id === old.id);+ await removeFiles(storage, old.id, old.files, new Set(current.files.map((file) => file.sha256)));+ }+ return { published: built.map((bundle) => bundle.id), uploaded, skipped };+}++// Drops a bundle and its files. The last bundle takes the index with it, so an+// empty bucket reads as "no index" rather than as an index of nothing.+export async function removeBundle({ id, storage = r2Storage(), env = process.env, now = Date.now() }) {+ const index = await readIndex({ storage, env });+ const bundle = index?.bundles.find((row) => row.id === id);+ if (!bundle) return { removed: false, remaining: index?.bundles.length || 0 };+ const bundles = index.bundles.filter((row) => row !== bundle);+ if (bundles.length) await writeIndex({ bundles, storage, env, now });+ else await storage.remove(INDEX_KEY);+ await removeFiles(storage, id, bundle.files);+ return { removed: true, remaining: bundles.length };+}++export async function readIndex({ storage = r2Storage(), env = process.env }) {+ secretFrom(env);+ const bytes = await storage.get(INDEX_KEY);+ if (!bytes) return null;+ let index;+ try {+ index = JSON.parse(bytes.toString('utf8'));+ } catch {+ throw new OverlayError('bad_signature', 'the bundle index is not JSON, so there is nothing to verify');+ }+ verifyIndex(index, env);+ if (index.version !== INDEX_VERSION) throw new OverlayError('bad_bundle', `bundle index version ${index.version} is not ${INDEX_VERSION}`);+ if (!Array.isArray(index.bundles)) throw new OverlayError('bad_bundle', 'the bundle index has no bundle list');+ index.bundles.forEach(checkBundle);+ return index;+}++// Downloads every file the index names and checks it against its recorded hash.+export async function verifyBundles({ storage = r2Storage(), env = process.env }) {+ const index = await readIndex({ storage, env });+ if (!index) return { index: null, bundles: 0, files: 0, problems: [] };+ const problems = fileConflicts(index.bundles);+ let files = 0;+ for (const bundle of index.bundles) {+ for (const file of bundle.files) {+ files++;+ const bytes = await storage.get(fileKey(bundle.id, file.sha256));+ if (!bytes) problems.push(`${bundle.id}: ${file.path} is missing from storage`);+ else if (sha256(bytes) !== file.sha256) problems.push(`${bundle.id}: ${file.path} does not hash to ${file.sha256}`);+ }+ }+ return { index, bundles: index.bundles.length, files, problems };+}++// ── The overlay directory ───────────────────────────────────────────────────+function kindOf(path) {+ try {+ const stat = lstatSync(path);+ return stat.isSymbolicLink() ? 'link' : stat.isDirectory() ? 'directory' : 'file';+ } catch (err) {+ if (err.code === 'ENOENT') return null;+ throw err;+ }+}++const isDirectory = (path) => existsSync(path) && statSync(path).isDirectory();++// Removes what is at `path` and never what a symlink there points to.+function discard(path) {+ const kind = kindOf(path);+ if (kind === 'directory') rmSync(path, { recursive: true, force: true });+ else if (kind) unlinkSync(path);+}++function link(target, path) {+ if (kindOf(path) === 'link' && readlinkSync(path) === target) return;+ discard(path);+ symlinkSync(target, path);+}++// Written beside the destination and renamed over it. A rename replaces a+// symlink instead of following it, so a write can never land in the image.+function writeAtomic(path, bytes) {+ const partial = `${path}.${process.pid}.partial`;+ discard(partial);+ writeFileSync(partial, bytes);+ if (kindOf(path) === 'directory') discard(path);+ renameSync(partial, path);+}++const parentOf = (path) => (path.includes('/') ? path.slice(0, path.lastIndexOf('/')) : '');+const nameOf = (path) => path.slice(path.lastIndexOf('/') + 1);++// Every directory on the way to each path, nearest the root included.+function directoriesOf(paths) {+ const directories = new Set();+ for (const path of paths) for (let parent = parentOf(path); parent; parent = parentOf(parent)) directories.add(parent);+ return directories;+}++// Lays `dir` out as a view of the image with room for `writes`, the relative+// paths of the files the overlay is about to write. A directory on the way to+// any of those paths becomes a real directory whose image entries are linked+// one by one; every other directory is a single link to the image's own. Works+// for a write at any depth, and leaves nothing behind from an earlier tick+// that this one does not want.+function layout({ imageRoot, dir, writes }) {+ const owned = new Set(writes);+ const parents = directoriesOf(writes);++ const visit = (relative) => {+ const here = join(dir, relative);+ if (kindOf(here) !== 'directory') {+ discard(here);+ mkdirSync(here, { recursive: true });+ }+ const wanted = new Set(relative ? [] : [MARKER]);+ const source = join(imageRoot, relative);+ const names = isDirectory(source) ? readdirSync(source) : [];+ for (const name of relative ? names : names.filter((entry) => EXPOSED.includes(entry))) {+ const child = relative ? `${relative}/${name}` : name;+ wanted.add(name);+ if (!owned.has(child) && !parents.has(child)) link(join(source, name), join(here, name));+ }+ for (const path of [...owned, ...parents]) if (parentOf(path) === relative) wanted.add(nameOf(path));+ for (const path of parents) if (parentOf(path) === relative) visit(path);+ for (const name of readdirSync(here)) if (!wanted.has(name)) discard(join(here, name));+ };+ visit('');+}++const inside = (child, parent) => child === parent || child.startsWith(`${parent}${sep}`);++// The overlay prunes what it does not recognize, so it only ever takes a+// directory that is new, empty, or already its own.+function claimDirectory(dir, imageRoot) {+ if (inside(imageRoot, dir)) throw new OverlayError('bad_dir', `${dir} contains the image root`);+ if (EXPOSED.some((name) => inside(dir, join(imageRoot, name)))) throw new OverlayError('bad_dir', `${dir} is inside a directory the overlay mirrors`);+ const kind = kindOf(dir);+ if (kind && kind !== 'directory') throw new OverlayError('bad_dir', `${dir} is not a directory`);+ if (kind && readdirSync(dir).length && !existsSync(join(dir, MARKER))) throw new OverlayError('bad_dir', `${dir} holds files that are not an overlay; pick an empty directory`);+ mkdirSync(dir, { recursive: true, mode: 0o700 });+}++const hasBytes = (path, hash) => kindOf(path) === 'file' && sha256(readFileSync(path)) === hash;++// The image queue with the bundled items laid over it. A post the image records+// as posted is history and is never replaced; cadence and quality settings are+// always the image's.+export function mergeQueue(imageQueue, bundles) {+ const items = [...(imageQueue.items || [])];+ const applied = [];+ const superseded = [];+ for (const bundle of bundles) {+ const at = items.findIndex((item) => item.id === bundle.id);+ if (at >= 0 && items[at].status === 'posted') {+ superseded.push(bundle.id);+ continue;+ }+ if (at >= 0) items[at] = bundle.item;+ else items.push(bundle.item);+ applied.push(bundle);+ }+ return { queue: { ...imageQueue, items }, applied, superseded };+}++// Builds the overlay directory and returns its root. `files` counts the bundle+// files in the overlay, `reused` how many of them an earlier tick had already+// downloaded.+export async function materialize({ imageRoot, storage = r2Storage(), env = process.env, dir = DEFAULT_OVERLAY_DIR, now = Date.now(), index = null }) {+ const image = resolve(imageRoot);+ const root = resolve(dir);+ const verified = index ? verifyIndex(index, env) : await readIndex({ storage, env });+ if (!verified) throw new OverlayError('no_index', 'there is no bundle index in storage');+ verified.bundles.forEach(checkBundle);++ const { queue, applied, superseded } = mergeQueue(loadQueue(image), verified.bundles);+ const conflicts = fileConflicts(applied);+ if (conflicts.length) throw new OverlayError('bad_bundle', `bundles disagree about a file: ${conflicts.join('; ')}`);+ const files = new Map();+ for (const bundle of applied) for (const file of bundle.files) files.set(file.path, { ...file, id: bundle.id });++ const writes = [QUEUE_PATH, ...applied.map((bundle) => reviewPath(bundle.id)), ...files.keys()];+ const directories = directoriesOf(writes);+ for (const path of files.keys()) if (directories.has(path)) throw new OverlayError('bad_path', `${path} is both a file and a directory on the way to another file`);++ claimDirectory(root, image);+ writeAtomic(join(root, MARKER), `${JSON.stringify({ imageRoot: image, materializedAt: new Date(now).toISOString() })}\n`);+ layout({ imageRoot: image, dir: root, writes });++ let reused = 0;+ for (const file of files.values()) {+ const path = join(root, file.path);+ if (hasBytes(path, file.sha256)) {+ reused++;+ continue;+ }+ const bytes = await storage.get(fileKey(file.id, file.sha256));+ if (!bytes) throw new OverlayError('bad_file', `${file.id}: ${file.path} is missing from storage`);+ if (sha256(bytes) !== file.sha256) throw new OverlayError('bad_file', `${file.id}: ${file.path} does not hash to the ${file.sha256.slice(0, 12)} the index recorded`);+ writeAtomic(path, bytes);+ }+ for (const bundle of applied) writeAtomic(join(root, reviewPath(bundle.id)), `${JSON.stringify(bundle.review, null, '\t')}\n`);+ writeAtomic(join(root, QUEUE_PATH), `${JSON.stringify(queue, null, '\t')}\n`);++ return { root, bundles: applied.map((bundle) => bundle.id), files: files.size, reused, superseded };+}++// The one call the cron makes: the root this tick should run against, and why.+// It answers with the image root whenever the overlay is switched off or cannot+// be trusted, and it never throws.+export async function resolveRoot({ imageRoot, env = process.env, storage = null, dir = DEFAULT_OVERLAY_DIR, now = Date.now() }) {+ const image = (reason) => ({ root: imageRoot, overlay: null, reason });+ const unavailable = (reason) => {+ console.warn('[x-content] overlay unavailable, using the image queue', reason);+ return image(reason);+ };+ if (env.X_CONTENT_REMOTE_QUEUE !== 'true') return image('X_CONTENT_REMOTE_QUEUE is not true');+ try {+ secretFrom(env);+ if (!storage && !(await import('../r2.js')).objectStorageConfigured()) return unavailable('object storage is not configured');+ const store = storage || r2Storage();+ const index = await readIndex({ storage: store, env });+ if (!index) return image('there is no bundle index in storage');+ const { root, ...overlay } = await materialize({ imageRoot, storage: store, env, dir, now, index });+ if (!overlay.bundles.length) return image('every bundle in the index is already posted in the image queue');+ return { root, overlay, reason: `${overlay.bundles.length} bundle(s) laid over the image queue` };+ } catch (err) {+ return unavailable(String(err?.message || err));+ }+}diff --git a/api/cron/x-content.js b/api/cron/x-content.jsindex 31237106b..678377ee3 100644--- a/api/cron/x-content.js+++ b/api/cron/x-content.js@@ -7,6 +7,12 @@// or an X Article followed by posts that quote it. Pacing, jitter, rotation,// and the publish ledger live in api/_lib/x-content/.//+// With X_CONTENT_REMOTE_QUEUE=true the tick first lays the signed bundles from+// object storage over the image (api/_lib/x-content/overlay.js), so a post+// approved after the last deploy goes out without another one. The engine and+// every editorial rule are the same either way, and a tick whose overlay cannot+// be verified runs against the image queue instead of failing.+//// Posting is off until the owner turns it on. With X_CONTENT_AUTO_PUBLISH unset// the tick runs in preview mode and returns the exact X API calls it would make,// so the queue can be verified in production before a single post goes out.@@ -19,6 +25,7 @@ import { json, method, wrapCron } from '../_lib/http.js';import { requireCron } from '../_lib/cron-auth.js';import { runTick } from '../_lib/x-content/runner.js';import { dbStore } from '../_lib/x-content/state.js';+import { resolveRoot } from '../_lib/x-content/overlay.js';export default wrapCron(async (req, res) => {if (!method(req, res, ['GET'])) return;@@ -31,9 +38,10 @@ export default wrapCron(async (req, res) => {}const live = process.env.X_CONTENT_AUTO_PUBLISH === 'true';try {- const result = await runTick({ root: process.cwd(), store, dryRun: !live });+ const { root, overlay, reason: queueReason } = await resolveRoot({ imageRoot: process.cwd() });+ const result = await runTick({ root, store, dryRun: !live });if (result.blocked.length) console.warn('[cron] x-content approved items failing validation', result.blocked);- json(res, 200, { ok: true, mode: live ? 'publish' : 'preview', ...result });+ json(res, 200, { ok: true, mode: live ? 'publish' : 'preview', ...result, queue: overlay ? 'overlay' : 'image', queueReason, overlay });} catch (err) {console.error('[cron] x-content publish failed; progress is saved and the next tick resumes', err);json(res, 502, { ok: false, mode: live ? 'publish' : 'preview', error: String(/** @type {any} */ (err)?.message || err) });diff --git a/scripts/x-bundle.mjs b/scripts/x-bundle.mjsnew file mode 100644index 000000000..f4c6bf1e2--- /dev/null+++ b/scripts/x-bundle.mjs@@ -0,0 +1,176 @@+#!/usr/bin/env node+// Operator CLI for the content queue's bundles: how an approved post reaches+// production without a deploy (api/_lib/x-content/overlay.js).+//+// node scripts/x-bundle.mjs publish <id> [<id>...] bundle approved items and upload them+// node scripts/x-bundle.mjs publish --approved every approved item the ledger has not published+// node scripts/x-bundle.mjs list the verified index, one bundle per line+// node scripts/x-bundle.mjs pull [--dir <path>] build the root production will see, locally+// node scripts/x-bundle.mjs remove <id> drop a bundle and its files+// node scripts/x-bundle.mjs verify check the signature and every file hash+//+// `publish --replace-index` starts a new index when the stored one no longer+// verifies, which is the step that follows rotating the secret.+//+// Env: reads .env.local then .env. The storage credentials (S3_ENDPOINT,+// S3_BUCKET, S3_ACCESS_KEY_ID, S3_SECRET_ACCESS_KEY) and the signing secret+// (X_CONTENT_BUNDLE_SECRET) are taken from the Cloud Run service when they are+// not set locally, so a bundle is signed with the secret production verifies+// against. DATABASE_URL gives `publish --approved` the shared publish ledger.++import { existsSync, readFileSync } from 'node:fs';+import { resolve } from 'node:path';+import { spawnSync } from 'node:child_process';+import { fileURLToPath } from 'node:url';++import { loadQueue } from '../api/_lib/x-content/queue.js';+import { REVIEW_MAX_AGE_DAYS } from '../api/_lib/x-content/review.js';+import { dbStore, memoryStore } from '../api/_lib/x-content/state.js';+import { DEFAULT_OVERLAY_DIR, buildBundle, materialize, publishBundles, r2Storage, readIndex, removeBundle, verifyBundles } from '../api/_lib/x-content/overlay.js';++const root = resolve(fileURLToPath(new URL('..', import.meta.url)));+const DAY = 24 * 60 * 60_000;++function loadEnvFile(path) {+ if (!existsSync(path)) return;+ for (const line of readFileSync(path, 'utf8').split('\n')) {+ const match = line.match(/^\s*([A-Z0-9_]+)\s*=\s*(.*?)\s*$/i);+ if (match && !process.env[match[1]]) process.env[match[1]] = match[2].replace(/^["']|["']$/g, '');+ }+}+loadEnvFile(resolve(root, '.env.local'));+loadEnvFile(resolve(root, '.env'));++const args = process.argv.slice(2);+const VALUE_FLAGS = new Set(['--dir']);+const positional = args.filter((arg, index) => !arg.startsWith('--') && !VALUE_FLAGS.has(args[index - 1]));+const has = (flag) => args.includes(`--${flag}`);+const option = (name, fallback = null) => {+ const inline = args.find((arg) => arg.startsWith(`--${name}=`));+ if (inline) return inline.slice(name.length + 3);+ const index = args.indexOf(`--${name}`);+ return index >= 0 && args[index + 1] && !args[index + 1].startsWith('--') ? args[index + 1] : fallback;+};+const command = positional[0] || 'list';++function fail(message) {+ console.error(message);+ process.exit(1);+}++// What api/_lib/env.js reads for the storage client, plus the signing secret.+const BUNDLE_ENV = ['S3_ENDPOINT', 'S3_BUCKET', 'S3_ACCESS_KEY_ID', 'S3_SECRET_ACCESS_KEY', 'X_CONTENT_BUNDLE_SECRET'];++function hydrateBundleEnv() {+ if (!process.env.GOOGLE_CLOUD_PROJECT) process.env.GOOGLE_CLOUD_PROJECT = 'aerial-vehicle-466722-p5';+ for (const name of BUNDLE_ENV.filter((key) => !String(process.env[key] ?? '').trim())) {+ const read = spawnSync(process.execPath, [resolve(root, 'scripts/read-service-env.mjs'), `^${name}$`, '--raw'], { cwd: root, encoding: 'utf8' });+ const value = read.status === 0 ? read.stdout.trim() : '';+ if (value) process.env[name] = value;+ }+ const missing = BUNDLE_ENV.filter((key) => !String(process.env[key] ?? '').trim());+ if (missing.includes('X_CONTENT_BUNDLE_SECRET')) {+ fail('X_CONTENT_BUNDLE_SECRET is not set here or on the Cloud Run service. Generate one with `openssl rand -hex 32`, store it on the service as a Secret Manager reference, and run this again.');+ }+ if (missing.length) fail(`Object storage is not configured: ${missing.join(', ')} missing here and on the Cloud Run service.`);+}++const kb = (bytes) => `${(bytes / 1024).toFixed(1)} KB`;+const totalBytes = (bundle) => bundle.files.reduce((sum, file) => sum + file.bytes, 0);++async function approvedIds() {+ const store = process.env.DATABASE_URL ? dbStore() : memoryStore();+ const state = await store.load();+ const published = new Set((state.published || []).map((row) => row.id));+ console.log(`ledger: ${store.label}`);+ return loadQueue(root).items.filter((item) => item.status === 'approved' && !published.has(item.id)).map((item) => item.id);+}++async function publish() {+ const named = positional.slice(1);+ if (!named.length && !has('approved')) fail('Usage: publish <id> [<id>...] | publish --approved [--replace-index]');+ const ids = named.length ? named : await approvedIds();+ if (!ids.length) {+ console.log('Nothing to bundle: no approved item is waiting to be published.');+ return;+ }++ // Named items are all or nothing. A sweep of everything approved skips what+ // cannot be bundled, says why, and still ships the rest.+ const ready = [];+ const refused = [];+ for (const id of ids) {+ try {+ const bundle = buildBundle(root, id);+ ready.push(id);+ console.log(`ready ${id}: ${bundle.files.length} file(s), ${kb(totalBytes(bundle))}`);+ } catch (err) {+ refused.push(id);+ console.error(`refuse ${err.message}`);+ }+ }+ if (refused.length && named.length) fail(`Nothing was uploaded: ${refused.join(', ')} cannot be bundled.`);+ if (!ready.length) fail('Nothing was uploaded: no item could be bundled.');++ const result = await publishBundles({ root, ids: ready, storage: r2Storage(), replaceIndex: has('replace-index') });+ console.log(`\nPublished ${result.published.join(', ')}: ${result.uploaded} file(s) uploaded, ${result.skipped} already stored.`);+ console.log('Production reads the index at the start of its next tick when X_CONTENT_REMOTE_QUEUE is true.');+ if (refused.length) process.exit(1);+}++function reviewWindow(bundle, now) {+ const age = (now - Date.parse(bundle.review.reviewedAt)) / DAY;+ if (!(age <= REVIEW_MAX_AGE_DAYS)) return `review EXPIRED (${Math.floor(age)} days old, limit ${REVIEW_MAX_AGE_DAYS})`;+ return `review ${Math.floor(age)} day(s) old, ${Math.ceil(REVIEW_MAX_AGE_DAYS - age)} left`;+}++async function list() {+ const index = await readIndex({ storage: r2Storage() });+ if (!index) {+ console.log('No bundle index in storage.');+ return;+ }+ const now = Date.now();+ console.log(`index signed and verified, updated ${index.updatedAt}, ${index.bundles.length} bundle(s)\n`);+ for (const bundle of index.bundles) {+ console.log(` ${bundle.id.padEnd(30)} T${bundle.item.tier} ${String(bundle.item.kind).padEnd(7)} ${bundle.files.length} file(s), ${kb(totalBytes(bundle))} bundled ${bundle.bundledAt} ${reviewWindow(bundle, now)}`);+ }+}++async function pull() {+ const dir = resolve(option('dir', DEFAULT_OVERLAY_DIR));+ const result = await materialize({ imageRoot: root, storage: r2Storage(), dir });+ console.log(`bundles: ${result.bundles.join(', ') || 'none'}`);+ if (result.superseded.length) console.log(`already posted in this checkout's queue, so left alone: ${result.superseded.join(', ')}`);+ console.log(`files: ${result.files} (${result.reused} reused from an earlier pull)`);+ console.log(`root: ${result.root}`);+}++async function remove() {+ const id = positional[1];+ if (!id) fail('Usage: remove <id>');+ const result = await removeBundle({ id, storage: r2Storage() });+ if (!result.removed) fail(`${id} is not in the bundle index.`);+ console.log(`Removed ${id} and its files. ${result.remaining ? `${result.remaining} bundle(s) remain.` : 'The index was the last object, so it is gone too.'}`);+}++async function verify() {+ const result = await verifyBundles({ storage: r2Storage() });+ if (!result.index) {+ console.log('No bundle index in storage; nothing to verify.');+ return;+ }+ console.log(`signature: ok (index updated ${result.index.updatedAt})`);+ for (const problem of result.problems) console.error(`error ${problem}`);+ if (result.problems.length) fail(`x-bundle: ${result.problems.length} problem(s) across ${result.bundles} bundle(s)`);+ console.log(`files: ${result.files} downloaded across ${result.bundles} bundle(s), every hash matches`);+}++const commands = { publish, list, pull, remove, verify };+if (!commands[command]) fail(`Unknown command ${command}. Commands: ${Object.keys(commands).join(', ')}`);+hydrateBundleEnv();+try {+ await commands[command]();+} catch (err) {+ fail(`x-bundle: ${err.code ? `${err.code}: ` : ''}${err.message}`);+}diff --git a/tests/x-content-overlay.test.js b/tests/x-content-overlay.test.jsnew file mode 100644index 000000000..5b833516e--- /dev/null+++ b/tests/x-content-overlay.test.js@@ -0,0 +1,591 @@+import { afterEach, describe, it, expect, vi } from 'vitest';+import { existsSync, lstatSync, mkdtempSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs';+import { tmpdir } from 'node:os';+import { dirname, join } from 'node:path';++import {+ INDEX_KEY, OverlayError, buildBundle, bundleHash, bundlePathProblem, canonical, fileKey, materialize, mergeQueue, proofPath, publishBundles,+ readIndex, removeBundle, resolveRoot, signIndex, verifyBundles, verifyIndex,+} from '../api/_lib/x-content/overlay.js';+import { loadQueue, validateQueue } from '../api/_lib/x-content/queue.js';+import { contentHash, loadReview, reviewPath } from '../api/_lib/x-content/review.js';+import { readMedia } from '../api/_lib/x-content/media.js';+import { runTick } from '../api/_lib/x-content/runner.js';+import { memoryStore } from '../api/_lib/x-content/state.js';++const HOUR = 3_600_000;+const SECRET = 'a'.repeat(64);+const env = { X_CONTENT_BUNDLE_SECRET: SECRET, X_CONTENT_REMOTE_QUEUE: 'true' };+const CADENCE = { windowMinutes: 45, minimumMinutesApart: 240, dailyCap: 3, slots: [{ tier: 3, at: '08:00' }, { tier: 1, at: '16:00' }, { tier: 2, at: '22:00' }] };+const QUALITY = { queueSimilarityLimit: 0.34, archiveSimilarityLimit: 0.42 };++const FRESH = 'Rig Doctor names the skeleton convention of any humanoid GLB, then lists which bones animate and which stay put: three.ws/rig-doctor';+const OLDER = 'Materialize measures the true solid volume of a repaired mesh and prices a physical print from that measurement: three.ws/materialize';++function write(root, path, content) {+ mkdirSync(dirname(join(root, path)), { recursive: true });+ writeFileSync(join(root, path), content);+}++function writeQueue(root, items, over = {}) {+ write(root, 'data/x-content/queue.json', JSON.stringify({ account: 'trythreews', cadence: CADENCE, quality: QUALITY, items, ...over }));+}++function writeReview(root, item, over = {}) {+ const record = { id: item.id, contentHash: contentHash(item, root), reviewedAt: new Date(Date.now() - HOUR).toISOString(), passed: true, blockers: [], ...over };+ write(root, reviewPath(item.id), JSON.stringify(record));+ return record;+}++const post = (id, text, over = {}) => ({+ id, status: 'approved', kind: 'post', tier: 1, lane: id, pattern: id, notBefore: '2026-09-01T00:00:00Z',+ posts: [{ text, media: [{ path: `public/x-media/${id}/card.png`, alt: `The ${id} page` }] }],+ ...over,+});++// The operator's checkout: one approved, reviewed post the image has never seen.+function checkout(over = {}) {+ const root = mkdtempSync(join(tmpdir(), 'x-overlay-checkout-'));+ const item = post('fresh', FRESH, over);+ write(root, 'public/x-media/fresh/card.png', Buffer.from('the fresh card, byte for byte'));+ writeQueue(root, [item]);+ writeReview(root, item);+ return { root, item };+}++// The production image: an older deploy with its own queue, history, and media.+function image(items = null) {+ const root = mkdtempSync(join(tmpdir(), 'x-overlay-image-'));+ const older = post('older', OLDER, { status: 'posted' });+ write(root, 'public/x-media/older/card.png', Buffer.from('the older card'));+ write(root, 'public/announce/img/hero.webp', Buffer.from('a hero that only the image has'));+ write(root, 'data/x-content/volume-model.json', JSON.stringify({ intercept: -2, baseRate: 0.1, features: [] }));+ write(root, 'data/archives/trythreews_tweets_1.json', JSON.stringify([{ text: 'An unrelated earlier post about something else entirely.' }]));+ write(root, 'scripts/probe.mjs', 'process.exit(0);\n');+ write(root, 'package.json', '{"name":"image"}\n');+ writeQueue(root, items || [older]);+ writeReview(root, older);+ return root;+}++function memoryStorage() {+ const objects = new Map();+ const puts = [];+ return {+ objects,+ puts,+ async put(key, buffer) {+ puts.push(key);+ objects.set(key, Buffer.from(buffer));+ },+ async get(key) {+ return objects.has(key) ? Buffer.from(objects.get(key)) : null;+ },+ async remove(key) {+ objects.delete(key);+ },+ };+}++const scratch = () => join(mkdtempSync(join(tmpdir(), 'x-overlay-out-')), 'overlay');++async function published(over = {}) {+ const source = checkout(over);+ const storage = memoryStorage();+ await publishBundles({ root: source.root, ids: ['fresh'], storage, env });+ return { ...source, storage };+}++// A signed index around hand-built bundles, for the shapes buildBundle refuses+// to produce.+async function forge(storage, bundles) {+ const index = { version: 1, updatedAt: new Date().toISOString(), bundles: bundles.map((bundle) => ({ ...bundle, hash: bundleHash(bundle) })) };+ index.signature = signIndex(index, env);+ await storage.put(INDEX_KEY, Buffer.from(JSON.stringify(index)));+ return index;+}++const rejection = async (promise) => {+ try {+ await promise;+ } catch (err) {+ return err;+ }+ return null;+};++afterEach(() => {+ vi.restoreAllMocks();+ vi.unstubAllEnvs();+});++describe('canonical JSON and the signature', () => {+ it('does not depend on key order at any depth', () => {+ const left = { b: 1, a: { d: [1, { z: 1, y: 2 }], c: 'x' }, gone: undefined };+ const right = { a: { c: 'x', d: [1, { y: 2, z: 1 }] }, b: 1 };+ expect(canonical(left)).toBe(canonical(right));+ expect(canonical(left)).toBe('{"a":{"c":"x","d":[1,{"y":2,"z":1}]},"b":1}');+ expect(canonical([1, 2])).not.toBe(canonical([2, 1]));+ expect(canonical(JSON.parse(JSON.stringify(left)))).toBe(canonical(left));+ });++ it('signs an index and verifies it again', () => {+ const index = { version: 1, updatedAt: '2026-09-29T00:00:00.000Z', bundles: [{ id: 'fresh', item: { id: 'fresh', tier: 1 } }] };+ index.signature = signIndex(index, env);+ expect(index.signature).toMatch(/^[0-9a-f]{64}$/);+ expect(verifyIndex(index, env)).toBe(index);+ expect(verifyIndex({ signature: index.signature, bundles: [{ item: { tier: 1, id: 'fresh' }, id: 'fresh' }], updatedAt: index.updatedAt, version: 1 }, env)).toBeTruthy();+ });++ it('rejects a tampered index, a tampered item, and the wrong secret', async () => {+ const { storage } = await published();+ const stored = JSON.parse(storage.objects.get(INDEX_KEY).toString('utf8'));+ expect((await readIndex({ storage, env })).bundles.map((bundle) => bundle.id)).toEqual(['fresh']);++ const tampered = [+ { ...stored, updatedAt: '2030-01-01T00:00:00.000Z' },+ { ...stored, bundles: [{ ...stored.bundles[0], item: { ...stored.bundles[0].item, posts: [{ text: 'Buy this other coin now' }] } }] },+ { ...stored, bundles: [] },+ { ...stored, signature: 'not a signature' },+ { ...stored, signature: undefined },+ ];+ for (const index of tampered) {+ storage.objects.set(INDEX_KEY, Buffer.from(JSON.stringify(index)));+ const err = await rejection(readIndex({ storage, env }));+ expect(err).toBeInstanceOf(OverlayError);+ expect(err.code).toBe('bad_signature');+ }++ storage.objects.set(INDEX_KEY, Buffer.from(JSON.stringify(stored)));+ expect((await rejection(readIndex({ storage, env: { X_CONTENT_BUNDLE_SECRET: 'b'.repeat(64) } }))).code).toBe('bad_signature');+ expect((await rejection(readIndex({ storage, env: {} }))).code).toBe('unconfigured');+ expect((await rejection(readIndex({ storage, env: { X_CONTENT_BUNDLE_SECRET: 'short' } }))).code).toBe('unconfigured');+ expect(await readIndex({ storage: memoryStorage(), env })).toBeNull();+ });+});++describe('buildBundle', () => {+ it('packs the item, its review record, and the hash of every file', () => {+ const { root, item } = checkout();+ const bundle = buildBundle(root, 'fresh', { now: Date.parse('2026-09-29T12:00:00Z') });+ expect(bundle.item).toEqual(item);+ expect(bundle.review).toEqual(loadReview(root, 'fresh'));+ expect(bundle.files).toEqual([{ path: 'public/x-media/fresh/card.png', sha256: expect.stringMatching(/^[0-9a-f]{64}$/), bytes: 29 }]);+ expect(bundle.bundledAt).toBe('2026-09-29T12:00:00.000Z');+ expect(bundle.hash).toBe(bundleHash(bundle));+ });++ it('refuses an item that is not approved', () => {+ const { root } = checkout({ status: 'review' });+ expect(() => buildBundle(root, 'fresh')).toThrow(/fresh cannot be bundled: status is review/);+ expect(() => buildBundle(root, 'absent')).toThrow(/no queue item/);+ });++ it('refuses an item whose review record does not match its content', () => {+ const { root, item } = checkout();+ writeQueue(root, [{ ...item, posts: [{ ...item.posts[0], text: FRESH.replace('Rig Doctor', 'The Rig Doctor') }] }]);+ expect(() => buildBundle(root, 'fresh')).toThrow(/changed after the last review/);++ writeQueue(root, [item]);+ writeFileSync(join(root, 'public/x-media/fresh/card.png'), Buffer.from('a different card'));+ expect(() => buildBundle(root, 'fresh')).toThrow(/changed after the last review/);+ });++ it('refuses a review that is too old or did not pass', () => {+ const { root, item } = checkout();+ writeReview(root, item, { reviewedAt: new Date(Date.now() - 20 * 24 * HOUR).toISOString() });+ expect(() => buildBundle(root, 'fresh')).toThrow(/days old/);+ writeReview(root, item, { passed: false, blockers: ['link three.ws/rig-doctor: HTTP 404'] });+ expect(() => buildBundle(root, 'fresh')).toThrow(/did not pass/);+ });++ it('bundles the body, cover, and inline images of an Article', () => {+ const root = mkdtempSync(join(tmpdir(), 'x-overlay-checkout-'));+ write(root, 'public/x-media/long/cover.png', Buffer.from('cover'));+ write(root, 'public/x-media/long/inline.png', Buffer.from('inline'));+ write(root, 'data/x-content/articles/long.md', '## Why\n\nBecause.\n\n\n\nThe end.');+ const item = {+ id: 'long', status: 'approved', kind: 'article', tier: 1, lane: 'article', pattern: 'longform', notBefore: '2026-09-01T00:00:00Z',+ article: { title: 'How rigs get named', body: 'data/x-content/articles/long.md', cover: { path: 'public/x-media/long/cover.png' } },+ posts: [{ text: 'The long version of how Rig Doctor reads a skeleton.' }],+ };+ writeQueue(root, [item]);+ writeReview(root, item);+ expect(buildBundle(root, 'long').files.map((file) => file.path)).toEqual(['data/x-content/articles/long.md', 'public/x-media/long/cover.png', 'public/x-media/long/inline.png']);+ });+});++describe('publishBundles', () => {+ it('uploads each file once and is idempotent', async () => {+ const { root } = checkout();+ const storage = memoryStorage();+ const first = await publishBundles({ root, ids: ['fresh', 'fresh'], storage, env });+ expect(first).toEqual({ published: ['fresh'], uploaded: 1, skipped: 0 });+ const [file] = buildBundle(root, 'fresh').files;+ expect(storage.puts).toEqual([fileKey('fresh', file.sha256), INDEX_KEY]);+ expect(storage.objects.get(fileKey('fresh', file.sha256)).equals(readFileSync(join(root, file.path)))).toBe(true);++ const second = await publishBundles({ root, ids: ['fresh'], storage, env });+ expect(second).toEqual({ published: ['fresh'], uploaded: 0, skipped: 1 });+ expect(storage.puts.filter((key) => key !== INDEX_KEY)).toHaveLength(1);+ const index = await readIndex({ storage, env });+ expect(index.bundles.map((bundle) => bundle.id)).toEqual(['fresh']);+ expect([...storage.objects.keys()].sort()).toEqual([fileKey('fresh', file.sha256), INDEX_KEY].sort());+ });++ it('uploads nothing when any named item cannot be bundled', async () => {+ const { root, item } = checkout();+ writeQueue(root, [item, post('draft', OLDER, { status: 'draft' })]);+ const storage = memoryStorage();+ await expect(publishBundles({ root, ids: ['fresh', 'draft'], storage, env })).rejects.toThrow(/draft cannot be bundled/);+ expect(storage.objects.size).toBe(0);+ await expect(publishBundles({ root, ids: ['fresh'], storage, env: {} })).rejects.toThrow(/X_CONTENT_BUNDLE_SECRET/);+ });++ it('replaces a bundle with the same id and drops the file it no longer uses', async () => {+ const { root, item, storage } = await published();+ const before = buildBundle(root, 'fresh').files[0];+ writeFileSync(join(root, 'public/x-media/fresh/card.png'), Buffer.from('a reshot card'));+ writeReview(root, item);+ await publishBundles({ root, ids: ['fresh'], storage, env });+ const after = buildBundle(root, 'fresh').files[0];+ expect(after.sha256).not.toBe(before.sha256);+ expect((await readIndex({ storage, env })).bundles).toHaveLength(1);+ expect(storage.objects.has(fileKey('fresh', before.sha256))).toBe(false);+ expect(storage.objects.has(fileKey('fresh', after.sha256))).toBe(true);+ });++ it('refuses to build on an index it cannot verify unless told to replace it', async () => {+ const { root, storage } = await published();+ const rotated = { X_CONTENT_BUNDLE_SECRET: 'c'.repeat(64) };+ expect((await rejection(publishBundles({ root, ids: ['fresh'], storage, env: rotated }))).code).toBe('bad_signature');+ await publishBundles({ root, ids: ['fresh'], storage, env: rotated, replaceIndex: true });+ expect((await readIndex({ storage, env: rotated })).bundles.map((bundle) => bundle.id)).toEqual(['fresh']);+ });++ it('removes a bundle, its files, and the index once it is empty', async () => {+ const { storage } = await published();+ expect(await removeBundle({ id: 'absent', storage, env })).toEqual({ removed: false, remaining: 1 });+ expect(await removeBundle({ id: 'fresh', storage, env })).toEqual({ removed: true, remaining: 0 });+ expect(storage.objects.size).toBe(0);+ expect(await readIndex({ storage, env })).toBeNull();+ });++ it('verifies every stored file against the index', async () => {+ const { root, storage } = await published();+ expect(await verifyBundles({ storage, env })).toMatchObject({ bundles: 1, files: 1, problems: [] });+ const [file] = buildBundle(root, 'fresh').files;+ storage.objects.set(fileKey('fresh', file.sha256), Buffer.from('swapped'));+ expect((await verifyBundles({ storage, env })).problems.join('\n')).toMatch(/does not hash/);+ storage.objects.delete(fileKey('fresh', file.sha256));+ expect((await verifyBundles({ storage, env })).problems.join('\n')).toMatch(/missing from storage/);+ });+});++describe('materialize', () => {+ it('builds a root the unchanged engine reads as an approved, reviewed item', async () => {+ const { root: source, item, storage } = await published();+ const imageRoot = image();+ const result = await materialize({ imageRoot, storage, env, dir: scratch() });+ expect(result).toMatchObject({ bundles: ['fresh'], files: 1, reused: 0, superseded: [] });++ const queue = loadQueue(result.root);+ expect(queue.items.map((row) => `${row.id}:${row.status}`)).toEqual(['older:posted', 'fresh:approved']);+ expect(queue.items[1]).toEqual(item);+ expect(queue.cadence).toEqual(CADENCE);+ expect(queue.quality).toEqual(QUALITY);++ const { problems } = validateQueue(queue, result.root);+ expect(problems.fresh).toEqual([]);+ expect(problems.older).toEqual([]);+ expect(loadReview(result.root, 'fresh')).toEqual(loadReview(source, 'fresh'));+ expect(loadReview(result.root, 'fresh').contentHash).toBe(contentHash(item, result.root));++ const media = readMedia(item.posts[0].media[0], result.root);+ expect(media.buffer.equals(readFileSync(join(source, 'public/x-media/fresh/card.png')))).toBe(true);+ expect(media).toMatchObject({ mime: 'image/png', kind: 'image', category: 'tweet_image' });+ expect(lstatSync(join(result.root, 'public/x-media/fresh/card.png')).isFile()).toBe(true);+ });++ it('reaches everything the image holds through the overlay', async () => {+ const { storage } = await published();+ const imageRoot = image();+ const { root } = await materialize({ imageRoot, storage, env, dir: scratch() });++ expect(JSON.parse(readFileSync(join(root, 'data/x-content/volume-model.json'), 'utf8')).baseRate).toBe(0.1);+ expect(readFileSync(join(root, 'public/announce/img/hero.webp'), 'utf8')).toBe('a hero that only the image has');+ expect(readFileSync(join(root, 'public/x-media/older/card.png'), 'utf8')).toBe('the older card');+ expect(loadReview(root, 'older')).toEqual(loadReview(imageRoot, 'older'));+ expect(existsSync(join(root, 'scripts/probe.mjs'))).toBe(true);+ expect(existsSync(join(root, 'package.json'))).toBe(true);++ // A directory the overlay never writes into is one link; a directory on the+ // way to a bundle file is real, with the image's entries linked inside it.+ expect(lstatSync(join(root, 'data/archives')).isSymbolicLink()).toBe(true);+ expect(lstatSync(join(root, 'public/announce')).isSymbolicLink()).toBe(true);+ expect(lstatSync(join(root, 'public/x-media/older')).isSymbolicLink()).toBe(true);+ for (const path of ['data', 'data/x-content', 'data/x-content/reviews', 'public', 'public/x-media', 'public/x-media/fresh']) {+ expect(lstatSync(join(root, path)).isDirectory()).toBe(true);+ }+ expect(lstatSync(join(root, 'data/x-content/reviews/older.json')).isSymbolicLink()).toBe(true);+ });++ it('writes a bundle file at any depth, and never into the image', async () => {+ const source = checkout();+ const deep = 'public/announce/img/launch/2026/card.png';+ const item = post('fresh', FRESH, { posts: [{ text: FRESH, media: [{ path: deep, alt: 'The launch card' }] }] });+ write(source.root, deep, Buffer.from('the reviewed card'));+ writeQueue(source.root, [item]);+ writeReview(source.root, item);+ const storage = memoryStorage();+ await publishBundles({ root: source.root, ids: ['fresh'], storage, env });++ const imageRoot = image([post('fresh', OLDER), post('older', OLDER, { status: 'posted' })]);+ write(imageRoot, deep, Buffer.from('the card the last deploy shipped'));+ writeFileSync(join(imageRoot, reviewPath('fresh')), '{"id":"fresh","stale":true}');+ const before = readFileSync(join(imageRoot, 'data/x-content/queue.json'), 'utf8');++ const { root } = await materialize({ imageRoot, storage, env, dir: scratch() });+ expect(readFileSync(join(root, deep), 'utf8')).toBe('the reviewed card');+ expect(readFileSync(join(root, 'public/announce/img/hero.webp'), 'utf8')).toBe('a hero that only the image has');+ expect(loadQueue(root).items.map((row) => row.id)).toEqual(['fresh', 'older']);+ expect(loadQueue(root).items[0]).toEqual(item);+ expect(validateQueue(loadQueue(root), root).problems.fresh).toEqual([]);++ expect(readFileSync(join(imageRoot, deep), 'utf8')).toBe('the card the last deploy shipped');+ expect(readFileSync(join(imageRoot, reviewPath('fresh')), 'utf8')).toBe('{"id":"fresh","stale":true}');+ expect(readFileSync(join(imageRoot, 'data/x-content/queue.json'), 'utf8')).toBe(before);+ });++ it('never replaces an item the image records as posted', async () => {+ const { storage } = await published();+ const posted = post('fresh', OLDER, { status: 'posted' });+ const imageRoot = image([posted]);+ write(imageRoot, 'public/x-media/fresh/card.png', Buffer.from('what actually went out'));+ const result = await materialize({ imageRoot, storage, env, dir: scratch() });++ expect(result).toMatchObject({ bundles: [], files: 0, superseded: ['fresh'] });+ expect(loadQueue(result.root).items).toEqual([posted]);+ expect(readFileSync(join(result.root, 'public/x-media/fresh/card.png'), 'utf8')).toBe('what actually went out');+ expect(mergeQueue({ cadence: CADENCE, items: [posted] }, [{ id: 'fresh', item: post('fresh', FRESH) }]).queue).toEqual({ cadence: CADENCE, items: [posted] });+ });++ it('reuses what an earlier tick downloaded and drops what is no longer bundled', async () => {+ const { storage } = await published();+ const imageRoot = image();+ const dir = scratch();+ await materialize({ imageRoot, storage, env, dir });+ const gets = [];+ const get = storage.get;+ storage.get = async (key) => {+ gets.push(key);+ return get(key);+ };+ expect(await materialize({ imageRoot, storage, env, dir })).toMatchObject({ bundles: ['fresh'], files: 1, reused: 1 });+ expect(gets).toEqual([INDEX_KEY]);++ // A file that rotted on disk is fetched again instead of trusted.+ writeFileSync(join(dir, 'public/x-media/fresh/card.png'), 'rot');+ expect(await materialize({ imageRoot, storage, env, dir })).toMatchObject({ files: 1, reused: 0 });+ expect(readFileSync(join(dir, 'public/x-media/fresh/card.png'), 'utf8')).toBe('the fresh card, byte for byte');++ const other = checkout();+ const next = post('next', OLDER.replace('Materialize', 'The volume tool'));+ write(other.root, 'public/x-media/next/card.png', Buffer.from('the next card'));+ writeQueue(other.root, [next]);+ writeReview(other.root, next);+ await publishBundles({ root: other.root, ids: ['next'], storage, env });+ await removeBundle({ id: 'fresh', storage, env });++ const result = await materialize({ imageRoot, storage, env, dir });+ expect(result.bundles).toEqual(['next']);+ expect(existsSync(join(dir, 'public/x-media/fresh'))).toBe(false);+ expect(existsSync(join(dir, reviewPath('fresh')))).toBe(false);+ expect(loadQueue(dir).items.map((row) => row.id)).toEqual(['older', 'next']);+ expect(readFileSync(join(imageRoot, 'public/x-media/older/card.png'), 'utf8')).toBe('the older card');+ expect(readFileSync(join(imageRoot, 'data/archives/trythreews_tweets_1.json'), 'utf8')).toMatch(/unrelated earlier post/);+ });++ it('throws bad_file when the stored bytes do not match the recorded hash', async () => {+ const { root, storage } = await published();+ const [file] = buildBundle(root, 'fresh').files;+ storage.objects.set(fileKey('fresh', file.sha256), Buffer.from('swapped in the bucket'));+ const dir = scratch();+ const err = await rejection(materialize({ imageRoot: image(), storage, env, dir }));+ expect(err).toBeInstanceOf(OverlayError);+ expect(err.code).toBe('bad_file');+ expect(existsSync(join(dir, 'public/x-media/fresh/card.png'))).toBe(false);++ storage.objects.delete(fileKey('fresh', file.sha256));+ expect((await rejection(materialize({ imageRoot: image(), storage, env, dir: scratch() }))).code).toBe('bad_file');+ });++ it('rejects path traversal and every path outside the media roots', async () => {+ expect(bundlePathProblem('public/x-media/fresh/card.png')).toBeNull();+ expect(bundlePathProblem('data/x-content/articles/fresh.md')).toBeNull();+ expect(bundlePathProblem(proofPath('fresh'))).toBeNull();++ const { item, root } = checkout();+ const review = loadReview(root, 'fresh');+ const bytes = Buffer.from('payload');+ const sha256 = buildBundle(root, 'fresh').files[0].sha256;+ const paths = [+ '../outside.png', 'public/../../outside.png', 'public/x-media/..', '/etc/passwd', 'C:/windows/card.png', 'scripts/x-content.mjs', 'api/cron/x-content.js',+ 'public\\x-media\\card.png', 'public//card.png', 'public/./card.png', 'data/x-content/queue.json', 'data/x-content/reviews/fresh.json', '',+ ];+ for (const path of paths) {+ expect(bundlePathProblem(path)).toBeTruthy();+ const storage = memoryStorage();+ await storage.put(fileKey('fresh', sha256), bytes);+ await forge(storage, [{ id: 'fresh', item, review, files: [{ path, sha256, bytes: bytes.length }], bundledAt: new Date().toISOString() }]);+ const imageRoot = image();+ const dir = scratch();+ const err = await rejection(materialize({ imageRoot, storage, env, dir }));+ expect(err).toBeInstanceOf(OverlayError);+ expect(err.code).toBe('bad_path');+ expect(existsSync(dir)).toBe(false);+ expect(existsSync(join(dirname(dir), 'outside.png'))).toBe(false);+ }+ });++ it('rejects a bundle whose id, item, or review record do not agree', async () => {+ const { item, root } = checkout();+ const review = loadReview(root, 'fresh');+ const bundledAt = new Date().toISOString();+ for (const bundle of [+ { id: '../fresh', item: { ...item, id: '../fresh' }, review: { ...review, id: '../fresh' }, files: [], bundledAt },+ { id: 'fresh', item: { ...item, id: 'other' }, review, files: [], bundledAt },+ { id: 'fresh', item, review: { ...review, id: 'other' }, files: [], bundledAt },+ ]) {+ const storage = memoryStorage();+ await forge(storage, [bundle]);+ expect((await rejection(materialize({ imageRoot: image(), storage, env, dir: scratch() }))).code).toBe('bad_bundle');+ }+ });++ it('only takes a directory that is empty or already an overlay', async () => {+ const { storage } = await published();+ const imageRoot = image();+ const occupied = mkdtempSync(join(tmpdir(), 'x-overlay-occupied-'));+ writeFileSync(join(occupied, 'notes.txt'), 'someone else keeps files here');+ expect((await rejection(materialize({ imageRoot, storage, env, dir: occupied }))).code).toBe('bad_dir');+ expect(readFileSync(join(occupied, 'notes.txt'), 'utf8')).toBe('someone else keeps files here');+ expect((await rejection(materialize({ imageRoot, storage, env, dir: imageRoot }))).code).toBe('bad_dir');+ expect((await rejection(materialize({ imageRoot, storage, env, dir: dirname(imageRoot) }))).code).toBe('bad_dir');+ expect((await rejection(materialize({ imageRoot, storage, env, dir: join(imageRoot, 'public/overlay') }))).code).toBe('bad_dir');+ expect(existsSync(join(imageRoot, 'public/x-media/older/card.png'))).toBe(true);++ const empty = mkdtempSync(join(tmpdir(), 'x-overlay-empty-'));+ expect((await materialize({ imageRoot, storage, env, dir: empty })).bundles).toEqual(['fresh']);+ });++ it('carries the proof record of an item with a scenario', async () => {+ const source = checkout();+ const proof = Buffer.from(`${JSON.stringify({ id: 'fresh', scenario: 'rig-doctor', steps: [{ name: 'diagnose', ok: true }] }, null, '\t')}\n`);+ write(source.root, proofPath('fresh'), proof);+ const bundle = buildBundle(source.root, 'fresh');+ expect(bundle.files.map((file) => file.path)).toEqual(['public/x-media/fresh/card.png', 'data/x-content/proofs/fresh.json']);++ const storage = memoryStorage();+ expect(await publishBundles({ root: source.root, ids: ['fresh'], storage, env })).toMatchObject({ uploaded: 2 });+ const imageRoot = image();+ const { root, files } = await materialize({ imageRoot, storage, env, dir: scratch() });+ expect(files).toBe(2);+ expect(readFileSync(join(root, 'data/x-content/proofs/fresh.json')).equals(proof)).toBe(true);+ expect(lstatSync(join(root, 'data/x-content/proofs/fresh.json')).isFile()).toBe(true);+ expect(existsSync(join(imageRoot, 'data/x-content/proofs'))).toBe(false);+ });+});++describe('resolveRoot', () => {+ const quiet = () => vi.spyOn(console, 'warn').mockImplementation(() => {});++ it('uses the overlay when the index verifies', async () => {+ const { storage } = await published();+ const imageRoot = image();+ const dir = scratch();+ const result = await resolveRoot({ imageRoot, env, storage, dir });+ expect(result.root).toBe(dir);+ expect(result.overlay).toMatchObject({ bundles: ['fresh'], files: 1, reused: 0 });+ expect(result.overlay.root).toBeUndefined();+ });++ it('falls back to the image root for every failure and never throws', async () => {+ const warn = quiet();+ const { root, storage } = await published();+ const imageRoot = image();+ const fallback = (result) => {+ expect(result.root).toBe(imageRoot);+ expect(result.overlay).toBeNull();+ expect(result.reason).toBeTruthy();+ return result.reason;+ };++ expect(fallback(await resolveRoot({ imageRoot, env: { X_CONTENT_BUNDLE_SECRET: SECRET }, storage, dir: scratch() }))).toMatch(/X_CONTENT_REMOTE_QUEUE/);+ expect(fallback(await resolveRoot({ imageRoot, env: { ...env, X_CONTENT_REMOTE_QUEUE: 'TRUE' }, storage, dir: scratch() }))).toMatch(/X_CONTENT_REMOTE_QUEUE/);+ expect(fallback(await resolveRoot({ imageRoot, env: { ...env, X_CONTENT_REMOTE_QUEUE: '1' }, storage, dir: scratch() }))).toMatch(/X_CONTENT_REMOTE_QUEUE/);+ expect(warn).not.toHaveBeenCalled();++ expect(fallback(await resolveRoot({ imageRoot, env: { X_CONTENT_REMOTE_QUEUE: 'true' }, storage, dir: scratch() }))).toMatch(/X_CONTENT_BUNDLE_SECRET is not set/);+ expect(fallback(await resolveRoot({ imageRoot, env, storage: memoryStorage(), dir: scratch() }))).toMatch(/no bundle index/);+ expect(fallback(await resolveRoot({ imageRoot, env: { ...env, X_CONTENT_BUNDLE_SECRET: 'd'.repeat(64) }, storage, dir: scratch() }))).toMatch(/signature does not verify/);++ for (const name of ['S3_ENDPOINT', 'S3_BUCKET', 'S3_PUBLIC_DOMAIN', 'S3_ACCESS_KEY_ID', 'S3_SECRET_ACCESS_KEY']) vi.stubEnv(name, '');+ expect(fallback(await resolveRoot({ imageRoot, env, dir: scratch() }))).toMatch(/object storage is not configured/);++ const down = { ...memoryStorage(), get: async () => { throw new Error('socket hang up'); } };+ expect(fallback(await resolveRoot({ imageRoot, env, storage: down, dir: scratch() }))).toMatch(/socket hang up/);++ const posted = image([post('fresh', OLDER, { status: 'posted' })]);+ expect((await resolveRoot({ imageRoot: posted, env, storage, dir: scratch() })).root).toBe(posted);++ const [file] = buildBundle(root, 'fresh').files;+ storage.objects.set(fileKey('fresh', file.sha256), Buffer.from('swapped in the bucket'));+ expect(fallback(await resolveRoot({ imageRoot, env, storage, dir: scratch() }))).toMatch(/does not hash/);++ const missing = mkdtempSync(join(tmpdir(), 'x-overlay-image-'));+ const result = await resolveRoot({ imageRoot: missing, env, storage, dir: scratch() });+ expect(result).toMatchObject({ root: missing, overlay: null });++ expect(warn).toHaveBeenCalledWith('[x-content] overlay unavailable, using the image queue', expect.stringMatching(/does not hash/));+ });+});++describe('a tick against the overlay', () => {+ it('previews the bundled post with the bytes that were reviewed', async () => {+ const { root: source, storage } = await published();+ const imageRoot = image();+ const { root, overlay } = await resolveRoot({ imageRoot, env, storage, dir: scratch() });+ expect(overlay.bundles).toEqual(['fresh']);++ // 16:50 UTC today: the 16:00 flagship slot is open whatever its jitter.+ const now = Date.parse(`${new Date().toISOString().slice(0, 10)}T16:50:00Z`);+ const result = await runTick({ root, store: memoryStore(), dryRun: true, now, env: {} });+ expect(result.blocked).toEqual([]);+ expect(result.preview.id).toBe('fresh');+ expect(result.preview.calls.map((call) => call.call)).toEqual(['media.upload', 'media.metadata', 'tweets.create']);+ expect(result.preview.calls[0]).toMatchObject({ bytes: readFileSync(join(source, 'public/x-media/fresh/card.png')).length, media_type: 'image/png', media_category: 'tweet_image' });+ expect(result.preview.calls[1].alt).toBe('The fresh page');+ expect(result.preview.calls[2].text).toBe(FRESH);++ // The same tick against the image alone has nothing to send.+ const without = await runTick({ root: imageRoot, store: memoryStore(), dryRun: true, now, env: {} });+ expect(without.preview).toBeUndefined();+ });++ it('blocks a bundled post whose review has expired, exactly as the image queue would', async () => {+ const source = checkout();+ const storage = memoryStorage();+ await publishBundles({ root: source.root, ids: ['fresh'], storage, env });+ const index = await readIndex({ storage, env });+ const aged = { ...index.bundles[0], review: { ...index.bundles[0].review, reviewedAt: new Date(Date.now() - 15 * 24 * HOUR).toISOString() } };+ await forge(storage, [aged]);++ const { root } = await materialize({ imageRoot: image(), storage, env, dir: scratch() });+ const now = Date.parse(`${new Date().toISOString().slice(0, 10)}T16:50:00Z`);+ const result = await runTick({ root, store: memoryStore(), dryRun: true, now, env: {} });+ expect(result.preview).toBeUndefined();+ expect(result.blocked.map((row) => row.id)).toEqual(['fresh']);+ expect(result.blocked[0].problems.join('\n')).toMatch(/days old/);+ });+});