Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.
JavaScript 3d-generationai-agentsanimationavatardigital-humangenerative-aiglbgltfimage-to-3dllmmcpmcp-servermodel-context-protocolsolanatext-to-3dthreejsweb-componentswebglwebxrx402
feat(x-content): store signed post bundles on Google Cloud Storage when R2 is unavailable, so unattended runs reach production without a deploy
nirholas101d7b82516e
diff --git a/api/_lib/x-content/overlay.js b/api/_lib/x-content/overlay.jsindex 53ba4833b..848e6a20d 100644--- a/api/_lib/x-content/overlay.js+++ b/api/_lib/x-content/overlay.js@@ -172,6 +172,53 @@ export function r2Storage(env = process.env) {};}+// The same store on Google Cloud Storage, for when Cloudflare R2 is not+// available. Selected by X_CONTENT_BUNDLE_BUCKET; the Cloud Run service account+// and the partner job authenticate with their own identity, so no key is stored.+const GCS_API = 'https://storage.googleapis.com';++export function gcsStorage(env = process.env) {+ const bucket = String(env.X_CONTENT_BUNDLE_BUCKET || '').trim();+ if (!bucket) throw new OverlayError('unconfigured', 'X_CONTENT_BUNDLE_BUCKET is not set');+ const call = async (method, url, { body, contentType } = {}) => {+ const { getGcpAccessToken } = await import('../gcp-auth.js');+ const headers = { authorization: `Bearer ${await getGcpAccessToken()}` };+ if (contentType) headers['content-type'] = contentType;+ return fetch(url, { method, headers, body, signal: AbortSignal.timeout(60_000) });+ };+ const mediaUrl = (key) => `${GCS_API}/storage/v1/b/${bucket}/o/${encodeURIComponent(sealedKey(key, env))}`;+ const failed = async (res, what) => new Error(`${what}: Google Cloud Storage answered ${res.status} ${(await res.text()).slice(0, 200)}`);+ return {+ label: 'Google Cloud Storage',+ async put(key, buffer, contentType) {+ const url = `${GCS_API}/upload/storage/v1/b/${bucket}/o?uploadType=media&name=${encodeURIComponent(sealedKey(key, env))}`;+ const res = await call('POST', url, { body: buffer, contentType: contentType || 'application/octet-stream' });+ if (!res.ok) throw await failed(res, `put ${key}`);+ },+ async get(key) {+ const res = await call('GET', `${mediaUrl(key)}?alt=media`);+ if (res.status === 404) return null;+ if (!res.ok) throw await failed(res, `get ${key}`);+ return Buffer.from(await res.arrayBuffer());+ },+ async has(key) {+ const res = await call('GET', `${mediaUrl(key)}?fields=name`);+ if (res.status === 404) return false;+ if (!res.ok) throw await failed(res, `has ${key}`);+ return true;+ },+ async remove(key) {+ const res = await call('DELETE', mediaUrl(key));+ if (!res.ok && res.status !== 404) throw await failed(res, `remove ${key}`);+ },+ };+}++// The store the overlay really uses: GCS when a bucket is named, R2 otherwise.+export function bundleStorage(env = process.env) {+ return String(env.X_CONTENT_BUNDLE_BUCKET || '').trim() ? gcsStorage(env) : r2Storage(env);+}+// `has` is optional on a store: one that only has put, get, and remove answers// by reading the object.const stored = async (storage, key) => (storage.has ? storage.has(key) : (await storage.get(key)) !== null);@@ -244,7 +291,7 @@ async function removeFiles(storage, id, files, keep = new Set()) {// `replaceIndex` starts a new index when the stored one cannot be verified,// which is what an operator needs after rotating the secret.export async function publishBundles({ root, ids, storage = null, env = process.env, now = Date.now(), replaceIndex = false }) {- storage ||= r2Storage(env);+ storage ||= bundleStorage(env);secretFrom(env);const built = [];const failures = [];@@ -294,7 +341,7 @@ export async function publishBundles({ root, ids, storage = null, env = process.// Drops a bundle and its files. The last bundle takes the index with it, so an// empty bucket reads as "no index" rather than as an index of nothing.export async function removeBundle({ id, storage = null, env = process.env, now = Date.now() }) {- storage ||= r2Storage(env);+ storage ||= bundleStorage(env);const index = await readIndex({ storage, env });const bundle = index?.bundles.find((row) => row.id === id);if (!bundle) return { removed: false, remaining: index?.bundles.length || 0 };@@ -306,7 +353,7 @@ export async function removeBundle({ id, storage = null, env = process.env, now}export async function readIndex({ storage = null, env = process.env }) {- storage ||= r2Storage(env);+ storage ||= bundleStorage(env);secretFrom(env);const bytes = await storage.get(INDEX_KEY);if (!bytes) return null;@@ -325,7 +372,7 @@ export async function readIndex({ storage = null, env = process.env }) {// Downloads every file the index names and checks it against its recorded hash.export async function verifyBundles({ storage = null, env = process.env }) {- storage ||= r2Storage(env);+ storage ||= bundleStorage(env);const index = await readIndex({ storage, env });if (!index) return { index: null, bundles: 0, files: 0, problems: [] };const problems = fileConflicts(index.bundles);@@ -457,7 +504,7 @@ export function mergeQueue(imageQueue, bundles) {// files in the overlay, `reused` how many of them an earlier tick had already// downloaded.export async function materialize({ imageRoot, storage = null, env = process.env, dir = DEFAULT_OVERLAY_DIR, now = Date.now(), index = null }) {- storage ||= r2Storage(env);+ storage ||= bundleStorage(env);const image = resolve(imageRoot);const root = resolve(dir);const verified = index ? verifyIndex(index, env) : await readIndex({ storage, env });@@ -509,7 +556,7 @@ export async function resolveRoot({ imageRoot, env = process.env, storage = nulltry {secretFrom(env);if (!storage && !(await import('../r2.js')).objectStorageConfigured()) return unavailable('object storage is not configured');- const store = storage || r2Storage(env);+ const store = storage || bundleStorage(env);const index = await readIndex({ storage: store, env });if (!index) return image('there is no bundle index in storage');const { root, ...overlay } = await materialize({ imageRoot, storage: store, env, dir, now, index });diff --git a/scripts/x-bundle.mjs b/scripts/x-bundle.mjsindex f4c6bf1e2..4bbbac53e 100644--- a/scripts/x-bundle.mjs+++ b/scripts/x-bundle.mjs@@ -26,7 +26,7 @@ import { fileURLToPath } from 'node:url';import { loadQueue } from '../api/_lib/x-content/queue.js';import { REVIEW_MAX_AGE_DAYS } from '../api/_lib/x-content/review.js';import { dbStore, memoryStore } from '../api/_lib/x-content/state.js';-import { DEFAULT_OVERLAY_DIR, buildBundle, materialize, publishBundles, r2Storage, readIndex, removeBundle, verifyBundles } from '../api/_lib/x-content/overlay.js';+import { DEFAULT_OVERLAY_DIR, buildBundle, materialize, publishBundles, bundleStorage, readIndex, removeBundle, verifyBundles } from '../api/_lib/x-content/overlay.js';const root = resolve(fileURLToPath(new URL('..', import.meta.url)));const DAY = 24 * 60 * 60_000;@@ -59,16 +59,23 @@ function fail(message) {}// What api/_lib/env.js reads for the storage client, plus the signing secret.-const BUNDLE_ENV = ['S3_ENDPOINT', 'S3_BUCKET', 'S3_ACCESS_KEY_ID', 'S3_SECRET_ACCESS_KEY', 'X_CONTENT_BUNDLE_SECRET'];+const R2_ENV = ['S3_ENDPOINT', 'S3_BUCKET', 'S3_ACCESS_KEY_ID', 'S3_SECRET_ACCESS_KEY'];+// With X_CONTENT_BUNDLE_BUCKET the bundles live on Google Cloud Storage and the+// R2 credentials are not needed.+const gcsBucket = () => Boolean(String(process.env.X_CONTENT_BUNDLE_BUCKET ?? '').trim());function hydrateBundleEnv() {if (!process.env.GOOGLE_CLOUD_PROJECT) process.env.GOOGLE_CLOUD_PROJECT = 'aerial-vehicle-466722-p5';- for (const name of BUNDLE_ENV.filter((key) => !String(process.env[key] ?? '').trim())) {- const read = spawnSync(process.execPath, [resolve(root, 'scripts/read-service-env.mjs'), `^${name}$`, '--raw'], { cwd: root, encoding: 'utf8' });- const value = read.status === 0 ? read.stdout.trim() : '';- if (value) process.env[name] = value;- }- const missing = BUNDLE_ENV.filter((key) => !String(process.env[key] ?? '').trim());+ const hydrate = (names) => {+ for (const name of names.filter((key) => !String(process.env[key] ?? '').trim())) {+ const read = spawnSync(process.execPath, [resolve(root, 'scripts/read-service-env.mjs'), `^${name}$`, '--raw'], { cwd: root, encoding: 'utf8' });+ const value = read.status === 0 ? read.stdout.trim() : '';+ if (value) process.env[name] = value;+ }+ };+ hydrate(['X_CONTENT_BUNDLE_BUCKET', 'X_CONTENT_BUNDLE_SECRET']);+ if (!gcsBucket()) hydrate(R2_ENV);+ const missing = [...(gcsBucket() ? [] : R2_ENV), 'X_CONTENT_BUNDLE_SECRET'].filter((key) => !String(process.env[key] ?? '').trim());if (missing.includes('X_CONTENT_BUNDLE_SECRET')) {fail('X_CONTENT_BUNDLE_SECRET is not set here or on the Cloud Run service. Generate one with `openssl rand -hex 32`, store it on the service as a Secret Manager reference, and run this again.');}@@ -112,7 +119,7 @@ async function publish() {if (refused.length && named.length) fail(`Nothing was uploaded: ${refused.join(', ')} cannot be bundled.`);if (!ready.length) fail('Nothing was uploaded: no item could be bundled.');- const result = await publishBundles({ root, ids: ready, storage: r2Storage(), replaceIndex: has('replace-index') });+ const result = await publishBundles({ root, ids: ready, storage: bundleStorage(), replaceIndex: has('replace-index') });console.log(`\nPublished ${result.published.join(', ')}: ${result.uploaded} file(s) uploaded, ${result.skipped} already stored.`);console.log('Production reads the index at the start of its next tick when X_CONTENT_REMOTE_QUEUE is true.');if (refused.length) process.exit(1);@@ -125,7 +132,7 @@ function reviewWindow(bundle, now) {}async function list() {- const index = await readIndex({ storage: r2Storage() });+ const index = await readIndex({ storage: bundleStorage() });if (!index) {console.log('No bundle index in storage.');return;@@ -139,7 +146,7 @@ async function list() {async function pull() {const dir = resolve(option('dir', DEFAULT_OVERLAY_DIR));- const result = await materialize({ imageRoot: root, storage: r2Storage(), dir });+ const result = await materialize({ imageRoot: root, storage: bundleStorage(), dir });console.log(`bundles: ${result.bundles.join(', ') || 'none'}`);if (result.superseded.length) console.log(`already posted in this checkout's queue, so left alone: ${result.superseded.join(', ')}`);console.log(`files: ${result.files} (${result.reused} reused from an earlier pull)`);@@ -149,13 +156,13 @@ async function pull() {async function remove() {const id = positional[1];if (!id) fail('Usage: remove <id>');- const result = await removeBundle({ id, storage: r2Storage() });+ const result = await removeBundle({ id, storage: bundleStorage() });if (!result.removed) fail(`${id} is not in the bundle index.`);console.log(`Removed ${id} and its files. ${result.remaining ? `${result.remaining} bundle(s) remain.` : 'The index was the last object, so it is gone too.'}`);}async function verify() {- const result = await verifyBundles({ storage: r2Storage() });+ const result = await verifyBundles({ storage: bundleStorage() });if (!result.index) {console.log('No bundle index in storage; nothing to verify.');return;diff --git a/tests/x-content-overlay.test.js b/tests/x-content-overlay.test.jsindex 7b50a28ed..1df4daa3d 100644--- a/tests/x-content-overlay.test.js+++ b/tests/x-content-overlay.test.js@@ -617,3 +617,22 @@ describe('where bundles live in the bucket', () => {expect(() => sealedKey(INDEX_KEY, {})).toThrow(/X_CONTENT_BUNDLE_SECRET is not set/);});});++describe('bundle storage selection', () => {+ const secretEnv = { X_CONTENT_BUNDLE_SECRET: 'a'.repeat(64) };++ it('uses Google Cloud Storage when a bundle bucket is named', async () => {+ const { bundleStorage } = await import('../api/_lib/x-content/overlay.js');+ expect(bundleStorage({ ...secretEnv, X_CONTENT_BUNDLE_BUCKET: 'bundles' }).label).toBe('Google Cloud Storage');+ });++ it('keeps object storage (R2) when no bucket is named', async () => {+ const { bundleStorage } = await import('../api/_lib/x-content/overlay.js');+ expect(bundleStorage(secretEnv).label).toBe('object storage');+ });++ it('refuses a GCS store without a bucket', async () => {+ const { gcsStorage } = await import('../api/_lib/x-content/overlay.js');+ expect(() => gcsStorage(secretEnv)).toThrow(/X_CONTENT_BUNDLE_BUCKET/);+ });+});