Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.
JavaScript 3d-generationai-agentsanimationavatardigital-humangenerative-aiglbgltfimage-to-3dllmmcpmcp-servermodel-context-protocolsolanatext-to-3dthreejsweb-componentswebglwebxrx402
feat(ops): give a rejected object-storage credential its own health signal and graceful degradation
nirholas99c521446070
diff --git a/api/_lib/ops/object-storage-health.js b/api/_lib/ops/object-storage-health.jsnew file mode 100644index 000000000..c377e8028--- /dev/null+++ b/api/_lib/ops/object-storage-health.js@@ -0,0 +1,100 @@+// @ts-check+// Object storage (Cloudflare R2) credential + reachability sensor.+//+// Why this exists: on 2026-09-07 the bucket credential stopped verifying and+// EVERY signed operation began failing with `SignatureDoesNotMatch`: the forge+// could not park a reference image, so text→3D returned a 502 for every user on+// every surface, and /cdn/* answered `upstream_error` for every avatar,+// thumbnail and GLB on the site. Production healthz reported `forge_generation:+// ok, 89% success` throughout, because the outcome sensor reads rows in+// forge_creations and this failure happens BEFORE the row is written: a total+// outage looked exactly like a quiet hour. Nothing else in the platform holds a+// signal for "our storage credential is rejected", so it went unseen for hours+// and was reported by users, not by us.+//+// The probe is a signed, read-only ListObjectsV2 capped at one key. It touches+// no object in particular (so it cannot be fooled by a deleted canary), it costs+// one Class-B operation, and it exercises the exact thing that broke: the+// signature. A rejected credential fails the same way for reads and writes, so+// one read proves both.+//+// Verdicts:+// unknown : storage isn't configured on this deployment (local/dev): neutral.+// ok : the bucket answered a signed request.+// down : the credential was rejected, or the endpoint is unreachable.+// degraded : the bucket answered, but slowly enough to hurt every asset read.++import { ListObjectsV2Command } from '@aws-sdk/client-s3';+import { r2, objectStorageConfigured, isStorageInfrastructureError } from '../r2.js';+import { env } from '../env.js';++const PROBE_TIMEOUT_MS = 3_000;+// R2 normally answers a one-key list in well under 200ms from Cloud Run. A+// second is already slow enough that every gallery page feels it, so it is worth+// surfacing before it becomes an outage.+const SLOW_MS = 1_000;++/**+ * Classify a probe outcome. Pure, so the thresholds and the credential-vs-latency+ * distinction are unit-testable without a bucket.+ * @param {{ configured: boolean, ok?: boolean, latencyMs?: number, error?: unknown }} probe+ * @returns {{ status: 'ok'|'degraded'|'down'|'unknown', detail: string, hint?: string }}+ */+export function classifyObjectStorageProbe(probe) {+ if (!probe?.configured) {+ return { status: 'unknown', detail: 'object storage not configured on this deployment' };+ }+ if (probe.ok) {+ const ms = Math.round(Number(probe.latencyMs) || 0);+ if (ms >= SLOW_MS) {+ return {+ status: 'degraded',+ detail: `bucket reachable but slow (signed list ${ms}ms)`,+ hint: 'Every avatar, thumbnail and generated GLB is read through this bucket. Check the R2 endpoint region and Cloud Run egress before it turns into read timeouts.',+ };+ }+ return { status: 'ok', detail: `signed read ok (${ms}ms)` };+ }+ const message = String(probe.error?.message || probe.error || 'unknown error');+ const rejected = /signaturedoesnotmatch|invalidaccesskeyid|access denied|forbidden/i.test(message);+ return {+ status: 'down',+ detail: rejected+ ? `bucket rejected our credential: ${message.slice(0, 160)}`+ : `bucket unreachable: ${message.slice(0, 160)}`,+ hint: rejected+ ? 'The credential is wrong, not the code, and NO lane failover routes around it: the forge cannot park a reference image, uploads cannot land, and /cdn falls back to the rate-limited public bucket domain. `SignatureDoesNotMatch` with a working access key id means the SECRET is wrong: on Cloudflare R2 the secret access key is the token\'s SHA-256 digest, not the API token value, and a trailing newline in the stored secret fails identically (env.js trims it, so a padded value can only come from a store that is read elsewhere). Re-set S3_SECRET_ACCESS_KEY on the Cloud Run service and redeploy nothing else.'+ : 'The bucket endpoint is unreachable from Cloud Run. Check S3_ENDPOINT and Cloudflare R2 status; reads fail over to the public bucket domain meanwhile, writes do not fail over at all.',+ };+}++/**+ * Run the signed probe and classify it. Never throws.+ * @returns {Promise<{ name: 'object_storage', label: string, status: string, detail: string, hint?: string, metrics?: object }>}+ */+export async function gatherObjectStorageHealth() {+ const base = { name: /** @type {const} */ ('object_storage'), label: 'Object storage (Cloudflare R2)' };+ if (!objectStorageConfigured()) {+ const v = classifyObjectStorageProbe({ configured: false });+ return { ...base, status: v.status, detail: v.detail };+ }+ const started = Date.now();+ try {+ await r2.send(new ListObjectsV2Command({ Bucket: env.S3_BUCKET, MaxKeys: 1 }), {+ abortSignal: AbortSignal.timeout(PROBE_TIMEOUT_MS),+ });+ const latencyMs = Date.now() - started;+ const v = classifyObjectStorageProbe({ configured: true, ok: true, latencyMs });+ return { ...base, status: v.status, detail: v.detail, ...(v.hint ? { hint: v.hint } : {}), metrics: { latencyMs } };+ } catch (err) {+ const latencyMs = Date.now() - started;+ const v = classifyObjectStorageProbe({ configured: true, ok: false, latencyMs, error: err });+ return {+ ...base,+ status: v.status,+ detail: v.detail,+ ...(v.hint ? { hint: v.hint } : {}),+ metrics: { latencyMs, credentialFault: isStorageInfrastructureError(err) },+ };+ }+}diff --git a/api/_lib/ops/subsystem-health.js b/api/_lib/ops/subsystem-health.jsindex 551c4cb4d..7f3e70dbc 100644--- a/api/_lib/ops/subsystem-health.js+++ b/api/_lib/ops/subsystem-health.js@@ -37,6 +37,7 @@ import { rateLimiterHealth } from '../rate-limit.js';import { checkRingInvariants } from '../x402/ring-allowlist.js';import { gatherX402SettleHealth } from './x402-settle-health.js';import { gatherForgeHealth } from './forge-health-sensor.js';+import { gatherObjectStorageHealth } from './object-storage-health.js';import { gatherIndexLagHealth } from './index-lag.js';import { gatherHomeHealth } from './home-health.js';import { describeSolvency } from '../sniper-solvency.js';@@ -557,6 +558,12 @@ export async function gatherSubsystemHealth({ probeDb = true } = {}) {// Settle SUCCESS RATE, not just "armed" — reads x402_autonomous_log. Needs// the DB, so it shares the probeDb gate; skipped-DB callers get `unknown`.probeDb ? gatherX402SettleHealth() : Promise.resolve({ name: 'x402_settle', label: 'x402 settlement success', status: 'unknown', detail: 'settle read skipped' }),+ // Object storage credentials. Not DB-gated: it is a signed one-key list+ // against the bucket. A rejected credential takes down 3D generation AND+ // every avatar/thumbnail/GLB read at once, and on 2026-09-07 it did so with+ // no health signature at all, because the forge sensor below can only see+ // generations that got far enough to write a row.+ gatherObjectStorageHealth(),// Generation SUCCESS RATE, not just lane liveness — reads forge_creations.// DB-gated like the settle sensor; skipped-DB callers get `unknown`.probeDb ? gatherForgeHealth() : Promise.resolve({ name: 'forge_generation', label: 'Forge 3D generation', status: 'unknown', detail: 'forge read skipped' }),diff --git a/api/cdn-object.js b/api/cdn-object.jsindex da8df3f0c..dcb2f459d 100644--- a/api/cdn-object.js+++ b/api/cdn-object.js@@ -19,7 +19,7 @@import { GetObjectCommand } from '@aws-sdk/client-s3';import { cors, error, wrap } from './_lib/http.js';-import { r2 } from './_lib/r2.js';+import { r2, isStorageInfrastructureError, publicUrlOrNull } from './_lib/r2.js';import { env } from './_lib/env.js';// Object keys are caller-controlled path input — keep them boring. UUID-based@@ -162,6 +162,25 @@ export default wrap(async (req, res) => {if (code === 'InvalidRange' || status === 416) {return error(res, 416, 'invalid_range', 'requested range not satisfiable');}+ // The signed read itself is broken: the credentials are rejected, or the+ // endpoint is unreachable, which says nothing about the object. The same+ // bytes are readable, unauthenticated, on the bucket's public domain, so+ // hand the caller there instead of 502ing every avatar, thumbnail and GLB+ // on the site. On 2026-09-07 a rejected R2 secret took the signed read+ // down and this route answered `upstream_error` for every object it+ // serves; the public domain was serving those same keys with a 200+ // throughout. Redirect, never cache: 302 + no-store, so the moment the+ // credential is healthy again traffic returns to the signed path (which+ // exists to dodge the public domain's rate limit) without a stale hop+ // pinned in anyone's cache.+ const fallback = isStorageInfrastructureError(err) ? publicUrlOrNull(key) : null;+ if (fallback) {+ console.error('[cdn-object] signed read failed, serving public bucket domain:', key, err?.message);+ res.statusCode = 302;+ res.setHeader('location', fallback);+ res.setHeader('cache-control', 'no-store');+ return res.end();+ }console.error('[cdn-object] r2 fetch failed:', key, err?.message);return error(res, 502, 'upstream_error', 'failed to fetch object');}diff --git a/api/forge.js b/api/forge.jsindex 526f4f183..997f26478 100644--- a/api/forge.js+++ b/api/forge.js@@ -93,6 +93,7 @@ import {markSupersededBy,findByJob,} from './_lib/forge-store.js';+import { isStorageInfrastructureError } from './_lib/r2.js';import { getSessionUser } from './_lib/auth.js';import { constantTimeEquals } from './_lib/crypto.js';import {@@ -2506,6 +2507,21 @@ async function startJob(req, res) {retry_after: 30,});}+ // Object storage rejected us (a rotated/whitespace-padded credential, or an+ // unreachable endpoint). Nothing about the prompt is wrong and no lane+ // failover can route around it: every lane has to park a reference image or+ // a finished mesh in the bucket. Answer with the designed unavailable state+ // instead of forwarding the store's raw signing complaint, which read as+ // "Check your secret access key" in the user's browser on 2026-09-07.+ if (isStorageInfrastructureError(err)) {+ console.error(`[forge] object storage rejected the generation: ${err?.message || err}`);+ res.setHeader('retry-after', '60');+ return json(res, 503, {+ error: 'storage_unavailable',+ message: '3D generation is temporarily unavailable while our asset storage recovers. Please try again shortly.',+ retry_after: 60,+ });+ }return json(res, 502, {error: 'generation_failed',message: err?.message || 'The generator could not start this job.',diff --git a/api/gpt-forge.js b/api/gpt-forge.jsindex ae9f5ef1e..36c101a60 100644--- a/api/gpt-forge.js+++ b/api/gpt-forge.js@@ -98,6 +98,7 @@ import {markFailed,findByJob,} from './_lib/forge-store.js';+import { isStorageInfrastructureError } from './_lib/r2.js';import { getSessionUser } from './_lib/auth.js';import { constantTimeEquals } from './_lib/crypto.js';import {@@ -2491,6 +2492,21 @@ async function startJob(req, res) {retry_after: 30,});}+ // Object storage rejected us (a rotated/whitespace-padded credential, or an+ // unreachable endpoint). Nothing about the prompt is wrong and no lane+ // failover can route around it: every lane has to park a reference image or+ // a finished mesh in the bucket. Answer with the designed unavailable state+ // instead of forwarding the store's raw signing complaint, which read as+ // "Check your secret access key" in the user's browser on 2026-09-07.+ if (isStorageInfrastructureError(err)) {+ console.error(`[gpt-forge] object storage rejected the generation: ${err?.message || err}`);+ res.setHeader('retry-after', '60');+ return json(res, 503, {+ error: 'storage_unavailable',+ message: '3D generation is temporarily unavailable while our asset storage recovers. Please try again shortly.',+ retry_after: 60,+ });+ }return json(res, 502, {error: 'generation_failed',message: err?.message || 'The generator could not start this job.',