Skip to content

Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.

feat(ops): give a rejected object-storage credential its own health signal and graceful degradation

nirholas99c521446070
diff --git a/api/_lib/ops/object-storage-health.js b/api/_lib/ops/object-storage-health.js
new file mode 100644
index 000000000..c377e8028
--- /dev/null
+++ b/api/_lib/ops/object-storage-health.js
@@ -0,0 +1,100 @@
+// @ts-check
+// Object storage (Cloudflare R2) credential + reachability sensor.
+//
+// Why this exists: on 2026-09-07 the bucket credential stopped verifying and
+// EVERY signed operation began failing with `SignatureDoesNotMatch`: the forge
+// could not park a reference image, so text→3D returned a 502 for every user on
+// every surface, and /cdn/* answered `upstream_error` for every avatar,
+// thumbnail and GLB on the site. Production healthz reported `forge_generation:
+// ok, 89% success` throughout, because the outcome sensor reads rows in
+// forge_creations and this failure happens BEFORE the row is written: a total
+// outage looked exactly like a quiet hour. Nothing else in the platform holds a
+// signal for "our storage credential is rejected", so it went unseen for hours
+// and was reported by users, not by us.
+//
+// The probe is a signed, read-only ListObjectsV2 capped at one key. It touches
+// no object in particular (so it cannot be fooled by a deleted canary), it costs
+// one Class-B operation, and it exercises the exact thing that broke: the
+// signature. A rejected credential fails the same way for reads and writes, so
+// one read proves both.
+//
+// Verdicts:
+// unknown : storage isn't configured on this deployment (local/dev): neutral.
+// ok : the bucket answered a signed request.
+// down : the credential was rejected, or the endpoint is unreachable.
+// degraded : the bucket answered, but slowly enough to hurt every asset read.
+
+import { ListObjectsV2Command } from '@aws-sdk/client-s3';
+import { r2, objectStorageConfigured, isStorageInfrastructureError } from '../r2.js';
+import { env } from '../env.js';
+
+const PROBE_TIMEOUT_MS = 3_000;
+// R2 normally answers a one-key list in well under 200ms from Cloud Run. A
+// second is already slow enough that every gallery page feels it, so it is worth
+// surfacing before it becomes an outage.
+const SLOW_MS = 1_000;
+
+/**
+ * Classify a probe outcome. Pure, so the thresholds and the credential-vs-latency
+ * distinction are unit-testable without a bucket.
+ * @param {{ configured: boolean, ok?: boolean, latencyMs?: number, error?: unknown }} probe
+ * @returns {{ status: 'ok'|'degraded'|'down'|'unknown', detail: string, hint?: string }}
+ */
+export function classifyObjectStorageProbe(probe) {
+ if (!probe?.configured) {
+ return { status: 'unknown', detail: 'object storage not configured on this deployment' };
+ }
+ if (probe.ok) {
+ const ms = Math.round(Number(probe.latencyMs) || 0);
+ if (ms >= SLOW_MS) {
+ return {
+ status: 'degraded',
+ detail: `bucket reachable but slow (signed list ${ms}ms)`,
+ hint: 'Every avatar, thumbnail and generated GLB is read through this bucket. Check the R2 endpoint region and Cloud Run egress before it turns into read timeouts.',
+ };
+ }
+ return { status: 'ok', detail: `signed read ok (${ms}ms)` };
+ }
+ const message = String(probe.error?.message || probe.error || 'unknown error');
+ const rejected = /signaturedoesnotmatch|invalidaccesskeyid|access denied|forbidden/i.test(message);
+ return {
+ status: 'down',
+ detail: rejected
+ ? `bucket rejected our credential: ${message.slice(0, 160)}`
+ : `bucket unreachable: ${message.slice(0, 160)}`,
+ hint: rejected
+ ? 'The credential is wrong, not the code, and NO lane failover routes around it: the forge cannot park a reference image, uploads cannot land, and /cdn falls back to the rate-limited public bucket domain. `SignatureDoesNotMatch` with a working access key id means the SECRET is wrong: on Cloudflare R2 the secret access key is the token\'s SHA-256 digest, not the API token value, and a trailing newline in the stored secret fails identically (env.js trims it, so a padded value can only come from a store that is read elsewhere). Re-set S3_SECRET_ACCESS_KEY on the Cloud Run service and redeploy nothing else.'
+ : 'The bucket endpoint is unreachable from Cloud Run. Check S3_ENDPOINT and Cloudflare R2 status; reads fail over to the public bucket domain meanwhile, writes do not fail over at all.',
+ };
+}
+
+/**
+ * Run the signed probe and classify it. Never throws.
+ * @returns {Promise<{ name: 'object_storage', label: string, status: string, detail: string, hint?: string, metrics?: object }>}
+ */
+export async function gatherObjectStorageHealth() {
+ const base = { name: /** @type {const} */ ('object_storage'), label: 'Object storage (Cloudflare R2)' };
+ if (!objectStorageConfigured()) {
+ const v = classifyObjectStorageProbe({ configured: false });
+ return { ...base, status: v.status, detail: v.detail };
+ }
+ const started = Date.now();
+ try {
+ await r2.send(new ListObjectsV2Command({ Bucket: env.S3_BUCKET, MaxKeys: 1 }), {
+ abortSignal: AbortSignal.timeout(PROBE_TIMEOUT_MS),
+ });
+ const latencyMs = Date.now() - started;
+ const v = classifyObjectStorageProbe({ configured: true, ok: true, latencyMs });
+ return { ...base, status: v.status, detail: v.detail, ...(v.hint ? { hint: v.hint } : {}), metrics: { latencyMs } };
+ } catch (err) {
+ const latencyMs = Date.now() - started;
+ const v = classifyObjectStorageProbe({ configured: true, ok: false, latencyMs, error: err });
+ return {
+ ...base,
+ status: v.status,
+ detail: v.detail,
+ ...(v.hint ? { hint: v.hint } : {}),
+ metrics: { latencyMs, credentialFault: isStorageInfrastructureError(err) },
+ };
+ }
+}
diff --git a/api/_lib/ops/subsystem-health.js b/api/_lib/ops/subsystem-health.js
index 551c4cb4d..7f3e70dbc 100644
--- a/api/_lib/ops/subsystem-health.js
+++ b/api/_lib/ops/subsystem-health.js
@@ -37,6 +37,7 @@ import { rateLimiterHealth } from '../rate-limit.js';
import { checkRingInvariants } from '../x402/ring-allowlist.js';
import { gatherX402SettleHealth } from './x402-settle-health.js';
import { gatherForgeHealth } from './forge-health-sensor.js';
+import { gatherObjectStorageHealth } from './object-storage-health.js';
import { gatherIndexLagHealth } from './index-lag.js';
import { gatherHomeHealth } from './home-health.js';
import { describeSolvency } from '../sniper-solvency.js';
@@ -557,6 +558,12 @@ export async function gatherSubsystemHealth({ probeDb = true } = {}) {
// Settle SUCCESS RATE, not just "armed" — reads x402_autonomous_log. Needs
// the DB, so it shares the probeDb gate; skipped-DB callers get `unknown`.
probeDb ? gatherX402SettleHealth() : Promise.resolve({ name: 'x402_settle', label: 'x402 settlement success', status: 'unknown', detail: 'settle read skipped' }),
+ // Object storage credentials. Not DB-gated: it is a signed one-key list
+ // against the bucket. A rejected credential takes down 3D generation AND
+ // every avatar/thumbnail/GLB read at once, and on 2026-09-07 it did so with
+ // no health signature at all, because the forge sensor below can only see
+ // generations that got far enough to write a row.
+ gatherObjectStorageHealth(),
// Generation SUCCESS RATE, not just lane liveness — reads forge_creations.
// DB-gated like the settle sensor; skipped-DB callers get `unknown`.
probeDb ? gatherForgeHealth() : Promise.resolve({ name: 'forge_generation', label: 'Forge 3D generation', status: 'unknown', detail: 'forge read skipped' }),
diff --git a/api/cdn-object.js b/api/cdn-object.js
index da8df3f0c..dcb2f459d 100644
--- a/api/cdn-object.js
+++ b/api/cdn-object.js
@@ -19,7 +19,7 @@
import { GetObjectCommand } from '@aws-sdk/client-s3';
import { cors, error, wrap } from './_lib/http.js';
-import { r2 } from './_lib/r2.js';
+import { r2, isStorageInfrastructureError, publicUrlOrNull } from './_lib/r2.js';
import { env } from './_lib/env.js';
// Object keys are caller-controlled path input — keep them boring. UUID-based
@@ -162,6 +162,25 @@ export default wrap(async (req, res) => {
if (code === 'InvalidRange' || status === 416) {
return error(res, 416, 'invalid_range', 'requested range not satisfiable');
}
+ // The signed read itself is broken: the credentials are rejected, or the
+ // endpoint is unreachable, which says nothing about the object. The same
+ // bytes are readable, unauthenticated, on the bucket's public domain, so
+ // hand the caller there instead of 502ing every avatar, thumbnail and GLB
+ // on the site. On 2026-09-07 a rejected R2 secret took the signed read
+ // down and this route answered `upstream_error` for every object it
+ // serves; the public domain was serving those same keys with a 200
+ // throughout. Redirect, never cache: 302 + no-store, so the moment the
+ // credential is healthy again traffic returns to the signed path (which
+ // exists to dodge the public domain's rate limit) without a stale hop
+ // pinned in anyone's cache.
+ const fallback = isStorageInfrastructureError(err) ? publicUrlOrNull(key) : null;
+ if (fallback) {
+ console.error('[cdn-object] signed read failed, serving public bucket domain:', key, err?.message);
+ res.statusCode = 302;
+ res.setHeader('location', fallback);
+ res.setHeader('cache-control', 'no-store');
+ return res.end();
+ }
console.error('[cdn-object] r2 fetch failed:', key, err?.message);
return error(res, 502, 'upstream_error', 'failed to fetch object');
}
diff --git a/api/forge.js b/api/forge.js
index 526f4f183..997f26478 100644
--- a/api/forge.js
+++ b/api/forge.js
@@ -93,6 +93,7 @@ import {
markSupersededBy,
findByJob,
} from './_lib/forge-store.js';
+import { isStorageInfrastructureError } from './_lib/r2.js';
import { getSessionUser } from './_lib/auth.js';
import { constantTimeEquals } from './_lib/crypto.js';
import {
@@ -2506,6 +2507,21 @@ async function startJob(req, res) {
retry_after: 30,
});
}
+ // Object storage rejected us (a rotated/whitespace-padded credential, or an
+ // unreachable endpoint). Nothing about the prompt is wrong and no lane
+ // failover can route around it: every lane has to park a reference image or
+ // a finished mesh in the bucket. Answer with the designed unavailable state
+ // instead of forwarding the store's raw signing complaint, which read as
+ // "Check your secret access key" in the user's browser on 2026-09-07.
+ if (isStorageInfrastructureError(err)) {
+ console.error(`[forge] object storage rejected the generation: ${err?.message || err}`);
+ res.setHeader('retry-after', '60');
+ return json(res, 503, {
+ error: 'storage_unavailable',
+ message: '3D generation is temporarily unavailable while our asset storage recovers. Please try again shortly.',
+ retry_after: 60,
+ });
+ }
return json(res, 502, {
error: 'generation_failed',
message: err?.message || 'The generator could not start this job.',
diff --git a/api/gpt-forge.js b/api/gpt-forge.js
index ae9f5ef1e..36c101a60 100644
--- a/api/gpt-forge.js
+++ b/api/gpt-forge.js
@@ -98,6 +98,7 @@ import {
markFailed,
findByJob,
} from './_lib/forge-store.js';
+import { isStorageInfrastructureError } from './_lib/r2.js';
import { getSessionUser } from './_lib/auth.js';
import { constantTimeEquals } from './_lib/crypto.js';
import {
@@ -2491,6 +2492,21 @@ async function startJob(req, res) {
retry_after: 30,
});
}
+ // Object storage rejected us (a rotated/whitespace-padded credential, or an
+ // unreachable endpoint). Nothing about the prompt is wrong and no lane
+ // failover can route around it: every lane has to park a reference image or
+ // a finished mesh in the bucket. Answer with the designed unavailable state
+ // instead of forwarding the store's raw signing complaint, which read as
+ // "Check your secret access key" in the user's browser on 2026-09-07.
+ if (isStorageInfrastructureError(err)) {
+ console.error(`[gpt-forge] object storage rejected the generation: ${err?.message || err}`);
+ res.setHeader('retry-after', '60');
+ return json(res, 503, {
+ error: 'storage_unavailable',
+ message: '3D generation is temporarily unavailable while our asset storage recovers. Please try again shortly.',
+ retry_after: 60,
+ });
+ }
return json(res, 502, {
error: 'generation_failed',
message: err?.message || 'The generator could not start this job.',