Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.
JavaScript 3d-generationai-agentsanimationavatardigital-humangenerative-aiglbgltfimage-to-3dllmmcpmcp-servermodel-context-protocolsolanatext-to-3dthreejsweb-componentswebglwebxrx402
fix(storage): recognise the credential rejection the SDK words as a sentence, not a code
nirholas2ab1cb56ab98
diff --git a/api/_lib/ops/object-storage-health.js b/api/_lib/ops/object-storage-health.jsindex c377e8028..b4e42496e 100644--- a/api/_lib/ops/object-storage-health.js+++ b/api/_lib/ops/object-storage-health.js@@ -55,8 +55,14 @@ export function classifyObjectStorageProbe(probe) {}return { status: 'ok', detail: `signed read ok (${ms}ms)` };}- const message = String(probe.error?.message || probe.error || 'unknown error');- const rejected = /signaturedoesnotmatch|invalidaccesskeyid|access denied|forbidden/i.test(message);+ const err = /** @type {any} */ (probe.error);+ const message = String(err?.message || err || 'unknown error');+ // The compact code lives in name/Code and the sentence in message, so read+ // both: `SignatureDoesNotMatch` never appears in the text a user or a log+ // line sees (see isStorageInfrastructureError in ../r2.js).+ const rejected = /signaturedoesnotmatch|does not match the signature|invalidaccesskeyid|access denied|forbidden/i.test(+ [err?.name, err?.Code, message].filter(Boolean).join(' '),+ );return {status: 'down',detail: rejecteddiff --git a/api/_lib/r2.js b/api/_lib/r2.jsindex 24d6343ab..e3c369b66 100644--- a/api/_lib/r2.js+++ b/api/_lib/r2.js@@ -70,13 +70,26 @@ export function objectStorageConfigured() {// Exported as a pattern string so the SQL repair path can ask the same question// with `~*` instead of maintaining a second, drifting copy (see// resetInfrastructureFailures in avatar-thumbs.js). JS/POSIX-ERE compatible.+// `does not match the signature` is the S3 error's human MESSAGE; the compact+// `SignatureDoesNotMatch` form only ever appears in err.name / err.Code, never in+// the sentence the SDK throws. Matching the name alone (as this did until+// 2026-09-07) meant the single most common credential fault sailed past every+// caller of this helper, including the batch runners that use it to decide+// whether an asset is blameless, and surfaced to users as a raw "Check your+// secret access key" in the browser.export const STORAGE_ERROR_PATTERN =- 'missing required env var: s3_|invalidaccesskeyid|signaturedoesnotmatch|nosuchbucket|access denied|econnrefused|enotfound|socket hang up|econnreset';+ 'missing required env var: s3_|invalidaccesskeyid|signaturedoesnotmatch|does not match the signature|nosuchbucket|access denied|econnrefused|enotfound|socket hang up|econnreset';const STORAGE_ERROR_RE = new RegExp(STORAGE_ERROR_PATTERN, 'i');export function isStorageInfrastructureError(err) {- return STORAGE_ERROR_RE.test(String(err?.message || err || ''));+ // The SDK splits one fault across three fields: name/Code carry the compact+ // code, message carries the sentence. Judge all of them, so neither shape+ // escapes.+ const text = [err?.name, err?.Code, err?.message ?? (err == null ? '' : String(err))]+ .filter(Boolean)+ .join(' ');+ return STORAGE_ERROR_RE.test(text);}// Short-lived signed URL for direct browser upload (PUT).diff --git a/tests/env-s3-cred.test.js b/tests/env-s3-cred.test.jsindex 238876143..b05285534 100644--- a/tests/env-s3-cred.test.js+++ b/tests/env-s3-cred.test.js@@ -12,7 +12,6 @@import { describe, it, expect, beforeEach, afterEach } from 'vitest';import { env } from '../api/_lib/env.js';-import { objectStorageConfigured } from '../api/_lib/r2.js';const KEYS = ['S3_ACCESS_KEY_ID', 'S3_SECRET_ACCESS_KEY', 'S3_ENDPOINT', 'S3_BUCKET', 'S3_PUBLIC_DOMAIN'];let saved;@@ -49,23 +48,3 @@ describe('S3 credential getters', () => {expect(() => env.S3_SECRET_ACCESS_KEY).toThrow(/S3_SECRET_ACCESS_KEY/);});});--describe('objectStorageConfigured', () => {- it('is false when a credential is present but blank', () => {- process.env.S3_ENDPOINT = 'https://acct.r2.cloudflarestorage.com';- process.env.S3_BUCKET = 'three-ws';- process.env.S3_PUBLIC_DOMAIN = 'https://pub-example.r2.dev';- process.env.S3_ACCESS_KEY_ID = 'AKIAEXAMPLE';- process.env.S3_SECRET_ACCESS_KEY = '\n';- expect(objectStorageConfigured()).toBe(false);- });-- it('is true when every var carries a real value', () => {- process.env.S3_ENDPOINT = 'https://acct.r2.cloudflarestorage.com';- process.env.S3_BUCKET = 'three-ws';- process.env.S3_PUBLIC_DOMAIN = 'https://pub-example.r2.dev';- process.env.S3_ACCESS_KEY_ID = 'AKIAEXAMPLE';- process.env.S3_SECRET_ACCESS_KEY = 'abc123secret';- expect(objectStorageConfigured()).toBe(true);- });-});diff --git a/tests/forge-health-stall.test.js b/tests/forge-health-stall.test.jsnew file mode 100644index 000000000..67d1ab059--- /dev/null+++ b/tests/forge-health-stall.test.js@@ -0,0 +1,44 @@+/**+ * Forge stall detection, unit test.+ *+ * The success-rate sensor can only judge generations that got far enough to+ * write a forge_creations row. On 2026-09-07 object storage rejected the+ * reference-image upload, which happens BEFORE that row exists, so the forge+ * produced zero rows for nearly five hours while healthz kept reporting+ * "89% success (118/133 finished)" off the rows from before the outage. Silence+ * now has to prove it is innocent: quiet is fine on a quiet lane, and an outage+ * on a busy one.+ */++import { describe, it, expect } from 'vitest';+import { classifyForgeStall } from '../api/_lib/ops/forge-health-sensor.js';++const MIN = 60_000;++describe('classifyForgeStall', () => {+ it('flags a busy forge that has started nothing for hours', () => {+ const v = classifyForgeStall({ lastCreatedAgeMs: 289 * MIN, hourlyRate: 12 });+ expect(v.stalled).toBe(true);+ expect(v.ageMinutes).toBe(289);+ expect(v.detail).toMatch(/no generation has STARTED in 289m/);+ expect(v.hint).toMatch(/object_storage/);+ });++ it('does not flag a genuinely quiet lane', () => {+ expect(classifyForgeStall({ lastCreatedAgeMs: 300 * MIN, hourlyRate: 1 }).stalled).toBe(false);+ });++ it('does not flag a busy forge that is still starting jobs', () => {+ expect(classifyForgeStall({ lastCreatedAgeMs: 4 * MIN, hourlyRate: 50 }).stalled).toBe(false);+ });++ it('tolerates a gap shorter than the stall window', () => {+ expect(classifyForgeStall({ lastCreatedAgeMs: 30 * MIN, hourlyRate: 20 }).stalled).toBe(false);+ });++ it('says nothing when the table is empty', () => {+ const v = classifyForgeStall({ lastCreatedAgeMs: null, hourlyRate: 0 });+ expect(v.stalled).toBe(false);+ expect(v.ageMinutes).toBe(null);+ });+});diff --git a/tests/object-storage-health.test.js b/tests/object-storage-health.test.jsnew file mode 100644index 000000000..510b2a1b9--- /dev/null+++ b/tests/object-storage-health.test.js@@ -0,0 +1,55 @@+/**+ * Object storage (R2) health sensor, unit test.+ *+ * On 2026-09-07 the bucket credential stopped verifying and every signed+ * operation failed with `SignatureDoesNotMatch`: text→3D returned an error for+ * every user on both the website and the ChatGPT surface, and /cdn answered 502+ * for every avatar and thumbnail on the site. Production healthz reported the+ * forge as `ok` throughout, because no subsystem held a signal for "our storage+ * credential is rejected". This pins the sensor that now does, including the+ * distinction that decides the runbook: a rejected credential (a human has to+ * re-set it) versus an unreachable endpoint (it may clear itself).+ */++import { describe, it, expect } from 'vitest';+import { classifyObjectStorageProbe } from '../api/_lib/ops/object-storage-health.js';++describe('classifyObjectStorageProbe', () => {+ it('stays neutral when storage is not configured on the deployment', () => {+ const v = classifyObjectStorageProbe({ configured: false });+ expect(v.status).toBe('unknown');+ });++ it('reads ok on a fast signed list', () => {+ const v = classifyObjectStorageProbe({ configured: true, ok: true, latencyMs: 120 });+ expect(v.status).toBe('ok');+ expect(v.detail).toContain('120ms');+ });++ it('degrades on a slow but working bucket', () => {+ const v = classifyObjectStorageProbe({ configured: true, ok: true, latencyMs: 1500 });+ expect(v.status).toBe('degraded');+ });++ it('reports a rejected credential as down, and says the secret is wrong', () => {+ const v = classifyObjectStorageProbe({+ configured: true,+ ok: false,+ error: new Error('The request signature we calculated does not match the signature you provided. Check your secret access key and signing method.'),+ });+ expect(v.status).toBe('down');+ expect(v.detail).toMatch(/rejected our credential/);+ expect(v.hint).toMatch(/S3_SECRET_ACCESS_KEY/);+ });++ it('separates an unreachable endpoint from a rejected credential', () => {+ const v = classifyObjectStorageProbe({+ configured: true,+ ok: false,+ error: new Error('getaddrinfo ENOTFOUND acct.r2.cloudflarestorage.com'),+ });+ expect(v.status).toBe('down');+ expect(v.detail).toMatch(/unreachable/);+ expect(v.hint).toMatch(/S3_ENDPOINT/);+ });+});diff --git a/tests/thumbnail-blameless-failures.test.js b/tests/thumbnail-blameless-failures.test.jsindex a8bdf942e..f16845357 100644--- a/tests/thumbnail-blameless-failures.test.js+++ b/tests/thumbnail-blameless-failures.test.js@@ -107,4 +107,14 @@ describe('objectStorageConfigured', () => {expect(withEnv(partial, objectStorageConfigured), `missing ${missing}`).toBe(false);}});++ // A credential that is nothing but a newline is not configured storage: it+ // signs every request straight into a SignatureDoesNotMatch while a bare+ // truthiness check reads healthy.+ it('is false when any single var is blank', () => {+ for (const blank of KEYS) {+ const padded = { ...full, [blank]: ' \n' };+ expect(withEnv(padded, objectStorageConfigured), `blank ${blank}`).toBe(false);+ }+ });});