Skip to content

Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.

fix(storage): recognise the credential rejection the SDK words as a sentence, not a code

nirholas2ab1cb56ab98
diff --git a/api/_lib/ops/object-storage-health.js b/api/_lib/ops/object-storage-health.js
index c377e8028..b4e42496e 100644
--- a/api/_lib/ops/object-storage-health.js
+++ b/api/_lib/ops/object-storage-health.js
@@ -55,8 +55,14 @@ export function classifyObjectStorageProbe(probe) {
}
return { status: 'ok', detail: `signed read ok (${ms}ms)` };
}
- const message = String(probe.error?.message || probe.error || 'unknown error');
- const rejected = /signaturedoesnotmatch|invalidaccesskeyid|access denied|forbidden/i.test(message);
+ const err = /** @type {any} */ (probe.error);
+ const message = String(err?.message || err || 'unknown error');
+ // The compact code lives in name/Code and the sentence in message, so read
+ // both: `SignatureDoesNotMatch` never appears in the text a user or a log
+ // line sees (see isStorageInfrastructureError in ../r2.js).
+ const rejected = /signaturedoesnotmatch|does not match the signature|invalidaccesskeyid|access denied|forbidden/i.test(
+ [err?.name, err?.Code, message].filter(Boolean).join(' '),
+ );
return {
status: 'down',
detail: rejected
diff --git a/api/_lib/r2.js b/api/_lib/r2.js
index 24d6343ab..e3c369b66 100644
--- a/api/_lib/r2.js
+++ b/api/_lib/r2.js
@@ -70,13 +70,26 @@ export function objectStorageConfigured() {
// Exported as a pattern string so the SQL repair path can ask the same question
// with `~*` instead of maintaining a second, drifting copy (see
// resetInfrastructureFailures in avatar-thumbs.js). JS/POSIX-ERE compatible.
+// `does not match the signature` is the S3 error's human MESSAGE; the compact
+// `SignatureDoesNotMatch` form only ever appears in err.name / err.Code, never in
+// the sentence the SDK throws. Matching the name alone (as this did until
+// 2026-09-07) meant the single most common credential fault sailed past every
+// caller of this helper, including the batch runners that use it to decide
+// whether an asset is blameless, and surfaced to users as a raw "Check your
+// secret access key" in the browser.
export const STORAGE_ERROR_PATTERN =
- 'missing required env var: s3_|invalidaccesskeyid|signaturedoesnotmatch|nosuchbucket|access denied|econnrefused|enotfound|socket hang up|econnreset';
+ 'missing required env var: s3_|invalidaccesskeyid|signaturedoesnotmatch|does not match the signature|nosuchbucket|access denied|econnrefused|enotfound|socket hang up|econnreset';
const STORAGE_ERROR_RE = new RegExp(STORAGE_ERROR_PATTERN, 'i');
export function isStorageInfrastructureError(err) {
- return STORAGE_ERROR_RE.test(String(err?.message || err || ''));
+ // The SDK splits one fault across three fields: name/Code carry the compact
+ // code, message carries the sentence. Judge all of them, so neither shape
+ // escapes.
+ const text = [err?.name, err?.Code, err?.message ?? (err == null ? '' : String(err))]
+ .filter(Boolean)
+ .join(' ');
+ return STORAGE_ERROR_RE.test(text);
}
// Short-lived signed URL for direct browser upload (PUT).
diff --git a/tests/env-s3-cred.test.js b/tests/env-s3-cred.test.js
index 238876143..b05285534 100644
--- a/tests/env-s3-cred.test.js
+++ b/tests/env-s3-cred.test.js
@@ -12,7 +12,6 @@
import { describe, it, expect, beforeEach, afterEach } from 'vitest';
import { env } from '../api/_lib/env.js';
-import { objectStorageConfigured } from '../api/_lib/r2.js';
const KEYS = ['S3_ACCESS_KEY_ID', 'S3_SECRET_ACCESS_KEY', 'S3_ENDPOINT', 'S3_BUCKET', 'S3_PUBLIC_DOMAIN'];
let saved;
@@ -49,23 +48,3 @@ describe('S3 credential getters', () => {
expect(() => env.S3_SECRET_ACCESS_KEY).toThrow(/S3_SECRET_ACCESS_KEY/);
});
});
-
-describe('objectStorageConfigured', () => {
- it('is false when a credential is present but blank', () => {
- process.env.S3_ENDPOINT = 'https://acct.r2.cloudflarestorage.com';
- process.env.S3_BUCKET = 'three-ws';
- process.env.S3_PUBLIC_DOMAIN = 'https://pub-example.r2.dev';
- process.env.S3_ACCESS_KEY_ID = 'AKIAEXAMPLE';
- process.env.S3_SECRET_ACCESS_KEY = '\n';
- expect(objectStorageConfigured()).toBe(false);
- });
-
- it('is true when every var carries a real value', () => {
- process.env.S3_ENDPOINT = 'https://acct.r2.cloudflarestorage.com';
- process.env.S3_BUCKET = 'three-ws';
- process.env.S3_PUBLIC_DOMAIN = 'https://pub-example.r2.dev';
- process.env.S3_ACCESS_KEY_ID = 'AKIAEXAMPLE';
- process.env.S3_SECRET_ACCESS_KEY = 'abc123secret';
- expect(objectStorageConfigured()).toBe(true);
- });
-});
diff --git a/tests/forge-health-stall.test.js b/tests/forge-health-stall.test.js
new file mode 100644
index 000000000..67d1ab059
--- /dev/null
+++ b/tests/forge-health-stall.test.js
@@ -0,0 +1,44 @@
+/**
+ * Forge stall detection, unit test.
+ *
+ * The success-rate sensor can only judge generations that got far enough to
+ * write a forge_creations row. On 2026-09-07 object storage rejected the
+ * reference-image upload, which happens BEFORE that row exists, so the forge
+ * produced zero rows for nearly five hours while healthz kept reporting
+ * "89% success (118/133 finished)" off the rows from before the outage. Silence
+ * now has to prove it is innocent: quiet is fine on a quiet lane, and an outage
+ * on a busy one.
+ */
+
+import { describe, it, expect } from 'vitest';
+import { classifyForgeStall } from '../api/_lib/ops/forge-health-sensor.js';
+
+const MIN = 60_000;
+
+describe('classifyForgeStall', () => {
+ it('flags a busy forge that has started nothing for hours', () => {
+ const v = classifyForgeStall({ lastCreatedAgeMs: 289 * MIN, hourlyRate: 12 });
+ expect(v.stalled).toBe(true);
+ expect(v.ageMinutes).toBe(289);
+ expect(v.detail).toMatch(/no generation has STARTED in 289m/);
+ expect(v.hint).toMatch(/object_storage/);
+ });
+
+ it('does not flag a genuinely quiet lane', () => {
+ expect(classifyForgeStall({ lastCreatedAgeMs: 300 * MIN, hourlyRate: 1 }).stalled).toBe(false);
+ });
+
+ it('does not flag a busy forge that is still starting jobs', () => {
+ expect(classifyForgeStall({ lastCreatedAgeMs: 4 * MIN, hourlyRate: 50 }).stalled).toBe(false);
+ });
+
+ it('tolerates a gap shorter than the stall window', () => {
+ expect(classifyForgeStall({ lastCreatedAgeMs: 30 * MIN, hourlyRate: 20 }).stalled).toBe(false);
+ });
+
+ it('says nothing when the table is empty', () => {
+ const v = classifyForgeStall({ lastCreatedAgeMs: null, hourlyRate: 0 });
+ expect(v.stalled).toBe(false);
+ expect(v.ageMinutes).toBe(null);
+ });
+});
diff --git a/tests/object-storage-health.test.js b/tests/object-storage-health.test.js
new file mode 100644
index 000000000..510b2a1b9
--- /dev/null
+++ b/tests/object-storage-health.test.js
@@ -0,0 +1,55 @@
+/**
+ * Object storage (R2) health sensor, unit test.
+ *
+ * On 2026-09-07 the bucket credential stopped verifying and every signed
+ * operation failed with `SignatureDoesNotMatch`: text→3D returned an error for
+ * every user on both the website and the ChatGPT surface, and /cdn answered 502
+ * for every avatar and thumbnail on the site. Production healthz reported the
+ * forge as `ok` throughout, because no subsystem held a signal for "our storage
+ * credential is rejected". This pins the sensor that now does, including the
+ * distinction that decides the runbook: a rejected credential (a human has to
+ * re-set it) versus an unreachable endpoint (it may clear itself).
+ */
+
+import { describe, it, expect } from 'vitest';
+import { classifyObjectStorageProbe } from '../api/_lib/ops/object-storage-health.js';
+
+describe('classifyObjectStorageProbe', () => {
+ it('stays neutral when storage is not configured on the deployment', () => {
+ const v = classifyObjectStorageProbe({ configured: false });
+ expect(v.status).toBe('unknown');
+ });
+
+ it('reads ok on a fast signed list', () => {
+ const v = classifyObjectStorageProbe({ configured: true, ok: true, latencyMs: 120 });
+ expect(v.status).toBe('ok');
+ expect(v.detail).toContain('120ms');
+ });
+
+ it('degrades on a slow but working bucket', () => {
+ const v = classifyObjectStorageProbe({ configured: true, ok: true, latencyMs: 1500 });
+ expect(v.status).toBe('degraded');
+ });
+
+ it('reports a rejected credential as down, and says the secret is wrong', () => {
+ const v = classifyObjectStorageProbe({
+ configured: true,
+ ok: false,
+ error: new Error('The request signature we calculated does not match the signature you provided. Check your secret access key and signing method.'),
+ });
+ expect(v.status).toBe('down');
+ expect(v.detail).toMatch(/rejected our credential/);
+ expect(v.hint).toMatch(/S3_SECRET_ACCESS_KEY/);
+ });
+
+ it('separates an unreachable endpoint from a rejected credential', () => {
+ const v = classifyObjectStorageProbe({
+ configured: true,
+ ok: false,
+ error: new Error('getaddrinfo ENOTFOUND acct.r2.cloudflarestorage.com'),
+ });
+ expect(v.status).toBe('down');
+ expect(v.detail).toMatch(/unreachable/);
+ expect(v.hint).toMatch(/S3_ENDPOINT/);
+ });
+});
diff --git a/tests/thumbnail-blameless-failures.test.js b/tests/thumbnail-blameless-failures.test.js
index a8bdf942e..f16845357 100644
--- a/tests/thumbnail-blameless-failures.test.js
+++ b/tests/thumbnail-blameless-failures.test.js
@@ -107,4 +107,14 @@ describe('objectStorageConfigured', () => {
expect(withEnv(partial, objectStorageConfigured), `missing ${missing}`).toBe(false);
}
});
+
+ // A credential that is nothing but a newline is not configured storage: it
+ // signs every request straight into a SignatureDoesNotMatch while a bare
+ // truthiness check reads healthy.
+ it('is false when any single var is blank', () => {
+ for (const blank of KEYS) {
+ const padded = { ...full, [blank]: ' \n' };
+ expect(withEnv(padded, objectStorageConfigured), `blank ${blank}`).toBe(false);
+ }
+ });
});