Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.
JavaScript 3d-generationai-agentsanimationavatardigital-humangenerative-aiglbgltfimage-to-3dllmmcpmcp-servermodel-context-protocolsolanatext-to-3dthreejsweb-componentswebglwebxrx402
fix(ops): retry the R2 health probe once before classifying it down
nirholasba53f5086c3f
diff --git a/api/_lib/ops/object-storage-health.js b/api/_lib/ops/object-storage-health.jsindex b4e42496e..fcf30b448 100644--- a/api/_lib/ops/object-storage-health.js+++ b/api/_lib/ops/object-storage-health.js@@ -33,6 +33,12 @@ const PROBE_TIMEOUT_MS = 3_000;// second is already slow enough that every gallery page feels it, so it is worth// surfacing before it becomes an outage.const SLOW_MS = 1_000;+// A single retry, inside one invocation, with a short backoff. This absorbs a+// one-off transient network blip so it never pages as DOWN, but it cannot and+// must not mask a sustained outage: a bucket that is still down 200-500ms later+// stays classified down, exactly as a bucket that has been down for days does.+// See prompts/finish/_context/production-100-OWNER-ACTIONS.md row 36.+const RETRY_BACKOFF_MS = 300;/*** Classify a probe outcome. Pure, so the thresholds and the credential-vs-latency@@ -75,32 +81,66 @@ export function classifyObjectStorageProbe(probe) {}/**- * Run the signed probe and classify it. Never throws.+ * One signed `ListObjectsV2Command` probe. Split out (rather than inlined in+ * gatherObjectStorageHealth) so tests can inject a synthetic probe without+ * mocking the AWS SDK or the thing under test's public contract.+ * @returns {Promise<void>}+ */+async function probeOnce() {+ await r2.send(new ListObjectsV2Command({ Bucket: env.S3_BUCKET, MaxKeys: 1 }), {+ abortSignal: AbortSignal.timeout(PROBE_TIMEOUT_MS),+ });+}++/**+ * @param {() => Promise<void>} probe+ * @returns {Promise<{ ok: true, latencyMs: number } | { ok: false, latencyMs: number, error: unknown }>}+ */+async function timeProbe(probe) {+ const started = Date.now();+ try {+ await probe();+ return { ok: true, latencyMs: Date.now() - started };+ } catch (error) {+ return { ok: false, latencyMs: Date.now() - started, error };+ }+}++/**+ * Run the signed probe, retrying once after a short backoff on failure, and+ * classify the outcome. Never throws.+ * @param {{ probe?: () => Promise<void> }} [opts] test-only probe override* @returns {Promise<{ name: 'object_storage', label: string, status: string, detail: string, hint?: string, metrics?: object }>}*/-export async function gatherObjectStorageHealth() {+export async function gatherObjectStorageHealth({ probe = probeOnce } = {}) {const base = { name: /** @type {const} */ ('object_storage'), label: 'Object storage (Cloudflare R2)' };if (!objectStorageConfigured()) {const v = classifyObjectStorageProbe({ configured: false });return { ...base, status: v.status, detail: v.detail };}- const started = Date.now();- try {- await r2.send(new ListObjectsV2Command({ Bucket: env.S3_BUCKET, MaxKeys: 1 }), {- abortSignal: AbortSignal.timeout(PROBE_TIMEOUT_MS),- });- const latencyMs = Date.now() - started;- const v = classifyObjectStorageProbe({ configured: true, ok: true, latencyMs });- return { ...base, status: v.status, detail: v.detail, ...(v.hint ? { hint: v.hint } : {}), metrics: { latencyMs } };- } catch (err) {- const latencyMs = Date.now() - started;- const v = classifyObjectStorageProbe({ configured: true, ok: false, latencyMs, error: err });+ let result = await timeProbe(probe);+ let retried = false;+ if (!result.ok) {+ await new Promise((resolve) => setTimeout(resolve, RETRY_BACKOFF_MS));+ result = await timeProbe(probe);+ retried = true;+ }+ if (result.ok) {+ const v = classifyObjectStorageProbe({ configured: true, ok: true, latencyMs: result.latencyMs });return {...base,status: v.status,detail: v.detail,...(v.hint ? { hint: v.hint } : {}),- metrics: { latencyMs, credentialFault: isStorageInfrastructureError(err) },+ metrics: { latencyMs: result.latencyMs, ...(retried ? { retried: true } : {}) },};}+ const v = classifyObjectStorageProbe({ configured: true, ok: false, latencyMs: result.latencyMs, error: result.error });+ return {+ ...base,+ status: v.status,+ detail: v.detail,+ ...(v.hint ? { hint: v.hint } : {}),+ metrics: { latencyMs: result.latencyMs, retried, credentialFault: isStorageInfrastructureError(result.error) },+ };}diff --git a/tests/object-storage-health.test.js b/tests/object-storage-health.test.jsindex 510b2a1b9..409b4c22f 100644--- a/tests/object-storage-health.test.js+++ b/tests/object-storage-health.test.js@@ -11,8 +11,27 @@* re-set it) versus an unreachable endpoint (it may clear itself).*/-import { describe, it, expect } from 'vitest';-import { classifyObjectStorageProbe } from '../api/_lib/ops/object-storage-health.js';+import { describe, it, expect, beforeEach, afterEach } from 'vitest';+import { classifyObjectStorageProbe, gatherObjectStorageHealth } from '../api/_lib/ops/object-storage-health.js';++const STORAGE_KEYS = ['S3_ENDPOINT', 'S3_BUCKET', 'S3_PUBLIC_DOMAIN', 'S3_ACCESS_KEY_ID', 'S3_SECRET_ACCESS_KEY'];+let savedEnv;++beforeEach(() => {+ savedEnv = Object.fromEntries(STORAGE_KEYS.map((k) => [k, process.env[k]]));+ process.env.S3_ENDPOINT = 'https://acct.r2.cloudflarestorage.com';+ process.env.S3_BUCKET = 'three-ws';+ process.env.S3_PUBLIC_DOMAIN = 'https://pub-example.r2.dev';+ process.env.S3_ACCESS_KEY_ID = 'AKIAEXAMPLE';+ process.env.S3_SECRET_ACCESS_KEY = 'secret';+});++afterEach(() => {+ for (const k of STORAGE_KEYS) {+ if (savedEnv[k] === undefined) delete process.env[k];+ else process.env[k] = savedEnv[k];+ }+});describe('classifyObjectStorageProbe', () => {it('stays neutral when storage is not configured on the deployment', () => {@@ -53,3 +72,48 @@ describe('classifyObjectStorageProbe', () => {expect(v.hint).toMatch(/S3_ENDPOINT/);});});++describe('gatherObjectStorageHealth retry', () => {+ /**+ * On 2026-10-11/12 row 36 traced the DEGRADED/DOWN flap to a single-probe+ * sensor with no retry: a genuinely transient one-off network blip is+ * indistinguishable from a sustained outage. This proves one retry absorbs+ * the blip without masking a real, repeated failure.+ */+ it('does not page DOWN on a single transient failure followed by a success', async () => {+ let calls = 0;+ const probe = async () => {+ calls += 1;+ if (calls === 1) throw new Error('getaddrinfo ENOTFOUND acct.r2.cloudflarestorage.com');+ };+ const result = await gatherObjectStorageHealth({ probe });+ expect(calls).toBe(2);+ expect(result.status).not.toBe('down');+ expect(result.status).toBe('ok');+ expect(result.metrics.retried).toBe(true);+ });++ it('still classifies down on two consecutive failures', async () => {+ let calls = 0;+ const probe = async () => {+ calls += 1;+ throw new Error('getaddrinfo ENOTFOUND acct.r2.cloudflarestorage.com');+ };+ const result = await gatherObjectStorageHealth({ probe });+ expect(calls).toBe(2);+ expect(result.status).toBe('down');+ expect(result.detail).toMatch(/unreachable/);+ expect(result.metrics.retried).toBe(true);+ });++ it('does not retry on a first-try success', async () => {+ let calls = 0;+ const probe = async () => {+ calls += 1;+ };+ const result = await gatherObjectStorageHealth({ probe });+ expect(calls).toBe(1);+ expect(result.status).toBe('ok');+ expect(result.metrics.retried).toBeUndefined();+ });+});