Skip to content

Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.

fix(ops): retry the R2 health probe once before classifying it down

nirholasba53f5086c3f
diff --git a/api/_lib/ops/object-storage-health.js b/api/_lib/ops/object-storage-health.js
index b4e42496e..fcf30b448 100644
--- a/api/_lib/ops/object-storage-health.js
+++ b/api/_lib/ops/object-storage-health.js
@@ -33,6 +33,12 @@ const PROBE_TIMEOUT_MS = 3_000;
// second is already slow enough that every gallery page feels it, so it is worth
// surfacing before it becomes an outage.
const SLOW_MS = 1_000;
+// A single retry, inside one invocation, with a short backoff. This absorbs a
+// one-off transient network blip so it never pages as DOWN, but it cannot and
+// must not mask a sustained outage: a bucket that is still down 200-500ms later
+// stays classified down, exactly as a bucket that has been down for days does.
+// See prompts/finish/_context/production-100-OWNER-ACTIONS.md row 36.
+const RETRY_BACKOFF_MS = 300;
/**
* Classify a probe outcome. Pure, so the thresholds and the credential-vs-latency
@@ -75,32 +81,66 @@ export function classifyObjectStorageProbe(probe) {
}
/**
- * Run the signed probe and classify it. Never throws.
+ * One signed `ListObjectsV2Command` probe. Split out (rather than inlined in
+ * gatherObjectStorageHealth) so tests can inject a synthetic probe without
+ * mocking the AWS SDK or the thing under test's public contract.
+ * @returns {Promise<void>}
+ */
+async function probeOnce() {
+ await r2.send(new ListObjectsV2Command({ Bucket: env.S3_BUCKET, MaxKeys: 1 }), {
+ abortSignal: AbortSignal.timeout(PROBE_TIMEOUT_MS),
+ });
+}
+
+/**
+ * @param {() => Promise<void>} probe
+ * @returns {Promise<{ ok: true, latencyMs: number } | { ok: false, latencyMs: number, error: unknown }>}
+ */
+async function timeProbe(probe) {
+ const started = Date.now();
+ try {
+ await probe();
+ return { ok: true, latencyMs: Date.now() - started };
+ } catch (error) {
+ return { ok: false, latencyMs: Date.now() - started, error };
+ }
+}
+
+/**
+ * Run the signed probe, retrying once after a short backoff on failure, and
+ * classify the outcome. Never throws.
+ * @param {{ probe?: () => Promise<void> }} [opts] test-only probe override
* @returns {Promise<{ name: 'object_storage', label: string, status: string, detail: string, hint?: string, metrics?: object }>}
*/
-export async function gatherObjectStorageHealth() {
+export async function gatherObjectStorageHealth({ probe = probeOnce } = {}) {
const base = { name: /** @type {const} */ ('object_storage'), label: 'Object storage (Cloudflare R2)' };
if (!objectStorageConfigured()) {
const v = classifyObjectStorageProbe({ configured: false });
return { ...base, status: v.status, detail: v.detail };
}
- const started = Date.now();
- try {
- await r2.send(new ListObjectsV2Command({ Bucket: env.S3_BUCKET, MaxKeys: 1 }), {
- abortSignal: AbortSignal.timeout(PROBE_TIMEOUT_MS),
- });
- const latencyMs = Date.now() - started;
- const v = classifyObjectStorageProbe({ configured: true, ok: true, latencyMs });
- return { ...base, status: v.status, detail: v.detail, ...(v.hint ? { hint: v.hint } : {}), metrics: { latencyMs } };
- } catch (err) {
- const latencyMs = Date.now() - started;
- const v = classifyObjectStorageProbe({ configured: true, ok: false, latencyMs, error: err });
+ let result = await timeProbe(probe);
+ let retried = false;
+ if (!result.ok) {
+ await new Promise((resolve) => setTimeout(resolve, RETRY_BACKOFF_MS));
+ result = await timeProbe(probe);
+ retried = true;
+ }
+ if (result.ok) {
+ const v = classifyObjectStorageProbe({ configured: true, ok: true, latencyMs: result.latencyMs });
return {
...base,
status: v.status,
detail: v.detail,
...(v.hint ? { hint: v.hint } : {}),
- metrics: { latencyMs, credentialFault: isStorageInfrastructureError(err) },
+ metrics: { latencyMs: result.latencyMs, ...(retried ? { retried: true } : {}) },
};
}
+ const v = classifyObjectStorageProbe({ configured: true, ok: false, latencyMs: result.latencyMs, error: result.error });
+ return {
+ ...base,
+ status: v.status,
+ detail: v.detail,
+ ...(v.hint ? { hint: v.hint } : {}),
+ metrics: { latencyMs: result.latencyMs, retried, credentialFault: isStorageInfrastructureError(result.error) },
+ };
}
diff --git a/tests/object-storage-health.test.js b/tests/object-storage-health.test.js
index 510b2a1b9..409b4c22f 100644
--- a/tests/object-storage-health.test.js
+++ b/tests/object-storage-health.test.js
@@ -11,8 +11,27 @@
* re-set it) versus an unreachable endpoint (it may clear itself).
*/
-import { describe, it, expect } from 'vitest';
-import { classifyObjectStorageProbe } from '../api/_lib/ops/object-storage-health.js';
+import { describe, it, expect, beforeEach, afterEach } from 'vitest';
+import { classifyObjectStorageProbe, gatherObjectStorageHealth } from '../api/_lib/ops/object-storage-health.js';
+
+const STORAGE_KEYS = ['S3_ENDPOINT', 'S3_BUCKET', 'S3_PUBLIC_DOMAIN', 'S3_ACCESS_KEY_ID', 'S3_SECRET_ACCESS_KEY'];
+let savedEnv;
+
+beforeEach(() => {
+ savedEnv = Object.fromEntries(STORAGE_KEYS.map((k) => [k, process.env[k]]));
+ process.env.S3_ENDPOINT = 'https://acct.r2.cloudflarestorage.com';
+ process.env.S3_BUCKET = 'three-ws';
+ process.env.S3_PUBLIC_DOMAIN = 'https://pub-example.r2.dev';
+ process.env.S3_ACCESS_KEY_ID = 'AKIAEXAMPLE';
+ process.env.S3_SECRET_ACCESS_KEY = 'secret';
+});
+
+afterEach(() => {
+ for (const k of STORAGE_KEYS) {
+ if (savedEnv[k] === undefined) delete process.env[k];
+ else process.env[k] = savedEnv[k];
+ }
+});
describe('classifyObjectStorageProbe', () => {
it('stays neutral when storage is not configured on the deployment', () => {
@@ -53,3 +72,48 @@ describe('classifyObjectStorageProbe', () => {
expect(v.hint).toMatch(/S3_ENDPOINT/);
});
});
+
+describe('gatherObjectStorageHealth retry', () => {
+ /**
+ * On 2026-10-11/12 row 36 traced the DEGRADED/DOWN flap to a single-probe
+ * sensor with no retry: a genuinely transient one-off network blip is
+ * indistinguishable from a sustained outage. This proves one retry absorbs
+ * the blip without masking a real, repeated failure.
+ */
+ it('does not page DOWN on a single transient failure followed by a success', async () => {
+ let calls = 0;
+ const probe = async () => {
+ calls += 1;
+ if (calls === 1) throw new Error('getaddrinfo ENOTFOUND acct.r2.cloudflarestorage.com');
+ };
+ const result = await gatherObjectStorageHealth({ probe });
+ expect(calls).toBe(2);
+ expect(result.status).not.toBe('down');
+ expect(result.status).toBe('ok');
+ expect(result.metrics.retried).toBe(true);
+ });
+
+ it('still classifies down on two consecutive failures', async () => {
+ let calls = 0;
+ const probe = async () => {
+ calls += 1;
+ throw new Error('getaddrinfo ENOTFOUND acct.r2.cloudflarestorage.com');
+ };
+ const result = await gatherObjectStorageHealth({ probe });
+ expect(calls).toBe(2);
+ expect(result.status).toBe('down');
+ expect(result.detail).toMatch(/unreachable/);
+ expect(result.metrics.retried).toBe(true);
+ });
+
+ it('does not retry on a first-try success', async () => {
+ let calls = 0;
+ const probe = async () => {
+ calls += 1;
+ };
+ const result = await gatherObjectStorageHealth({ probe });
+ expect(calls).toBe(1);
+ expect(result.status).toBe('ok');
+ expect(result.metrics.retried).toBeUndefined();
+ });
+});