Skip to content

Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.

fix(cdn): stream the R2 failover instead of redirecting, so /cdn keeps its CORS grant

nirholas64833a35894d
diff --git a/api/cdn-object.js b/api/cdn-object.js
index dcb2f459d..cd1c6ec75 100644
--- a/api/cdn-object.js
+++ b/api/cdn-object.js
@@ -17,9 +17,12 @@
// safe to render inline is sent as an attachment, and `sandbox` puts whatever
// does render into an opaque origin. See `isInlineSafe` below.
+import { Readable } from 'node:stream';
+import { pipeline } from 'node:stream/promises';
import { GetObjectCommand } from '@aws-sdk/client-s3';
import { cors, error, wrap } from './_lib/http.js';
import { r2, isStorageInfrastructureError, publicUrlOrNull } from './_lib/r2.js';
+import { fetchUpstream } from './_lib/upstream-fetch.js';
import { env } from './_lib/env.js';
// Object keys are caller-controlled path input — keep them boring. UUID-based
@@ -165,21 +168,87 @@ export default wrap(async (req, res) => {
// The signed read itself is broken: the credentials are rejected, or the
// endpoint is unreachable, which says nothing about the object. The same
// bytes are readable, unauthenticated, on the bucket's public domain, so
- // hand the caller there instead of 502ing every avatar, thumbnail and GLB
+ // serve them from there instead of 502ing every avatar, thumbnail and GLB
// on the site. On 2026-09-07 a rejected R2 secret took the signed read
// down and this route answered `upstream_error` for every object it
// serves; the public domain was serving those same keys with a 200
- // throughout. Redirect, never cache: 302 + no-store, so the moment the
- // credential is healthy again traffic returns to the signed path (which
- // exists to dodge the public domain's rate limit) without a stale hop
- // pinned in anyone's cache.
+ // throughout.
+ //
+ // STREAM the bytes, never redirect to them. This rung shipped as a 302 and
+ // that quietly broke a second contract while fixing the first: r2.dev sends
+ // no access-control-allow-origin at all, and a browser re-runs the CORS
+ // check on a redirect's target, so the wildcard grant set at the top of
+ // this handler is discarded the moment the hop is followed. Measured on
+ // 2026-09-09 from a foreign origin, every /cdn read threw "Failed to fetch"
+ // while the same URL curled 200, which is how the model-viewer stages on
+ // /spatial-mcp went dark. Streaming keeps the response on three.ws, so the
+ // CORS and CORP headers, the server-chosen content type and the attachment
+ // rule below all survive the degraded path unchanged. api/avatars/[id]/
+ // [action].js takes the same route for the same reason.
+ //
+ // It also stops the failover defeating its own purpose: a redirect points
+ // every browser straight at the rate-limited public domain this route was
+ // built to dodge, whereas a proxied read is absorbed by our own edge cache.
+ // That cache stays short (60s) rather than no-store, so a burst of gallery
+ // thumbnails costs r2.dev one fetch instead of one per viewer, and traffic
+ // still returns to the signed path within a minute of the credential being
+ // healthy again.
const fallback = isStorageInfrastructureError(err) ? publicUrlOrNull(key) : null;
if (fallback) {
- console.error('[cdn-object] signed read failed, serving public bucket domain:', key, err?.message);
- res.statusCode = 302;
- res.setHeader('location', fallback);
- res.setHeader('cache-control', 'no-store');
- return res.end();
+ console.error('[cdn-object] signed read failed, streaming public bucket domain:', key, err?.message);
+ try {
+ const upstream = await fetchUpstream(
+ fallback,
+ { headers: range ? { range } : {} },
+ {
+ name: 'r2:public-object',
+ timeoutMs: 20_000,
+ attempts: 2,
+ // A 404 on the public domain is the object genuinely being
+ // absent, not an outage: the signed error never got far
+ // enough to say so. Let it through and answer 404, so a
+ // caller that treats a missing thumbnail as an empty state
+ // is not handed a 502 it will retry forever.
+ okWhen: (r) => r.ok || r.status === 404,
+ },
+ );
+ if (upstream.status === 404) {
+ upstream.body?.cancel?.();
+ return error(res, 404, 'not_found', 'object not found');
+ }
+ const type = contentTypeFor(key, upstream.headers.get('content-type'));
+ res.statusCode = upstream.status === 206 ? 206 : 200;
+ res.setHeader('content-type', type);
+ res.setHeader('content-security-policy', "default-src 'none'; sandbox");
+ res.setHeader('cross-origin-resource-policy', 'cross-origin');
+ if (!isInlineSafe(type)) res.setHeader('content-disposition', 'attachment');
+ res.setHeader('accept-ranges', 'bytes');
+ res.setHeader('cache-control', 'public, max-age=60, s-maxage=60');
+ const len = upstream.headers.get('content-length');
+ if (len) res.setHeader('content-length', len);
+ const contentRange = upstream.headers.get('content-range');
+ if (contentRange) res.setHeader('content-range', contentRange);
+ const etag = upstream.headers.get('etag');
+ if (etag) res.setHeader('etag', etag);
+ if (req.method === 'HEAD') {
+ upstream.body?.cancel?.();
+ return res.end();
+ }
+ await pipeline(Readable.fromWeb(upstream.body), res);
+ return;
+ } catch (fallbackErr) {
+ console.error('[cdn-object] public bucket domain failed too:', key, fallbackErr?.message);
+ if (res.headersSent) return res.destroy(fallbackErr);
+ // Nothing is on the wire yet, so the 502 below still gets to
+ // answer, but only once the object's own length and range come
+ // back off: error() refuses to touch a committed response and
+ // never clears headers, so a leftover content-length would pin the
+ // JSON body to the object's byte count and hang the client waiting
+ // for megabytes that are never coming.
+ res.removeHeader('content-length');
+ res.removeHeader('content-range');
+ res.removeHeader('etag');
+ }
}
console.error('[cdn-object] r2 fetch failed:', key, err?.message);
return error(res, 502, 'upstream_error', 'failed to fetch object');
diff --git a/tests/cdn-object-storage-failover.test.js b/tests/cdn-object-storage-failover.test.js
index cfb288283..495a62298 100644
--- a/tests/cdn-object-storage-failover.test.js
+++ b/tests/cdn-object-storage-failover.test.js
@@ -6,14 +6,18 @@
* unauthenticated, the whole time. On 2026-09-07 the R2 credential stopped
* verifying and this route answered `upstream_error` for every avatar, thumbnail
* and GLB on the site while `pub-….r2.dev` was serving those same keys with a
- * 200. A credential fault now redirects there instead of 502ing the page.
+ * 200. A credential fault now serves those bytes instead of 502ing the page.
*
- * The redirect is deliberately uncacheable: the moment the credential is healthy
- * again, traffic has to return to the signed path rather than sit pinned on the
- * rate-limited domain by a cached hop.
+ * It STREAMS them; it must never redirect to them. `pub-….r2.dev` sends no
+ * access-control-allow-origin, and a browser re-runs the CORS check on a
+ * redirect's target, so a 302 discards the wildcard grant this route sets and
+ * turns a 502 into a cross-origin "Failed to fetch" for every embed, every
+ * GLTFLoader and every model-viewer off three.ws. Measured from a foreign origin
+ * on 2026-09-09, which is what took the stages on /spatial-mcp dark.
*/
import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';
+import { Writable } from 'node:stream';
const send = vi.fn();
vi.mock('../api/_lib/r2.js', async (importOriginal) => {
@@ -21,31 +25,45 @@ vi.mock('../api/_lib/r2.js', async (importOriginal) => {
return { ...actual, r2: { send: (...args) => send(...args) } };
});
+const fetchUpstream = vi.fn();
+vi.mock('../api/_lib/upstream-fetch.js', async (importOriginal) => {
+ const actual = await importOriginal();
+ return { ...actual, fetchUpstream: (...args) => fetchUpstream(...args) };
+});
+
const { default: handler } = await import('../api/cdn-object.js');
const KEYS = ['S3_ENDPOINT', 'S3_BUCKET', 'S3_PUBLIC_DOMAIN', 'S3_ACCESS_KEY_ID', 'S3_SECRET_ACCESS_KEY'];
let saved;
+/** A response sink that is a real Writable, so `pipeline()` drives it for real. */
function makeRes() {
- return {
- statusCode: 200,
- headers: {},
- body: '',
- setHeader(k, v) {
- this.headers[k.toLowerCase()] = String(v);
- },
- getHeader(k) {
- return this.headers[k.toLowerCase()];
- },
- end(chunk) {
- if (chunk) this.body += chunk;
- this.ended = true;
- return this;
+ const chunks = [];
+ const res = new Writable({
+ write(chunk, _enc, cb) {
+ chunks.push(Buffer.from(chunk));
+ cb();
},
+ });
+ res.statusCode = 200;
+ res.headers = {};
+ res.setHeader = (k, v) => {
+ res.headers[k.toLowerCase()] = String(v);
+ };
+ res.getHeader = (k) => res.headers[k.toLowerCase()];
+ res.removeHeader = (k) => {
+ delete res.headers[k.toLowerCase()];
};
+ res.written = () => Buffer.concat(chunks);
+ const end = res.end.bind(res);
+ res.end = (chunk) => {
+ res.ended = true;
+ return end(chunk);
+ };
+ return res;
}
-const req = (key) => ({ method: 'GET', url: `/api/cdn-object?key=${key}`, headers: {}, query: { key } });
+const req = (key, headers = {}, method = 'GET') => ({ method, url: `/api/cdn-object?key=${key}`, headers, query: { key } });
function s3Error(name, message) {
const err = new Error(message);
@@ -55,6 +73,22 @@ function s3Error(name, message) {
return err;
}
+/** A `fetch` Response as the handler consumes it: web stream body plus headers. */
+function upstreamResponse(bytes, { status = 200, headers = {} } = {}) {
+ const body = new ReadableStream({
+ start(controller) {
+ controller.enqueue(new Uint8Array(bytes));
+ controller.close();
+ },
+ });
+ return { status, ok: status >= 200 && status < 300, body, headers: new Headers(headers) };
+}
+
+const CREDENTIAL_FAULT = s3Error(
+ 'SignatureDoesNotMatch',
+ 'The request signature we calculated does not match the signature you provided. Check your secret access key and signing method.',
+);
+
beforeEach(() => {
saved = Object.fromEntries(KEYS.map((k) => [k, process.env[k]]));
process.env.S3_ENDPOINT = 'https://acct.r2.cloudflarestorage.com';
@@ -63,6 +97,7 @@ beforeEach(() => {
process.env.S3_ACCESS_KEY_ID = 'AKIAEXAMPLE';
process.env.S3_SECRET_ACCESS_KEY = 'secret';
send.mockReset();
+ fetchUpstream.mockReset();
});
afterEach(() => {
@@ -73,25 +108,69 @@ afterEach(() => {
});
describe('/cdn on a storage credential fault', () => {
- it('redirects to the public bucket domain instead of 502ing the object', async () => {
- send.mockRejectedValue(
- s3Error(
- 'SignatureDoesNotMatch',
- 'The request signature we calculated does not match the signature you provided. Check your secret access key and signing method.',
- ),
- );
+ it('streams the public bucket domain instead of 502ing the object', async () => {
+ send.mockRejectedValue(CREDENTIAL_FAULT);
+ fetchUpstream.mockResolvedValue(upstreamResponse([1, 2, 3, 4], { headers: { 'content-length': '4' } }));
const res = makeRes();
await handler(req('thumb/abc.png'), res);
- expect(res.statusCode).toBe(302);
- expect(res.getHeader('location')).toBe('https://pub-example.r2.dev/thumb/abc.png');
+ expect(fetchUpstream.mock.calls[0][0]).toBe('https://pub-example.r2.dev/thumb/abc.png');
+ expect(res.statusCode).toBe(200);
+ expect(res.written()).toEqual(Buffer.from([1, 2, 3, 4]));
});
- it('never lets the fallback hop be cached', async () => {
+ it('never redirects, because the public domain answers with no CORS header', async () => {
send.mockRejectedValue(s3Error('InvalidAccessKeyId', 'The AWS Access Key Id you provided does not exist in our records.'));
+ fetchUpstream.mockResolvedValue(upstreamResponse([9]));
+ const res = makeRes();
+ await handler(req('u/1/model.glb'), res);
+ expect(res.statusCode).toBe(200);
+ expect(res.getHeader('location')).toBeUndefined();
+ });
+
+ it('keeps serving the server-chosen content type on the degraded path', async () => {
+ send.mockRejectedValue(CREDENTIAL_FAULT);
+ fetchUpstream.mockResolvedValue(upstreamResponse([0], { headers: { 'content-type': 'text/html' } }));
const res = makeRes();
await handler(req('u/1/model.glb'), res);
- expect(res.statusCode).toBe(302);
- expect(res.getHeader('cache-control')).toBe('no-store');
+ expect(res.getHeader('content-type')).toBe('model/gltf-binary');
+ expect(res.getHeader('cross-origin-resource-policy')).toBe('cross-origin');
+ });
+
+ it('forwards a range request and answers 206 with the upstream range', async () => {
+ send.mockRejectedValue(CREDENTIAL_FAULT);
+ fetchUpstream.mockResolvedValue(
+ upstreamResponse([7, 8], { status: 206, headers: { 'content-range': 'bytes 0-1/100', 'content-length': '2' } }),
+ );
+ const res = makeRes();
+ await handler(req('u/1/model.glb', { range: 'bytes=0-1' }), res);
+ expect(fetchUpstream.mock.calls[0][1].headers.range).toBe('bytes=0-1');
+ expect(res.statusCode).toBe(206);
+ expect(res.getHeader('content-range')).toBe('bytes 0-1/100');
+ });
+
+ it('caches the degraded hop briefly so a burst costs the rate-limited domain one read', async () => {
+ send.mockRejectedValue(CREDENTIAL_FAULT);
+ fetchUpstream.mockResolvedValue(upstreamResponse([1]));
+ const res = makeRes();
+ await handler(req('thumb/abc.png'), res);
+ expect(res.getHeader('cache-control')).toBe('public, max-age=60, s-maxage=60');
+ });
+
+ it('answers 404 when the public domain says the object is genuinely absent', async () => {
+ send.mockRejectedValue(CREDENTIAL_FAULT);
+ fetchUpstream.mockResolvedValue(upstreamResponse([], { status: 404 }));
+ const res = makeRes();
+ await handler(req('thumb/gone.png'), res);
+ expect(res.statusCode).toBe(404);
+ });
+
+ it('502s once the public domain fails too, with no length left pinned to the object', async () => {
+ send.mockRejectedValue(CREDENTIAL_FAULT);
+ fetchUpstream.mockRejectedValue(new Error('r2.dev unreachable'));
+ const res = makeRes();
+ await handler(req('thumb/abc.png'), res);
+ expect(res.statusCode).toBe(502);
+ expect(res.getHeader('content-length')).toBeUndefined();
});
it('still 404s a missing object rather than bouncing it to the public domain', async () => {
@@ -99,6 +178,7 @@ describe('/cdn on a storage credential fault', () => {
const res = makeRes();
await handler(req('thumb/gone.png'), res);
expect(res.statusCode).toBe(404);
+ expect(fetchUpstream).not.toHaveBeenCalled();
});
it('keeps 502ing a fault that says nothing about our credentials', async () => {
@@ -108,5 +188,6 @@ describe('/cdn on a storage credential fault', () => {
const res = makeRes();
await handler(req('thumb/abc.png'), res);
expect(res.statusCode).toBe(502);
+ expect(fetchUpstream).not.toHaveBeenCalled();
});
});