Skip to content

Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.

fix(forge): survive a bucket that refuses the reference-view write

nirholas427ef2505889
diff --git a/api/_lib/forge-tiers.js b/api/_lib/forge-tiers.js
index ad82f1957..deec667ff 100644
--- a/api/_lib/forge-tiers.js
+++ b/api/_lib/forge-tiers.js
@@ -670,6 +670,21 @@ export function selfhostQualityForTier(tierId) {
return SELFHOST_TRELLIS_QUALITY[tierId] || SELFHOST_TRELLIS_QUALITY[DEFAULT_TIER];
}
+// Can this backend read a reference view supplied inline as a data URI, instead
+// of a public https URL it has to fetch? True for our own GPU workers and only
+// them: every workers/model-* service declares `images: [data-uri|url, ...]` and
+// base64-decodes the payload directly. A third-party reconstructor (NIM, HF
+// Spaces, Replicate, and the BYOK vendors) takes a URL it fetches itself, so an
+// inline view must never be handed to one.
+//
+// This is what makes the object-storage failover in _lib/image-persist.js safe:
+// when the bucket refuses the reference-view write, the generation can still run
+// on a gcp lane, and the guard in api/forge.js uses this to refuse the lanes
+// where it could not.
+export function backendAcceptsInlineViews(backendId) {
+ return BACKENDS[backendId]?.provider === 'gcp';
+}
+
// A platform backend is "live" only when its required env is present. BYOK
// backends are always selectable — liveness depends on the caller's key, which
// is resolved per-request, not here.
diff --git a/api/_lib/image-persist.js b/api/_lib/image-persist.js
index d84d3d5f4..cccd015d2 100644
--- a/api/_lib/image-persist.js
+++ b/api/_lib/image-persist.js
@@ -10,12 +10,17 @@
// every synthesized image is persisted to R2 and handed on as a durable
// public https URL.
//
+// The one exception is a storage outage: see persistImageBytes below, which
+// falls back to an inline data URI our own workers can read rather than letting
+// a rejected bucket credential fail the whole generation. Use isInlineImageRef()
+// before handing a reference view to a third-party provider.
+//
// The format is sniffed from the magic bytes so the object key extension and
// Content-Type always match the real payload: NIM FLUX returns JPEG artifacts,
// Vertex Imagen and Gemini return PNG, and the Livepeer gateway can serve
// either. Unknown bytes keep the PNG label (the historical default).
-import { putObject, publicUrl } from './r2.js';
+import { putObject, publicUrl, isStorageInfrastructureError } from './r2.js';
// 'jpg' | 'png' - sniffed from magic bytes. JPEG: FF D8 FF. Anything else is
// labeled png, matching the legacy behavior of every prior copy of this code.
@@ -34,11 +39,53 @@ export function looksLikeImageBytes(body) {
return body[0] === 0x89 && body[1] === 0x50 && body[2] === 0x4e && body[3] === 0x47; // PNG
}
-// Persist raw image bytes to R2 and return a durable public https URL.
+// Largest payload we will inline as a data URI when object storage is down.
+// A synthesized reference view is 0.2-1.5 MB; base64 inflates it by 4/3, so 4 MB
+// of source bytes stays a sane JSON request body for our own workers while
+// refusing to turn a pathological payload into a multi-megabyte POST.
+const INLINE_FALLBACK_MAX_BYTES = 4 * 1024 * 1024;
+
+// True for a reference view we synthesized inline rather than parked in the
+// bucket. Callers that hand the view to a THIRD-PARTY reconstructor must refuse
+// it (those APIs take a URL they can fetch); our own workers accept it, because
+// every one of them declares `images: [data-uri|url, ...]` and decodes the
+// base64 directly (workers/model-{trellis,hunyuan3d,triposg}/main.py).
+export function isInlineImageRef(url) {
+ return typeof url === 'string' && url.startsWith('data:image/');
+}
+
+// Persist raw image bytes and return a reference the reconstruction lane can
+// read: a durable public https URL normally, and an inline data URI when the
+// bucket refuses the write.
+//
+// The fallback exists because this one write is a single point of failure for
+// the whole text->3D flow. On 2026-09-07 the R2 credential stopped verifying and
+// every signed operation began answering SignatureDoesNotMatch; this function
+// threw, and because it runs BEFORE the reconstructor is ever called, 100% of
+// text->3D returned 502 while image->3D (which supplies its own view URLs and
+// never touches this path) kept working. Storage held the flagship flow hostage
+// with no failover of any kind.
+//
+// Our own GPU workers do not need the bucket at all: they accept the view inline.
+// So a rejected credential now costs a durable copy of the reference view, not
+// the generation. Only a storage-INFRASTRUCTURE fault (credential rejected or
+// revoked, bucket missing, endpoint unreachable) takes this path; a programming
+// error still throws, and an oversized payload still throws rather than being
+// smuggled into a request body.
export async function persistImageBytes(body) {
const ext = sniffImageFormat(body);
const key = `forge/refs/${globalThis.crypto.randomUUID()}.${ext}`;
- await putObject({ key, body, contentType: ext === 'jpg' ? 'image/jpeg' : 'image/png' });
+ const contentType = ext === 'jpg' ? 'image/jpeg' : 'image/png';
+ try {
+ await putObject({ key, body, contentType });
+ } catch (err) {
+ if (!isStorageInfrastructureError(err)) throw err;
+ if (!body || body.length > INLINE_FALLBACK_MAX_BYTES) throw err;
+ console.warn(
+ `[image-persist] object storage rejected the reference view (${String(err?.name || err?.Code || 'storage error')}); inlining ${body.length} B as a data URI so the generation continues`,
+ );
+ return `data:${contentType};base64,${Buffer.from(body).toString('base64')}`;
+ }
return publicUrl(key);
}
diff --git a/tests/forge-inline-views.test.js b/tests/forge-inline-views.test.js
new file mode 100644
index 000000000..cdf5bf8b1
--- /dev/null
+++ b/tests/forge-inline-views.test.js
@@ -0,0 +1,49 @@
+// backendAcceptsInlineViews — api/_lib/forge-tiers.js.
+//
+// The predicate that makes the object-storage failover safe. When the bucket
+// refuses to park a synthesized reference view, _lib/image-persist.js hands back
+// an inline data URI instead of throwing, and api/forge.js uses this to decide
+// which lanes may receive it. Our own GPU workers declare
+// `images: [data-uri|url, ...]` and base64-decode the payload directly
+// (workers/model-{trellis,hunyuan3d,triposg}/main.py); every third-party
+// reconstructor fetches a URL and would reject or mishandle inline bytes.
+//
+// A wrong answer here is not cosmetic: a false positive sends a multi-megabyte
+// data URI to a vendor API, and a false negative refuses the only lanes still
+// capable of delivering a mesh during a storage outage.
+
+import { describe, it, expect } from 'vitest';
+import { backendAcceptsInlineViews, BACKENDS } from '../api/_lib/forge-tiers.js';
+
+describe('backendAcceptsInlineViews', () => {
+ it('accepts every self-hosted worker lane', () => {
+ for (const id of ['trellis_selfhost', 'hunyuan3d', 'triposg']) {
+ expect(backendAcceptsInlineViews(id), id).toBe(true);
+ }
+ });
+
+ it('refuses every third-party reconstructor', () => {
+ for (const id of ['nvidia', 'huggingface', 'trellis', 'meshy', 'tripo', 'rodin', 'stability', 'replicate_byok']) {
+ expect(backendAcceptsInlineViews(id), id).toBe(false);
+ }
+ });
+
+ it('refuses an unknown or absent backend rather than assuming inline support', () => {
+ expect(backendAcceptsInlineViews('does_not_exist')).toBe(false);
+ expect(backendAcceptsInlineViews(undefined)).toBe(false);
+ expect(backendAcceptsInlineViews(null)).toBe(false);
+ });
+
+ // The predicate is defined as "provider is our own gcp worker fleet". Pin that
+ // to the registry so a backend added later is classified by its provider
+ // rather than by anyone remembering to edit a hand-written list here.
+ it('tracks the registry: exactly the gcp-provider backends accept inline views', () => {
+ const gcp = Object.values(BACKENDS)
+ .filter((b) => b?.provider === 'gcp')
+ .map((b) => b.id);
+ expect(gcp.length).toBeGreaterThan(0);
+ for (const id of Object.keys(BACKENDS)) {
+ expect(backendAcceptsInlineViews(id), id).toBe(gcp.includes(id));
+ }
+ });
+});
diff --git a/tests/image-persist.test.js b/tests/image-persist.test.js
index d39ab6b10..887775fc8 100644
--- a/tests/image-persist.test.js
+++ b/tests/image-persist.test.js
@@ -5,12 +5,26 @@
import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';
-import { looksLikeImageBytes, sniffImageFormat, persistImageBase64, persistImageBytes } from '../api/_lib/image-persist.js';
+import {
+ looksLikeImageBytes,
+ sniffImageFormat,
+ persistImageBase64,
+ persistImageBytes,
+ isInlineImageRef,
+} from '../api/_lib/image-persist.js';
import { putObject, publicUrl } from '../api/_lib/r2.js';
+// The real classifier, not a stub: the fallback is only correct if it fires on
+// the exact error shapes S3 actually throws, and that mapping lives in r2.js.
+const STORAGE_ERROR_RE =
+ /missing required env var: s3_|invalidaccesskeyid|signaturedoesnotmatch|does not match the signature|unauthorized|nosuchbucket|access denied|econnrefused|enotfound|socket hang up|econnreset/i;
+
vi.mock('../api/_lib/r2.js', () => ({
putObject: vi.fn(async () => ({})),
publicUrl: vi.fn((key) => `https://cdn.example.com/${key}`),
+ isStorageInfrastructureError: vi.fn((err) =>
+ STORAGE_ERROR_RE.test([err?.name, err?.Code, err?.message].filter(Boolean).join(' ')),
+ ),
}));
const PNG_SIG = [0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a];
@@ -76,3 +90,67 @@ describe('persistImageBase64', () => {
expect(Buffer.compare(call.body, body)).toBe(0);
});
});
+
+
+// The 2026-09-07 outage: the R2 credential stopped verifying, this write threw,
+// and because it runs before the reconstructor is ever called, 100% of text->3D
+// answered 502 while image->3D kept working. Our own workers accept the view
+// inline, so storage must no longer be able to hold the whole flow hostage.
+describe('persistImageBytes when object storage is down', () => {
+ const PNG = Buffer.from([...PNG_SIG, 1, 2, 3, 4, 5]);
+
+ it('inlines the view as a data URI when the credential is rejected', async () => {
+ putObject.mockRejectedValueOnce(
+ Object.assign(new Error('The request signature we calculated does not match the signature you provided.'), {
+ name: 'SignatureDoesNotMatch',
+ }),
+ );
+ const url = await persistImageBytes(PNG);
+ expect(url).toBe(`data:image/png;base64,${PNG.toString('base64')}`);
+ expect(isInlineImageRef(url)).toBe(true);
+ });
+
+ it('inlines a revoked key and an unreachable endpoint too', async () => {
+ for (const err of [
+ Object.assign(new Error('Unauthorized'), { name: 'Unauthorized' }),
+ Object.assign(new Error('getaddrinfo ENOTFOUND bucket.example.com'), { name: 'Error' }),
+ ]) {
+ putObject.mockRejectedValueOnce(err);
+ expect(isInlineImageRef(await persistImageBytes(PNG))).toBe(true);
+ }
+ });
+
+ it('keeps the JPEG content type in the inlined URI', async () => {
+ const jpeg = Buffer.from([...JPEG_SIG, 7, 7, 7]);
+ putObject.mockRejectedValueOnce(Object.assign(new Error('nope'), { name: 'SignatureDoesNotMatch' }));
+ expect(await persistImageBytes(jpeg)).toBe(`data:image/jpeg;base64,${jpeg.toString('base64')}`);
+ });
+
+ it('rethrows a programming error instead of hiding it behind a data URI', async () => {
+ putObject.mockRejectedValueOnce(new TypeError('body.pipe is not a function'));
+ await expect(persistImageBytes(PNG)).rejects.toThrow('body.pipe is not a function');
+ });
+
+ it('refuses to inline a payload too large to be a request body', async () => {
+ const huge = Buffer.alloc(5 * 1024 * 1024, 0x41);
+ PNG_SIG.forEach((b, i) => (huge[i] = b));
+ putObject.mockRejectedValueOnce(Object.assign(new Error('nope'), { name: 'SignatureDoesNotMatch' }));
+ await expect(persistImageBytes(huge)).rejects.toThrow(/nope/);
+ });
+
+ it('still returns the durable URL when the bucket is healthy', async () => {
+ const url = await persistImageBytes(PNG);
+ expect(url).toMatch(/^https:\/\/cdn\.example\.com\/forge\/refs\//);
+ expect(isInlineImageRef(url)).toBe(false);
+ });
+});
+
+describe('isInlineImageRef', () => {
+ it('recognises only an inline image payload', () => {
+ expect(isInlineImageRef('data:image/png;base64,AAAA')).toBe(true);
+ expect(isInlineImageRef('https://cdn.example.com/a.png')).toBe(false);
+ expect(isInlineImageRef('data:text/plain;base64,AAAA')).toBe(false);
+ expect(isInlineImageRef(null)).toBe(false);
+ expect(isInlineImageRef(undefined)).toBe(false);
+ });
+});