Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.
JavaScript 3d-generationai-agentsanimationavatardigital-humangenerative-aiglbgltfimage-to-3dllmmcpmcp-servermodel-context-protocolsolanatext-to-3dthreejsweb-componentswebglwebxrx402
fix(api): answer object-storage (R2) faults with a clean 503 and Retry-After instead of a raw 500 plus an unhandled-5xx alert
nirholasa90873157e9a
diff --git a/api/_lib/http.js b/api/_lib/http.jsindex 54e97cac4..187e721ba 100644--- a/api/_lib/http.js+++ b/api/_lib/http.js@@ -7,6 +7,7 @@ import { captureException } from './sentry.js';import { sendOpsAlert } from './alerts.js';import { instrument as zauthInstrument, drain as zauthDrain } from './zauth.js';import { isDbUnavailableError, isDbCapacityError, isStoragePressured } from './db.js';+import { isStorageInfrastructureError } from './r2.js';import { redactUrlSecrets } from './scrub-secrets.js';import { provenanceHeaders, withProvenance } from './brownout/provenance.js';import { CHAOS_HEADER, CHAOS_STATUS_HEADER, chaosDecision, withChaos } from './brownout/chaos.js';@@ -222,6 +223,14 @@ export function reportServerError(err, { code = 'internal_error', status = 500,} catch { /* alerts best-effort */ }return ref;}+ // Object storage (R2/S3) outage: same class of fault, own deduped alert.+ if (isStorageInfrastructureError(err)) {+ logDbUnavailableOnce(code, `object storage unavailable: ${detail}`);+ try {+ sendOpsAlert('object storage unavailable', `${detail}\nref ${ref}`, { signature: 'storage:unavailable' });+ } catch { /* alerts best-effort */ }+ return ref;+ }console.error(`[server-error ${ref}] ${code} (${status}): ${detail}`);try {// Capture the original Error (preserving its stack + class) only when its@@ -267,6 +276,16 @@ export function serverError(res, status, code, err, extra = {}) {...extra,}, { 'cache-control': 'no-store' });}+ if (isStorageInfrastructureError(err)) {+ const ref = reportServerError(err, { code, status: 503 });+ if (typeof res.setHeader === 'function') res.setHeader('retry-after', '30');+ return json(res, 503, {+ error: 'service_unavailable',+ error_description: 'object storage temporarily unavailable, retry shortly',+ ref,+ ...extra,+ }, { 'cache-control': 'no-store' });+ }const ref = reportServerError(err, { code, status });return json(res, status, {error: code,@@ -631,7 +650,12 @@ export function wrap(handler) {// (the July 2026 siwe/siws nonce incident: JWT_SECRET unset → hundreds of// anonymous 500s that never said which var was missing).const missingEnv = !dbDegraded && /^Missing required env var: /.test(err?.message || '');- const status = dbDegraded || missingEnv ? 503 : (err.status || 500);+ // An object-storage (R2/S3) fault is infrastructure too. Deliberate+ // author-written errors (`expose`) and 4xx throws keep their own contract+ // even when their text happens to match the storage pattern ("unauthorized").+ const storageDown = !dbDegraded && !missingEnv && !err?.expose && !(err?.status && err.status < 500)+ && isStorageInfrastructureError(err);+ const status = dbDegraded || missingEnv || storageDown ? 503 : (err.status || 500);if (status >= 500) {const ref = correlationId();if (dbDegraded) {@@ -644,6 +668,13 @@ export function wrap(handler) {sendOpsAlert(dbFull ? 'database at storage cap — retention needed' : 'database unavailable', `${err?.message || String(err)}\nref ${ref}`, {signature: dbFull ? 'db:capacity' : 'db:unavailable',});+ } else if (storageDown) {+ // Every route that writes to R2 without its own try/catch lands here+ // during a bucket outage: throttle the log, skip Sentry, one deduped alert.+ logDbUnavailableOnce(`${req.method} ${redactUrl(req.url)}`, `object storage unavailable: ${err?.message || String(err)}`);+ sendOpsAlert('object storage unavailable', `${err?.message || String(err)}\nref ${ref}`, {+ signature: 'storage:unavailable',+ });} else if (missingEnv) {// One concise line + one deduped alert per (route, var) — the fix is// always the same: set the named var in the deployment env.@@ -665,7 +696,7 @@ export function wrap(handler) {// so err.message would leak HELIUS_API_KEY to the client. Hand back a// sanitized envelope keyed to the same ref we just logged.if (!res.headersSent && !res.writableEnded) {- if (dbDegraded) {+ if (dbDegraded || storageDown) {// 503 with Retry-After so clients and CDNs know to back off.res.setHeader('retry-after', '30');}@@ -706,9 +737,11 @@ export function wrap(handler) {// mapping our internal topology. 4xx keeps its code below, where// it is a deliberate, documented part of each endpoint's contract.json(res, status, {- error: dbDegraded ? 'service_unavailable' : 'internal_error',+ error: dbDegraded || storageDown ? 'service_unavailable' : 'internal_error',error_description: dbDegraded? 'database temporarily unavailable, retry shortly'+ : storageDown+ ? 'object storage temporarily unavailable, retry shortly': `internal error, quote ref ${ref} to support`,ref,}, { 'cache-control': 'no-store' });@@ -832,6 +865,14 @@ export function wrapCron(handler, { requireWriteCapacity = false, maxDurationMs}return;}+ if (isStorageInfrastructureError(err) && !err?.expose && !(err?.status && err.status < 500)) {+ console.warn('[cron] object storage unavailable, skipping tick:', err.message);+ sendOpsAlert('object storage unavailable', `${err.message}\ncron tick skipped`, { signature: 'storage:unavailable' });+ if (!res.headersSent && !res.writableEnded) {+ json(res, 200, { ok: false, reason: 'storage_unavailable' });+ }+ return;+ }throw err;}})();diff --git a/data/changelog.json b/data/changelog.jsonindex de82083e7..a0fa0a077 100644--- a/data/changelog.json+++ b/data/changelog.json@@ -1,14 +1,6 @@{"$comment": "Curated changelog entries for three.ws, the editorial layer of the public changelog. New PAGE launches are tracked automatically via the `added` field in data/pages.json; this file is for everything else holders care about: improvements, fixes, SDK releases, trading/payments capabilities, security work. scripts/build-page-index.mjs merges both sources into CHANGELOG.md, public/changelog.json, and public/changelog.xml. Newest first. Allowed tags: feature, improvement, fix, sdk, infra, docs, security, desktop (desktop entries become the desktop app's release notes). `link` is optional and must be a live three.ws path.","entries": [- {- "date": "2026-10-25",- "title": "Commit feed cron skips quietly on a lock-check fault",- "summary": "The commit feed cron now skips a tick with a 200 when the database lock check hits a transient fault, instead of answering a raw 500 and raising an ops alert. The next tick retries.",- "tags": [- "fix"- ]- },{"date": "2026-10-25","title": "Object-storage outages now answer a clean 503 with Retry-After instead of a raw 500",@@ -110,113 +102,82 @@"date": "2026-10-24","title": "Chat no longer fails when its default NVIDIA model is retired","summary": "NVIDIA stopped serving the Nemotron 3 Super function while still listing it, so chat requests that fell through to it got a 404 and, with the other providers throttled, ended in a 503. Chat, the tool chain, the brain and the CAD writer now use the Nemotron 3.5 Lightning model (with Ultra as its fallback), both verified live.",- "tags": [- "fix"- ],+ "tags": ["fix"],"link": "/chat"},{"date": "2026-10-24","title": "Fixed three more legacy thumbnail links that browsers refused to load","summary": "A handful of forge creations and 3D props stored from before an earlier URL fix still pointed straight at the storage bucket's own domain instead of three.ws's own CDN link, which Chrome blocks as an opaque cross-origin response. Model-page posters, a creation's related-models sidebar, forge share-card previews (the image X and Discord show when a link is pasted), and the Forged gallery's model-viewer thumbnails now all resolve through the same CDN link as everything else.",- "tags": [- "fix"- ],+ "tags": ["fix"],"link": "/forge"},{"date": "2026-10-23","title": "Bonding-curve status API no longer hard-fails when pump.fun blips","summary": "GET /api/crypto/bonding was returning a 503 to every caller during a pump.fun outage, even for mints it had already fetched successfully. It now caches each mint's bonding-curve status and keeps a longer last-known-good copy, so a failed upstream fetch serves that cached data (flagged with an x-pump-stale header) instead of a hard error. A 503 with a retry hint is now only returned for a mint that has no cached data at all.",- "tags": [- "fix"- ],+ "tags": ["fix"],"link": "/api/crypto/bonding"},{"date": "2026-10-24","title": "Hundreds of dependency security advisories cleared across the repo","summary": "Every committed lockfile was audited and patched: same-line security releases everywhere, vetted moves to patched majors (vitest 4, zod 4, helia 7, Svelte 5 and Vite 7 in the chat app, transformers 4 in HQ, gltf-transform 4 for the meshopt worker), and each package's tests and builds were rerun before and after. What remains are advisories with no patched release published anywhere (bigint-buffer, elliptic, node-forge) and pins that upstream SDKs still require.",- "tags": [- "security",- "infra"- ]+ "tags": ["security", "infra"]},{"date": "2026-10-24","title": "Desktop app moves to Electron 45","summary": "The desktop app now runs on Electron 45 and electron-builder 26, clearing 61 security advisories from the old Electron 33 toolchain. Launch at login still starts the app hidden in the tray; on Windows, toggle the setting once if the console window opens at login.",- "tags": [- "desktop",- "security"- ]+ "tags": ["desktop", "security"]},{"date": "2026-10-24","title": "Math renders in chat again","summary": "The chat app moved to Svelte 5, and KaTeX formulas in messages now render instead of showing raw TeX.",- "tags": [- "fix"- ],+ "tags": ["fix"],"link": "/chat"},{"date": "2026-10-24","title": "Long MCP answers arrive as a summary, and agents can search the docs","summary": "list_my_agents, recall and crypto_data now answer with a short summary, a count, a handle and the first page when the list is long, and the new expand tool reads the rest on request. A 100-coin market list drops from about 47,800 to 4,400 tokens. Handles are signed, account-bound and last 15 minutes. The new free search_docs tool answers how-to questions from the docs index on the main and studio servers.",- "tags": [- "feature",- "docs"- ],+ "tags": ["feature", "docs"],"link": "/docs/mcp"},{"date": "2026-10-24","title": "See what a prediction strategy would have earned before it trades","summary": "Agent Predictions records every strategy decision at the live order-book price in shadow mode, marks it at resolution, and shows shadow beside live results. A fee-aware scanner reads Solana event contracts for YES plus NO pairs under $1. Live stays owner-only.",- "tags": [- "feature"- ],+ "tags": ["feature"],"link": "/agent-predictions"},{"date": "2026-10-24","title": "Pay for a model request with a wallet, no account needed","summary": "POST /api/v1/chat/completions now answers an unauthenticated request with an x402 upto challenge on Solana, sized to the model's worst case for that request. Sign it, retry, and only the metered tokens are charged at the published per-token price; the rest of the cap is refunded. Streaming settles when the stream closes, and a dropped stream is charged only for the tokens already sent. GET /api/v1/models now lists the price sheet and round limits.",- "tags": [- "feature",- "docs"- ],+ "tags": ["feature", "docs"],"link": "/docs/metered-router"},{"date": "2026-10-24","title": "Skills are scanned for hidden behaviour before they are listed","summary": "Every marketplace skill version is now checked for secret access, calls to hosts it never declared, hidden or obfuscated code and instructions that skip the owner's confirmation. Clean skills list as before, suspicious ones are held for a moderator, and dangerous ones are refused. Authors see the file, the line and how to fix each finding in the publish form, and the existing catalog was rescanned.",- "tags": [- "security",- "feature",- "docs"- ],+ "tags": ["security", "feature", "docs"],"link": "/docs/skill-supply-scan"},{"date": "2026-10-24","title": "See what a Forge generation costs before you press Generate","summary": "The Forge page now shows a live cost line for the tier, engine and options you picked, read from the same pricing code that bills you, with nothing spent. A pay-to-keep mode (free preview, pay in $THREE to download, holders keep free) is built and switched off, so downloads stay free until it is turned on.",- "tags": [- "feature",- "docs"- ],+ "tags": ["feature", "docs"],"link": "/docs/forge"},{"date": "2026-10-24","title": "Agents join the open Solana agent registry and hires read trust tiers","summary": "Minted agents are enrolled in the open Solana agent registry (8004) with the platform authority signing, so owners pay nothing. The marketplace ranks providers by registry tier after completed hires, a hire can require a minimum tier, and agent pages show an 8004 tier badge when registered. A rate-limited back-fill covers existing agents.",- "tags": [- "feature"- ],+ "tags": ["feature"],"link": "/docs/solana-agent-registry"},{@@ -233,10 +194,7 @@"date": "2026-10-24","title": "Drive the CLI from scripts and agents, and download a run report","summary": "Every three-ws command now takes --json (one JSON document on stdout, diagnostics on stderr) and --non-interactive (never prompts), with a stable exit-code table: 2 usage, 3 auth, 4 refused by a guard, 5 payment required, 6 network. New commands cover agents, runs, wallet balance, skill install and orders. `three-ws report run <id> --out run.html` writes one self-contained offline file with the goal, tool calls, decisions, trades, outcome and the decision-trail head hash, and the run page has a matching Download report button.",- "tags": [- "feature",- "sdk"- ],+ "tags": ["feature", "sdk"],"link": "/docs/cli"},{diff --git a/tests/http-storage-unavailable-boundary.test.js b/tests/http-storage-unavailable-boundary.test.jsnew file mode 100644index 000000000..f7d2be5cf--- /dev/null+++ b/tests/http-storage-unavailable-boundary.test.js@@ -0,0 +1,147 @@+/**+ * http.js error boundary: object-storage-unavailable degradation.+ *+ * An R2 outage (disabled bucket, rotated keys, unreachable endpoint) makes every+ * putObject/getObjectBuffer/headObject call reject. Like a DB outage this is+ * infrastructure, not a code bug, so the shared boundaries must answer a clean+ * 503 (wrap/serverError) or a 200 skip (wrapCron) with a throttled warn and no+ * Sentry capture, instead of a raw 500 plus an `unhandled 5xx` alert per request.+ */++import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';+import { wrap, serverError, wrapCron } from '../api/_lib/http.js';+import { captureException } from '../api/_lib/sentry.js';++vi.mock('../api/_lib/alerts.js', () => ({+ sendOpsAlert: async () => {},+}));+vi.mock('../api/_lib/cache.js', () => ({+ cacheGet: async () => null,+ cacheSet: async () => {},+}));+vi.mock('../api/_lib/sentry.js', () => ({+ captureException: vi.fn(),+}));++function fakeRes() {+ const headers = {};+ return {+ statusCode: 0,+ body: undefined,+ ended: false,+ headersSent: false,+ writableEnded: false,+ setHeader(k, v) { headers[String(k).toLowerCase()] = v; },+ getHeader(k) { return headers[String(k).toLowerCase()]; },+ end(b) { this.body = b; this.ended = true; this.writableEnded = true; },+ _headers: headers,+ };+}++function storageDownError() {+ // The exact message production's disabled R2 account returns.+ return new Error('Please enable R2 through the Cloudflare Dashboard.');+}++function storageSigError() {+ return Object.assign(+ new Error('The request signature we calculated does not match the signature you provided.'),+ { name: 'SignatureDoesNotMatch' },+ );+}++let errSpy, warnSpy;+beforeEach(() => {+ errSpy = vi.spyOn(console, 'error').mockImplementation(() => {});+ warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {});+ captureException.mockClear();+});+afterEach(() => {+ errSpy.mockRestore();+ warnSpy.mockRestore();+});++const unhandledLogged = () => errSpy.mock.calls.some((c) => String(c[0]).includes('[api] unhandled'));++describe('wrap() under an object-storage outage', () => {+ for (const [label, make] of [['disabled R2 account', storageDownError], ['SignatureDoesNotMatch', storageSigError]]) {+ it(`returns 503 + Retry-After without unhandled log or Sentry capture (${label})`, async () => {+ const handler = wrap(async () => { throw make(); });+ const res = fakeRes();+ await handler({ method: 'POST', url: '/api/avatars/upload', headers: {} }, res);++ expect(res.statusCode).toBe(503);+ expect(res.getHeader('retry-after')).toBe('30');+ const body = JSON.parse(res.body);+ expect(body.error).toBe('service_unavailable');+ expect(body.error_description).toMatch(/storage/);+ expect(unhandledLogged()).toBe(false);+ expect(captureException).not.toHaveBeenCalled();+ });+ }++ it('still logs, captures and 500s a genuine handler bug', async () => {+ const handler = wrap(async () => { throw new Error('boom'); });+ const res = fakeRes();+ await handler({ method: 'POST', url: '/api/whatever', headers: {} }, res);++ expect(res.statusCode).toBe(500);+ expect(unhandledLogged()).toBe(true);+ expect(captureException).toHaveBeenCalledTimes(1);+ });++ it('does not reclassify a deliberate 4xx whose message happens to match the storage pattern', async () => {+ const handler = wrap(async () => {+ throw Object.assign(new Error('unauthorized'), { status: 401 });+ });+ const res = fakeRes();+ await handler({ method: 'GET', url: '/api/me', headers: {} }, res);++ expect(res.statusCode).toBe(401);+ expect(res.getHeader('retry-after')).toBeUndefined();+ });+});++describe('wrapCron() under an object-storage outage', () => {+ it('skips the tick with 200 storage_unavailable', async () => {+ const handler = wrapCron(async () => { throw storageDownError(); }, { maxDurationMs: 30_000 });+ const res = fakeRes();+ await handler({ method: 'GET', url: '/api/cron/some-job', headers: {} }, res);++ expect(res.statusCode).toBe(200);+ expect(JSON.parse(res.body)).toMatchObject({ ok: false, reason: 'storage_unavailable' });+ expect(unhandledLogged()).toBe(false);+ expect(captureException).not.toHaveBeenCalled();+ });++ it('still surfaces a genuine cron bug as a 500', async () => {+ const handler = wrapCron(async () => { throw new Error('boom'); }, { maxDurationMs: 30_000 });+ const res = fakeRes();+ await handler({ method: 'GET', url: '/api/cron/some-job', headers: {} }, res);++ expect(res.statusCode).toBe(500);+ expect(unhandledLogged()).toBe(true);+ });+});++describe('serverError() under an object-storage outage', () => {+ it('coerces a storage fault to 503 + Retry-After regardless of the nominal status', () => {+ const res = fakeRes();+ serverError(res, 500, 'upload_failed', storageDownError());++ expect(res.statusCode).toBe(503);+ expect(res.getHeader('retry-after')).toBe('30');+ expect(JSON.parse(res.body).error).toBe('service_unavailable');+ expect(errSpy.mock.calls.some((c) => String(c[0]).includes('[server-error'))).toBe(false);+ expect(captureException).not.toHaveBeenCalled();+ });++ it('keeps a genuine 5xx as-is with the [server-error] log line', () => {+ const res = fakeRes();+ serverError(res, 502, 'pulse_failed', new Error('upstream exploded'));++ expect(res.statusCode).toBe(502);+ expect(JSON.parse(res.body).error).toBe('pulse_failed');+ expect(errSpy.mock.calls.some((c) => String(c[0]).includes('[server-error'))).toBe(true);+ });+});