Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.
JavaScript 3d-generationai-agentsanimationavatardigital-humangenerative-aiglbgltfimage-to-3dllmmcpmcp-servermodel-context-protocolsolanatext-to-3dthreejsweb-componentswebglwebxrx402
// /api/ibm/vision — IBM Granite Vision, the multimodal eye of the watsonx suite.
//
// POST /api/ibm/vision { image | imageUrl, subject?, hint? }
// Shows a single image to a Granite Vision model on watsonx.ai and returns a
// structured "read" of it. The headline demo points it at a three.ws 3D avatar
// (a rendered snapshot, or its thumbnail) and Granite Vision invents a complete
// agent identity from how the avatar *looks* — appearance, vibe, persona, a name,
// a bio, tone tags, a fitting voice. `subject:"token"` reads a coin's image; the
// default reads any picture.
//
// image — a data: URL (client canvas capture or an uploaded file).
// imageUrl — an https URL we fetch server-side (SSRF-allowlisted) so the browser
// never has to fight canvas cross-origin tainting to analyse an avatar.
//
// GET /api/ibm/vision → a handful of real public avatars to show Granite, so the
// demo works for anonymous visitors without an upload.
//
// No mock path: every read is a real Granite Vision inference. When watsonx is
// unconfigured the endpoint says so (503) instead of inventing a description.
import { sql } from '../_lib/db.js';
import { cors, method, readJson, error, json, wrap, rateLimited } from '../_lib/http.js';
import { limits, clientIp } from '../_lib/rate-limit.js';
import { publicUrl, thumbnailUrl } from '../_lib/r2.js';
import { watsonxConfig, watsonxChatComplete } from '../_lib/watsonx.js';
import { assertSafePublicUrl } from '../_lib/ssrf-guard.js';
import { normalizeGatewayURL } from '../../src/ipfs.js';
// Granite Vision 3.2 (2B) is the default multimodal model on watsonx.ai. Override
// per account/region with WATSONX_VISION_MODEL_ID.
export const VISION_MODEL =
process.env.WATSONX_VISION_MODEL_ID?.trim() || 'ibm/granite-vision-3-2-2b';
// The exact watsonx.ai chat payload for a Granite Vision read: a system message
// plus a user message whose content is the multimodal block array Granite expects
// (a text instruction followed by the image as a data/base64 URL). Extracted so the
// verification script can assert this wire shape without a live call.
export function buildVisionMessages(subject, hint, dataUrl) {
const { system, user } = buildPrompt(subject, hint);
return [
{ role: 'system', content: system },
{
role: 'user',
content: [
{ type: 'text', text: user },
{ type: 'image_url', image_url: { url: dataUrl } },
],
},
];
}
// Server-side image-fetch limits. Granite accepts png/jpeg/webp/gif; we cap bytes
// so a hostile URL can't stream us an unbounded body, and only follow https.
const MAX_IMAGE_BYTES = 6 * 1024 * 1024;
const FETCH_TIMEOUT_MS = 8000;
// SSRF allowlist for imageUrl. We only fetch from our own asset host and a small
// set of public, content-addressed media CDNs the platform already serves images
// from — never an arbitrary host, so this can't be turned into an internal probe.
export function allowedImageHost(host) {
host = host.toLowerCase();
const ours = [process.env.S3_PUBLIC_DOMAIN, process.env.APP_ORIGIN, 'https://three.ws']
.map(hostOf)
.filter(Boolean);
if (ours.includes(host)) return true;
const suffixes = [
'.r2.dev',
'.r2.cloudflarestorage.com',
'three.ws',
'.mypinata.cloud',
'.pinata.cloud',
'ipfs.io',
'.ipfs.dweb.link',
'ipfs.filebase.io',
'4everland.io',
'.4everland.io',
'cf-ipfs.com',
'arweave.net',
'.arweave.net',
'pump.mypinata.cloud',
'image-cdn.solana.fm',
'.githubusercontent.com',
];
return suffixes.some((s) => (s.startsWith('.') ? host.endsWith(s) : host === s));
}
function hostOf(u) {
try {
return new URL(u).host.toLowerCase();
} catch {
return null;
}
}
export default wrap(async (req, res) => {
if (cors(req, res, { methods: 'GET,POST,OPTIONS', origins: '*' })) return;
if (!method(req, res, ['GET', 'POST'])) return;
const rl = await limits.publicIp(clientIp(req));
if (!rl.success) return rateLimited(res, rl);
// Global hourly ceiling on the shared watsonx server key (same bucket as
// api/watsonx/embed) — a hard aggregate cost cap independent of any one IP.
// Only consumed when watsonx is configured, i.e. when a request can
// actually bill Granite inference.
if (watsonxConfig().configured) {
const global = await limits.watsonxEmbedGlobal();
if (!global.success)
return rateLimited(res, global, 'watsonx capacity reached — try again shortly');
}
if (req.method === 'GET') return handleSubjects(req, res);
return handleVision(req, res);
});
export const maxDuration = 60;
// ── GET: featured subjects to show Granite ───────────────────────────────────
async function handleSubjects(req, res) {
// Public avatars that have both a thumbnail (for the image read) and a stored
// GLB (for the live 3D render). Featured + most-viewed first so the lineup looks
// intentional. Read-only; safe for anonymous visitors.
let rows = [];
try {
rows = await sql`
SELECT id, slug, name, storage_key, thumbnail_key,
COALESCE(featured, false) AS featured,
COALESCE(view_count, 0) AS view_count
FROM avatars
WHERE deleted_at IS NULL
AND visibility = 'public'
AND thumbnail_key IS NOT NULL
AND storage_key IS NOT NULL
ORDER BY COALESCE(featured, false) DESC, COALESCE(view_count, 0) DESC, created_at DESC
LIMIT 18
`;
} catch (err) {
// The lineup is a convenience, not the feature: a DB blip degrades to an
// empty gallery rather than a 500. Keep the same response shape as the
// success path so a client reading visionModel never sees undefined.
console.error('[ibm/vision] subjects query failed', err);
return json(res, 200, { subjects: [], visionModel: VISION_MODEL });
}
const subjects = rows.map((r) => ({
id: r.id,
name: r.name || 'Avatar',
slug: r.slug || null,
thumbnail: thumbnailUrl(r.thumbnail_key),
model_url: publicUrl(r.storage_key),
}));
// Subjects list changes at most once an hour; cache aggressively so the demo
// loads instantly for repeat visitors while keeping CDN cost negligible.
res.setHeader('cache-control', 's-maxage=3600, stale-while-revalidate=86400');
return json(res, 200, { subjects, visionModel: VISION_MODEL });
}
// ── POST: read an image with Granite Vision ──────────────────────────────────
async function handleVision(req, res) {
const cfg = watsonxConfig();
// Granite Vision leads on watsonx, but a vision-capable OpenRouter model is a
// full backstop (watsonxChatComplete falls over to it). So the endpoint is
// available when EITHER watsonx is configured OR an OpenRouter key is set —
// only both-absent is a true 503.
const openrouterAvailable = Boolean(
process.env.OPENROUTER_API_KEY?.trim() || (process.env.OPENROUTER_FALLBACK_KEYS || '').trim(),
);
if (!cfg.configured && !openrouterAvailable) {
return error(
res,
503,
'watsonx_unavailable',
'IBM Granite Vision runs on watsonx.ai. Set WATSONX_API_KEY and WATSONX_PROJECT_ID ' +
'(or WATSONX_SPACE_ID), or an OPENROUTER_API_KEY backstop, to read an image.',
);
}
let body;
try {
// a base64 data URL of a snapshot can be large
const parsed = await readJson(req, 9_000_000);
// `null` / a bare string is valid JSON that readJson returns as-is on the
// raw-stream path, and reading .image off it throws a TypeError that surfaces
// as a 500. Malformed input takes the documented 400 bad_image path instead.
body = parsed && typeof parsed === 'object' ? parsed : {};
} catch (e) {
return error(res, e.status || 400, 'bad_request', e.message);
}
const subject = ['avatar', 'token', 'image'].includes(body.subject) ? body.subject : 'avatar';
const hint = typeof body.hint === 'string' ? body.hint.trim().slice(0, 200) : '';
let dataUrl;
try {
dataUrl = await resolveImage(body);
} catch (e) {
return error(res, e.status || 400, e.code || 'bad_image', e.message);
}
let reply;
try {
reply = await watsonxChatComplete(cfg, {
model: VISION_MODEL,
maxTokens: 460,
temperature: 0.4,
messages: buildVisionMessages(subject, hint, dataUrl),
});
} catch (e) {
// Surface the real upstream cause (auth, quota, model not deployed in region).
// model_unavailable gives the operator a clear signal to check WATSONX_VISION_MODEL_ID
// or the regional model catalogue; vision_failed covers everything else.
const msg = String(e?.message || 'watsonx vision request failed');
const code = /not.*found|not.*deployed|unsupported.*model/i.test(msg)
? 'model_unavailable'
: 'vision_failed';
return error(res, 502, code, msg);
}
const vision = parseVision(reply.text);
return json(res, 200, {
subject,
model: reply.model || VISION_MODEL,
vision,
raw: vision.structured ? undefined : reply.text,
usage: reply.usage || null,
});
}
// Resolve the request into a base64 data URL Granite can consume. Accepts a client
// data URL directly, or fetches an allowlisted https image server-side.
async function resolveImage(body) {
const image = typeof body.image === 'string' ? body.image : '';
const imageUrl = typeof body.imageUrl === 'string' ? body.imageUrl : '';
if (image.startsWith('data:image/')) {
// Guard the decoded size (base64 is ~4/3 of the bytes).
const commaAt = image.indexOf(',');
const b64 = commaAt >= 0 ? image.slice(commaAt + 1) : '';
if (!b64) throw fail(400, 'bad_image', 'image data URL is empty');
if (b64.length * 0.75 > MAX_IMAGE_BYTES)
throw fail(413, 'image_too_large', 'image exceeds 6MB');
return image;
}
if (/^https:\/\//i.test(imageUrl)) {
// Token art pinned to a retired IPFS gateway (pump.fun still hands out
// ipfs.io) is re-pointed at a live one before it is fetched.
const target = normalizeGatewayURL(imageUrl);
const host = hostOf(target);
if (!host || !allowedImageHost(host)) {
throw fail(400, 'image_host_not_allowed', 'imageUrl host is not allowed');
}
return await fetchImageAsDataUrl(target);
}
throw fail(400, 'bad_image', 'provide image (data URL) or imageUrl (https)');
}
const MAX_IMAGE_REDIRECTS = 5;
// Re-validate every redirect hop before following it. An allowlisted host (the
// list includes broad CDNs and open-redirect-prone providers) could otherwise
// 3xx us toward an internal/metadata address. Each hop must stay https, keep an
// allowlisted host, AND pass the DNS-resolving SSRF guard (private/loopback/
// link-local/metadata IPs blocked). Bounded by MAX_IMAGE_REDIRECTS.
async function fetchImageAsDataUrl(url) {
const controller = new AbortController();
// The timer stays armed until the BODY is fully read, not just the headers:
// several allowlisted hosts serve user-supplied content, and a host that
// answers headers promptly then trickles the body would otherwise hold the
// invocation open for as long as it liked.
const timer = setTimeout(() => controller.abort(), FETCH_TIMEOUT_MS);
try {
let resp;
let current = url;
let redirects = 0;
try {
while (true) {
const host = hostOf(current);
if (!/^https:\/\//i.test(current) || !host || !allowedImageHost(host)) {
throw fail(
400,
'image_host_not_allowed',
'image redirect target host is not allowed',
);
}
await assertSafePublicUrl(current);
resp = await fetch(current, { signal: controller.signal, redirect: 'manual' });
if (resp.status >= 300 && resp.status < 400 && resp.headers.get('location')) {
if (++redirects > MAX_IMAGE_REDIRECTS) {
throw fail(502, 'image_fetch_failed', 'too many redirects');
}
current = new URL(resp.headers.get('location'), current).toString();
continue;
}
break;
}
} catch (e) {
if (e?.status && e?.code) throw e; // already a structured fail()
throw fail(502, 'image_fetch_failed', `could not fetch image: ${e.message}`);
}
if (!resp.ok) throw fail(502, 'image_fetch_failed', `image fetch returned ${resp.status}`);
const ct = (resp.headers.get('content-type') || '').split(';')[0].trim().toLowerCase();
if (!ct.startsWith('image/'))
throw fail(415, 'not_an_image', `unexpected content-type: ${ct || 'unknown'}`);
const len = Number(resp.headers.get('content-length') || 0);
if (len && len > MAX_IMAGE_BYTES) throw fail(413, 'image_too_large', 'image exceeds 6MB');
const buf = await readCappedBody(resp);
return `data:${ct};base64,${buf.toString('base64')}`;
} finally {
clearTimeout(timer);
}
}
// Read a response body under a hard byte ceiling. content-length is a hint a
// host can simply omit (or lie about) on a chunked response, so the cap is
// enforced on the stream itself: buffering the whole body first and checking
// its size afterwards means an oversized response is already resident in memory
// by the time it is rejected.
async function readCappedBody(resp) {
if (!resp.body) {
const buf = Buffer.from(await resp.arrayBuffer());
if (buf.length > MAX_IMAGE_BYTES) throw fail(413, 'image_too_large', 'image exceeds 6MB');
return buf;
}
const chunks = [];
let total = 0;
// Throwing out of the loop closes the iterator, which cancels the body, so
// the rest of an oversized response is never pulled off the socket.
for await (const chunk of resp.body) {
total += chunk.length;
if (total > MAX_IMAGE_BYTES) throw fail(413, 'image_too_large', 'image exceeds 6MB');
chunks.push(Buffer.from(chunk));
}
return Buffer.concat(chunks);
}
function fail(status, code, message) {
const e = new Error(message);
e.status = status;
e.code = code;
return e;
}
// ── Prompting ────────────────────────────────────────────────────────────────
export function buildPrompt(subject, hint) {
const hintLine = hint
? `\nThe person who made it adds: "${hint}". Take it into account but trust your eyes first.`
: '';
if (subject === 'token') {
return {
system:
'You are a sharp, neutral brand analyst looking at a single cryptocurrency / memecoin image (its logo or art). ' +
'Describe only what is visible and the vibe it projects. Never give financial advice or price opinions. ' +
'Reply with ONLY a JSON object and nothing else.',
user:
'Read this coin image.' +
hintLine +
'\nReturn JSON exactly like: {"appearance":"2-3 sentences on what is in the image","vibe":"3-6 comma-separated adjectives",' +
'"persona":"1-2 sentences on the character/brand this projects","bio":"a punchy one-line tagline",' +
'"tone_tags":["3-6","lowercase","tags"],"voice":"a short phrase for a fitting narrator voice"}',
};
}
if (subject === 'image') {
return {
system:
'You are a perceptive art director describing a single image precisely and only from what is visible. ' +
'Reply with ONLY a JSON object and nothing else.',
user:
'Read this image.' +
hintLine +
'\nReturn JSON exactly like: {"appearance":"2-3 sentences on what is visible","vibe":"3-6 comma-separated adjectives",' +
'"persona":"1-2 sentences on the character/mood it suggests","suggested_name":"a fitting short name",' +
'"bio":"a punchy one-line bio","tone_tags":["3-6","lowercase","tags"],"voice":"a short phrase for a fitting voice"}',
};
}
// avatar (default)
return {
system:
'You are a casting director giving a 3D avatar a personality. You are shown one rendered 3D character. ' +
'Look only at what is visible — features, clothing, colors, style, posture — and infer a personality that fits the look. ' +
'Be specific and vivid, never generic. Reply with ONLY a JSON object and nothing else.',
user:
'Read this 3D avatar and give it an identity.' +
hintLine +
'\nReturn JSON exactly like: {"appearance":"2-3 sentences on how the avatar looks","vibe":"3-6 comma-separated adjectives",' +
'"persona":"1-2 sentences on the personality this look suggests","suggested_name":"a fitting character name",' +
'"bio":"a punchy one-line bio for this character","tone_tags":["3-6","lowercase","tags"],' +
'"voice":"a short phrase describing a fitting speaking voice"}',
};
}
// Parse Granite's reply into a normalized identity. Tolerant of code fences and
// stray prose; falls back to { structured:false } so the UI shows the raw read
// rather than nothing — never fabricates fields the model didn't return.
export function parseVision(text) {
const out = {
structured: false,
appearance: '',
vibe: '',
persona: '',
suggested_name: '',
bio: '',
tone_tags: [],
voice: '',
};
if (!text) return out;
const m = String(text).match(/\{[\s\S]*\}/);
if (!m) return out;
let obj;
try {
obj = JSON.parse(m[0]);
} catch {
return out;
}
out.structured = true;
out.appearance = str(obj.appearance);
out.vibe = str(obj.vibe);
out.persona = str(obj.persona);
out.suggested_name = str(obj.suggested_name || obj.name).slice(0, 60);
out.bio = str(obj.bio).slice(0, 200);
out.voice = str(obj.voice).slice(0, 120);
out.tone_tags = Array.isArray(obj.tone_tags)
? obj.tone_tags
.map((t) => str(t).toLowerCase().slice(0, 24))
.filter(Boolean)
.slice(0, 8)
: typeof obj.tone_tags === 'string'
? obj.tone_tags
.split(',')
.map((t) => t.trim().toLowerCase())
.filter(Boolean)
.slice(0, 8)
: [];
return out;
}
function str(v) {
return typeof v === 'string' ? v.replace(/\s+/g, ' ').trim() : '';
}