Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.
JavaScript 3d-generationai-agentsanimationavatardigital-humangenerative-aiglbgltfimage-to-3dllmmcpmcp-servermodel-context-protocolsolanatext-to-3dthreejsweb-componentswebglwebxrx402
// Reference-image builder for the text→3D forge (NEW, additive lane).
//
// ── Why this module exists ───────────────────────────────────────────────────
// The text→3D pipeline is: user prompt → prompt director rewrite → a reference
// IMAGE is generated → TRELLIS/Hunyuan3D reconstructs a textured GLB from that
// image. The realism of the final mesh is set almost entirely by the realism of
// that one reference image: a photoreal, single-subject, plain-seamless-
// background, evenly-lit image reconstructs into a real-looking mesh; a
// cartoonish or cluttered one does not.
//
// This module generates that reference image on the Vertex Gemini 2.5 Flash
// Image lane (GCP-credit funded, owner-approved spend), tuned specifically to
// produce the IDEAL reconstruction input:
// - a single centered subject on a plain seamless studio background,
// - soft even shadowless lighting,
// - real-world materials and surface micro-detail (the photoreal look),
// - the directed prompt's positive cues PLUS the director's negative prompt
// folded in as explicit natural-language avoidance (Gemini honors "Do not
// include: …" reliably; generateContent has no negative_prompt parameter),
// - the highest resolution the model tier serves (imageConfig.imageSize "2K").
//
// It is ADDITIVE and never a gate: if the Vertex lane is unconfigured or fails
// for any reason, it falls through to the existing reference-image provider
// (api/_mcp3d/text-to-image.js - NIM FLUX free lane, Vertex Imagen, Replicate
// backstop), so nothing regresses relative to today's behavior.
//
// ── Router seam ──────────────────────────────────────────────────────────────
// api/forge.js currently synthesizes the text→3D reference view at ~line 1557:
//
// const synthesized = await textToImage(directedPrompt, {
// aspectRatio: aspect,
// skipNim: nimGatewayDegraded,
// seed: opts.seed ?? undefined,
// });
//
// The router agent (which owns forge.js) can drop this module in as a direct
// replacement - the return shape { imageUrl, model } is identical, so nothing
// else in that function changes:
//
// import { generateReferenceImage } from './_lib/forge-reference-image.js';
// const synthesized = await generateReferenceImage(directedPrompt, {
// aspectRatio: aspect,
// skipNim: nimGatewayDegraded,
// seed: opts.seed ?? undefined,
// negativePrompt: directorNegativePrompt, // optional; auto-derived if omitted
// });
//
// `negativePrompt` is optional: when omitted, a subject-aware default is derived
// from the prompt via subjectNegativePrompt() (the same assembly the
// /api/forge-enhance response uses), so the router gets subject-aware negatives
// for free without having to run the enhance endpoint first.
//
// Output: { imageUrl, model, lane } where imageUrl is a durable public https URL
// (persisted to R2, same as textToImage), model is a provider/model label, and
// lane is 'vertex-reference' or the fallthrough provider's own label.
import { createHash } from 'node:crypto';
import { getGcpAccessToken } from './gcp-auth.js';
import { persistImageBase64 } from './image-persist.js';
import { subjectNegativePrompt } from '../forge-enhance.js';
import { isProviderRefusal } from './ai-image-lanes.js';
import { textToImage, ladderBudgetMs, laneTimeoutMs, VERTEX_IMAGE_COOLDOWN_KEY } from '../_mcp3d/text-to-image.js';
import { parseJsonLoose } from './vision.js';
import { getRedis } from './redis.js';
import { markProviderCooldown, providersInCooldown, AUTH_COOLDOWN_SECONDS } from './provider-health.js';
// Reference-set cache: the SAME enhanced prompt (already director-rewritten,
// so two users typing differently-worded ideas that converge on one directed
// spec both benefit) within 24h reuses its generated reference image set
// instead of re-spending a Vertex image (and QA-scoring) call. Keyed on a hash
// of everything that changes the output — prompt, aspect, negatives — mirroring
// the content-addressing idiom in forge-cache.js. Fail-open in every direction:
// no Redis, or any command error, degrades to "miss", never blocking generation.
const REF_CACHE_PREFIX = 'fr:ref:';
const REF_CACHE_TTL_S = Math.max(0, Number(process.env.FORGE_REFERENCE_CACHE_TTL_S) || 24 * 3600);
function referenceCacheKey(prompt, { aspectRatio, negativePrompt }) {
const text = String(prompt || '').trim().toLowerCase();
if (!text) return null;
const basis = JSON.stringify([text, aspectRatio || '1:1', String(negativePrompt || '').trim().toLowerCase()]);
return createHash('sha256').update(basis).digest('hex').slice(0, 40);
}
function referenceCacheEnabled() {
if (/^(0|false|off|no)$/i.test(String(process.env.FORGE_REFERENCE_CACHE ?? '').trim())) return false;
return Boolean(getRedis());
}
async function getCachedReference(key) {
const r = getRedis();
if (!r || !key || !referenceCacheEnabled()) return null;
try {
const raw = await r.get(`${REF_CACHE_PREFIX}${key}`);
if (!raw) return null;
const value = typeof raw === 'string' ? JSON.parse(raw) : raw;
return value && typeof value.imageUrl === 'string' && value.imageUrl ? value : null;
} catch {
return null;
}
}
async function putCachedReference(key, value) {
const r = getRedis();
if (!r || !key || !referenceCacheEnabled()) return;
if (!value?.imageUrl) return;
try {
await r.set(`${REF_CACHE_PREFIX}${key}`, JSON.stringify(value), { ex: REF_CACHE_TTL_S });
} catch {
/* best-effort */
}
}
// Gemini image model on Vertex ("Nano Banana"): live, credit-billed, uses the
// :generateContent shape. Overridable without a code change.
const DEFAULT_MODEL = process.env.VERTEX_IMAGEN_MODEL || 'gemini-2.5-flash-image';
// The Gemini image lane is served on the un-prefixed global host by default; a
// regional value ('us-central1') uses the region-prefixed host. Both verified
// live 2026-07-16. Separate from the text lanes so the image lane can be pinned
// independently.
const DEFAULT_LOCATION =
process.env.GOOGLE_CLOUD_LOCATION_IMAGE ||
process.env.VERTEX_IMAGEN_LOCATION ||
process.env.GOOGLE_CLOUD_LOCATION ||
'global';
// Request the highest resolution the model tier serves. "2K" is accepted today
// and currently renders at 1024px on this project's tier (harmless, and it
// upgrades automatically when the tier serves true 2K). Force 1K with
// VERTEX_IMAGE_SIZE=1K if a deployment wants the smaller/faster output.
const DEFAULT_IMAGE_SIZE = process.env.VERTEX_IMAGE_SIZE || '2K';
// Aspect ratios Gemini image generation accepts via generationConfig.imageConfig.
const GEMINI_ASPECTS = new Set([
'1:1', '3:2', '2:3', '3:4', '4:3', '4:5', '5:4', '9:16', '16:9', '21:9',
]);
// Match the private gate in text-to-image.js / ai-image-lanes.js: the Vertex
// image lane is active when GOOGLE_CLOUD_PROJECT is set unless VERTEX_IMAGEN_ENABLED
// explicitly disables it.
function vertexImageEnabled() {
if (!process.env.GOOGLE_CLOUD_PROJECT) return false;
const raw = process.env.VERTEX_IMAGEN_ENABLED;
if (raw == null) return true;
return !/^(0|false|no|off)$/i.test(String(raw).trim());
}
function aiplatformHost(location) {
return location === 'global'
? 'https://aiplatform.googleapis.com'
: `https://${location}-aiplatform.googleapis.com`;
}
// Wrap the directed prompt into a reference-image instruction that pins the
// exact composition a reconstructor needs: one centered subject, plain seamless
// studio sweep, soft even shadowless light, photoreal real-world materials, and
// the negative prompt as explicit avoidance. This is the difference between
// "draw the thing" and "shoot the thing the way a photogrammetry rig needs it".
export function buildReferenceInstruction(prompt, negativePrompt) {
const subject = String(prompt || '').trim();
// The instruction never names the downstream use ("for 3D reconstruction"):
// telling an image model its output feeds a reconstructor measurably nudges it
// toward a CGI/render aesthetic. It only ever describes the desired LOOK - a
// real studio photograph - and the composition a reconstructor happens to need.
const lines = [
'A single photorealistic studio photograph of the following subject, shot on a professional camera.',
`Subject: ${subject}.`,
'Composition: exactly one subject, centered and fully in frame, filling most of the frame, ' +
'shot straight-on at eye level like a product or portrait catalog photo.',
'Background: a plain, seamless, uncluttered light-grey studio sweep with nothing else in the scene.',
'Lighting: soft, even, neutral studio lighting with no harsh shadows and no colored gels, so the ' +
'subject is evenly lit from all sides.',
'Look: this must read as a genuine photograph taken with a real camera, with true-to-life ' +
'materials, natural surface micro-detail (grain, weave, pores, wear, brush marks), real depth ' +
'of field and sharp focus. Absolutely not a 3D render, not CGI, not an illustration, not a cartoon.',
];
if (negativePrompt && String(negativePrompt).trim()) {
lines.push(`Do NOT include: ${String(negativePrompt).trim()}.`);
}
return lines.join(' ');
}
// One Vertex Gemini image generation. Returns { b64, mime } on success; throws a
// designed error on any failure so the caller can fall through. Retries once
// without imageConfig.imageSize if the tier rejects that field (defensive: the
// field is accepted today, but a tier change must degrade to a valid request
// rather than a hard failure that skips the whole Vertex lane).
// `deadline` (epoch ms) bounds EVERY request this makes, the retry-without-size
// and the no-image re-roll included. Before it, each request had its own 60 s
// window and a single call could chain up to four of them.
const VERTEX_REQUEST_CEILING_MS = 60_000;
function msLeft(deadline) {
return deadline ? deadline - Date.now() : VERTEX_REQUEST_CEILING_MS;
}
async function generateViaVertex({ instruction, aspectRatio, deadline = null }) {
const project = process.env.GOOGLE_CLOUD_PROJECT;
const location = DEFAULT_LOCATION;
const model = DEFAULT_MODEL;
const aspect = GEMINI_ASPECTS.has(aspectRatio) ? aspectRatio : '1:1';
const endpoint =
`${aiplatformHost(location)}/v1/projects/${project}` +
`/locations/${location}/publishers/google/models/${model}:generateContent`;
const token = await getGcpAccessToken();
const request = async (withSize) => {
const imageConfig = withSize
? { aspectRatio: aspect, imageSize: DEFAULT_IMAGE_SIZE }
: { aspectRatio: aspect };
const res = await fetch(endpoint, {
method: 'POST',
headers: { authorization: `Bearer ${token}`, 'content-type': 'application/json' },
body: JSON.stringify({
contents: [{ role: 'user', parts: [{ text: instruction }] }],
generationConfig: { responseModalities: ['IMAGE'], imageConfig },
}),
signal: AbortSignal.timeout(Math.max(1_000, Math.min(VERTEX_REQUEST_CEILING_MS, msLeft(deadline)))),
});
return res;
};
const attempt = async () => {
let res = await request(true);
if (res.status === 400) {
// Peek at the error: only retry-without-size when it is specifically about
// imageConfig/imageSize, so a genuine bad request still fails fast.
const detail = await res.text().catch(() => '');
if (/imageSize|imageConfig/i.test(detail) && msLeft(deadline) > 5_000) {
res = await request(false);
} else {
throw Object.assign(new Error(`Vertex reference image 400: ${detail.slice(0, 200)}`), {
providerStatus: 400,
});
}
}
if (!res.ok) {
const detail = await res.text().catch(() => '');
const err = Object.assign(
new Error(`Vertex reference image returned ${res.status}: ${detail.slice(0, 200)}`),
{ providerStatus: res.status },
);
if (res.status === 429) err.code = 'rate_limited';
throw err;
}
const data = await res.json();
const parts = data?.candidates?.[0]?.content?.parts || [];
const imgPart = parts.find((p) => p?.inlineData?.data);
const b64 = imgPart?.inlineData?.data;
if (!b64) {
const reason = data?.candidates?.[0]?.finishReason;
throw new Error(`Vertex reference image produced no image${reason ? ` (finishReason: ${reason})` : ''}`);
}
return { b64, mime: imgPart.inlineData.mimeType || 'image/png', model: `vertex-ai/${model}` };
};
try {
return await attempt();
} catch (err) {
// A no-image response with a transient finishReason (NO_IMAGE, OTHER, but not
// the refusal classes) is a known Gemini flake worth exactly one re-roll:
// falling through to the generic text→image ladder loses the art-directed
// instruction, the negatives, and the QA gate, which is a far worse image
// than a second attempt at this lane. Refusals and HTTP errors re-throw
// unchanged so genuine blocks and outages still fail fast.
const noImage = /produced no image/i.test(String(err?.message || ''));
if (!noImage || isProviderRefusal(err) || msLeft(deadline) < 10_000) throw err;
return await attempt();
}
}
// ── Reference-image QA gate ──────────────────────────────────────────────────
// GPU reconstruction time (self-host TRELLIS/Hunyuan3D, or a paid vendor) is
// far more expensive than one more Gemini vision call, so before handing a
// reference off to the mesh step we score it with a cheap Vertex flash vision
// pass: is the subject complete, centered, photorealistic, and the background
// clean? A single retry with corrective feedback folded into the instruction
// gets a second roll; whichever scored higher ships (best-of-2), never a hard
// gate — any scoring failure just ships the first image unscored.
const QA_PASS_SCORE = Number(process.env.FORGE_REFERENCE_QA_PASS_SCORE) || 70;
const QA_MODEL = process.env.VERTEX_QUALITY_MODEL || 'gemini-2.5-flash';
function qaRubric(prompt) {
return [
'You are a quality inspector for a text-to-3D pipeline. You are shown ONE reference image that will be ' +
'reconstructed into a 3D mesh — its quality directly determines the mesh quality.',
prompt ? `It was generated for this subject: "${String(prompt).slice(0, 300)}".` : '',
'Judge four things: (1) is exactly ONE subject present, fully in frame, not cropped or cut off; ' +
'(2) is it centered; (3) does it read as a real photograph (not a cartoon, illustration, or CG render); ' +
'(4) is the background a plain, clean, uncluttered studio sweep with nothing else in the scene.',
'Reply with ONLY this JSON object, nothing else:',
'{"score": <int 0-100 overall>, "complete": <bool>, "centered": <bool>, "photoreal": <bool>, ' +
'"clean_background": <bool>, "issue": "<one short phrase naming the main problem, or empty string>"}',
].filter(Boolean).join(' ');
}
// Score one generated reference image via the same Vertex Gemini vision lane.
// Returns a normalized verdict or null on any failure (fail-open: the caller
// ships the image unscored rather than blocking on a QA outage).
async function scoreReferenceImage({ b64, mime, prompt }) {
try {
const project = process.env.GOOGLE_CLOUD_PROJECT;
if (!project) return null;
const location = process.env.GOOGLE_CLOUD_LOCATION_QUALITY || process.env.GOOGLE_CLOUD_LOCATION || 'us-central1';
const token = await getGcpAccessToken();
const host = location === 'global' ? 'https://aiplatform.googleapis.com' : `https://${location}-aiplatform.googleapis.com`;
const endpoint = `${host}/v1/projects/${project}/locations/${location}/publishers/google/models/${QA_MODEL}:generateContent`;
const res = await fetch(endpoint, {
method: 'POST',
headers: { authorization: `Bearer ${token}`, 'content-type': 'application/json' },
body: JSON.stringify({
contents: [{ role: 'user', parts: [{ text: qaRubric(prompt) }, { inlineData: { mimeType: mime, data: b64 } }] }],
generationConfig: {
temperature: 0,
responseMimeType: 'application/json',
maxOutputTokens: 300,
thinkingConfig: { thinkingBudget: 0 },
},
}),
signal: AbortSignal.timeout(15_000),
});
if (!res.ok) return null;
const data = await res.json();
const text = (data?.candidates?.[0]?.content?.parts || []).map((p) => p?.text || '').join('').trim();
if (!text) return null;
const raw = parseJsonLoose(text);
const score = Number(raw?.score);
if (!Number.isFinite(score)) return null;
return {
score: Math.max(0, Math.min(100, Math.round(score))),
complete: raw?.complete !== false,
centered: raw?.centered !== false,
photoreal: raw?.photoreal !== false,
cleanBackground: raw?.clean_background !== false,
issue: typeof raw?.issue === 'string' ? raw.issue.slice(0, 160) : '',
};
} catch (err) {
console.warn(`[forge-reference-image] QA scoring failed, shipping unscored: ${err?.message}`);
return null;
}
}
// Generate the reconstruction reference image for a directed text→3D prompt.
//
// generateReferenceImage(directedPrompt, {
// aspectRatio = '1:1', // Gemini aspect; unknown values → '1:1'
// negativePrompt = null, // director negatives; auto-derived from the prompt if omitted
// seed, // honored only on the fallthrough lanes (Gemini image has no seed)
// skipNim = false, // forwarded to the fallthrough textToImage lane
// })
//
// Vertex-first (high-res, reference-tuned), then automatic fallthrough to the
// existing provider chain. Returns { imageUrl, model, lane }. Never throws for a
// recoverable Vertex issue - only the fallthrough's terminal error surfaces when
// every lane is exhausted, exactly as textToImage does today.
export async function generateReferenceImage(
prompt,
{ aspectRatio = '1:1', negativePrompt = null, seed, skipNim = false, budgetMs } = {},
) {
// One budget for the whole reference step, shared with the fallthrough
// ladder: the Vertex lane gets the leader's share (it has a fallback behind
// it), QA scoring and the corrective retry only run while there is real time
// left, and whatever remains is handed to textToImage. Measured 2026-08-25:
// with no budget this step held a submit open for minutes before any job
// existed, on every text prompt.
const budget = ladderBudgetMs(budgetMs);
const startedAt = Date.now();
const remaining = () => budget - (Date.now() - startedAt);
const negatives =
negativePrompt && String(negativePrompt).trim()
? String(negativePrompt).trim()
: subjectNegativePrompt(prompt, { realistic: true });
// Same directed prompt (+aspect +negatives) within 24h reuses its reference
// set instead of re-spending a Vertex image generation + QA-scoring pass.
// Credits are approved, but latency is a UX cost this avoids for free.
const cacheKey = referenceCacheKey(prompt, { aspectRatio, negativePrompt: negatives });
const cached = await getCachedReference(cacheKey);
if (cached) {
console.log(`[forge-reference-image] cache hit (24h) for key ${cacheKey}`);
return { ...cached, lane: 'vertex-reference-cached' };
}
// A project-wide billing hold or safety block fails this lane identically on
// every call until ops fixes it. Share the breaker with text-to-image.js's
// own Vertex rung (VERTEX_IMAGE_COOLDOWN_KEY): either call site marking it
// silences both for AUTH_COOLDOWN_SECONDS, instead of this step paying the
// full Vertex timeout only to fall through to a textToImage() that pays it
// again for the exact same failure.
const vertexCooling =
vertexImageEnabled() && (await providersInCooldown([VERTEX_IMAGE_COOLDOWN_KEY])).has(VERTEX_IMAGE_COOLDOWN_KEY);
if (vertexImageEnabled() && !vertexCooling) {
try {
const instruction = buildReferenceInstruction(prompt, negatives);
const leaderMs = laneTimeoutMs(VERTEX_REQUEST_CEILING_MS, { budgetMs: budget, remainingMs: remaining(), laneRemains: true });
let generated = await generateViaVertex({ instruction, aspectRatio, deadline: Date.now() + leaderMs });
// Scoring is cheap insurance, not a gate: with the budget mostly spent
// the image ships unscored rather than blocking on one more call.
let verdict =
remaining() > budget * 0.25
? await scoreReferenceImage({ b64: generated.b64, mime: generated.mime, prompt })
: null;
// One retry with corrective feedback on a scoring failure: fold the
// vision model's own named issue into the instruction as an explicit
// "the previous attempt failed on X, fix it" line, then keep whichever
// of the two attempts scored higher (best-of-2). GPU reconstruction
// time dwarfs the cost of one extra vision call, so this is cheap
// insurance against a bad reference wasting the expensive step downstream.
if (verdict && verdict.score < QA_PASS_SCORE && remaining() > budget * 0.5) {
const issue = verdict.issue || 'the subject was incomplete, off-center, not photoreal, or the background was cluttered';
const retryInstruction = `${instruction} IMPORTANT: a previous attempt at this shot failed on: ${issue}. Fix that specifically this time.`;
try {
const retried = await generateViaVertex({ instruction: retryInstruction, aspectRatio, deadline: Date.now() + Math.floor(remaining() * 0.6) });
const retryVerdict = await scoreReferenceImage({ b64: retried.b64, mime: retried.mime, prompt });
if (!verdict || (retryVerdict && retryVerdict.score > verdict.score)) {
generated = retried;
verdict = retryVerdict;
}
} catch (err) {
console.warn(`[forge-reference-image] QA retry generation failed, keeping first attempt: ${err?.message}`);
}
}
if (verdict) {
console.log(
`[forge-reference-image] qa score=${verdict.score} complete=${verdict.complete} ` +
`centered=${verdict.centered} photoreal=${verdict.photoreal} clean_bg=${verdict.cleanBackground}` +
(verdict.issue ? ` issue="${verdict.issue}"` : ''),
);
}
const imageUrl = await persistImageBase64(generated.b64);
console.log(`[forge-reference-image] served by ${generated.model} (vertex-reference lane)`);
const result = { imageUrl, model: generated.model, qaScore: verdict?.score ?? null };
await putCachedReference(cacheKey, result);
return { ...result, lane: 'vertex-reference' };
} catch (err) {
// Any Vertex failure (unconfigured token, safety block, throttle, tier
// change) degrades to the existing chain - the reference image still gets
// made, just on the prior path. Warn, don't error.
const status = err?.providerStatus;
if (status === 401 || status === 402 || status === 403) {
markProviderCooldown(VERTEX_IMAGE_COOLDOWN_KEY, AUTH_COOLDOWN_SECONDS, 'auth').catch(() => {});
}
console.warn(`[forge-reference-image] vertex lane failed, falling through: ${err?.message}`);
}
}
// Fallthrough: the current reference-image provider (NIM FLUX free → Vertex
// Imagen → Replicate). Same call the router makes today, so this can only ever
// add the Vertex-reference lane in front, never remove a path.
const served = await textToImage(prompt, { aspectRatio, skipNim, seed, budgetMs: Math.max(5_000, remaining()) });
return { ...served, lane: served.lane || (served.predictionId ? 'replicate' : 'fallthrough') };
}