Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.
JavaScript 3d-generationai-agentsanimationavatardigital-humangenerative-aiglbgltfimage-to-3dllmmcpmcp-servermodel-context-protocolsolanatext-to-3dthreejsweb-componentswebglwebxrx402
fix(forge): hand bounded callers a submit ticket and reject provider billing notices as completions
nirholas6288b59dbca3
diff --git a/api/_lib/forge-submit-ticket.js b/api/_lib/forge-submit-ticket.jsnew file mode 100644index 000000000..1067d0c37--- /dev/null+++ b/api/_lib/forge-submit-ticket.js@@ -0,0 +1,160 @@+// Submit tickets: a job handle that exists before the job does.+//+// Why: /api/gpt-forge answers a text submit only after it has painted the+// reference image, and with Vertex Imagen billing-denied that paint falls+// through a ladder of providers. Measured on production 2026-09-30, text submits+// took 30 to 42 s. ChatGPT drops any tool call still open at 60 s, so the+// ChatGPT surface bounds every call (CHATGPT_CALL_BUDGET_MS) and aborts a submit+// that runs past it. The server finished those submits anyway and created real+// jobs, but the caller never learned their ids: the tool answered "Generation is+// taking longer than expected" and the model was orphaned. Test cases 1 and 2 of+// the OpenAI App Directory resubmission failed exactly this way.+//+// The fix hands the caller a handle up front. The client mints a ticket and+// sends it with the submit (x-forge-ticket). The server marks the ticket+// `submitting` before it starts work and records the submit's response under it+// when it answers, whether or not the caller is still listening. A caller whose+// submit ran out of time returns the ticket handle (`t1.<ticket>`) as its+// pending job id, and polling that handle on /api/gpt-forge resolves it to the+// real job once the submit lands. check_job and the viewer widget need no+// changes: they already poll whatever handle they were given.+//+// Stored in the shared cache (Redis, with per-instance memory as its degraded+// fallback), keyed by a hash of the ticket so the raw value never appears in a+// key listing.++import { createHash, randomUUID } from 'node:crypto';+import { cacheGetFresh, cacheSet } from './cache.js';++export const TICKET_HEADER = 'x-forge-ticket';+export const TICKET_PREFIX = 't1.';+// Outlives any client's collection loop, matching the done-frame cache.+export const TICKET_TTL_S = 6 * 3600;+// A submit that has not reported back in this long died with the instance that+// was running it. The slowest measured submit is 42 s; five minutes is far past+// any live one, so a ticket this old is reported as failed rather than left to+// spin forever.+export const TICKET_STALE_MS = 5 * 60_000;++// A poll can reach the cache before the server's `submitting` mark is visible+// to it (the mark is written when the submit arrives, and the cache degrades to+// per-instance memory when Redis is unhealthy). A ticket younger than this with+// no record yet is still being submitted, not unknown.+export const TICKET_MARK_GRACE_MS = 90_000;++const TICKET_RE = /^[A-Za-z0-9-]{16,64}$/;++// `<mint time base36>-<uuid>`: the mint time lets a poll tell a ticket that has+// not been marked yet from one that never existed. It carries nothing else.+export function newTicket(now = Date.now()) {+ return `${now.toString(36)}-${randomUUID()}`;+}++/** When a ticket was minted (epoch ms), or null for a ticket without a mint time. */+export function ticketMintedAt(ticket) {+ const m = /^([0-9a-z]{6,11})-/.exec(String(ticket || ''));+ if (!m) return null;+ const at = parseInt(m[1], 36);+ return Number.isFinite(at) ? at : null;+}++export function isValidTicket(ticket) {+ return typeof ticket === 'string' && TICKET_RE.test(ticket);+}++export function ticketHandle(ticket) {+ return `${TICKET_PREFIX}${ticket}`;+}++/** The ticket inside a `t1.` handle, or null when the handle is an ordinary job id. */+export function ticketFromHandle(jobId) {+ const s = String(jobId || '');+ if (!s.startsWith(TICKET_PREFIX)) return null;+ const ticket = s.slice(TICKET_PREFIX.length);+ return isValidTicket(ticket) ? ticket : null;+}++/** The ticket a submit request carries, or null. */+export function ticketFromRequest(req) {+ const raw = req?.headers?.[TICKET_HEADER];+ const ticket = Array.isArray(raw) ? raw[0] : raw;+ return isValidTicket(ticket) ? ticket : null;+}++export function ticketKey(ticket) {+ return `forge:ticket:${createHash('sha256').update(String(ticket)).digest('hex').slice(0, 40)}`;+}++export async function markTicketSubmitting(ticket, now = Date.now()) {+ await cacheSet(ticketKey(ticket), { state: 'submitting', at: now }, TICKET_TTL_S);+}++export async function recordTicketOutcome(ticket, status, body, now = Date.now()) {+ await cacheSet(ticketKey(ticket), { state: 'settled', status, body: body ?? null, at: now }, TICKET_TTL_S);+}++export async function recallTicket(ticket) {+ // Fresh read: a poll is waiting for this exact record to change.+ return cacheGetFresh(ticketKey(ticket)).catch(() => null);+}++/**+ * Record whatever the handler answers under the ticket before the response+ * leaves. `end` is deferred by one cache write so the record exists by the time+ * any poll can ask for it, and it stays idempotent so a second send behaves+ * exactly as it would have without the capture.+ */+export function captureTicketOutcome(res, ticket) {+ const end = res.end.bind(res);+ let captured = false;+ res.end = (chunk, ...rest) => {+ if (captured) return end(chunk, ...rest);+ captured = true;+ let body = null;+ try {+ if (chunk != null) body = JSON.parse(Buffer.isBuffer(chunk) ? chunk.toString('utf8') : String(chunk));+ } catch {+ body = null;+ }+ recordTicketOutcome(ticket, res.statusCode, body)+ .catch((err) => console.warn(`[forge] submit ticket record failed: ${err?.message || err}`))+ .finally(() => end(chunk, ...rest));+ return res;+ };+}++const START_AGAIN = 'Start a new generation.';++/**+ * Pure: what a poll of a ticket handle should answer, given its record (and+ * the ticket itself, whose mint time covers the moment before it is marked).+ * unknown no record: a mistyped or expired handle.+ * submitting the submit is still running server-side.+ * job the submit was accepted; poll `jobId` for the model.+ * done the submit finished the model outright; `body` is its frame.+ * failed the submit was refused, errored, or died; `message` says why.+ */+export function resolveTicket(record, now = Date.now(), ticket = null) {+ if (!record || typeof record !== 'object') {+ const mintedAt = ticketMintedAt(ticket);+ if (mintedAt != null && now - mintedAt >= 0 && now - mintedAt < TICKET_MARK_GRACE_MS) {+ return { kind: 'submitting', elapsedMs: now - mintedAt };+ }+ return { kind: 'unknown' };+ }+ if (record.state === 'submitting') {+ const elapsedMs = Math.max(0, now - Number(record.at || now));+ if (elapsedMs > TICKET_STALE_MS) {+ return { kind: 'failed', message: `The generation request never completed. ${START_AGAIN}` };+ }+ return { kind: 'submitting', elapsedMs };+ }+ const status = Number(record.status);+ const body = record.body && typeof record.body === 'object' ? record.body : {};+ if (status >= 200 && status < 300) {+ if (body.status === 'done' && body.glb_url) return { kind: 'done', body };+ if (typeof body.job_id === 'string' && body.job_id) return { kind: 'job', jobId: body.job_id };+ }+ const reason = typeof body.message === 'string' && body.message.trim() ? body.message.trim() : 'The generation could not start.';+ return { kind: 'failed', message: /start a new generation/i.test(reason) ? reason : `${reason.replace(/[.\s]*$/, '.')} ${START_AGAIN}` };+}diff --git a/api/_lib/llm.js b/api/_lib/llm.jsindex 4526ab533..4b2881b16 100644--- a/api/_lib/llm.js+++ b/api/_lib/llm.js@@ -241,6 +241,22 @@ export function resetLlmCooldowns() {COOLDOWNS.clear();}+// Some hosts answer an exhausted key with HTTP 200 and put the account notice in+// message.content, so the chain took it as the answer. Production forge rows+// from 2026-09-12 on carry "The API key used for this request has reached its+// budget. Please [raise the key budget](https://enter.pollinations.ai/...)" as+// their prompt: the prompt director accepted the notice, and a 3D model was+// reconstructed from it. Matched narrowly: a short reply whose whole content is+// a key or account running out of budget, quota or credits, not any answer that+// happens to discuss API keys.+const ACCOUNT_NOTICE_MAX_CHARS = 600;+const ACCOUNT_NOTICE = /\b(?:api key|key|account|token)\b[^.]{0,80}?\b(?:has reached|reached|exceeded|exhausted|run out of|ran out of|is out of|insufficient)\b[^.]{0,40}?\b(?:budget|quota|credits?|balance|spend(?:ing)? limit|usage limit)\b/i;++export function isProviderAccountNotice(text) {+ const t = String(text || '').trim();+ return t.length > 0 && t.length <= ACCOUNT_NOTICE_MAX_CHARS && ACCOUNT_NOTICE.test(t);+}+// Per-provider wall clock for one rung of the chain. Shared so a rung that wants// "the same budget as everyone else" cannot drift from the number llmComplete// actually enforces.@@ -1017,6 +1033,14 @@ export async function llmComplete({ system, user, maxTokens = 1024, anthropicKeyattempts.push({ provider: p.name, ms: Date.now() - startedAt, error: 'empty completion' });continue;}+ if (isProviderAccountNotice(text)) {+ // A billing or quota notice delivered as the completion itself. It is a+ // failed call wearing a 200, so it fails over like one and is never+ // returned: a caller cannot tell it from an answer.+ lastErr = Object.assign(new Error(`${p.name} answered with an account notice instead of a completion`), { status: 502, code: 'provider_notice' });+ attempts.push({ provider: p.name, ms: Date.now() - startedAt, error: 'account notice as completion' });+ continue;+ }COOLDOWNS.delete(rungKey(p));recordLlmSpend(p, usage, Date.now() - startedAt, track);return {diff --git a/api/_mcp-studio/gpt-forge-client.js b/api/_mcp-studio/gpt-forge-client.jsindex 1cbd682a6..630d5f0d6 100644--- a/api/_mcp-studio/gpt-forge-client.js+++ b/api/_mcp-studio/gpt-forge-client.js@@ -18,6 +18,7 @@import { env } from '../_lib/env.js';import { watsonxConfig, watsonxChatComplete } from '../_lib/watsonx.js';import { llmComplete } from '../_lib/llm.js';+import { TICKET_HEADER, newTicket, ticketHandle } from '../_lib/forge-submit-ticket.js';const DEFAULT_TIMEOUT_MS = 180_000;const DEFAULT_POLL_MS = 3_000;@@ -30,10 +31,13 @@ const RIG_SUBMIT_TIMEOUT_MS = 30_000;// the deadline: without an accepted job there is nothing to hand back, and the// surface's budget leaves room for it under the host's limit.const SUBMIT_FLOOR_MS = 8_000;+// A ticketed submit has a handle to hand back whatever happens, so it needs no+// room past the deadline, only enough to get the request onto the wire.+const TICKET_SUBMIT_FLOOR_MS = 3_000;-function submitWindow(deadline, capMs) {+function submitWindow(deadline, capMs, floorMs = SUBMIT_FLOOR_MS) {if (!deadline) return capMs;- return Math.min(capMs, Math.max(SUBMIT_FLOOR_MS, deadline - Date.now()));+ return Math.min(capMs, Math.max(floorMs, deadline - Date.now()));}function envNum(key, def) {@@ -114,7 +118,19 @@ function internalHeaders() {// and silently received standard has been overcharged, and the seller side// settles on success, so refusing here is what keeps their money in their// wallet.-export async function startForge(base, { prompt, imageUrls, aspect, backend, path, tier, internal, strictTier = false }, { deadline } = {}) {+//+// `director: false` tells the server not to run its own prompt director: the+// studio tools direct the prompt before they submit, and a second pass rewrote+// an already-directed brief (with the mesh director, even for an avatar) while+// spending up to another 15 s of the caller's budget.+//+// A bounded call (`deadline`) sends a submit ticket (see+// ../_lib/forge-submit-ticket.js). If the submit is still running at the+// deadline, it resolves to the ticket's handle, `{ status: 'submitting',+// job_id: 't1.…' }`, instead of a timeout: the server records the job under the+// ticket when it lands, and polling that handle collects it.+export async function startForge(base, { prompt, imageUrls, aspect, backend, path, tier, internal, director, strictTier = false }, { deadline } = {}) {+ const ticket = deadline ? newTicket() : null;const attempt = async (tierId, withInternal) => {const payload = {...(prompt ? { prompt } : {}),@@ -123,14 +139,19 @@ export async function startForge(base, { prompt, imageUrls, aspect, backend, pat...(backend ? { backend } : {}),...(path ? { path } : {}),...(tierId ? { tier: tierId } : {}),+ ...(director === false ? { director: false } : {}),};let res;try {res = await fetch(`${base}/api/gpt-forge`, {method: 'POST',- headers: { 'content-type': 'application/json', ...(withInternal ? internalHeaders() : {}) },+ headers: {+ 'content-type': 'application/json',+ ...(withInternal ? internalHeaders() : {}),+ ...(ticket ? { [TICKET_HEADER]: ticket } : {}),+ },body: JSON.stringify(payload),- signal: AbortSignal.timeout(submitWindow(deadline, SUBMIT_TIMEOUT_MS)),+ signal: AbortSignal.timeout(submitWindow(deadline, SUBMIT_TIMEOUT_MS, ticket ? TICKET_SUBMIT_FLOOR_MS : SUBMIT_FLOOR_MS)),});} catch (err) {if (err?.name === 'TimeoutError' || err?.name === 'AbortError')@@ -147,6 +168,10 @@ export async function startForge(base, { prompt, imageUrls, aspect, backend, pat({ res, data } = await attempt(tier, !!internal));} catch (err) {if (err?.code !== 'timeout') throw err;+ // The submit is still running server-side and will record its job under+ // the ticket, so hand back the ticket's handle rather than submit a+ // second job or report a timeout.+ if (ticket) return { status: 'submitting', job_id: ticketHandle(ticket) };// A bounded call that spent its budget on the first submit has no time// left for a second one; the caller reports the timeout instead.if (deadline && deadline - Date.now() < SUBMIT_FLOOR_MS) throw err;@@ -284,6 +309,8 @@ export async function pollOnce(base, jobId) {export async function generate(base, submitArgs, { timeoutEnv, deadline } = {}) {const job = await startForge(base, submitArgs, { deadline });if (job.status === 'done' && job.glb_url) return job;+ // The submit outlived the call budget: its ticket handle is the job id.+ if (job.status === 'submitting') return { _timedOut: true, job_id: job.job_id };const out = await pollJob(base, job.job_id, {timeoutMs: timeoutEnv ? envNum(timeoutEnv, DEFAULT_TIMEOUT_MS) : DEFAULT_TIMEOUT_MS,intervalMs: envNum('STUDIO_POLL_MS', DEFAULT_POLL_MS),@@ -351,6 +378,11 @@ export function isUsableDirectorRewrite(refined, rawPrompt) {const text = refined.trim();if (text.length < 3 || text.length > DIRECTOR_MAX_CHARS) return false;if (!ENDS_COMPLETE.test(text)) return false;+ // A brief describes an object; it never carries a link. A URL or a markdown+ // link means the reply is a provider notice ("raise the key budget at+ // https://...") or chatter, and reconstructing a mesh from it is how 22+ // production generations came out of a billing message.+ if (/https?:\/\/|\]\(|\bwww\./i.test(text)) return false;// The director's contract is to ENRICH a rough idea into a denser spec. A// result no longer than what the caller typed has added nothing, and is more// likely a clipped opening clause than a genuine tightening, so the user'sdiff --git a/api/gpt-forge.js b/api/gpt-forge.jsindex 74b0ff90a..854fb35d1 100644--- a/api/gpt-forge.js+++ b/api/gpt-forge.js@@ -130,6 +130,15 @@ import {import { markProviderCooldown, providersInCooldown } from './_lib/provider-health.js';import { acquireLock, releaseLock, cacheGet, cacheSet } from './_lib/cache.js';import { recallDoneFrame, rememberDoneFrame } from './_lib/forge-done-cache.js';+import {+ captureTicketOutcome,+ markTicketSubmitting,+ recallTicket,+ resolveTicket,+ ticketFromHandle,+ ticketFromRequest,+ ticketHandle,+} from './_lib/forge-submit-ticket.js';import { sanitizeJobError } from './_lib/provider-job-error.js';import { normalizeForgeOptions, providerReconstructParams, summarizeForgeOptions } from './_lib/forge-options.js';import { bindJobToOptions, optionsForJob } from './_lib/forge-job-options.js';@@ -2777,6 +2786,29 @@ async function pollGcpStatus({ gcp, upstreamId, clientKey, createdAt }) {return result; // genuinely orphaned → existing failover path acts on it}+// Poll a submit-ticket handle (`t1.<ticket>`): the job a bounded caller could+// not wait for. Once the submit has answered, the ticket resolves to its real+// job and the poll is served exactly as a poll of that job would be.+async function pollTicket(req, res, ticket) {+ const out = resolveTicket(await recallTicket(ticket), Date.now(), ticket);+ if (out.kind === 'job') return pollJob(req, res, out.jobId);+ const rl = await limits.mcp3dStatus(clientIp(req));+ if (!rl.success) return rateLimited(res, rl);+ const handle = ticketHandle(ticket);+ if (out.kind === 'done') return json(res, 200, { ...out.body, job_id: handle });+ if (out.kind === 'submitting') {+ return json(res, 200, {+ job_id: handle,+ status: 'queued',+ stage: 'submit',+ elapsed_seconds: Math.round(out.elapsedMs / 1000),+ message: 'The generation request is still being accepted.',+ });+ }+ if (out.kind === 'failed') return json(res, 200, { job_id: handle, status: 'failed', error: out.message });+ return json(res, 404, { error: 'unknown_job', message: 'That job id is not recognized (it may be mistyped or expired).' });+}+async function pollJob(req, res, jobId) {// A job handle is either a bare Replicate prediction id (legacy / image-// TRELLIS path) or a forge token encoding the geometry/GCP provider + the@@ -3136,6 +3168,14 @@ export default wrap(async (req, res) => {if ((url.searchParams.get('action') || '').trim() === 'rig') {return startRigJob(req, res);}+ // A bounded caller (the ChatGPT surface) sends a ticket so a submit that+ // outlives its call budget still yields a job it can collect. See+ // _lib/forge-submit-ticket.js.+ const ticket = ticketFromRequest(req);+ if (ticket) {+ await markTicketSubmitting(ticket).catch((err) => console.warn(`[forge] submit ticket mark failed: ${err?.message || err}`));+ captureTicketOutcome(res, ticket);+ }return startJob(req, res);}@@ -3175,5 +3215,7 @@ export default wrap(async (req, res) => {if (!jobId) {return json(res, 400, { error: 'missing_job', message: 'Pass ?job=<id> to poll a job.' });}+ const ticket = ticketFromHandle(jobId);+ if (ticket) return pollTicket(req, res, ticket);return pollJob(req, res, jobId);});