Skip to content

Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.

fix(forge): hand bounded callers a submit ticket and reject provider billing notices as completions

nirholas6288b59dbca3
diff --git a/api/_lib/forge-submit-ticket.js b/api/_lib/forge-submit-ticket.js
new file mode 100644
index 000000000..1067d0c37
--- /dev/null
+++ b/api/_lib/forge-submit-ticket.js
@@ -0,0 +1,160 @@
+// Submit tickets: a job handle that exists before the job does.
+//
+// Why: /api/gpt-forge answers a text submit only after it has painted the
+// reference image, and with Vertex Imagen billing-denied that paint falls
+// through a ladder of providers. Measured on production 2026-09-30, text submits
+// took 30 to 42 s. ChatGPT drops any tool call still open at 60 s, so the
+// ChatGPT surface bounds every call (CHATGPT_CALL_BUDGET_MS) and aborts a submit
+// that runs past it. The server finished those submits anyway and created real
+// jobs, but the caller never learned their ids: the tool answered "Generation is
+// taking longer than expected" and the model was orphaned. Test cases 1 and 2 of
+// the OpenAI App Directory resubmission failed exactly this way.
+//
+// The fix hands the caller a handle up front. The client mints a ticket and
+// sends it with the submit (x-forge-ticket). The server marks the ticket
+// `submitting` before it starts work and records the submit's response under it
+// when it answers, whether or not the caller is still listening. A caller whose
+// submit ran out of time returns the ticket handle (`t1.<ticket>`) as its
+// pending job id, and polling that handle on /api/gpt-forge resolves it to the
+// real job once the submit lands. check_job and the viewer widget need no
+// changes: they already poll whatever handle they were given.
+//
+// Stored in the shared cache (Redis, with per-instance memory as its degraded
+// fallback), keyed by a hash of the ticket so the raw value never appears in a
+// key listing.
+
+import { createHash, randomUUID } from 'node:crypto';
+import { cacheGetFresh, cacheSet } from './cache.js';
+
+export const TICKET_HEADER = 'x-forge-ticket';
+export const TICKET_PREFIX = 't1.';
+// Outlives any client's collection loop, matching the done-frame cache.
+export const TICKET_TTL_S = 6 * 3600;
+// A submit that has not reported back in this long died with the instance that
+// was running it. The slowest measured submit is 42 s; five minutes is far past
+// any live one, so a ticket this old is reported as failed rather than left to
+// spin forever.
+export const TICKET_STALE_MS = 5 * 60_000;
+
+// A poll can reach the cache before the server's `submitting` mark is visible
+// to it (the mark is written when the submit arrives, and the cache degrades to
+// per-instance memory when Redis is unhealthy). A ticket younger than this with
+// no record yet is still being submitted, not unknown.
+export const TICKET_MARK_GRACE_MS = 90_000;
+
+const TICKET_RE = /^[A-Za-z0-9-]{16,64}$/;
+
+// `<mint time base36>-<uuid>`: the mint time lets a poll tell a ticket that has
+// not been marked yet from one that never existed. It carries nothing else.
+export function newTicket(now = Date.now()) {
+ return `${now.toString(36)}-${randomUUID()}`;
+}
+
+/** When a ticket was minted (epoch ms), or null for a ticket without a mint time. */
+export function ticketMintedAt(ticket) {
+ const m = /^([0-9a-z]{6,11})-/.exec(String(ticket || ''));
+ if (!m) return null;
+ const at = parseInt(m[1], 36);
+ return Number.isFinite(at) ? at : null;
+}
+
+export function isValidTicket(ticket) {
+ return typeof ticket === 'string' && TICKET_RE.test(ticket);
+}
+
+export function ticketHandle(ticket) {
+ return `${TICKET_PREFIX}${ticket}`;
+}
+
+/** The ticket inside a `t1.` handle, or null when the handle is an ordinary job id. */
+export function ticketFromHandle(jobId) {
+ const s = String(jobId || '');
+ if (!s.startsWith(TICKET_PREFIX)) return null;
+ const ticket = s.slice(TICKET_PREFIX.length);
+ return isValidTicket(ticket) ? ticket : null;
+}
+
+/** The ticket a submit request carries, or null. */
+export function ticketFromRequest(req) {
+ const raw = req?.headers?.[TICKET_HEADER];
+ const ticket = Array.isArray(raw) ? raw[0] : raw;
+ return isValidTicket(ticket) ? ticket : null;
+}
+
+export function ticketKey(ticket) {
+ return `forge:ticket:${createHash('sha256').update(String(ticket)).digest('hex').slice(0, 40)}`;
+}
+
+export async function markTicketSubmitting(ticket, now = Date.now()) {
+ await cacheSet(ticketKey(ticket), { state: 'submitting', at: now }, TICKET_TTL_S);
+}
+
+export async function recordTicketOutcome(ticket, status, body, now = Date.now()) {
+ await cacheSet(ticketKey(ticket), { state: 'settled', status, body: body ?? null, at: now }, TICKET_TTL_S);
+}
+
+export async function recallTicket(ticket) {
+ // Fresh read: a poll is waiting for this exact record to change.
+ return cacheGetFresh(ticketKey(ticket)).catch(() => null);
+}
+
+/**
+ * Record whatever the handler answers under the ticket before the response
+ * leaves. `end` is deferred by one cache write so the record exists by the time
+ * any poll can ask for it, and it stays idempotent so a second send behaves
+ * exactly as it would have without the capture.
+ */
+export function captureTicketOutcome(res, ticket) {
+ const end = res.end.bind(res);
+ let captured = false;
+ res.end = (chunk, ...rest) => {
+ if (captured) return end(chunk, ...rest);
+ captured = true;
+ let body = null;
+ try {
+ if (chunk != null) body = JSON.parse(Buffer.isBuffer(chunk) ? chunk.toString('utf8') : String(chunk));
+ } catch {
+ body = null;
+ }
+ recordTicketOutcome(ticket, res.statusCode, body)
+ .catch((err) => console.warn(`[forge] submit ticket record failed: ${err?.message || err}`))
+ .finally(() => end(chunk, ...rest));
+ return res;
+ };
+}
+
+const START_AGAIN = 'Start a new generation.';
+
+/**
+ * Pure: what a poll of a ticket handle should answer, given its record (and
+ * the ticket itself, whose mint time covers the moment before it is marked).
+ * unknown no record: a mistyped or expired handle.
+ * submitting the submit is still running server-side.
+ * job the submit was accepted; poll `jobId` for the model.
+ * done the submit finished the model outright; `body` is its frame.
+ * failed the submit was refused, errored, or died; `message` says why.
+ */
+export function resolveTicket(record, now = Date.now(), ticket = null) {
+ if (!record || typeof record !== 'object') {
+ const mintedAt = ticketMintedAt(ticket);
+ if (mintedAt != null && now - mintedAt >= 0 && now - mintedAt < TICKET_MARK_GRACE_MS) {
+ return { kind: 'submitting', elapsedMs: now - mintedAt };
+ }
+ return { kind: 'unknown' };
+ }
+ if (record.state === 'submitting') {
+ const elapsedMs = Math.max(0, now - Number(record.at || now));
+ if (elapsedMs > TICKET_STALE_MS) {
+ return { kind: 'failed', message: `The generation request never completed. ${START_AGAIN}` };
+ }
+ return { kind: 'submitting', elapsedMs };
+ }
+ const status = Number(record.status);
+ const body = record.body && typeof record.body === 'object' ? record.body : {};
+ if (status >= 200 && status < 300) {
+ if (body.status === 'done' && body.glb_url) return { kind: 'done', body };
+ if (typeof body.job_id === 'string' && body.job_id) return { kind: 'job', jobId: body.job_id };
+ }
+ const reason = typeof body.message === 'string' && body.message.trim() ? body.message.trim() : 'The generation could not start.';
+ return { kind: 'failed', message: /start a new generation/i.test(reason) ? reason : `${reason.replace(/[.\s]*$/, '.')} ${START_AGAIN}` };
+}
diff --git a/api/_lib/llm.js b/api/_lib/llm.js
index 4526ab533..4b2881b16 100644
--- a/api/_lib/llm.js
+++ b/api/_lib/llm.js
@@ -241,6 +241,22 @@ export function resetLlmCooldowns() {
COOLDOWNS.clear();
}
+// Some hosts answer an exhausted key with HTTP 200 and put the account notice in
+// message.content, so the chain took it as the answer. Production forge rows
+// from 2026-09-12 on carry "The API key used for this request has reached its
+// budget. Please [raise the key budget](https://enter.pollinations.ai/...)" as
+// their prompt: the prompt director accepted the notice, and a 3D model was
+// reconstructed from it. Matched narrowly: a short reply whose whole content is
+// a key or account running out of budget, quota or credits, not any answer that
+// happens to discuss API keys.
+const ACCOUNT_NOTICE_MAX_CHARS = 600;
+const ACCOUNT_NOTICE = /\b(?:api key|key|account|token)\b[^.]{0,80}?\b(?:has reached|reached|exceeded|exhausted|run out of|ran out of|is out of|insufficient)\b[^.]{0,40}?\b(?:budget|quota|credits?|balance|spend(?:ing)? limit|usage limit)\b/i;
+
+export function isProviderAccountNotice(text) {
+ const t = String(text || '').trim();
+ return t.length > 0 && t.length <= ACCOUNT_NOTICE_MAX_CHARS && ACCOUNT_NOTICE.test(t);
+}
+
// Per-provider wall clock for one rung of the chain. Shared so a rung that wants
// "the same budget as everyone else" cannot drift from the number llmComplete
// actually enforces.
@@ -1017,6 +1033,14 @@ export async function llmComplete({ system, user, maxTokens = 1024, anthropicKey
attempts.push({ provider: p.name, ms: Date.now() - startedAt, error: 'empty completion' });
continue;
}
+ if (isProviderAccountNotice(text)) {
+ // A billing or quota notice delivered as the completion itself. It is a
+ // failed call wearing a 200, so it fails over like one and is never
+ // returned: a caller cannot tell it from an answer.
+ lastErr = Object.assign(new Error(`${p.name} answered with an account notice instead of a completion`), { status: 502, code: 'provider_notice' });
+ attempts.push({ provider: p.name, ms: Date.now() - startedAt, error: 'account notice as completion' });
+ continue;
+ }
COOLDOWNS.delete(rungKey(p));
recordLlmSpend(p, usage, Date.now() - startedAt, track);
return {
diff --git a/api/_mcp-studio/gpt-forge-client.js b/api/_mcp-studio/gpt-forge-client.js
index 1cbd682a6..630d5f0d6 100644
--- a/api/_mcp-studio/gpt-forge-client.js
+++ b/api/_mcp-studio/gpt-forge-client.js
@@ -18,6 +18,7 @@
import { env } from '../_lib/env.js';
import { watsonxConfig, watsonxChatComplete } from '../_lib/watsonx.js';
import { llmComplete } from '../_lib/llm.js';
+import { TICKET_HEADER, newTicket, ticketHandle } from '../_lib/forge-submit-ticket.js';
const DEFAULT_TIMEOUT_MS = 180_000;
const DEFAULT_POLL_MS = 3_000;
@@ -30,10 +31,13 @@ const RIG_SUBMIT_TIMEOUT_MS = 30_000;
// the deadline: without an accepted job there is nothing to hand back, and the
// surface's budget leaves room for it under the host's limit.
const SUBMIT_FLOOR_MS = 8_000;
+// A ticketed submit has a handle to hand back whatever happens, so it needs no
+// room past the deadline, only enough to get the request onto the wire.
+const TICKET_SUBMIT_FLOOR_MS = 3_000;
-function submitWindow(deadline, capMs) {
+function submitWindow(deadline, capMs, floorMs = SUBMIT_FLOOR_MS) {
if (!deadline) return capMs;
- return Math.min(capMs, Math.max(SUBMIT_FLOOR_MS, deadline - Date.now()));
+ return Math.min(capMs, Math.max(floorMs, deadline - Date.now()));
}
function envNum(key, def) {
@@ -114,7 +118,19 @@ function internalHeaders() {
// and silently received standard has been overcharged, and the seller side
// settles on success, so refusing here is what keeps their money in their
// wallet.
-export async function startForge(base, { prompt, imageUrls, aspect, backend, path, tier, internal, strictTier = false }, { deadline } = {}) {
+//
+// `director: false` tells the server not to run its own prompt director: the
+// studio tools direct the prompt before they submit, and a second pass rewrote
+// an already-directed brief (with the mesh director, even for an avatar) while
+// spending up to another 15 s of the caller's budget.
+//
+// A bounded call (`deadline`) sends a submit ticket (see
+// ../_lib/forge-submit-ticket.js). If the submit is still running at the
+// deadline, it resolves to the ticket's handle, `{ status: 'submitting',
+// job_id: 't1.…' }`, instead of a timeout: the server records the job under the
+// ticket when it lands, and polling that handle collects it.
+export async function startForge(base, { prompt, imageUrls, aspect, backend, path, tier, internal, director, strictTier = false }, { deadline } = {}) {
+ const ticket = deadline ? newTicket() : null;
const attempt = async (tierId, withInternal) => {
const payload = {
...(prompt ? { prompt } : {}),
@@ -123,14 +139,19 @@ export async function startForge(base, { prompt, imageUrls, aspect, backend, pat
...(backend ? { backend } : {}),
...(path ? { path } : {}),
...(tierId ? { tier: tierId } : {}),
+ ...(director === false ? { director: false } : {}),
};
let res;
try {
res = await fetch(`${base}/api/gpt-forge`, {
method: 'POST',
- headers: { 'content-type': 'application/json', ...(withInternal ? internalHeaders() : {}) },
+ headers: {
+ 'content-type': 'application/json',
+ ...(withInternal ? internalHeaders() : {}),
+ ...(ticket ? { [TICKET_HEADER]: ticket } : {}),
+ },
body: JSON.stringify(payload),
- signal: AbortSignal.timeout(submitWindow(deadline, SUBMIT_TIMEOUT_MS)),
+ signal: AbortSignal.timeout(submitWindow(deadline, SUBMIT_TIMEOUT_MS, ticket ? TICKET_SUBMIT_FLOOR_MS : SUBMIT_FLOOR_MS)),
});
} catch (err) {
if (err?.name === 'TimeoutError' || err?.name === 'AbortError')
@@ -147,6 +168,10 @@ export async function startForge(base, { prompt, imageUrls, aspect, backend, pat
({ res, data } = await attempt(tier, !!internal));
} catch (err) {
if (err?.code !== 'timeout') throw err;
+ // The submit is still running server-side and will record its job under
+ // the ticket, so hand back the ticket's handle rather than submit a
+ // second job or report a timeout.
+ if (ticket) return { status: 'submitting', job_id: ticketHandle(ticket) };
// A bounded call that spent its budget on the first submit has no time
// left for a second one; the caller reports the timeout instead.
if (deadline && deadline - Date.now() < SUBMIT_FLOOR_MS) throw err;
@@ -284,6 +309,8 @@ export async function pollOnce(base, jobId) {
export async function generate(base, submitArgs, { timeoutEnv, deadline } = {}) {
const job = await startForge(base, submitArgs, { deadline });
if (job.status === 'done' && job.glb_url) return job;
+ // The submit outlived the call budget: its ticket handle is the job id.
+ if (job.status === 'submitting') return { _timedOut: true, job_id: job.job_id };
const out = await pollJob(base, job.job_id, {
timeoutMs: timeoutEnv ? envNum(timeoutEnv, DEFAULT_TIMEOUT_MS) : DEFAULT_TIMEOUT_MS,
intervalMs: envNum('STUDIO_POLL_MS', DEFAULT_POLL_MS),
@@ -351,6 +378,11 @@ export function isUsableDirectorRewrite(refined, rawPrompt) {
const text = refined.trim();
if (text.length < 3 || text.length > DIRECTOR_MAX_CHARS) return false;
if (!ENDS_COMPLETE.test(text)) return false;
+ // A brief describes an object; it never carries a link. A URL or a markdown
+ // link means the reply is a provider notice ("raise the key budget at
+ // https://...") or chatter, and reconstructing a mesh from it is how 22
+ // production generations came out of a billing message.
+ if (/https?:\/\/|\]\(|\bwww\./i.test(text)) return false;
// The director's contract is to ENRICH a rough idea into a denser spec. A
// result no longer than what the caller typed has added nothing, and is more
// likely a clipped opening clause than a genuine tightening, so the user's
diff --git a/api/gpt-forge.js b/api/gpt-forge.js
index 74b0ff90a..854fb35d1 100644
--- a/api/gpt-forge.js
+++ b/api/gpt-forge.js
@@ -130,6 +130,15 @@ import {
import { markProviderCooldown, providersInCooldown } from './_lib/provider-health.js';
import { acquireLock, releaseLock, cacheGet, cacheSet } from './_lib/cache.js';
import { recallDoneFrame, rememberDoneFrame } from './_lib/forge-done-cache.js';
+import {
+ captureTicketOutcome,
+ markTicketSubmitting,
+ recallTicket,
+ resolveTicket,
+ ticketFromHandle,
+ ticketFromRequest,
+ ticketHandle,
+} from './_lib/forge-submit-ticket.js';
import { sanitizeJobError } from './_lib/provider-job-error.js';
import { normalizeForgeOptions, providerReconstructParams, summarizeForgeOptions } from './_lib/forge-options.js';
import { bindJobToOptions, optionsForJob } from './_lib/forge-job-options.js';
@@ -2777,6 +2786,29 @@ async function pollGcpStatus({ gcp, upstreamId, clientKey, createdAt }) {
return result; // genuinely orphaned → existing failover path acts on it
}
+// Poll a submit-ticket handle (`t1.<ticket>`): the job a bounded caller could
+// not wait for. Once the submit has answered, the ticket resolves to its real
+// job and the poll is served exactly as a poll of that job would be.
+async function pollTicket(req, res, ticket) {
+ const out = resolveTicket(await recallTicket(ticket), Date.now(), ticket);
+ if (out.kind === 'job') return pollJob(req, res, out.jobId);
+ const rl = await limits.mcp3dStatus(clientIp(req));
+ if (!rl.success) return rateLimited(res, rl);
+ const handle = ticketHandle(ticket);
+ if (out.kind === 'done') return json(res, 200, { ...out.body, job_id: handle });
+ if (out.kind === 'submitting') {
+ return json(res, 200, {
+ job_id: handle,
+ status: 'queued',
+ stage: 'submit',
+ elapsed_seconds: Math.round(out.elapsedMs / 1000),
+ message: 'The generation request is still being accepted.',
+ });
+ }
+ if (out.kind === 'failed') return json(res, 200, { job_id: handle, status: 'failed', error: out.message });
+ return json(res, 404, { error: 'unknown_job', message: 'That job id is not recognized (it may be mistyped or expired).' });
+}
+
async function pollJob(req, res, jobId) {
// A job handle is either a bare Replicate prediction id (legacy / image-
// TRELLIS path) or a forge token encoding the geometry/GCP provider + the
@@ -3136,6 +3168,14 @@ export default wrap(async (req, res) => {
if ((url.searchParams.get('action') || '').trim() === 'rig') {
return startRigJob(req, res);
}
+ // A bounded caller (the ChatGPT surface) sends a ticket so a submit that
+ // outlives its call budget still yields a job it can collect. See
+ // _lib/forge-submit-ticket.js.
+ const ticket = ticketFromRequest(req);
+ if (ticket) {
+ await markTicketSubmitting(ticket).catch((err) => console.warn(`[forge] submit ticket mark failed: ${err?.message || err}`));
+ captureTicketOutcome(res, ticket);
+ }
return startJob(req, res);
}
@@ -3175,5 +3215,7 @@ export default wrap(async (req, res) => {
if (!jobId) {
return json(res, 400, { error: 'missing_job', message: 'Pass ?job=<id> to poll a job.' });
}
+ const ticket = ticketFromHandle(jobId);
+ if (ticket) return pollTicket(req, res, ticket);
return pollJob(req, res, jobId);
});