Skip to content

Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.

fix(mcp-chatgpt): let a pending refinement, avatar or slow status check still finish in the ChatGPT viewer

nirholas79803cdf9a6d
diff --git a/api/_mcp-studio/component.js b/api/_mcp-studio/component.js
index 5a6a58491..d87563c45 100644
--- a/api/_mcp-studio/component.js
+++ b/api/_mcp-studio/component.js
@@ -331,7 +331,7 @@ export const COMPONENT_HTML = `<!doctype html>
stopPolling();
fail('This model is taking much longer than usual. It is still running, and you can keep waiting for it.', {
title: 'Still rendering',
- retry: function () { beginPolling(resume.jobId, resume.stage, resume.next, resume.origin, Date.now()); },
+ retry: function () { beginPolling(resume.jobId, resume.stage, resume.next, resume.origin, Date.now(), resume.refine, resume.prompt); },
});
return;
}
@@ -341,52 +341,65 @@ export const COMPONENT_HTML = `<!doctype html>
function rigThenFinish(mesh) {
var p = poll;
+ p.prompt = p.prompt || mesh.prompt;
p.stage = 'rig';
p.next = null;
+ p.refine = null;
paintPending();
window.openai.callTool('rig_mesh', { glb_url: mesh.glbUrl }).then(function (r) {
if (poll !== p) return;
var out = structuredOf(r);
if (out && out.status === 'pending' && out.jobId) { p.jobId = out.jobId; p.delay = POLL_FIRST_MS; schedule(); return; }
- if (out && out.glbUrl) { finish(out); return; }
+ if (out && out.glbUrl) { finish(Object.assign({}, out, { prompt: p.prompt || out.prompt })); return; }
// The mesh is real and saved; a failed rig must not throw it away.
finish(mesh);
}, function () { if (poll === p) finish(mesh); });
}
+ // A check that failed while the job itself is fine (timeout, busy) is retried
+ // on the normal cadence; only repeated failures surface, and even then the
+ // user can resume.
+ function retryLater(p) {
+ if (++p.failures >= MAX_POLL_FAILURES) {
+ var resume = p;
+ stopPolling();
+ fail('Lost contact with the job while it was rendering. It keeps running, so checking again usually finds it.', {
+ title: 'Connection interrupted',
+ retry: function () { beginPolling(resume.jobId, resume.stage, resume.next, resume.origin, resume.since, resume.refine, resume.prompt); },
+ });
+ return;
+ }
+ schedule();
+ }
+
function tick() {
var p = poll;
if (!p) return;
- window.openai.callTool('check_job', { job_id: p.jobId }).then(function (r) {
+ var args = { job_id: p.jobId };
+ if (p.refine) args.refine = p.refine;
+ window.openai.callTool('check_job', args).then(function (r) {
if (poll !== p) return;
- p.failures = 0;
var out = structuredOf(r);
+ if (out && out.error && out.retryable) { retryLater(p); return; }
+ p.failures = 0;
if (out && out.status === 'pending') { paintPending(); schedule(); return; }
if (out && out.glbUrl) {
if (p.next === 'rig' && !out.rigged) { rigThenFinish(out); return; }
- finish(p.stage === 'rig' ? Object.assign({}, out, { rigged: true }) : out);
+ // A rig job knows only its input mesh, so its result carries the
+ // rigger's own label; keep the words the user asked for.
+ finish(p.stage === 'rig' ? Object.assign({}, out, { rigged: true, prompt: p.prompt || out.prompt }) : out);
return;
}
stopPolling();
fail((out && out.message) || 'Generation did not return a model. Try generating it again.');
}, function () {
- if (poll !== p) return;
- if (++p.failures >= MAX_POLL_FAILURES) {
- var resume = p;
- stopPolling();
- fail('Lost contact with the job while it was rendering. It keeps running, so checking again usually finds it.', {
- title: 'Connection interrupted',
- retry: function () { beginPolling(resume.jobId, resume.stage, resume.next, resume.origin, resume.since); },
- });
- return;
- }
- schedule();
+ if (poll === p) retryLater(p);
});
}
- function beginPolling(jobId, stage, next, origin, since) {
+ function beginPolling(jobId, stage, next, origin, since, refine, prompt) {
stopPolling();
- poll = { jobId: jobId, origin: origin || jobId, stage: stage, next: next, since: since, delay: POLL_FIRST_MS, failures: 0, timer: null, ticker: null };
+ poll = { jobId: jobId, origin: origin || jobId, stage: stage, next: next, refine: refine || null, prompt: prompt || null, since: since, delay: POLL_FIRST_MS, failures: 0, timer: null, ticker: null };
mv.classList.add('veiled'); bar.classList.add('hidden'); versionsEl.classList.add('hidden');
paintPending();
show(loading);
@@ -399,7 +412,7 @@ export const COMPONENT_HTML = `<!doctype html>
if (poll && poll.origin === out.jobId) return;
var elapsed = Number(out.elapsedSeconds);
var since = Date.now() - (isFinite(elapsed) && elapsed > 0 ? elapsed * 1000 : 0);
- if (canCallTools()) { beginPolling(out.jobId, out.stage, out.next, out.jobId, since); return; }
+ if (canCallTools()) { beginPolling(out.jobId, out.stage, out.next, out.jobId, since, out.refine, out.prompt); return; }
// A host without widget tool calls: say what is happening and how to collect it.
mv.classList.add('veiled'); bar.classList.add('hidden'); versionsEl.classList.add('hidden');
loadingTitle.textContent = 'Still rendering your 3D model…';
diff --git a/api/_mcp-studio/tools.js b/api/_mcp-studio/tools.js
index 77779b1f5..3ef10e9f7 100644
--- a/api/_mcp-studio/tools.js
+++ b/api/_mcp-studio/tools.js
@@ -139,10 +139,12 @@ function ok({ glbUrl, base, kind, prompt, rigged, referenceImageUrl }) {
};
}
-function toolError(message) {
+// `extra` carries machine-readable facts about the failure, such as
+// `retryable: true` when the job is fine and only the check itself failed.
+function toolError(message, extra = {}) {
return {
content: [{ type: 'text', text: message }],
- structuredContent: { error: true, message },
+ structuredContent: { error: true, message, ...extra },
isError: true,
};
}
@@ -172,7 +174,12 @@ function pendingTiming(job) {
// ran out of budget before the rig stage). The viewer widget reads it and runs
// rig_mesh itself once the mesh lands, so the user still ends with the rigged
// avatar they asked for rather than a bare mesh.
-function pendingResult({ base, jobId, what, prompt, etaRemainingSeconds, stage = 'mesh', cold = null, next = null }) {
+//
+// `refine` rides along on a pending refinement: the version history the new
+// model joins. A client hands it back to check_job, which appends the finished
+// version the same way refine_model would have, so the version strip survives
+// the wait.
+function pendingResult({ base, jobId, what, prompt, etaRemainingSeconds, stage = 'mesh', cold = null, next = null, refine = null }) {
// The ChatGPT pipeline's own endpoint, not /api/forge: the whole point of
// the clone is that this surface can evolve independently.
const pollUrl = `${base}/api/gpt-forge?job=${encodeURIComponent(jobId)}`;
@@ -234,6 +241,7 @@ function pendingResult({ base, jobId, what, prompt, etaRemainingSeconds, stage =
: {}),
...(elapsed != null ? { elapsedSeconds: elapsed } : {}),
...(prompt ? { prompt } : {}),
+ ...(refine ? { refine } : {}),
},
};
}
@@ -355,6 +363,17 @@ function looksNonHumanoid(prompt) {
// and a coffee cup gains nothing from the portrait lane.
const AVATAR_TIER = 'high';
+// The avatar director is fail-soft, and when it fails the raw words ("a knight
+// in silver plate armor") go to a portrait-leaning lane that answers with a
+// head-and-shoulders bust: a model that cannot be rigged into anything posable.
+// The fallback therefore still states the framing the director would have, so
+// an outage on the LLM chain costs polish, never the body.
+export function avatarFallbackBrief(prompt, subject) {
+ return subject === 'animal'
+ ? `${prompt}, full body, the whole animal in frame from head to tail and feet, standing, centered on a plain neutral background`
+ : `${prompt}, full-body character standing in a neutral A-pose, arms slightly away from the body, legs slightly apart, the entire figure in frame head to toe, centered on a plain neutral background`;
+}
+
// ── handlers ────────────────────────────────────────────────────────────────
async function handleForgeFree(args, _auth, req, ctx = {}) {
@@ -428,7 +447,7 @@ async function handleTextToAvatar(args, _auth, req, ctx = {}) {
if (prompt && !imageUrl) {
const subject = classifySubject(prompt) === 'animal' ? 'animal' : 'person';
const directed = await directPrompt(avatarDirectorFor(subject), prompt);
- if (directed) effective = directed;
+ effective = directed || avatarFallbackBrief(prompt, subject);
}
let job;
try {
@@ -541,7 +560,7 @@ async function handleForgeAvatar(args, _auth, req, ctx = {}) {
if (prompt && !imageUrl) {
const subject = classifySubject(prompt) === 'animal' ? 'animal' : 'person';
const directed = await directPrompt(avatarDirectorFor(subject), prompt);
- if (directed) effective = directed;
+ effective = directed || avatarFallbackBrief(prompt, subject);
}
let gen;
try {
@@ -587,6 +606,25 @@ async function handleForgeAvatar(args, _auth, req, ctx = {}) {
return ok({ glbUrl: rigged.glb_url, base, kind: 'avatar', prompt: prompt || undefined, rigged: true, referenceImageUrl: gen.preview_image_url });
}
+// Rebuild a client-supplied lineage (refine_model's parent_lineage, or the
+// `refine.lineage` a pending refinement handed out) into the internal shape. It
+// is UNTRUSTED: buildLineageChain checks contiguous indices, a single root and no
+// cycles, and a malformed array returns null so the caller starts fresh rather
+// than corrupting history.
+function rehydrateLineage(clientLineage) {
+ if (!Array.isArray(clientLineage) || clientLineage.length === 0) return null;
+ const rehydrated = clientLineage.map((v, i) => ({
+ index: Number.isInteger(v?.index) ? v.index : i,
+ parentIndex: v?.parentIndex ?? (i > 0 ? i - 1 : null),
+ glbUrl: v?.glbUrl,
+ viewerUrl: v?.viewerUrl || null,
+ prompt: v?.prompt || null,
+ instruction: v?.instruction || null,
+ refKind: v?.refKind || (i === 0 ? 'origin' : 'text'),
+ }));
+ return buildLineageChain(rehydrated).ok ? rehydrated : null;
+}
+
// Conversational refinement — carry a prior model forward with a natural-language
// change ("make it metallic", "bigger helmet"). REAL anchored re-generation: the
// parent prompt is folded into the new prompt so form/subject/materials carry
@@ -624,23 +662,9 @@ async function handleRefineModel(args, _auth, req, ctx = {}) {
// root, no cycles) with buildLineageChain before extending it. A malformed
// array (buggy client, tampering) falls back to a fresh lineage rooted at the
// parent model rather than corrupting history.
- const freshLineage = () => seedLineage({ glbUrl, viewerUrl: viewerUrl(base, glbUrl), prompt: parentPrompt || null });
- const clientLineage = Array.isArray(args.parent_lineage) ? args.parent_lineage : null;
- let baseLineage;
- if (clientLineage && clientLineage.length > 0) {
- const rehydrated = clientLineage.map((v, i) => ({
- index: Number.isInteger(v.index) ? v.index : i,
- parentIndex: v.parentIndex ?? (i > 0 ? i - 1 : null),
- glbUrl: v.glbUrl,
- viewerUrl: v.viewerUrl || null,
- prompt: v.prompt || null,
- instruction: v.instruction || null,
- refKind: v.refKind || (i === 0 ? 'origin' : 'text'),
- }));
- baseLineage = buildLineageChain(rehydrated).ok ? rehydrated : freshLineage();
- } else {
- baseLineage = freshLineage();
- }
+ const baseLineage =
+ rehydrateLineage(args.parent_lineage) ||
+ seedLineage({ glbUrl, viewerUrl: viewerUrl(base, glbUrl), prompt: parentPrompt || null });
// Branch point: refine off an earlier version instead of the leaf. branchFrom
// validates the index against the lineage; an out-of-range index falls back to
@@ -673,6 +697,12 @@ async function handleRefineModel(args, _auth, req, ctx = {}) {
what: 'refined model',
prompt: composed || undefined,
...pendingTiming(job),
+ refine: {
+ lineage: baseLineage,
+ instruction,
+ refKind: refImageUrl ? 'image' : 'text',
+ ...(parentIndex !== undefined ? { parentIndex } : {}),
+ },
});
if (job._timedOut || !job.glb_url) return toolError('Refinement is taking longer than expected. Please try again.');
@@ -687,6 +717,33 @@ async function handleRefineModel(args, _auth, req, ctx = {}) {
return refineOk({ glbUrl: job.glb_url, base, prompt: composed, instruction, lineage, activeIndex: lineage.length - 1 });
}
+// A refinement collected by check_job joins the version history its pending
+// result carried, exactly as refine_model appends it inline. Anything malformed
+// falls back to the plain model envelope; the model itself is never lost.
+function finishRefinement(base, data, refine) {
+ const instruction = typeof refine?.instruction === 'string' ? refine.instruction.trim() : '';
+ const lineage = instruction ? rehydrateLineage(refine.lineage) : null;
+ if (!lineage) return null;
+ let parentIndex;
+ if (Number.isInteger(refine.parentIndex)) {
+ try {
+ parentIndex = branchFrom(lineage, refine.parentIndex);
+ } catch {
+ parentIndex = undefined;
+ }
+ }
+ const prompt = typeof data.prompt === 'string' && data.prompt ? data.prompt : null;
+ const next = appendVersion(lineage, {
+ glbUrl: data.glb_url,
+ viewerUrl: viewerUrl(base, data.glb_url),
+ prompt,
+ instruction,
+ refKind: refine.refKind === 'image' ? 'image' : 'text',
+ ...(parentIndex !== undefined ? { parentIndex } : {}),
+ });
+ return refineOk({ glbUrl: data.glb_url, base, prompt: prompt || undefined, instruction, lineage: next, activeIndex: next.length - 1 });
+}
+
// Collect a generation that outlived a tool call's inline wait. One status
// probe, no loop: done renders the full result envelope in the widget, still
// running returns a fresh pending envelope with updated timing, failed returns
@@ -700,9 +757,15 @@ async function handleCheckJob(args, _auth, req) {
try {
data = await pollOnce(base, jobId);
} catch (err) {
- return toolError(failureMessage(err));
+ // Only an unrecognized handle is final. A timeout, a busy status bucket or
+ // an upstream blip leaves the job running, and the first check of a
+ // finished job does the slow save-and-score work, so checking again
+ // usually returns the model at once.
+ return toolError(failureMessage(err), err?.code === 'unknown_job' ? {} : { retryable: true });
}
if (data.status === 'done' && data.glb_url) {
+ const refined = finishRefinement(base, data, args.refine);
+ if (refined) return refined;
return ok({
glbUrl: data.glb_url,
base,
@@ -983,6 +1046,19 @@ const DEFS = [
maxLength: 4096,
description: 'The job_id (or jobId) a pending generation returned.',
},
+ refine: {
+ type: 'object',
+ additionalProperties: true,
+ description:
+ 'Optional: the refine object a pending refine_model result carried. Pass it back unchanged so the ' +
+ 'finished model joins that version history.',
+ properties: {
+ lineage: { type: 'array', items: { type: 'object', additionalProperties: true } },
+ instruction: { type: 'string', maxLength: 500 },
+ parentIndex: { type: 'integer', minimum: 0 },
+ refKind: { type: 'string', enum: ['text', 'image'] },
+ },
+ },
},
},
annotations: {
diff --git a/tests/fixtures/mcp-golden-tools.json b/tests/fixtures/mcp-golden-tools.json
index 8f548339d..d802fbdd8 100644
--- a/tests/fixtures/mcp-golden-tools.json
+++ b/tests/fixtures/mcp-golden-tools.json
@@ -37,7 +37,7 @@
"idempotentHint": false,
"openWorldHint": true
},
- "schemaHash": "06fcfd29d4c4"
+ "schemaHash": "5ec2056d2880"
},
{
"name": "forge_avatar",
diff --git a/tests/mcp-chatgpt-budget.test.js b/tests/mcp-chatgpt-budget.test.js
index f1c5e40cf..15e855bf9 100644
--- a/tests/mcp-chatgpt-budget.test.js
+++ b/tests/mcp-chatgpt-budget.test.js
@@ -132,3 +132,91 @@ describe('viewer widget collects pending jobs', () => {
expect(COMPONENT_HTML).toContain('canCallTools()');
});
});
+
+describe('check_job keeps a pending job collectable', () => {
+ function jsonFetch(status, body) {
+ return vi.fn(async () => ({ ok: status < 400, status, json: async () => body }));
+ }
+
+ it('flags a failed check as retryable while the job itself is fine', async () => {
+ const timeout = Object.assign(new Error('timed out'), { name: 'TimeoutError' });
+ globalThis.fetch = vi.fn(async () => {
+ throw timeout;
+ });
+ const r = await dispatch(callMsg('check_job', { job_id: 'job-slow-first-done' }), auth, req, { surface: 'chatgpt' });
+ expect(r.result.isError).toBe(true);
+ expect(r.result.structuredContent.retryable).toBe(true);
+ });
+
+ it('does not ask for a retry on a handle that will never resolve', async () => {
+ globalThis.fetch = jsonFetch(404, { message: 'unknown job' });
+ const r = await dispatch(callMsg('check_job', { job_id: 'job-expired-1' }), auth, req, { surface: 'chatgpt' });
+ expect(r.result.isError).toBe(true);
+ expect(r.result.structuredContent.retryable).toBeUndefined();
+ });
+
+ it('hands a pending refinement its version history', async () => {
+ vi.mocked(generate).mockResolvedValue({ _timedOut: true, job_id: 'job-refine-1', status: 'running' });
+ const r = await dispatch(
+ callMsg('refine_model', { glb_url: 'https://three.ws/cdn/a.glb', parent_prompt: 'a brass telescope', instruction: 'make it tarnished' }),
+ auth,
+ req,
+ { surface: 'chatgpt' },
+ );
+ const sc = r.result.structuredContent;
+ expect(sc.status).toBe('pending');
+ expect(sc.refine.instruction).toBe('make it tarnished');
+ expect(sc.refine.lineage).toHaveLength(1);
+ expect(sc.refine.lineage[0].glbUrl).toBe('https://three.ws/cdn/a.glb');
+ });
+
+ it('appends the collected model to that history, as refine_model would inline', async () => {
+ vi.mocked(generate).mockResolvedValue({ _timedOut: true, job_id: 'job-refine-2', status: 'running' });
+ const pending = await dispatch(
+ callMsg('refine_model', { glb_url: 'https://three.ws/cdn/a.glb', parent_prompt: 'a brass telescope', instruction: 'make it tarnished' }),
+ auth,
+ req,
+ { surface: 'chatgpt' },
+ );
+ globalThis.fetch = jsonFetch(200, { status: 'done', glb_url: 'https://three.ws/cdn/b.glb', prompt: 'a brass telescope, tarnished' });
+ const done = await dispatch(
+ callMsg('check_job', { job_id: 'job-refine-2', refine: pending.result.structuredContent.refine }),
+ auth,
+ req,
+ { surface: 'chatgpt' },
+ );
+ const sc = done.result.structuredContent;
+ expect(sc.kind).toBe('refined model');
+ expect(sc.glbUrl).toBe('https://three.ws/cdn/b.glb');
+ expect(sc.lineage.map((v) => v.label)).toEqual(['Original', 'make it tarnished']);
+ expect(sc.activeIndex).toBe(1);
+ });
+
+ it('falls back to the plain model when the refine context is malformed', async () => {
+ globalThis.fetch = jsonFetch(200, { status: 'done', glb_url: 'https://three.ws/cdn/c.glb' });
+ const done = await dispatch(
+ callMsg('check_job', { job_id: 'job-refine-3', refine: { instruction: 'x', lineage: [{ index: 5, parentIndex: 9 }] } }),
+ auth,
+ req,
+ { surface: 'chatgpt' },
+ );
+ expect(done.result.structuredContent.glbUrl).toBe('https://three.ws/cdn/c.glb');
+ expect(done.result.structuredContent.lineage).toBeUndefined();
+ });
+});
+
+describe('avatar prompt when the director is down', () => {
+ it('still asks for a full body, so the rigger gets a figure and not a bust', async () => {
+ vi.mocked(generate).mockResolvedValue({ status: 'done', glb_url: 'https://three.ws/cdn/k.glb' });
+ vi.mocked(generate).mockResolvedValueOnce({ _timedOut: true, job_id: 'job-knight', status: 'running' });
+ await dispatch(callMsg('forge_avatar', { prompt: 'a knight in silver plate armor' }), auth, req, { surface: 'chatgpt' });
+ const sent = vi.mocked(generate).mock.calls[0][1].prompt;
+ expect(sent).toMatch(/^a knight in silver plate armor, full-body character/);
+ expect(sent).toMatch(/head to toe/);
+ });
+
+ it('frames an animal as a whole animal', async () => {
+ const { avatarFallbackBrief } = await import('../api/_mcp-studio/tools.js');
+ expect(avatarFallbackBrief('a red fox', 'animal')).toMatch(/whole animal in frame from head to tail/);
+ });
+});