Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.
JavaScript 3d-generationai-agentsanimationavatardigital-humangenerative-aiglbgltfimage-to-3dllmmcpmcp-servermodel-context-protocolsolanatext-to-3dthreejsweb-componentswebglwebxrx402
fix(mcp-chatgpt): let a pending refinement, avatar or slow status check still finish in the ChatGPT viewer
nirholas79803cdf9a6d
diff --git a/api/_mcp-studio/component.js b/api/_mcp-studio/component.jsindex 5a6a58491..d87563c45 100644--- a/api/_mcp-studio/component.js+++ b/api/_mcp-studio/component.js@@ -331,7 +331,7 @@ export const COMPONENT_HTML = `<!doctype html>stopPolling();fail('This model is taking much longer than usual. It is still running, and you can keep waiting for it.', {title: 'Still rendering',- retry: function () { beginPolling(resume.jobId, resume.stage, resume.next, resume.origin, Date.now()); },+ retry: function () { beginPolling(resume.jobId, resume.stage, resume.next, resume.origin, Date.now(), resume.refine, resume.prompt); },});return;}@@ -341,52 +341,65 @@ export const COMPONENT_HTML = `<!doctype html>function rigThenFinish(mesh) {var p = poll;+ p.prompt = p.prompt || mesh.prompt;p.stage = 'rig';p.next = null;+ p.refine = null;paintPending();window.openai.callTool('rig_mesh', { glb_url: mesh.glbUrl }).then(function (r) {if (poll !== p) return;var out = structuredOf(r);if (out && out.status === 'pending' && out.jobId) { p.jobId = out.jobId; p.delay = POLL_FIRST_MS; schedule(); return; }- if (out && out.glbUrl) { finish(out); return; }+ if (out && out.glbUrl) { finish(Object.assign({}, out, { prompt: p.prompt || out.prompt })); return; }// The mesh is real and saved; a failed rig must not throw it away.finish(mesh);}, function () { if (poll === p) finish(mesh); });}+ // A check that failed while the job itself is fine (timeout, busy) is retried+ // on the normal cadence; only repeated failures surface, and even then the+ // user can resume.+ function retryLater(p) {+ if (++p.failures >= MAX_POLL_FAILURES) {+ var resume = p;+ stopPolling();+ fail('Lost contact with the job while it was rendering. It keeps running, so checking again usually finds it.', {+ title: 'Connection interrupted',+ retry: function () { beginPolling(resume.jobId, resume.stage, resume.next, resume.origin, resume.since, resume.refine, resume.prompt); },+ });+ return;+ }+ schedule();+ }+function tick() {var p = poll;if (!p) return;- window.openai.callTool('check_job', { job_id: p.jobId }).then(function (r) {+ var args = { job_id: p.jobId };+ if (p.refine) args.refine = p.refine;+ window.openai.callTool('check_job', args).then(function (r) {if (poll !== p) return;- p.failures = 0;var out = structuredOf(r);+ if (out && out.error && out.retryable) { retryLater(p); return; }+ p.failures = 0;if (out && out.status === 'pending') { paintPending(); schedule(); return; }if (out && out.glbUrl) {if (p.next === 'rig' && !out.rigged) { rigThenFinish(out); return; }- finish(p.stage === 'rig' ? Object.assign({}, out, { rigged: true }) : out);+ // A rig job knows only its input mesh, so its result carries the+ // rigger's own label; keep the words the user asked for.+ finish(p.stage === 'rig' ? Object.assign({}, out, { rigged: true, prompt: p.prompt || out.prompt }) : out);return;}stopPolling();fail((out && out.message) || 'Generation did not return a model. Try generating it again.');}, function () {- if (poll !== p) return;- if (++p.failures >= MAX_POLL_FAILURES) {- var resume = p;- stopPolling();- fail('Lost contact with the job while it was rendering. It keeps running, so checking again usually finds it.', {- title: 'Connection interrupted',- retry: function () { beginPolling(resume.jobId, resume.stage, resume.next, resume.origin, resume.since); },- });- return;- }- schedule();+ if (poll === p) retryLater(p);});}- function beginPolling(jobId, stage, next, origin, since) {+ function beginPolling(jobId, stage, next, origin, since, refine, prompt) {stopPolling();- poll = { jobId: jobId, origin: origin || jobId, stage: stage, next: next, since: since, delay: POLL_FIRST_MS, failures: 0, timer: null, ticker: null };+ poll = { jobId: jobId, origin: origin || jobId, stage: stage, next: next, refine: refine || null, prompt: prompt || null, since: since, delay: POLL_FIRST_MS, failures: 0, timer: null, ticker: null };mv.classList.add('veiled'); bar.classList.add('hidden'); versionsEl.classList.add('hidden');paintPending();show(loading);@@ -399,7 +412,7 @@ export const COMPONENT_HTML = `<!doctype html>if (poll && poll.origin === out.jobId) return;var elapsed = Number(out.elapsedSeconds);var since = Date.now() - (isFinite(elapsed) && elapsed > 0 ? elapsed * 1000 : 0);- if (canCallTools()) { beginPolling(out.jobId, out.stage, out.next, out.jobId, since); return; }+ if (canCallTools()) { beginPolling(out.jobId, out.stage, out.next, out.jobId, since, out.refine, out.prompt); return; }// A host without widget tool calls: say what is happening and how to collect it.mv.classList.add('veiled'); bar.classList.add('hidden'); versionsEl.classList.add('hidden');loadingTitle.textContent = 'Still rendering your 3D model…';diff --git a/api/_mcp-studio/tools.js b/api/_mcp-studio/tools.jsindex 77779b1f5..3ef10e9f7 100644--- a/api/_mcp-studio/tools.js+++ b/api/_mcp-studio/tools.js@@ -139,10 +139,12 @@ function ok({ glbUrl, base, kind, prompt, rigged, referenceImageUrl }) {};}-function toolError(message) {+// `extra` carries machine-readable facts about the failure, such as+// `retryable: true` when the job is fine and only the check itself failed.+function toolError(message, extra = {}) {return {content: [{ type: 'text', text: message }],- structuredContent: { error: true, message },+ structuredContent: { error: true, message, ...extra },isError: true,};}@@ -172,7 +174,12 @@ function pendingTiming(job) {// ran out of budget before the rig stage). The viewer widget reads it and runs// rig_mesh itself once the mesh lands, so the user still ends with the rigged// avatar they asked for rather than a bare mesh.-function pendingResult({ base, jobId, what, prompt, etaRemainingSeconds, stage = 'mesh', cold = null, next = null }) {+//+// `refine` rides along on a pending refinement: the version history the new+// model joins. A client hands it back to check_job, which appends the finished+// version the same way refine_model would have, so the version strip survives+// the wait.+function pendingResult({ base, jobId, what, prompt, etaRemainingSeconds, stage = 'mesh', cold = null, next = null, refine = null }) {// The ChatGPT pipeline's own endpoint, not /api/forge: the whole point of// the clone is that this surface can evolve independently.const pollUrl = `${base}/api/gpt-forge?job=${encodeURIComponent(jobId)}`;@@ -234,6 +241,7 @@ function pendingResult({ base, jobId, what, prompt, etaRemainingSeconds, stage =: {}),...(elapsed != null ? { elapsedSeconds: elapsed } : {}),...(prompt ? { prompt } : {}),+ ...(refine ? { refine } : {}),},};}@@ -355,6 +363,17 @@ function looksNonHumanoid(prompt) {// and a coffee cup gains nothing from the portrait lane.const AVATAR_TIER = 'high';+// The avatar director is fail-soft, and when it fails the raw words ("a knight+// in silver plate armor") go to a portrait-leaning lane that answers with a+// head-and-shoulders bust: a model that cannot be rigged into anything posable.+// The fallback therefore still states the framing the director would have, so+// an outage on the LLM chain costs polish, never the body.+export function avatarFallbackBrief(prompt, subject) {+ return subject === 'animal'+ ? `${prompt}, full body, the whole animal in frame from head to tail and feet, standing, centered on a plain neutral background`+ : `${prompt}, full-body character standing in a neutral A-pose, arms slightly away from the body, legs slightly apart, the entire figure in frame head to toe, centered on a plain neutral background`;+}+// ── handlers ────────────────────────────────────────────────────────────────async function handleForgeFree(args, _auth, req, ctx = {}) {@@ -428,7 +447,7 @@ async function handleTextToAvatar(args, _auth, req, ctx = {}) {if (prompt && !imageUrl) {const subject = classifySubject(prompt) === 'animal' ? 'animal' : 'person';const directed = await directPrompt(avatarDirectorFor(subject), prompt);- if (directed) effective = directed;+ effective = directed || avatarFallbackBrief(prompt, subject);}let job;try {@@ -541,7 +560,7 @@ async function handleForgeAvatar(args, _auth, req, ctx = {}) {if (prompt && !imageUrl) {const subject = classifySubject(prompt) === 'animal' ? 'animal' : 'person';const directed = await directPrompt(avatarDirectorFor(subject), prompt);- if (directed) effective = directed;+ effective = directed || avatarFallbackBrief(prompt, subject);}let gen;try {@@ -587,6 +606,25 @@ async function handleForgeAvatar(args, _auth, req, ctx = {}) {return ok({ glbUrl: rigged.glb_url, base, kind: 'avatar', prompt: prompt || undefined, rigged: true, referenceImageUrl: gen.preview_image_url });}+// Rebuild a client-supplied lineage (refine_model's parent_lineage, or the+// `refine.lineage` a pending refinement handed out) into the internal shape. It+// is UNTRUSTED: buildLineageChain checks contiguous indices, a single root and no+// cycles, and a malformed array returns null so the caller starts fresh rather+// than corrupting history.+function rehydrateLineage(clientLineage) {+ if (!Array.isArray(clientLineage) || clientLineage.length === 0) return null;+ const rehydrated = clientLineage.map((v, i) => ({+ index: Number.isInteger(v?.index) ? v.index : i,+ parentIndex: v?.parentIndex ?? (i > 0 ? i - 1 : null),+ glbUrl: v?.glbUrl,+ viewerUrl: v?.viewerUrl || null,+ prompt: v?.prompt || null,+ instruction: v?.instruction || null,+ refKind: v?.refKind || (i === 0 ? 'origin' : 'text'),+ }));+ return buildLineageChain(rehydrated).ok ? rehydrated : null;+}+// Conversational refinement — carry a prior model forward with a natural-language// change ("make it metallic", "bigger helmet"). REAL anchored re-generation: the// parent prompt is folded into the new prompt so form/subject/materials carry@@ -624,23 +662,9 @@ async function handleRefineModel(args, _auth, req, ctx = {}) {// root, no cycles) with buildLineageChain before extending it. A malformed// array (buggy client, tampering) falls back to a fresh lineage rooted at the// parent model rather than corrupting history.- const freshLineage = () => seedLineage({ glbUrl, viewerUrl: viewerUrl(base, glbUrl), prompt: parentPrompt || null });- const clientLineage = Array.isArray(args.parent_lineage) ? args.parent_lineage : null;- let baseLineage;- if (clientLineage && clientLineage.length > 0) {- const rehydrated = clientLineage.map((v, i) => ({- index: Number.isInteger(v.index) ? v.index : i,- parentIndex: v.parentIndex ?? (i > 0 ? i - 1 : null),- glbUrl: v.glbUrl,- viewerUrl: v.viewerUrl || null,- prompt: v.prompt || null,- instruction: v.instruction || null,- refKind: v.refKind || (i === 0 ? 'origin' : 'text'),- }));- baseLineage = buildLineageChain(rehydrated).ok ? rehydrated : freshLineage();- } else {- baseLineage = freshLineage();- }+ const baseLineage =+ rehydrateLineage(args.parent_lineage) ||+ seedLineage({ glbUrl, viewerUrl: viewerUrl(base, glbUrl), prompt: parentPrompt || null });// Branch point: refine off an earlier version instead of the leaf. branchFrom// validates the index against the lineage; an out-of-range index falls back to@@ -673,6 +697,12 @@ async function handleRefineModel(args, _auth, req, ctx = {}) {what: 'refined model',prompt: composed || undefined,...pendingTiming(job),+ refine: {+ lineage: baseLineage,+ instruction,+ refKind: refImageUrl ? 'image' : 'text',+ ...(parentIndex !== undefined ? { parentIndex } : {}),+ },});if (job._timedOut || !job.glb_url) return toolError('Refinement is taking longer than expected. Please try again.');@@ -687,6 +717,33 @@ async function handleRefineModel(args, _auth, req, ctx = {}) {return refineOk({ glbUrl: job.glb_url, base, prompt: composed, instruction, lineage, activeIndex: lineage.length - 1 });}+// A refinement collected by check_job joins the version history its pending+// result carried, exactly as refine_model appends it inline. Anything malformed+// falls back to the plain model envelope; the model itself is never lost.+function finishRefinement(base, data, refine) {+ const instruction = typeof refine?.instruction === 'string' ? refine.instruction.trim() : '';+ const lineage = instruction ? rehydrateLineage(refine.lineage) : null;+ if (!lineage) return null;+ let parentIndex;+ if (Number.isInteger(refine.parentIndex)) {+ try {+ parentIndex = branchFrom(lineage, refine.parentIndex);+ } catch {+ parentIndex = undefined;+ }+ }+ const prompt = typeof data.prompt === 'string' && data.prompt ? data.prompt : null;+ const next = appendVersion(lineage, {+ glbUrl: data.glb_url,+ viewerUrl: viewerUrl(base, data.glb_url),+ prompt,+ instruction,+ refKind: refine.refKind === 'image' ? 'image' : 'text',+ ...(parentIndex !== undefined ? { parentIndex } : {}),+ });+ return refineOk({ glbUrl: data.glb_url, base, prompt: prompt || undefined, instruction, lineage: next, activeIndex: next.length - 1 });+}+// Collect a generation that outlived a tool call's inline wait. One status// probe, no loop: done renders the full result envelope in the widget, still// running returns a fresh pending envelope with updated timing, failed returns@@ -700,9 +757,15 @@ async function handleCheckJob(args, _auth, req) {try {data = await pollOnce(base, jobId);} catch (err) {- return toolError(failureMessage(err));+ // Only an unrecognized handle is final. A timeout, a busy status bucket or+ // an upstream blip leaves the job running, and the first check of a+ // finished job does the slow save-and-score work, so checking again+ // usually returns the model at once.+ return toolError(failureMessage(err), err?.code === 'unknown_job' ? {} : { retryable: true });}if (data.status === 'done' && data.glb_url) {+ const refined = finishRefinement(base, data, args.refine);+ if (refined) return refined;return ok({glbUrl: data.glb_url,base,@@ -983,6 +1046,19 @@ const DEFS = [maxLength: 4096,description: 'The job_id (or jobId) a pending generation returned.',},+ refine: {+ type: 'object',+ additionalProperties: true,+ description:+ 'Optional: the refine object a pending refine_model result carried. Pass it back unchanged so the ' ++ 'finished model joins that version history.',+ properties: {+ lineage: { type: 'array', items: { type: 'object', additionalProperties: true } },+ instruction: { type: 'string', maxLength: 500 },+ parentIndex: { type: 'integer', minimum: 0 },+ refKind: { type: 'string', enum: ['text', 'image'] },+ },+ },},},annotations: {diff --git a/tests/fixtures/mcp-golden-tools.json b/tests/fixtures/mcp-golden-tools.jsonindex 8f548339d..d802fbdd8 100644--- a/tests/fixtures/mcp-golden-tools.json+++ b/tests/fixtures/mcp-golden-tools.json@@ -37,7 +37,7 @@"idempotentHint": false,"openWorldHint": true},- "schemaHash": "06fcfd29d4c4"+ "schemaHash": "5ec2056d2880"},{"name": "forge_avatar",diff --git a/tests/mcp-chatgpt-budget.test.js b/tests/mcp-chatgpt-budget.test.jsindex f1c5e40cf..15e855bf9 100644--- a/tests/mcp-chatgpt-budget.test.js+++ b/tests/mcp-chatgpt-budget.test.js@@ -132,3 +132,91 @@ describe('viewer widget collects pending jobs', () => {expect(COMPONENT_HTML).toContain('canCallTools()');});});++describe('check_job keeps a pending job collectable', () => {+ function jsonFetch(status, body) {+ return vi.fn(async () => ({ ok: status < 400, status, json: async () => body }));+ }++ it('flags a failed check as retryable while the job itself is fine', async () => {+ const timeout = Object.assign(new Error('timed out'), { name: 'TimeoutError' });+ globalThis.fetch = vi.fn(async () => {+ throw timeout;+ });+ const r = await dispatch(callMsg('check_job', { job_id: 'job-slow-first-done' }), auth, req, { surface: 'chatgpt' });+ expect(r.result.isError).toBe(true);+ expect(r.result.structuredContent.retryable).toBe(true);+ });++ it('does not ask for a retry on a handle that will never resolve', async () => {+ globalThis.fetch = jsonFetch(404, { message: 'unknown job' });+ const r = await dispatch(callMsg('check_job', { job_id: 'job-expired-1' }), auth, req, { surface: 'chatgpt' });+ expect(r.result.isError).toBe(true);+ expect(r.result.structuredContent.retryable).toBeUndefined();+ });++ it('hands a pending refinement its version history', async () => {+ vi.mocked(generate).mockResolvedValue({ _timedOut: true, job_id: 'job-refine-1', status: 'running' });+ const r = await dispatch(+ callMsg('refine_model', { glb_url: 'https://three.ws/cdn/a.glb', parent_prompt: 'a brass telescope', instruction: 'make it tarnished' }),+ auth,+ req,+ { surface: 'chatgpt' },+ );+ const sc = r.result.structuredContent;+ expect(sc.status).toBe('pending');+ expect(sc.refine.instruction).toBe('make it tarnished');+ expect(sc.refine.lineage).toHaveLength(1);+ expect(sc.refine.lineage[0].glbUrl).toBe('https://three.ws/cdn/a.glb');+ });++ it('appends the collected model to that history, as refine_model would inline', async () => {+ vi.mocked(generate).mockResolvedValue({ _timedOut: true, job_id: 'job-refine-2', status: 'running' });+ const pending = await dispatch(+ callMsg('refine_model', { glb_url: 'https://three.ws/cdn/a.glb', parent_prompt: 'a brass telescope', instruction: 'make it tarnished' }),+ auth,+ req,+ { surface: 'chatgpt' },+ );+ globalThis.fetch = jsonFetch(200, { status: 'done', glb_url: 'https://three.ws/cdn/b.glb', prompt: 'a brass telescope, tarnished' });+ const done = await dispatch(+ callMsg('check_job', { job_id: 'job-refine-2', refine: pending.result.structuredContent.refine }),+ auth,+ req,+ { surface: 'chatgpt' },+ );+ const sc = done.result.structuredContent;+ expect(sc.kind).toBe('refined model');+ expect(sc.glbUrl).toBe('https://three.ws/cdn/b.glb');+ expect(sc.lineage.map((v) => v.label)).toEqual(['Original', 'make it tarnished']);+ expect(sc.activeIndex).toBe(1);+ });++ it('falls back to the plain model when the refine context is malformed', async () => {+ globalThis.fetch = jsonFetch(200, { status: 'done', glb_url: 'https://three.ws/cdn/c.glb' });+ const done = await dispatch(+ callMsg('check_job', { job_id: 'job-refine-3', refine: { instruction: 'x', lineage: [{ index: 5, parentIndex: 9 }] } }),+ auth,+ req,+ { surface: 'chatgpt' },+ );+ expect(done.result.structuredContent.glbUrl).toBe('https://three.ws/cdn/c.glb');+ expect(done.result.structuredContent.lineage).toBeUndefined();+ });+});++describe('avatar prompt when the director is down', () => {+ it('still asks for a full body, so the rigger gets a figure and not a bust', async () => {+ vi.mocked(generate).mockResolvedValue({ status: 'done', glb_url: 'https://three.ws/cdn/k.glb' });+ vi.mocked(generate).mockResolvedValueOnce({ _timedOut: true, job_id: 'job-knight', status: 'running' });+ await dispatch(callMsg('forge_avatar', { prompt: 'a knight in silver plate armor' }), auth, req, { surface: 'chatgpt' });+ const sent = vi.mocked(generate).mock.calls[0][1].prompt;+ expect(sent).toMatch(/^a knight in silver plate armor, full-body character/);+ expect(sent).toMatch(/head to toe/);+ });++ it('frames an animal as a whole animal', async () => {+ const { avatarFallbackBrief } = await import('../api/_mcp-studio/tools.js');+ expect(avatarFallbackBrief('a red fox', 'animal')).toMatch(/whole animal in frame from head to tail/);+ });+});