Skip to content

Open source CapCut alternative.

TypeScript

fix(chatgpt): an editable project card is the inline editor from its first paint

David Lec6d04932be42
diff --git a/site/scripts/eval-chatgpt-widget.ts b/site/scripts/eval-chatgpt-widget.ts
index cea76c0e..1cf3f4b0 100644
--- a/site/scripts/eval-chatgpt-widget.ts
+++ b/site/scripts/eval-chatgpt-widget.ts
@@ -22,20 +22,24 @@ const hostHtml = `<!doctype html><html><body><iframe title="Donkey Cut preview"
const frame = document.querySelector('iframe');
const project = {id:'project',name:'Launch film',revision:'cloud:2',url:'https://donkeycut.com/app/p/project'};
let count = 0, renewals = 0;
-window.calls = []; window.links = []; window.modes = []; window.editable = false;
+window.calls = []; window.links = []; window.modes = []; window.editable = false; window.holdMs = 0; window.freshCard = false;
const view = (selected, preview = null) => ({view:selected?'project':'projects',projects:selected?[]:[project],nextCursor:null,project:selected?project:null,preview,canRender:true,canEdit:window.editable,export:null,job:null,results:[],changed:false,account:null});
// ChatGPT's sandbox drops null-valued keys before the widget sees a result.
const dropNulls = (value) => Array.isArray(value) ? value.map(dropNulls) : value && typeof value === 'object' ? Object.fromEntries(Object.entries(value).filter(([, v]) => v !== null).map(([k, v]) => [k, dropNulls(v)])) : value;
const result = (data, playback = null, editor = null) => dropNulls({content:[{type:'text',text:'Preview'}],structuredContent:data,_meta:{playback,editor,pollMs:1000}});
const reply = (id, result) => frame.contentWindow.postMessage({jsonrpc:'2.0',id,result},'*');
+// A card ChatGPT shows again after a reload replays its own result: a stale
+// preview with an expired playback link, and an editor link whose minute is
+// spent unless the fixture says the card was just opened.
+const cardResult = () => !window.editable ? result(view(false)) : result(view(true,{id:'job',status:'done',progress:1,revision:'cloud:1'}),{url:location.origin+'/preview.mp4?stale',expiresAt:Date.now()-1},window.freshCard ? {url:location.origin+'/embed?code=card&project=project',expiresAt:Date.now()+60000} : {url:location.origin+'/embed?code=spent&project=project',expiresAt:Date.now()-1});
window.addEventListener('message', ({source,data}) => {
if(source !== frame.contentWindow || !data.method) return;
if(data.method === 'ui/initialize') reply(data.id,{protocolVersion:data.params.protocolVersion,hostInfo:{name:'fixture',version:'1'},hostCapabilities:{serverTools:{},openLinks:{}},hostContext:{theme:'light',displayMode:'inline',availableDisplayModes:['inline','fullscreen'],safeAreaInsets:{top:0,right:0,bottom:120,left:0}}});
- if(data.method === 'ui/notifications/initialized') frame.contentWindow.postMessage({jsonrpc:'2.0',method:'ui/notifications/tool-result',params:window.editable ? result(view(true),null,{url:location.origin+'/embed?code=spent&project=project',expiresAt:Date.now()-1}) : result(view(false))},'*');
+ if(data.method === 'ui/notifications/initialized') frame.contentWindow.postMessage({jsonrpc:'2.0',method:'ui/notifications/tool-result',params:cardResult()},'*');
if(data.method === 'tools/call') {
const name = data.params.name; window.calls.push(name);
if(name === 'list_projects') reply(data.id,result(view(false)));
- if(name === 'open_project') reply(data.id, window.editable ? result(view(true),null,{url:location.origin+'/embed?code=one-use&project=project',expiresAt:Date.now()+60000}) : result(view(true)));
+ if(name === 'open_project') setTimeout(() => reply(data.id, window.editable ? result(view(true),null,{url:location.origin+'/embed?code=one-use&project=project',expiresAt:Date.now()+60000}) : result(view(true))), window.holdMs);
if(name === 'render_preview') { count=0; reply(data.id,result(view(true,{id:'job',status:'queued',progress:0,revision:'cloud:2'}))); }
if(name === 'get_preview_status') {
count++; const done = count >= 3;
@@ -43,7 +47,7 @@ window.addEventListener('message', ({source,data}) => {
}
}
if(data.method === 'ui/open-link') { window.links.push(data.params.url); reply(data.id,{}); }
- if(data.method === 'ui/request-display-mode') { window.modes.push(data.params.mode); reply(data.id,{mode:window.modes.length > 1 ? data.params.mode : 'inline'}); }
+ if(data.method === 'ui/request-display-mode') { window.modes.push(data.params.mode); reply(data.id,{mode:data.params.mode}); }
});
</script></body></html>`;
const browser = await chromium.launch({ headless: true });
@@ -78,23 +82,38 @@ try {
await page.evaluate(() => { (window as unknown as { editable: boolean }).editable = true; });
await app.getByRole("button", { name: "Launch film" }).click();
await app.frameLocator("iframe.editor").locator("#editor").waitFor();
- assert.deepEqual(await page.evaluate(() => (window as unknown as { modes: string[] }).modes), ["fullscreen"], "the editor asks for the whole window on open");
+ assert.deepEqual(await page.evaluate(() => (window as unknown as { modes: string[] }).modes), [], "an opened editor stays inline");
assert.equal(await app.locator("button").count(), 0, "the editor card has no controls of its own");
await app.frameLocator("iframe.editor").locator("#inset").filter({ hasText: /^0$/ }).waitFor();
await app.frameLocator("iframe.editor").locator("#mode").filter({ hasText: /^inline$/ }).waitFor();
await app.frameLocator("iframe.editor").getByRole("button", { name: "Fullscreen" }).click();
await app.frameLocator("iframe.editor").locator("#mode").filter({ hasText: /^fullscreen$/ }).waitFor();
await app.frameLocator("iframe.editor").locator("#inset").filter({ hasText: /^120$/ }).waitFor();
- assert.deepEqual(await page.evaluate(() => (window as unknown as { modes: string[] }).modes), ["fullscreen", "fullscreen"], "the button asks again and the granted mode hides it");
+ assert.deepEqual(await page.evaluate(() => (window as unknown as { modes: string[] }).modes), ["fullscreen"], "the editor's button asks for the whole window");
await page.screenshot({ path: "/tmp/donkey-chatgpt-widget-editor.png" });
- // A card rehydrated with a spent link asks for a fresh one instead of showing the preview.
- await page.evaluate(() => { (window as unknown as { calls: string[] }).calls = []; });
+ // A card rehydrated with a spent link holds the editor's space, inline and
+ // without controls, while it mints a fresh link. The held reply outlasts the
+ // stale playback's one-second renewal, so a poll would show up in the calls.
+ type Host = { calls: string[]; modes: string[]; holdMs: number; freshCard: boolean };
+ await page.evaluate(() => { const host = window as unknown as Host; host.calls = []; host.holdMs = 1500; });
await frame.evaluate(() => { location.reload(); });
- await app.frameLocator("iframe.editor").locator("#editor").waitFor();
- const rehydrated = await page.evaluate(() => (window as unknown as { calls: string[] }).calls);
- assert.deepEqual(rehydrated, ["open_project"], "the rehydrated card mints one fresh link");
+ await page.waitForFunction(() => (window as unknown as Host).calls.length === 1);
+ await app.locator("div.editor").waitFor();
+ assert.equal(await app.locator("button").count(), 0, "a waking editor card shows no controls");
+ assert.equal(await app.getByText("Render preview").count(), 0, "a waking editor card shows no preview card");
+ await app.frameLocator("iframe.editor").locator("#editor").filter({ hasText: "one-use" }).waitFor();
+ await app.frameLocator("iframe.editor").locator("#mode").filter({ hasText: /^inline$/ }).waitFor();
+ assert.deepEqual(await page.evaluate(() => (window as unknown as Host).calls), ["open_project"], "the rehydrated card mints one fresh link and polls nothing");
+ assert.deepEqual(await page.evaluate(() => (window as unknown as Host).modes), ["fullscreen"], "a card that wakes up stays inline");
+ // A card whose own result carries a live link opens with it, inline.
+ await page.evaluate(() => { const host = window as unknown as Host; host.calls = []; host.holdMs = 0; host.freshCard = true; });
+ await frame.evaluate(() => { location.reload(); });
+ await app.frameLocator("iframe.editor").locator("#editor").filter({ hasText: "via card" }).waitFor();
+ await app.frameLocator("iframe.editor").locator("#mode").filter({ hasText: /^inline$/ }).waitFor();
+ assert.deepEqual(await page.evaluate(() => (window as unknown as Host).modes), ["fullscreen"], "a newly opened card opens inline");
+ assert.deepEqual(await page.evaluate(() => (window as unknown as Host).calls), [], "a live link needs no second call");
assert.deepEqual(errors, []);
- console.log("PASS: project selection, repeated polling, native playback, URL recovery, hidden pause, Open in Donkey Cut, editor in the card, decoder teardown, lazy HLS.");
+ console.log("PASS: project selection, repeated polling, native playback, URL recovery, hidden pause, Open in Donkey Cut, editor in the card, waking editor card, decoder teardown, lazy HLS.");
} finally {
await browser.close(); server.stop(true); await rm(scratch, { recursive: true, force: true });
}
diff --git a/site/src/clients/chatgpt/ui/PreviewApp.tsx b/site/src/clients/chatgpt/ui/PreviewApp.tsx
index 5f047f40..067cb069 100644
--- a/site/src/clients/chatgpt/ui/PreviewApp.tsx
+++ b/site/src/clients/chatgpt/ui/PreviewApp.tsx
@@ -4,16 +4,21 @@ import { useProjectApp } from "./useProjectApp";
import "./preview.css";
export function PreviewApp() {
- const { ready, view, playback, download, editor, fullscreen, hostInset, error, busy, visible, run, playbackFailed, openDonkey, openDownload, openFromFrame, saveFromFrame, requestFullscreen } = useProjectApp();
+ const { ready, view, playback, editor, fullscreen, hostInset, error, busy, visible, run, playbackFailed, openDonkey, openFromFrame, saveFromFrame, requestFullscreen, retryEditor } = useProjectApp();
const project = view?.project;
const preview = view?.preview;
const rendering = preview?.status === "queued" || preview?.status === "running";
- const exp = view?.export;
- const exporting = exp?.status === "queued" || exp?.status === "running";
- // The editor is the card: its own chrome carries every control. Inline, one
- // control asks the host for the whole window.
- if (editor && project) return <main className="editing" data-mode={fullscreen ? "fullscreen" : "inline"}>
- <EditorFrame url={editor.url} name={project.name} inset={hostInset} fullscreen={fullscreen} onFullscreen={requestFullscreen} onOpen={openFromFrame} onSave={saveFromFrame} />
+ // An editable project is the editor, and its own chrome carries every
+ // control. The card holds the editor's inline space from the first paint,
+ // so a card ChatGPT shows again after a reload is the editor while it
+ // fetches a fresh link.
+ if (view ? project && view.canEdit : !error) return <main className="editing" data-mode={fullscreen ? "fullscreen" : "inline"} aria-busy={!editor}>
+ {editor && project
+ ? <EditorFrame url={editor.url} name={project.name} inset={hostInset} fullscreen={fullscreen} onFullscreen={requestFullscreen} onOpen={openFromFrame} onSave={saveFromFrame} />
+ : <div className="editor waiting">{error && project && <>
+ <p role="alert">{error}</p>
+ <button disabled={busy || !ready} onClick={retryEditor}>Try again</button>
+ </>}</div>}
</main>;
return <main aria-busy={busy}>
<header><span className="brand">Donkey Cut</span><span>Cloud projects</span></header>
@@ -37,13 +42,8 @@ export function PreviewApp() {
{preview?.status === "expired" && <p>This preview has expired. Render it again to watch.</p>}
{preview?.status === "error" && <p role="alert">{preview.error}</p>}
{preview?.status === "done" && preview.revision !== project.revision && <p className="muted">This preview may be from an earlier edit.</p>}
- {exporting && <div role="status"><p>{exp.status === "queued" ? "Waiting to export…" : `Exporting ${exp.name ?? "video"}… ${Math.round(exp.progress * 100)}%`}</p><progress max={1} value={exp.progress} /></div>}
- {exp?.status === "done" && !download && <p className="muted">Export {exp.name ?? ""} finished. Renewing the download link…</p>}
- {(exp?.status === "error" || exp?.status === "expired") && <p role="alert">{exp.error ?? "The export did not finish."}</p>}
<nav>
{view.canRender && <button className="primary" disabled={busy || rendering || !ready} onClick={() => void run("render_preview", { projectId: project.id })}>Render preview</button>}
- {download && <button className="primary" disabled={!ready} onClick={() => openDownload()}>Download {download.name}</button>}
- {view.canEdit && <button disabled={busy || exporting || !ready} onClick={() => void run("export_video", { projectId: project.id })}>Export video</button>}
<button disabled={!ready} onClick={() => openDonkey()}>Open in Donkey Cut ↗</button>
<button disabled={busy || !ready} onClick={() => void run("list_projects", {})}>Projects</button>
{error && <button disabled={busy || !ready} onClick={() => void run("open_project", { projectId: project.id })}>Try again</button>}
diff --git a/site/src/clients/chatgpt/ui/preview.css b/site/src/clients/chatgpt/ui/preview.css
index fa988a04..0a6dd33c 100644
--- a/site/src/clients/chatgpt/ui/preview.css
+++ b/site/src/clients/chatgpt/ui/preview.css
@@ -2,3 +2,4 @@
:root[data-theme=dark]{color:#eeede8;background:#20201e;color-scheme:dark;--line:#45443f;--muted:#b6b3a8;--panel:#292925}
*{box-sizing:border-box}body{margin:0}main{padding:20px;max-width:840px;margin:auto}header{display:flex;justify-content:space-between;color:var(--muted);font-size:12px}.brand{font-weight:800;font-size:16px;color:inherit}h1{font-size:22px;line-height:1.2;margin:24px 0 16px;overflow-wrap:anywhere}video{width:100%;max-height:65vh;display:block;border-radius:12px;background:#000}button{font:inherit;color:inherit;background:var(--panel);border:1px solid var(--line);border-radius:10px;padding:10px 14px;cursor:pointer}button:hover{filter:brightness(.95)}button:disabled{opacity:.5;cursor:default}button:focus-visible{outline:2px solid #5d80cc;outline-offset:3px}.primary{background:#eac36a;color:#242018;border-color:#eac36a}nav{display:flex;flex-wrap:wrap;gap:8px;margin-top:16px}.projects{display:grid;gap:8px}.projects button{display:flex;justify-content:space-between;gap:16px;text-align:left}.projects button span:first-child{overflow-wrap:anywhere;min-width:0}.muted{color:var(--muted);font-size:12px}.error{color:#ba453b}progress{width:100%;accent-color:#d8a743}p{overflow-wrap:anywhere}
.editing{padding:0;max-width:none;position:relative}.editor{display:block;width:100%;height:560px;border:0;background:#000}[data-mode=fullscreen] .editor{height:100vh}
+.waiting{--panel:#292925;--line:#45443f;color:#eeede8;display:flex;flex-direction:column;align-items:center;justify-content:center;gap:12px;padding:20px;text-align:center}
diff --git a/site/src/clients/chatgpt/ui/useProjectApp.ts b/site/src/clients/chatgpt/ui/useProjectApp.ts
index 4e4ec992..78d384a8 100644
--- a/site/src/clients/chatgpt/ui/useProjectApp.ts
+++ b/site/src/clients/chatgpt/ui/useProjectApp.ts
@@ -1,8 +1,12 @@
import { useCallback, useEffect, useRef, useState } from "react";
import { App } from "@modelcontextprotocol/ext-apps";
-import { downloadSchema, editorSchema, playbackSchema, viewSchema, type Download, type Editor, type Playback, type ProjectView } from "../contracts";
+import { editorSchema, playbackSchema, viewSchema, type Editor, type Playback, type ProjectView } from "../contracts";
type ToolResult = { isError?: boolean; structuredContent?: unknown; _meta?: Record<string, unknown>; content?: unknown[] };
+/** Where a result came from. The card's own result can be old, its editor
+ * link long spent, when ChatGPT shows the card again after a reload; a call
+ * the card makes gets a link minted for it. */
+type Origin = "card" | "call";
/** The host carries authentication; the iframe only calls the MCP Apps bridge. */
export function useProjectApp() {
@@ -11,7 +15,6 @@ export function useProjectApp() {
const [ready, setReady] = useState(false);
const [view, setView] = useState<ProjectView | null>(null);
const [playback, setPlayback] = useState<Playback | null>(null);
- const [download, setDownload] = useState<Download | null>(null);
const [editor, setEditor] = useState<Editor | null>(null);
const [fullscreen, setFullscreen] = useState(false);
const [hostInset, setHostInset] = useState(0);
@@ -23,23 +26,23 @@ export function useProjectApp() {
const [pollMs, setPollMs] = useState(2000);
const recoveries = useRef(0);
- const receive = useCallback((result: ToolResult) => {
+ const receive = useCallback((result: ToolResult, origin: Origin) => {
if (result.isError) throw new Error("Donkey Cut could not complete this request. Try again.");
const parsed = viewSchema.safeParse(result.structuredContent);
if (!parsed.success) throw new Error("The preview response could not be read.");
+ // An editable project is the editor. Its link is good for one minute, so
+ // the card's own result counts it only inside that minute.
+ const editable = parsed.data.canEdit && parsed.data.project ? parsed.data.project.id : null;
+ const embed = editorSchema.safeParse(result._meta?.editor);
+ const link = embed.success && (origin !== "card" || embed.data.expiresAt > Date.now()) ? embed.data : null;
+ if (origin === "call" && editable && !link) throw new Error("Donkey Cut could not open the editor. Try again.");
const media = playbackSchema.safeParse(result._meta?.playback);
- const file = downloadSchema.safeParse(result._meta?.download);
const interval = result._meta?.pollMs;
if (typeof interval === "number" && interval >= 1000 && interval <= 30000) setPollMs(interval);
setView(parsed.data);
setPlayback(media.success ? media.data : null);
- setDownload(file.success ? file.data : null);
- // A one-use link that has already had its minute is spent: a card
- // rehydrated later asks the server for a fresh one, once.
- const embed = editorSchema.safeParse(result._meta?.editor);
- const fresh = embed.success && embed.data.expiresAt > Date.now();
- if (fresh) setEditor(embed.data);
- setEditorRequest(fresh || !parsed.data.canEdit || !parsed.data.project ? null : parsed.data.project.id);
+ if (editable && link) setEditor((current) => current ?? link);
+ setEditorRequest(editable && !link ? editable : null);
setError(null);
}, []);
@@ -52,7 +55,7 @@ export function useProjectApp() {
epoch.current++;
recoveries.current = 0;
setBusy(false);
- try { receive(result); } catch (e) { setError((e as Error).message); }
+ try { receive(result, "card"); } catch (e) { setError((e as Error).message); }
};
// ChatGPT overlays its composer on the bottom of a fullscreen card and
// reports the covered band as a safe-area inset; the editor frame gets
@@ -83,43 +86,34 @@ export function useProjectApp() {
};
}, [receive]);
- const run = useCallback(async (name: string, args: Record<string, string>, manual = true) => {
+ const run = useCallback(async (name: string, args: Record<string, string>, manual = true, origin: Origin = "call") => {
const app = appRef.current;
if (!app || !ready) return;
const current = ++epoch.current;
if (manual) { recoveries.current = 0; setBusy(true); setError(null); }
try {
const result = await app.callServerTool({ name, arguments: args });
- if (current === epoch.current && appRef.current === app) receive(result);
+ if (current === epoch.current && appRef.current === app) receive(result, origin);
} catch (e) {
if (current === epoch.current && appRef.current === app) setError(e instanceof Error ? e.message : "Request failed.");
} finally { if (current === epoch.current && appRef.current === app) setBusy(false); }
}, [ready, receive]);
+ // Only the preview card polls. An editable project is the editor, which
+ // plays and exports on its own; a poll there would also drop the answer
+ // to the editor's link request.
+ const framed = Boolean(view?.canEdit && view.project);
const projectId = view?.project?.id;
const jobId = view?.preview?.id;
const status = view?.preview?.status;
useEffect(() => {
- if (!ready || !visible || busy || error || editor || !projectId || !jobId) return;
+ if (!ready || !visible || busy || error || framed || !projectId || !jobId) return;
const waiting = status === "queued" || status === "running";
if (!waiting && !playback) return;
const delay = waiting ? pollMs : Math.max(1000, playback!.expiresAt - Date.now() - 60000);
const timeout = window.setTimeout(() => { void run("get_preview_status", { projectId, jobId }, false); }, delay);
return () => window.clearTimeout(timeout);
- }, [ready, visible, busy, error, editor, projectId, jobId, status, playback, pollMs, run, view]);
-
- // An export in flight polls the same way; a finished one renews its
- // download link before it expires.
- const exportId = view?.export?.id;
- const exportStatus = view?.export?.status;
- useEffect(() => {
- if (!ready || !visible || busy || error || editor || !exportId) return;
- const waiting = exportStatus === "queued" || exportStatus === "running";
- if (!waiting && !download) return;
- const delay = waiting ? pollMs : Math.max(1000, download!.expiresAt - Date.now() - 60000);
- const timeout = window.setTimeout(() => { void run("get_export_status", { jobId: exportId }, false); }, delay);
- return () => window.clearTimeout(timeout);
- }, [ready, visible, busy, error, editor, exportId, exportStatus, download, pollMs, run, view]);
+ }, [ready, visible, busy, error, framed, projectId, jobId, status, playback, pollMs, run, view]);
const playbackFailed = useCallback(() => {
if (!projectId || !jobId) return;
@@ -134,7 +128,6 @@ export function useProjectApp() {
catch { setError(failure); }
};
const openDonkey = () => { if (view?.project) void openLink(view.project.url, "Could not open Donkey Cut."); };
- const openDownload = () => { if (download) void openLink(download.url, "Could not open the download."); };
/** The framed editor asking for a tab: opening the project on donkeycut.com,
* and saving an export, which the card's sandbox blocks in the frame. */
const openFromFrame = useCallback((url: string) => {
@@ -158,8 +151,10 @@ export function useProjectApp() {
requested.current = editorRequest;
void run("open_project", { projectId: editorRequest }, false);
}, [ready, editorRequest, editor, run]);
+ /** Asks again for the link a waking card could not get. */
+ const retryEditor = () => { if (view?.project) void run("open_project", { projectId: view.project.id }); };
- // The editor wants the whole window; the host grants what it supports.
+ // The editor's own button asks for the whole window; the host grants what it supports.
const requestFullscreen = useCallback(async () => {
const app = appRef.current;
if (!app) return;
@@ -170,6 +165,5 @@ export function useProjectApp() {
setHostInset(full ? (app.getHostContext()?.safeAreaInsets?.bottom ?? 120) : 0);
} catch { /* the inline card still works */ }
}, []);
- useEffect(() => { if (editor) void requestFullscreen(); }, [editor, requestFullscreen]);
- return { ready, view, playback, download, editor, fullscreen, hostInset, error, busy, visible, run, playbackFailed, openDonkey, openDownload, openFromFrame, saveFromFrame, requestFullscreen };
+ return { ready, view, playback, editor, fullscreen, hostInset, error, busy, visible, run, playbackFailed, openDonkey, openFromFrame, saveFromFrame, requestFullscreen, retryEditor };
}