Open source CapCut alternative.
TypeScript
fix(chatgpt): an editable project card is the inline editor from its first paint
David Lec6d04932be42
diff --git a/site/scripts/eval-chatgpt-widget.ts b/site/scripts/eval-chatgpt-widget.tsindex cea76c0e..1cf3f4b0 100644--- a/site/scripts/eval-chatgpt-widget.ts+++ b/site/scripts/eval-chatgpt-widget.ts@@ -22,20 +22,24 @@ const hostHtml = `<!doctype html><html><body><iframe title="Donkey Cut preview"const frame = document.querySelector('iframe');const project = {id:'project',name:'Launch film',revision:'cloud:2',url:'https://donkeycut.com/app/p/project'};let count = 0, renewals = 0;-window.calls = []; window.links = []; window.modes = []; window.editable = false;+window.calls = []; window.links = []; window.modes = []; window.editable = false; window.holdMs = 0; window.freshCard = false;const view = (selected, preview = null) => ({view:selected?'project':'projects',projects:selected?[]:[project],nextCursor:null,project:selected?project:null,preview,canRender:true,canEdit:window.editable,export:null,job:null,results:[],changed:false,account:null});// ChatGPT's sandbox drops null-valued keys before the widget sees a result.const dropNulls = (value) => Array.isArray(value) ? value.map(dropNulls) : value && typeof value === 'object' ? Object.fromEntries(Object.entries(value).filter(([, v]) => v !== null).map(([k, v]) => [k, dropNulls(v)])) : value;const result = (data, playback = null, editor = null) => dropNulls({content:[{type:'text',text:'Preview'}],structuredContent:data,_meta:{playback,editor,pollMs:1000}});const reply = (id, result) => frame.contentWindow.postMessage({jsonrpc:'2.0',id,result},'*');+// A card ChatGPT shows again after a reload replays its own result: a stale+// preview with an expired playback link, and an editor link whose minute is+// spent unless the fixture says the card was just opened.+const cardResult = () => !window.editable ? result(view(false)) : result(view(true,{id:'job',status:'done',progress:1,revision:'cloud:1'}),{url:location.origin+'/preview.mp4?stale',expiresAt:Date.now()-1},window.freshCard ? {url:location.origin+'/embed?code=card&project=project',expiresAt:Date.now()+60000} : {url:location.origin+'/embed?code=spent&project=project',expiresAt:Date.now()-1});window.addEventListener('message', ({source,data}) => {if(source !== frame.contentWindow || !data.method) return;if(data.method === 'ui/initialize') reply(data.id,{protocolVersion:data.params.protocolVersion,hostInfo:{name:'fixture',version:'1'},hostCapabilities:{serverTools:{},openLinks:{}},hostContext:{theme:'light',displayMode:'inline',availableDisplayModes:['inline','fullscreen'],safeAreaInsets:{top:0,right:0,bottom:120,left:0}}});- if(data.method === 'ui/notifications/initialized') frame.contentWindow.postMessage({jsonrpc:'2.0',method:'ui/notifications/tool-result',params:window.editable ? result(view(true),null,{url:location.origin+'/embed?code=spent&project=project',expiresAt:Date.now()-1}) : result(view(false))},'*');+ if(data.method === 'ui/notifications/initialized') frame.contentWindow.postMessage({jsonrpc:'2.0',method:'ui/notifications/tool-result',params:cardResult()},'*');if(data.method === 'tools/call') {const name = data.params.name; window.calls.push(name);if(name === 'list_projects') reply(data.id,result(view(false)));- if(name === 'open_project') reply(data.id, window.editable ? result(view(true),null,{url:location.origin+'/embed?code=one-use&project=project',expiresAt:Date.now()+60000}) : result(view(true)));+ if(name === 'open_project') setTimeout(() => reply(data.id, window.editable ? result(view(true),null,{url:location.origin+'/embed?code=one-use&project=project',expiresAt:Date.now()+60000}) : result(view(true))), window.holdMs);if(name === 'render_preview') { count=0; reply(data.id,result(view(true,{id:'job',status:'queued',progress:0,revision:'cloud:2'}))); }if(name === 'get_preview_status') {count++; const done = count >= 3;@@ -43,7 +47,7 @@ window.addEventListener('message', ({source,data}) => {}}if(data.method === 'ui/open-link') { window.links.push(data.params.url); reply(data.id,{}); }- if(data.method === 'ui/request-display-mode') { window.modes.push(data.params.mode); reply(data.id,{mode:window.modes.length > 1 ? data.params.mode : 'inline'}); }+ if(data.method === 'ui/request-display-mode') { window.modes.push(data.params.mode); reply(data.id,{mode:data.params.mode}); }});</script></body></html>`;const browser = await chromium.launch({ headless: true });@@ -78,23 +82,38 @@ try {await page.evaluate(() => { (window as unknown as { editable: boolean }).editable = true; });await app.getByRole("button", { name: "Launch film" }).click();await app.frameLocator("iframe.editor").locator("#editor").waitFor();- assert.deepEqual(await page.evaluate(() => (window as unknown as { modes: string[] }).modes), ["fullscreen"], "the editor asks for the whole window on open");+ assert.deepEqual(await page.evaluate(() => (window as unknown as { modes: string[] }).modes), [], "an opened editor stays inline");assert.equal(await app.locator("button").count(), 0, "the editor card has no controls of its own");await app.frameLocator("iframe.editor").locator("#inset").filter({ hasText: /^0$/ }).waitFor();await app.frameLocator("iframe.editor").locator("#mode").filter({ hasText: /^inline$/ }).waitFor();await app.frameLocator("iframe.editor").getByRole("button", { name: "Fullscreen" }).click();await app.frameLocator("iframe.editor").locator("#mode").filter({ hasText: /^fullscreen$/ }).waitFor();await app.frameLocator("iframe.editor").locator("#inset").filter({ hasText: /^120$/ }).waitFor();- assert.deepEqual(await page.evaluate(() => (window as unknown as { modes: string[] }).modes), ["fullscreen", "fullscreen"], "the button asks again and the granted mode hides it");+ assert.deepEqual(await page.evaluate(() => (window as unknown as { modes: string[] }).modes), ["fullscreen"], "the editor's button asks for the whole window");await page.screenshot({ path: "/tmp/donkey-chatgpt-widget-editor.png" });- // A card rehydrated with a spent link asks for a fresh one instead of showing the preview.- await page.evaluate(() => { (window as unknown as { calls: string[] }).calls = []; });+ // A card rehydrated with a spent link holds the editor's space, inline and+ // without controls, while it mints a fresh link. The held reply outlasts the+ // stale playback's one-second renewal, so a poll would show up in the calls.+ type Host = { calls: string[]; modes: string[]; holdMs: number; freshCard: boolean };+ await page.evaluate(() => { const host = window as unknown as Host; host.calls = []; host.holdMs = 1500; });await frame.evaluate(() => { location.reload(); });- await app.frameLocator("iframe.editor").locator("#editor").waitFor();- const rehydrated = await page.evaluate(() => (window as unknown as { calls: string[] }).calls);- assert.deepEqual(rehydrated, ["open_project"], "the rehydrated card mints one fresh link");+ await page.waitForFunction(() => (window as unknown as Host).calls.length === 1);+ await app.locator("div.editor").waitFor();+ assert.equal(await app.locator("button").count(), 0, "a waking editor card shows no controls");+ assert.equal(await app.getByText("Render preview").count(), 0, "a waking editor card shows no preview card");+ await app.frameLocator("iframe.editor").locator("#editor").filter({ hasText: "one-use" }).waitFor();+ await app.frameLocator("iframe.editor").locator("#mode").filter({ hasText: /^inline$/ }).waitFor();+ assert.deepEqual(await page.evaluate(() => (window as unknown as Host).calls), ["open_project"], "the rehydrated card mints one fresh link and polls nothing");+ assert.deepEqual(await page.evaluate(() => (window as unknown as Host).modes), ["fullscreen"], "a card that wakes up stays inline");+ // A card whose own result carries a live link opens with it, inline.+ await page.evaluate(() => { const host = window as unknown as Host; host.calls = []; host.holdMs = 0; host.freshCard = true; });+ await frame.evaluate(() => { location.reload(); });+ await app.frameLocator("iframe.editor").locator("#editor").filter({ hasText: "via card" }).waitFor();+ await app.frameLocator("iframe.editor").locator("#mode").filter({ hasText: /^inline$/ }).waitFor();+ assert.deepEqual(await page.evaluate(() => (window as unknown as Host).modes), ["fullscreen"], "a newly opened card opens inline");+ assert.deepEqual(await page.evaluate(() => (window as unknown as Host).calls), [], "a live link needs no second call");assert.deepEqual(errors, []);- console.log("PASS: project selection, repeated polling, native playback, URL recovery, hidden pause, Open in Donkey Cut, editor in the card, decoder teardown, lazy HLS.");+ console.log("PASS: project selection, repeated polling, native playback, URL recovery, hidden pause, Open in Donkey Cut, editor in the card, waking editor card, decoder teardown, lazy HLS.");} finally {await browser.close(); server.stop(true); await rm(scratch, { recursive: true, force: true });}diff --git a/site/src/clients/chatgpt/ui/PreviewApp.tsx b/site/src/clients/chatgpt/ui/PreviewApp.tsxindex 5f047f40..067cb069 100644--- a/site/src/clients/chatgpt/ui/PreviewApp.tsx+++ b/site/src/clients/chatgpt/ui/PreviewApp.tsx@@ -4,16 +4,21 @@ import { useProjectApp } from "./useProjectApp";import "./preview.css";export function PreviewApp() {- const { ready, view, playback, download, editor, fullscreen, hostInset, error, busy, visible, run, playbackFailed, openDonkey, openDownload, openFromFrame, saveFromFrame, requestFullscreen } = useProjectApp();+ const { ready, view, playback, editor, fullscreen, hostInset, error, busy, visible, run, playbackFailed, openDonkey, openFromFrame, saveFromFrame, requestFullscreen, retryEditor } = useProjectApp();const project = view?.project;const preview = view?.preview;const rendering = preview?.status === "queued" || preview?.status === "running";- const exp = view?.export;- const exporting = exp?.status === "queued" || exp?.status === "running";- // The editor is the card: its own chrome carries every control. Inline, one- // control asks the host for the whole window.- if (editor && project) return <main className="editing" data-mode={fullscreen ? "fullscreen" : "inline"}>- <EditorFrame url={editor.url} name={project.name} inset={hostInset} fullscreen={fullscreen} onFullscreen={requestFullscreen} onOpen={openFromFrame} onSave={saveFromFrame} />+ // An editable project is the editor, and its own chrome carries every+ // control. The card holds the editor's inline space from the first paint,+ // so a card ChatGPT shows again after a reload is the editor while it+ // fetches a fresh link.+ if (view ? project && view.canEdit : !error) return <main className="editing" data-mode={fullscreen ? "fullscreen" : "inline"} aria-busy={!editor}>+ {editor && project+ ? <EditorFrame url={editor.url} name={project.name} inset={hostInset} fullscreen={fullscreen} onFullscreen={requestFullscreen} onOpen={openFromFrame} onSave={saveFromFrame} />+ : <div className="editor waiting">{error && project && <>+ <p role="alert">{error}</p>+ <button disabled={busy || !ready} onClick={retryEditor}>Try again</button>+ </>}</div>}</main>;return <main aria-busy={busy}><header><span className="brand">Donkey Cut</span><span>Cloud projects</span></header>@@ -37,13 +42,8 @@ export function PreviewApp() {{preview?.status === "expired" && <p>This preview has expired. Render it again to watch.</p>}{preview?.status === "error" && <p role="alert">{preview.error}</p>}{preview?.status === "done" && preview.revision !== project.revision && <p className="muted">This preview may be from an earlier edit.</p>}- {exporting && <div role="status"><p>{exp.status === "queued" ? "Waiting to export…" : `Exporting ${exp.name ?? "video"}… ${Math.round(exp.progress * 100)}%`}</p><progress max={1} value={exp.progress} /></div>}- {exp?.status === "done" && !download && <p className="muted">Export {exp.name ?? ""} finished. Renewing the download link…</p>}- {(exp?.status === "error" || exp?.status === "expired") && <p role="alert">{exp.error ?? "The export did not finish."}</p>}<nav>{view.canRender && <button className="primary" disabled={busy || rendering || !ready} onClick={() => void run("render_preview", { projectId: project.id })}>Render preview</button>}- {download && <button className="primary" disabled={!ready} onClick={() => openDownload()}>Download {download.name}</button>}- {view.canEdit && <button disabled={busy || exporting || !ready} onClick={() => void run("export_video", { projectId: project.id })}>Export video</button>}<button disabled={!ready} onClick={() => openDonkey()}>Open in Donkey Cut ↗</button><button disabled={busy || !ready} onClick={() => void run("list_projects", {})}>Projects</button>{error && <button disabled={busy || !ready} onClick={() => void run("open_project", { projectId: project.id })}>Try again</button>}diff --git a/site/src/clients/chatgpt/ui/preview.css b/site/src/clients/chatgpt/ui/preview.cssindex fa988a04..0a6dd33c 100644--- a/site/src/clients/chatgpt/ui/preview.css+++ b/site/src/clients/chatgpt/ui/preview.css@@ -2,3 +2,4 @@:root[data-theme=dark]{color:#eeede8;background:#20201e;color-scheme:dark;--line:#45443f;--muted:#b6b3a8;--panel:#292925}*{box-sizing:border-box}body{margin:0}main{padding:20px;max-width:840px;margin:auto}header{display:flex;justify-content:space-between;color:var(--muted);font-size:12px}.brand{font-weight:800;font-size:16px;color:inherit}h1{font-size:22px;line-height:1.2;margin:24px 0 16px;overflow-wrap:anywhere}video{width:100%;max-height:65vh;display:block;border-radius:12px;background:#000}button{font:inherit;color:inherit;background:var(--panel);border:1px solid var(--line);border-radius:10px;padding:10px 14px;cursor:pointer}button:hover{filter:brightness(.95)}button:disabled{opacity:.5;cursor:default}button:focus-visible{outline:2px solid #5d80cc;outline-offset:3px}.primary{background:#eac36a;color:#242018;border-color:#eac36a}nav{display:flex;flex-wrap:wrap;gap:8px;margin-top:16px}.projects{display:grid;gap:8px}.projects button{display:flex;justify-content:space-between;gap:16px;text-align:left}.projects button span:first-child{overflow-wrap:anywhere;min-width:0}.muted{color:var(--muted);font-size:12px}.error{color:#ba453b}progress{width:100%;accent-color:#d8a743}p{overflow-wrap:anywhere}.editing{padding:0;max-width:none;position:relative}.editor{display:block;width:100%;height:560px;border:0;background:#000}[data-mode=fullscreen] .editor{height:100vh}+.waiting{--panel:#292925;--line:#45443f;color:#eeede8;display:flex;flex-direction:column;align-items:center;justify-content:center;gap:12px;padding:20px;text-align:center}diff --git a/site/src/clients/chatgpt/ui/useProjectApp.ts b/site/src/clients/chatgpt/ui/useProjectApp.tsindex 4e4ec992..78d384a8 100644--- a/site/src/clients/chatgpt/ui/useProjectApp.ts+++ b/site/src/clients/chatgpt/ui/useProjectApp.ts@@ -1,8 +1,12 @@import { useCallback, useEffect, useRef, useState } from "react";import { App } from "@modelcontextprotocol/ext-apps";-import { downloadSchema, editorSchema, playbackSchema, viewSchema, type Download, type Editor, type Playback, type ProjectView } from "../contracts";+import { editorSchema, playbackSchema, viewSchema, type Editor, type Playback, type ProjectView } from "../contracts";type ToolResult = { isError?: boolean; structuredContent?: unknown; _meta?: Record<string, unknown>; content?: unknown[] };+/** Where a result came from. The card's own result can be old, its editor+ * link long spent, when ChatGPT shows the card again after a reload; a call+ * the card makes gets a link minted for it. */+type Origin = "card" | "call";/** The host carries authentication; the iframe only calls the MCP Apps bridge. */export function useProjectApp() {@@ -11,7 +15,6 @@ export function useProjectApp() {const [ready, setReady] = useState(false);const [view, setView] = useState<ProjectView | null>(null);const [playback, setPlayback] = useState<Playback | null>(null);- const [download, setDownload] = useState<Download | null>(null);const [editor, setEditor] = useState<Editor | null>(null);const [fullscreen, setFullscreen] = useState(false);const [hostInset, setHostInset] = useState(0);@@ -23,23 +26,23 @@ export function useProjectApp() {const [pollMs, setPollMs] = useState(2000);const recoveries = useRef(0);- const receive = useCallback((result: ToolResult) => {+ const receive = useCallback((result: ToolResult, origin: Origin) => {if (result.isError) throw new Error("Donkey Cut could not complete this request. Try again.");const parsed = viewSchema.safeParse(result.structuredContent);if (!parsed.success) throw new Error("The preview response could not be read.");+ // An editable project is the editor. Its link is good for one minute, so+ // the card's own result counts it only inside that minute.+ const editable = parsed.data.canEdit && parsed.data.project ? parsed.data.project.id : null;+ const embed = editorSchema.safeParse(result._meta?.editor);+ const link = embed.success && (origin !== "card" || embed.data.expiresAt > Date.now()) ? embed.data : null;+ if (origin === "call" && editable && !link) throw new Error("Donkey Cut could not open the editor. Try again.");const media = playbackSchema.safeParse(result._meta?.playback);- const file = downloadSchema.safeParse(result._meta?.download);const interval = result._meta?.pollMs;if (typeof interval === "number" && interval >= 1000 && interval <= 30000) setPollMs(interval);setView(parsed.data);setPlayback(media.success ? media.data : null);- setDownload(file.success ? file.data : null);- // A one-use link that has already had its minute is spent: a card- // rehydrated later asks the server for a fresh one, once.- const embed = editorSchema.safeParse(result._meta?.editor);- const fresh = embed.success && embed.data.expiresAt > Date.now();- if (fresh) setEditor(embed.data);- setEditorRequest(fresh || !parsed.data.canEdit || !parsed.data.project ? null : parsed.data.project.id);+ if (editable && link) setEditor((current) => current ?? link);+ setEditorRequest(editable && !link ? editable : null);setError(null);}, []);@@ -52,7 +55,7 @@ export function useProjectApp() {epoch.current++;recoveries.current = 0;setBusy(false);- try { receive(result); } catch (e) { setError((e as Error).message); }+ try { receive(result, "card"); } catch (e) { setError((e as Error).message); }};// ChatGPT overlays its composer on the bottom of a fullscreen card and// reports the covered band as a safe-area inset; the editor frame gets@@ -83,43 +86,34 @@ export function useProjectApp() {};}, [receive]);- const run = useCallback(async (name: string, args: Record<string, string>, manual = true) => {+ const run = useCallback(async (name: string, args: Record<string, string>, manual = true, origin: Origin = "call") => {const app = appRef.current;if (!app || !ready) return;const current = ++epoch.current;if (manual) { recoveries.current = 0; setBusy(true); setError(null); }try {const result = await app.callServerTool({ name, arguments: args });- if (current === epoch.current && appRef.current === app) receive(result);+ if (current === epoch.current && appRef.current === app) receive(result, origin);} catch (e) {if (current === epoch.current && appRef.current === app) setError(e instanceof Error ? e.message : "Request failed.");} finally { if (current === epoch.current && appRef.current === app) setBusy(false); }}, [ready, receive]);+ // Only the preview card polls. An editable project is the editor, which+ // plays and exports on its own; a poll there would also drop the answer+ // to the editor's link request.+ const framed = Boolean(view?.canEdit && view.project);const projectId = view?.project?.id;const jobId = view?.preview?.id;const status = view?.preview?.status;useEffect(() => {- if (!ready || !visible || busy || error || editor || !projectId || !jobId) return;+ if (!ready || !visible || busy || error || framed || !projectId || !jobId) return;const waiting = status === "queued" || status === "running";if (!waiting && !playback) return;const delay = waiting ? pollMs : Math.max(1000, playback!.expiresAt - Date.now() - 60000);const timeout = window.setTimeout(() => { void run("get_preview_status", { projectId, jobId }, false); }, delay);return () => window.clearTimeout(timeout);- }, [ready, visible, busy, error, editor, projectId, jobId, status, playback, pollMs, run, view]);-- // An export in flight polls the same way; a finished one renews its- // download link before it expires.- const exportId = view?.export?.id;- const exportStatus = view?.export?.status;- useEffect(() => {- if (!ready || !visible || busy || error || editor || !exportId) return;- const waiting = exportStatus === "queued" || exportStatus === "running";- if (!waiting && !download) return;- const delay = waiting ? pollMs : Math.max(1000, download!.expiresAt - Date.now() - 60000);- const timeout = window.setTimeout(() => { void run("get_export_status", { jobId: exportId }, false); }, delay);- return () => window.clearTimeout(timeout);- }, [ready, visible, busy, error, editor, exportId, exportStatus, download, pollMs, run, view]);+ }, [ready, visible, busy, error, framed, projectId, jobId, status, playback, pollMs, run, view]);const playbackFailed = useCallback(() => {if (!projectId || !jobId) return;@@ -134,7 +128,6 @@ export function useProjectApp() {catch { setError(failure); }};const openDonkey = () => { if (view?.project) void openLink(view.project.url, "Could not open Donkey Cut."); };- const openDownload = () => { if (download) void openLink(download.url, "Could not open the download."); };/** The framed editor asking for a tab: opening the project on donkeycut.com,* and saving an export, which the card's sandbox blocks in the frame. */const openFromFrame = useCallback((url: string) => {@@ -158,8 +151,10 @@ export function useProjectApp() {requested.current = editorRequest;void run("open_project", { projectId: editorRequest }, false);}, [ready, editorRequest, editor, run]);+ /** Asks again for the link a waking card could not get. */+ const retryEditor = () => { if (view?.project) void run("open_project", { projectId: view.project.id }); };- // The editor wants the whole window; the host grants what it supports.+ // The editor's own button asks for the whole window; the host grants what it supports.const requestFullscreen = useCallback(async () => {const app = appRef.current;if (!app) return;@@ -170,6 +165,5 @@ export function useProjectApp() {setHostInset(full ? (app.getHostContext()?.safeAreaInsets?.bottom ?? 120) : 0);} catch { /* the inline card still works */ }}, []);- useEffect(() => { if (editor) void requestFullscreen(); }, [editor, requestFullscreen]);- return { ready, view, playback, download, editor, fullscreen, hostInset, error, busy, visible, run, playbackFailed, openDonkey, openDownload, openFromFrame, saveFromFrame, requestFullscreen };+ return { ready, view, playback, editor, fullscreen, hostInset, error, busy, visible, run, playbackFailed, openDonkey, openFromFrame, saveFromFrame, requestFullscreen, retryEditor };}