Skip to content

Record your screen, ship a demo. Free and open-source, GPU-accelerated, no watermarks, no subscriptions. Windows, macOS, Linux. Actively maintained.

fix(mcp): let getFrames read a GIF export

Etienne Lescot3823809a7eb9
diff --git a/electron/mcp/hidden-video.test.ts b/electron/mcp/hidden-video.test.ts
new file mode 100644
index 000000000..90bebab44
--- /dev/null
+++ b/electron/mcp/hidden-video.test.ts
@@ -0,0 +1,35 @@
+import { describe, expect, it, vi } from "vitest";
+
+vi.mock("electron", () => ({ BrowserWindow: class {} }));
+
+import { gifFrameIndexAt, isGifPath } from "./hidden-video";
+
+describe("isGifPath", () => {
+ it("picks a GIF by its extension, in any case", () => {
+ expect(isGifPath("/tmp/out.gif")).toBe(true);
+ expect(isGifPath("C:\\exports\\OUT.GIF")).toBe(true);
+ expect(isGifPath("/tmp/out.mp4")).toBe(false);
+ expect(isGifPath("/tmp/gif/out.webm")).toBe(false);
+ });
+});
+
+describe("gifFrameIndexAt", () => {
+ // A 15 fps GIF: delays are whole centiseconds, so frames alternate 70 and 60 ms.
+ const starts = [0, 0.07, 0.13, 0.2, 0.26];
+
+ it("shows the frame whose span holds the time", () => {
+ expect(gifFrameIndexAt(starts, 0)).toBe(0);
+ expect(gifFrameIndexAt(starts, 0.069)).toBe(0);
+ expect(gifFrameIndexAt(starts, 0.07)).toBe(1);
+ expect(gifFrameIndexAt(starts, 0.2)).toBe(3);
+ });
+
+ it("holds the last frame past the end and the first before the start", () => {
+ expect(gifFrameIndexAt(starts, 99)).toBe(4);
+ expect(gifFrameIndexAt(starts, -1)).toBe(0);
+ });
+
+ it("answers 0 for a one-frame GIF", () => {
+ expect(gifFrameIndexAt([0], 3)).toBe(0);
+ });
+});
diff --git a/electron/mcp/hidden-video.ts b/electron/mcp/hidden-video.ts
index a2f94e6c8..c217c9f3c 100644
--- a/electron/mcp/hidden-video.ts
+++ b/electron/mcp/hidden-video.ts
@@ -1,6 +1,7 @@
// A video decoded by Chromium's own <video> element, in a blank hidden page: its
-// duration and size, and still frames of it. No renderer bundle and no ffmpeg
-// (the Windows installer ships no ffmpeg executable).
+// duration and size, and still frames of it. A GIF goes through WebCodecs' ImageDecoder
+// instead (see openGif). No renderer bundle and no ffmpeg (the Windows installer ships no
+// ffmpeg executable).
import { pathToFileURL } from "node:url";
import { BrowserWindow } from "electron";
@@ -54,6 +55,7 @@ export async function openVideoInHiddenWindow(videoPath: string): Promise<Opened
const close = () => {
if (!win.isDestroyed()) win.destroy();
};
+ if (isGifPath(videoPath)) return openGif(win, videoPath, close);
try {
await win.loadURL("about:blank");
const meta = (await withTimeout(
@@ -104,3 +106,80 @@ export async function openVideoInHiddenWindow(videoPath: string): Promise<Opened
throw error;
}
}
+
+export function isGifPath(path: string): boolean {
+ return path.toLowerCase().endsWith(".gif");
+}
+
+/** The frame on screen at `timeSec`, from each frame's start time in seconds (ascending, from 0). */
+export function gifFrameIndexAt(frameStartsSec: readonly number[], timeSec: number): number {
+ let index = 0;
+ for (let i = 1; i < frameStartsSec.length && frameStartsSec[i] <= timeSec; i++) index = i;
+ return index;
+}
+
+// Chromium's <video> does not play a GIF, so a GIF export is decoded with WebCodecs'
+// ImageDecoder instead. That API needs a secure context, which about:blank is not, so the
+// page is the GIF itself: a file:// page is secure, and fetching its own URL is same-origin.
+async function openGif(
+ win: BrowserWindow,
+ gifPath: string,
+ close: () => void,
+): Promise<OpenedVideo> {
+ try {
+ const url = pathToFileURL(gifPath).href;
+ await win.loadURL(url);
+ const meta = (await withTimeout(
+ win.webContents.executeJavaScript(`(async () => {
+ if (typeof ImageDecoder === "undefined") throw new Error("Chromium cannot decode a GIF here (no ImageDecoder)");
+ const response = await fetch(${JSON.stringify(url)});
+ const decoder = new ImageDecoder({ data: response.body, type: "image/gif" });
+ await decoder.tracks.ready;
+ await decoder.completed;
+ const frameCount = decoder.tracks.selectedTrack.frameCount;
+ // Frame durations come with the decoded frames only, so every frame is decoded once.
+ const frameStartsSec = [];
+ let t = 0;
+ let width = 0;
+ let height = 0;
+ for (let i = 0; i < frameCount; i++) {
+ const { image } = await decoder.decode({ frameIndex: i });
+ frameStartsSec.push(t);
+ t += (image.duration ?? 0) / 1e6;
+ width = image.displayWidth;
+ height = image.displayHeight;
+ image.close();
+ }
+ window.__gif = decoder;
+ return { durationSec: t, width, height, frameStartsSec };
+ })()`),
+ "loading the GIF",
+ )) as { durationSec: number; width: number; height: number; frameStartsSec: number[] };
+ const { frameStartsSec, ...size } = meta;
+
+ return {
+ ...size,
+ close,
+ async capture(timeSec, maxWidth) {
+ const frameIndex = gifFrameIndexAt(frameStartsSec, timeSec);
+ const dataUrl = (await withTimeout(
+ win.webContents.executeJavaScript(`(async () => {
+ const { image } = await window.__gif.decode({ frameIndex: ${frameIndex} });
+ const scale = Math.min(1, ${maxWidth} / image.displayWidth);
+ const canvas = document.createElement("canvas");
+ canvas.width = Math.round(image.displayWidth * scale);
+ canvas.height = Math.round(image.displayHeight * scale);
+ canvas.getContext("2d").drawImage(image, 0, 0, canvas.width, canvas.height);
+ image.close();
+ return canvas.toDataURL("image/jpeg", 0.85);
+ })()`),
+ `decoding the GIF at ${timeSec}s`,
+ )) as string;
+ return { mimeType: "image/jpeg", data: dataUrl.slice(dataUrl.indexOf(",") + 1) };
+ },
+ };
+ } catch (error) {
+ close();
+ throw error;
+ }
+}