Record your screen, ship a demo. Free and open-source, GPU-accelerated, no watermarks, no subscriptions. Windows, macOS, Linux. Actively maintained.
TypeScript cross-platformelectronffmpegopen-sourceproduct-demorustscreen-capturescreen-recorderscreen-recordingscreen-studioscreencastspeech-to-texttypescriptvideo-editingvideo-editorwaylandwgpuwhisper
feat(mcp): openscreen mcp, the editing tools plus record, export and frames on stdio
EtienneLescotcfea91ef150a
diff --git a/electron/ai-edition/document-service.ts b/electron/ai-edition/document-service.tsindex 1228e728d..c1e0ea20b 100644--- a/electron/ai-edition/document-service.ts+++ b/electron/ai-edition/document-service.ts@@ -243,6 +243,11 @@ export class DocumentService {return path.join(this.projectsRoot, `${safe}${PROJECT_FILE_EXTENSION}`);}+ /** The project's own file, for a reader that takes a path (the export runner). */+ filePathFor(projectId: string): string {+ return this.fileFor(projectId);+ }+private legacyFileFor(projectId: string): string {const safe = safeProjectId(projectId);return path.join(this.projectsRoot, `${safe}${LEGACY_PROJECT_FILE_EXTENSION}`);diff --git a/electron/cli/args.test.ts b/electron/cli/args.test.tsindex ddf012321..dd4a373cc 100644--- a/electron/cli/args.test.ts+++ b/electron/cli/args.test.ts@@ -230,6 +230,11 @@ describe("parseCliArgs", () => {});});+ it("parses mcp, which takes no argument", () => {+ expect(parse(["mcp"])).toEqual({ kind: "mcp" });+ expect(parse(["mcp", "--port", "1"])).toMatchObject({ kind: "error" });+ });+it("parses info and help", () => {expect(parse(["info", "demo.openscreen", "--json"])).toMatchObject({kind: "info",diff --git a/electron/cli/args.ts b/electron/cli/args.tsindex 23926da47..9c6532d9d 100644--- a/electron/cli/args.ts+++ b/electron/cli/args.ts@@ -22,10 +22,16 @@ export interface CliErrorCommand {message: string;}+/** Serve the MCP tools on stdin/stdout, to the agent that started this process. */+export interface CliMcpCommand {+ kind: "mcp";+}+export type CliCommand = (| CliRequest| CliInfoCommand| CliPackCommand+ | CliMcpCommand| CliHelpCommand| CliErrorCommand) & {@@ -60,6 +66,7 @@ const SUBCOMMANDS = new Set(["pack","captions","info",+ "mcp","help","--help","-h",@@ -75,6 +82,7 @@ Usage:openscreen captions <project.openscreen> Add auto-captions (on-device Whisper) to a project[--min-words <1-12>] [--max-words <1-12>]openscreen info <project.openscreen> [--json] Inspect a project file+ openscreen mcp Serve the agent tools over MCP on stdioopenscreen help Show this helpExport options:@@ -165,6 +173,10 @@ export function parseCliArgs(if (sub === "sources") return parseSources(args.slice(1), cwd);if (sub === "pack") return parsePack(args.slice(1), cwd);if (sub === "captions") return parseCaptions(args.slice(1), cwd);+ if (sub === "mcp") {+ if (args.length > 1) throw new Error(`Unexpected argument for mcp: ${args[1]}`);+ return { kind: "mcp" };+ }return parseInfo(args.slice(1), cwd);} catch (error) {return { kind: "error", message: error instanceof Error ? error.message : String(error) };diff --git a/electron/cli/cliMain.ts b/electron/cli/cliMain.tsindex d64b27e5d..b3ed45792 100644--- a/electron/cli/cliMain.ts+++ b/electron/cli/cliMain.ts@@ -7,18 +7,17 @@ import path from "node:path";import readline from "node:readline";import { fileURLToPath } from "node:url";import { app, BrowserWindow, ipcMain, session, systemPreferences } from "electron";-import type {- CliDoneResult,- CliProgressEvent,- CliRequest,- CliSourcesResult,-} from "../../src/lib/cliContracts";+import type { CliProgressEvent, CliSourcesResult } from "../../src/lib/cliContracts";import { isDiagnosticModeEnabled } from "../diagnostics/main-log-buffer";import { getSelectedDesktopSource, registerIpcHandlers } from "../ipc/handlers";+import { openVideoInHiddenWindow } from "../mcp/hidden-video";+import { serveMcpOverStdio } from "../mcp/stdio-server";import { registerSttIpc } from "../stt";import { ASSET_BASE_URL_ARG } from "../windows";import { CLI_USAGE, type CliCommand } from "./args";import { runInfoCommand, runPackCommand } from "./projectCommands";+import { createRunnerHost } from "./runnerHost";+import { openStdin } from "./stdin";const __dirname = path.dirname(fileURLToPath(import.meta.url));const VITE_DEV_SERVER_URL = process.env["VITE_DEV_SERVER_URL"];@@ -157,7 +156,7 @@ function registerAppHandlersForCli(cliWindowRef: () => BrowserWindow | null) {const notAvailable = () => {throw new Error("Window not available in CLI mode");};- registerIpcHandlers(+ return registerIpcHandlers(noop, // createEditorWindow: recording finished; runner drives completionnotAvailable, // createSourceSelectorWindownotAvailable, // createCountdownOverlayWindow@@ -213,7 +212,7 @@ function setupRecordStopSignals(stop: (reason: string) => void): void {try {// Touching process.stdin can throw on Windows GUI-subsystem builds when// spawned with stdio "ignore"; signals and --duration still work then.- const rl = readline.createInterface({ input: process.stdin });+ const rl = readline.createInterface({ input: openStdin() });rl.on("line", (line) => {const trimmed = line.trim().toLowerCase();if (trimmed === "stop" || trimmed === "q" || trimmed === "quit") {@@ -299,8 +298,8 @@ export function runCli(command: CliCommand): void {app.on("window-all-closed", () => {// Completion is signalled via cli-done; a vanished window is a failure- // only while the run is still in flight.- if (finished) return;+ // only while the run is still in flight. The MCP server outlives its runners.+ if (finished || command.kind === "mcp") return;output.error("Renderer window closed unexpectedly");app.exit(1);});@@ -382,7 +381,7 @@ export function runCli(command: CliCommand): void {milestone("media access checked");let cliWindow: BrowserWindow | null = null;- registerAppHandlersForCli(() => cliWindow);+ const { agentServices } = registerAppHandlersForCli(() => cliWindow);milestone("app handlers registered");// Speech-to-text backs the captions command; registered by the GUI boot@@ -397,107 +396,24 @@ export function runCli(command: CliCommand): void {});ipcMain.handle("update-global-shortcut", () => ({ success: false }));- const request: CliRequest = command;- ipcMain.handle("cli-get-request", () => {- milestone("renderer asked for the request");- return request;+ // The latest runner is the app's "main window" for the recording handlers, as the+ // HUD is in the GUI.+ const runners = createRunnerHost(ipcMain, (windowType): BrowserWindow => {+ const runnerWindow = loadRunnerWindow(windowType);+ cliWindow = runnerWindow;+ return runnerWindow;});- ipcMain.on("cli-log", (_event, level: string, message: string) => {- if (level === "error") {- output.error(message);- } else {- output.info(message);- output.event("log", { message });- }- });- ipcMain.on("cli-progress", (_event, progress: CliProgressEvent) => {- output.progress(progress);- });-- ipcMain.handle("cli-done", async (_event, result: CliDoneResult) => {- milestone(`renderer reported done (success=${result.success})`);- if (finished) return;- finished = true;-- try {- if (result.success && command.kind === "record" && command.projectOut) {- if (result.projectData !== undefined) {- await writeProjectFile(command.projectOut, result.projectData);- result.projectPath = command.projectOut;- }- }- if (result.success && command.kind === "captions" && result.projectData !== undefined) {- await writeProjectFile(command.projectPath, result.projectData);- }- // -o writes where no wrapper can redirect. stdout itself is fine --- // Chromium logs to stderr -- but a caller does not control what wraps- // this process, and xvfb-run on Ubuntu merges the two, which is enough- // to make `sources --json | jq` fail under the very tool used to run a- // GUI binary headlessly.- if (- result.success &&- command.kind === "sources" &&- command.jsonOutPath &&- result.sources- ) {- // Same as writeProjectFile above: `record --project out/x` creates- // `out/`, so `sources -o out/x.json` has to as well. Without it a- // fully successful enumeration is lost on every channel at once --- // the ENOENT lands in the catch below, which clears result.success- // and so skips printSources and the `done` payload alike.- await fs.mkdir(path.dirname(command.jsonOutPath), { recursive: true });- // Write beside the target and rename over it. writeFile truncates- // first, so a failure part-way through -- a full disk is the easy- // case -- would leave a half-written file where website/docs/cli.md promises- // an earlier run's result is untouched by a later failure. rename is- // atomic within a directory, so the reader sees the old file or the- // new one and never a torn one.- const tmpOutPath = `${command.jsonOutPath}.${process.pid}.tmp`;- try {- await fs.writeFile(- tmpOutPath,- `${JSON.stringify(result.sources, null, 2)}\n`,- "utf8",- );- await fs.rename(tmpOutPath, command.jsonOutPath);- } catch (writeError) {- // Best-effort: the original error is what the caller needs to see.- await fs.rm(tmpOutPath, { force: true }).catch(() => undefined);- throw writeError;- }- }- } catch (error) {- result.success = false;- result.error = `Run succeeded but writing the output file failed: ${String(error)}`;- }- if (result.success) {- for (const warning of result.warnings ?? []) {- output.info(`Warning: ${warning}`);- output.event("warning", { message: warning });- }- if (command.kind === "sources" && result.sources) {- printSources(output, result.sources);- } else if (command.kind === "captions") {- output.info(- `Added ${result.captionCount ?? 0} caption annotation(s) → ${result.projectPath}`,- );- } else if (command.kind === "export") {- output.info(`Exported ${result.format ?? ""} → ${result.outputPath}`);- } else {- output.info(`Recording saved → ${result.screenVideoPath}`);- if (result.cursorDataPath) output.info(`Cursor data → ${result.cursorDataPath}`);- if (result.projectPath) output.info(`Project → ${result.projectPath}`);- }- output.event("done", { ...result });- } else {- output.error(result.error ?? "Unknown failure");- output.event("done", { success: false, error: result.error });- }-- // Give the renderer a beat to resolve the invoke before exiting.- setTimeout(() => app.exit(result.success ? 0 : 1), 50);- });+ if (command.kind === "mcp") {+ await serveMcpOverStdio({+ ...agentServices,+ runners,+ openVideo: openVideoInHiddenWindow,+ onClose: () => app.exit(0),+ });+ milestone("mcp server listening on stdio");+ return;+ }if (command.kind === "record") {const stop = (reason: string) => {@@ -508,39 +424,119 @@ export function runCli(command: CliCommand): void {setupRecordStopSignals(stop);}- milestone("cli ipc registered; resolving window type");- const windowType = {- export: "cli-export",- record: "cli-record",- sources: "cli-sources",- captions: "cli-captions",- }[command.kind];- cliWindow = loadRunnerWindow(windowType);- milestone(`runner window created (${windowType})`);- cliWindow.webContents.on("did-finish-load", () => milestone("renderer did-finish-load"));- cliWindow.webContents.on("dom-ready", () => milestone("renderer dom-ready"));+ milestone("cli ipc registered; starting the runner");+ const run = runners.start(command, {+ onRequest: () => milestone("renderer asked for the request"),+ onLog: (level, message) => {+ if (level === "error") {+ output.error(message);+ } else {+ output.info(message);+ output.event("log", { message });+ }+ },+ onProgress: (progress) => output.progress(progress),+ });+ const runnerWindow = run.window;+ milestone(`runner window created (${command.kind})`);+ runnerWindow.webContents.on("did-finish-load", () => milestone("renderer did-finish-load"));+ runnerWindow.webContents.on("dom-ready", () => milestone("renderer dom-ready"));// Surface renderer console errors/warnings on stderr — the hidden window// has no other way to show what went wrong (toasts are invisible).- cliWindow.webContents.on("console-message", (details) => {+ runnerWindow.webContents.on("console-message", (details) => {if (details.level === "error" || details.level === "warning") {safeWrite(process.stderr, `[renderer] ${details.message}\n`);}});- cliWindow.webContents.on("did-fail-load", (_e, code, description) => {+ runnerWindow.webContents.on("did-fail-load", (_e, code, description) => {output.error(`Failed to load runner window: ${description} (${code})`);app.exit(1);});app.on("child-process-gone", (_e, details) => {milestone(`child-process-gone: ${details.type} (${details.reason})`);});- cliWindow.webContents.on("render-process-gone", (_e, details) => {+ runnerWindow.webContents.on("render-process-gone", (_e, details) => {output.error(`Renderer crashed: ${details.reason}`);app.exit(1);});output.event("started", { command: command.kind });++ const result = await run.done;+ milestone(`renderer reported done (success=${result.success})`);+ finished = true;++ try {+ if (result.success && command.kind === "record" && command.projectOut) {+ if (result.projectData !== undefined) {+ await writeProjectFile(command.projectOut, result.projectData);+ result.projectPath = command.projectOut;+ }+ }+ if (result.success && command.kind === "captions" && result.projectData !== undefined) {+ await writeProjectFile(command.projectPath, result.projectData);+ }+ // -o writes where no wrapper can redirect. stdout itself is fine --+ // Chromium logs to stderr -- but a caller does not control what wraps+ // this process, and xvfb-run on Ubuntu merges the two, which is enough+ // to make `sources --json | jq` fail under the very tool used to run a+ // GUI binary headlessly.+ if (result.success && command.kind === "sources" && command.jsonOutPath && result.sources) {+ // Same as writeProjectFile above: `record --project out/x` creates+ // `out/`, so `sources -o out/x.json` has to as well. Without it a+ // fully successful enumeration is lost on every channel at once --+ // the ENOENT lands in the catch below, which clears result.success+ // and so skips printSources and the `done` payload alike.+ await fs.mkdir(path.dirname(command.jsonOutPath), { recursive: true });+ // Write beside the target and rename over it. writeFile truncates+ // first, so a failure part-way through -- a full disk is the easy+ // case -- would leave a half-written file where website/docs/cli.md promises+ // an earlier run's result is untouched by a later failure. rename is+ // atomic within a directory, so the reader sees the old file or the+ // new one and never a torn one.+ const tmpOutPath = `${command.jsonOutPath}.${process.pid}.tmp`;+ try {+ await fs.writeFile(tmpOutPath, `${JSON.stringify(result.sources, null, 2)}\n`, "utf8");+ await fs.rename(tmpOutPath, command.jsonOutPath);+ } catch (writeError) {+ // Best-effort: the original error is what the caller needs to see.+ await fs.rm(tmpOutPath, { force: true }).catch(() => undefined);+ throw writeError;+ }+ }+ } catch (error) {+ result.success = false;+ result.error = `Run succeeded but writing the output file failed: ${String(error)}`;+ }++ if (result.success) {+ for (const warning of result.warnings ?? []) {+ output.info(`Warning: ${warning}`);+ output.event("warning", { message: warning });+ }+ if (command.kind === "sources" && result.sources) {+ printSources(output, result.sources);+ } else if (command.kind === "captions") {+ output.info(+ `Added ${result.captionCount ?? 0} caption annotation(s) → ${result.projectPath}`,+ );+ } else if (command.kind === "export") {+ output.info(`Exported ${result.format ?? ""} → ${result.outputPath}`);+ } else {+ output.info(`Recording saved → ${result.screenVideoPath}`);+ if (result.cursorDataPath) output.info(`Cursor data → ${result.cursorDataPath}`);+ if (result.projectPath) output.info(`Project → ${result.projectPath}`);+ }+ output.event("done", { ...result });+ } else {+ output.error(result.error ?? "Unknown failure");+ output.event("done", { success: false, error: result.error });+ }++ // Give the renderer a beat to resolve the invoke before exiting.+ setTimeout(() => app.exit(result.success ? 0 : 1), 50);}).catch((error) => {output.error(error instanceof Error ? (error.stack ?? error.message) : String(error));diff --git a/electron/cli/runnerHost.test.ts b/electron/cli/runnerHost.test.tsnew file mode 100644index 000000000..9ae8c7472--- /dev/null+++ b/electron/cli/runnerHost.test.ts@@ -0,0 +1,100 @@+import { EventEmitter } from "node:events";+import { describe, expect, it } from "vitest";+import type { CliRequest } from "../../src/lib/cliContracts";+import { createRunnerHost, type RunnerIpc } from "./runnerHost";++type Listener = (event: { sender: { id: number } }, ...args: unknown[]) => unknown;++/** `ipcMain`, reduced to what a runner window can call on it. */+class FakeIpc {+ readonly listeners = new Map<string, Listener>();+ handle(channel: string, listener: Listener) {+ this.listeners.set(channel, listener);+ }+ on(channel: string, listener: Listener) {+ this.listeners.set(channel, listener);+ }+ /** What the renderer of window `id` does when it calls `channel`. */+ call(id: number, channel: string, ...args: unknown[]) {+ return this.listeners.get(channel)?.({ sender: { id } }, ...args);+ }+}++class FakeWindow {+ readonly events = new EventEmitter();+ readonly webContents;+ constructor(id: number) {+ this.webContents = {+ id,+ once: (event: "destroyed", listener: () => void) => this.events.once(event, listener),+ };+ }+}++function setup() {+ const ipc = new FakeIpc();+ const opened: { windowType: string; window: FakeWindow }[] = [];+ let nextId = 1;+ const host = createRunnerHost(ipc as unknown as RunnerIpc, (windowType) => {+ const window = new FakeWindow(nextId++);+ opened.push({ windowType, window });+ return window;+ });+ return { ipc, host, opened };+}++const sources: CliRequest = { kind: "sources" };++describe("createRunnerHost", () => {+ it("hands each runner its own request and result, several at once", async () => {+ const { ipc, host, opened } = setup();+ const first = host.start(sources);+ const second = host.start({+ kind: "captions",+ projectPath: "/p.openscreen",+ minWordsPerCaption: 2,+ maxWordsPerCaption: 7,+ });+ expect(opened.map((o) => o.windowType)).toEqual(["cli-sources", "cli-captions"]);++ expect(ipc.call(2, "cli-get-request")).toMatchObject({ kind: "captions" });+ expect(ipc.call(1, "cli-get-request")).toEqual(sources);++ ipc.call(2, "cli-done", { success: true, captionCount: 3 });+ ipc.call(1, "cli-done", { success: false, error: "no display" });+ await expect(second.done).resolves.toEqual({ success: true, captionCount: 3 });+ await expect(first.done).resolves.toEqual({ success: false, error: "no display" });+ });++ it("routes logs and progress to the run they came from", () => {+ const { ipc, host } = setup();+ const logs: string[] = [];+ const progress: number[] = [];+ let asked = 0;+ host.start(sources, {+ onRequest: () => asked++,+ onLog: (level, message) => logs.push(`${level}:${message}`),+ onProgress: (p) => progress.push(p.percentage),+ });+ host.start(sources, { onLog: () => logs.push("wrong run") });++ ipc.call(1, "cli-get-request");+ ipc.call(1, "cli-log", "info", "Recording started");+ ipc.call(1, "cli-progress", { percentage: 40 });+ expect({ asked, logs, progress }).toEqual({+ asked: 1,+ logs: ["info:Recording started"],+ progress: [40],+ });+ });++ it("fails a run whose window goes away, and ignores a second result", async () => {+ const { ipc, host, opened } = setup();+ const run = host.start(sources);+ opened[0].window.events.emit("destroyed");+ await expect(run.done).resolves.toMatchObject({ success: false });+ // Too late: the run is over, and nothing answers for an unknown window.+ expect(ipc.call(1, "cli-done", { success: true })).toBeUndefined();+ expect(ipc.call(1, "cli-get-request")).toBeNull();+ });+});diff --git a/electron/cli/runnerHost.ts b/electron/cli/runnerHost.tsnew file mode 100644index 000000000..22f7f0b6b--- /dev/null+++ b/electron/cli/runnerHost.ts@@ -0,0 +1,95 @@+// Runs the hidden renderer runners (record, export, sources, captions) and+// routes their IPC by window, so one process can run any number of them: the+// one-shot CLI runs one, the MCP server one per tool call.+//+// No `electron` import: the IPC surface and the window factory are handed in,+// so the routing stays unit-testable.++import type { IpcMain } from "electron";+import type { CliDoneResult, CliProgressEvent, CliRequest } from "../../src/lib/cliContracts";++/** The slice of a BrowserWindow this needs. */+export interface RunnerWindow {+ webContents: {+ id: number;+ once(event: "destroyed", listener: () => void): unknown;+ };+}++interface RunnerIpcEvent {+ sender: { id: number };+}++/** The slice of `ipcMain` this needs. */+export type RunnerIpc = Pick<IpcMain, "handle" | "on">;++export interface RunnerHandlers {+ /** The renderer asked for its request: it booted. */+ onRequest?(): void;+ onLog?(level: string, message: string): void;+ onProgress?(progress: CliProgressEvent): void;+}++export interface RunnerRun<W> {+ window: W;+ /** The runner's own result; a window that goes away first fails the run. */+ done: Promise<CliDoneResult>;+}++interface PendingRun {+ request: CliRequest;+ handlers: RunnerHandlers;+ resolve(result: CliDoneResult): void;+}++export const RUNNER_WINDOW_TYPES = {+ export: "cli-export",+ record: "cli-record",+ sources: "cli-sources",+ captions: "cli-captions",+} as const satisfies Record<CliRequest["kind"], string>;++export function createRunnerHost<W extends RunnerWindow>(+ ipc: RunnerIpc,+ openWindow: (windowType: string) => W,+) {+ const runs = new Map<number, PendingRun>();+ const runFor = (event: RunnerIpcEvent) => runs.get(event.sender.id);++ ipc.handle("cli-get-request", (event) => {+ const run = runFor(event);+ run?.handlers.onRequest?.();+ return run?.request ?? null;+ });+ ipc.on("cli-log", (event, level: string, message: string) => {+ runFor(event)?.handlers.onLog?.(level, message);+ });+ ipc.on("cli-progress", (event, progress: CliProgressEvent) => {+ runFor(event)?.handlers.onProgress?.(progress);+ });+ ipc.handle("cli-done", (event, result: CliDoneResult) => {+ const run = runFor(event);+ if (!run) return;+ runs.delete(event.sender.id);+ run.resolve(result);+ });++ return {+ start(request: CliRequest, handlers: RunnerHandlers = {}): RunnerRun<W> {+ const window = openWindow(RUNNER_WINDOW_TYPES[request.kind]);+ const id = window.webContents.id;+ // Registered in the same tick the load starts in: the renderer cannot ask+ // for its request before this line has run.+ const done = new Promise<CliDoneResult>((resolve) => {+ runs.set(id, { request, handlers, resolve });+ });+ window.webContents.once("destroyed", () => {+ const run = runs.get(id);+ if (!run) return;+ runs.delete(id);+ run.resolve({ success: false, error: "The runner window closed before it finished" });+ });+ return { window, done };+ },+ };+}diff --git a/electron/cli/stdin.test.ts b/electron/cli/stdin.test.tsnew file mode 100644index 000000000..23e2479e2--- /dev/null+++ b/electron/cli/stdin.test.ts@@ -0,0 +1,13 @@+import { afterEach, describe, expect, it, vi } from "vitest";+import { openStdin } from "./stdin";++describe("openStdin", () => {+ afterEach(() => {+ vi.unstubAllGlobals();+ });++ it("keeps Node's own stdin outside Windows", () => {+ vi.stubGlobal("process", { ...process, platform: "linux", stdin: "the-stdin" });+ expect(openStdin()).toBe("the-stdin");+ });+});diff --git a/electron/cli/stdin.ts b/electron/cli/stdin.tsnew file mode 100644index 000000000..6e8d886b1--- /dev/null+++ b/electron/cli/stdin.ts@@ -0,0 +1,16 @@+// The process's real stdin. On Windows, Electron's main process gets a+// `process.stdin` that ends at once whatever the parent piped in (measured with+// Electron 41), so a pipe has to be read from descriptor 0 directly.++import net from "node:net";+import type { Readable } from "node:stream";++export function openStdin(): Readable {+ if (process.platform !== "win32") return process.stdin;+ try {+ return new net.Socket({ fd: 0, readable: true, writable: false });+ } catch {+ // Not a pipe (a console, or nothing at all): no one can write to us anyway.+ return process.stdin;+ }+}diff --git a/electron/ipc/handlers.ts b/electron/ipc/handlers.tsindex fdc79c32a..c71996c1c 100644--- a/electron/ipc/handlers.ts+++ b/electron/ipc/handlers.ts@@ -79,6 +79,7 @@ import {import { EditorDocumentHost } from "../mcp/editor-document-host";import { McpController } from "../mcp/mcp-controller";import { McpSettingsStore } from "../mcp/mcp-settings-store";+import type { McpToolDeps } from "../mcp/openscreen-mcp-server";import { type AudioPeaksResult, getAudioPeaks } from "../media/audioPeaks";import {readCursorRecordingFile as readCursorRecordingFileFrom,@@ -5030,13 +5031,14 @@ export function registerIpcHandlers(// read on every call — separate from the in-app agent's "Project edits", so// turning the server on grants a client read access and nothing more.const mcpSettings = new McpSettingsStore(app.getPath("userData"), safeStorage);- const mcpController = new McpController(mcpSettings, {+ const mcpToolDeps: McpToolDeps = {host: new EditorDocumentHost(ipcMain),projects: aiEditionDocuments,editsAllowed: () => mcpSettings.getSettings().allowEdits,cursor: agentCursorTelemetryReader,version: app.getVersion(),- });+ };+ const mcpController = new McpController(mcpSettings, mcpToolDeps);registerNativeBridgeHandlers({getPlatform: () => process.platform,@@ -5096,5 +5098,15 @@ export function registerIpcHandlers(getMcpController: () => mcpController,});- return { mcpController };+ // `openscreen mcp` serves the same tools on stdio, from the CLI process.+ return {+ mcpController,+ agentServices: {+ mcpToolDeps,+ documents: aiEditionDocuments,+ // The reader the export runner's auto zoom reads through.+ readCursorTelemetry: async (videoPath: string) =>+ (await readCursorTelemetryFile(videoPath)).samples,+ },+ };}diff --git a/electron/mcp/hidden-video.ts b/electron/mcp/hidden-video.tsnew file mode 100644index 000000000..8cfa2b0b0--- /dev/null+++ b/electron/mcp/hidden-video.ts@@ -0,0 +1,100 @@+// A video decoded by Chromium's own <video> element, in a blank hidden page: its+// duration and size, and still frames of it. No renderer bundle and no ffmpeg+// (the Windows installer ships no ffmpeg executable).++import { pathToFileURL } from "node:url";+import { BrowserWindow } from "electron";++export interface VideoFrame {+ mimeType: "image/jpeg";+ /** Base64, no data-URL prefix: the shape an MCP image result takes. */+ data: string;+}++export interface OpenedVideo {+ durationSec: number;+ width: number;+ height: number;+ /** The frame shown at `timeSec`, scaled down to at most `maxWidth` pixels wide. */+ capture(timeSec: number, maxWidth: number): Promise<VideoFrame>;+ close(): void;+}++// Each step is bounded: a file Chromium cannot decode may never fire an event,+// and a crashed page never answers.+const STEP_TIMEOUT_MS = 15_000;++function withTimeout<T>(promise: Promise<T>, what: string): Promise<T> {+ return new Promise<T>((resolve, reject) => {+ const timer = setTimeout(() => reject(new Error(`Timed out ${what}`)), STEP_TIMEOUT_MS);+ promise.then(+ (value) => {+ clearTimeout(timer);+ resolve(value);+ },+ (error) => {+ clearTimeout(timer);+ reject(error);+ },+ );+ });+}++export async function openVideoInHiddenWindow(videoPath: string): Promise<OpenedVideo> {+ const win = new BrowserWindow({+ show: false,+ webPreferences: {+ nodeIntegration: false,+ contextIsolation: true,+ // A blank page may not load a file:// video otherwise; same relaxation as the runners.+ webSecurity: false,+ backgroundThrottling: false,+ },+ });+ const close = () => {+ if (!win.isDestroyed()) win.destroy();+ };+ try {+ await win.loadURL("about:blank");+ const meta = (await withTimeout(+ win.webContents.executeJavaScript(`new Promise((resolve, reject) => {+ const video = document.createElement("video");+ video.muted = true;+ video.preload = "auto";+ video.onloadeddata = () =>+ resolve({ durationSec: video.duration, width: video.videoWidth, height: video.videoHeight });+ video.onerror = () =>+ reject(new Error("Chromium could not decode the video (media error " + (video.error && video.error.code) + ")"));+ window.__video = video;+ video.src = ${JSON.stringify(pathToFileURL(videoPath).href)};+ })`),+ "loading the video",+ )) as { durationSec: number; width: number; height: number };++ return {+ ...meta,+ close,+ async capture(timeSec, maxWidth) {+ const dataUrl = (await withTimeout(+ win.webContents.executeJavaScript(`new Promise((resolve) => {+ const video = window.__video;+ video.onseeked = () => {+ const scale = Math.min(1, ${maxWidth} / video.videoWidth);+ const canvas = document.createElement("canvas");+ canvas.width = Math.round(video.videoWidth * scale);+ canvas.height = Math.round(video.videoHeight * scale);+ canvas.getContext("2d").drawImage(video, 0, 0, canvas.width, canvas.height);+ resolve(canvas.toDataURL("image/jpeg", 0.85));+ };+ video.currentTime = ${timeSec};+ })`),+ `seeking to ${timeSec}s`,+ )) as string;+ return { mimeType: "image/jpeg", data: dataUrl.slice(dataUrl.indexOf(",") + 1) };+ },+ };+ } catch (error) {+ close();+ throw error;+ }+}diff --git a/electron/mcp/media-tools.test.ts b/electron/mcp/media-tools.test.tsnew file mode 100644index 000000000..b1ac23d56--- /dev/null+++ b/electron/mcp/media-tools.test.ts@@ -0,0 +1,257 @@+// The media tools through the SDK's own client, beside the document tools they+// hand projects to: a real DocumentService on a temp directory, the real tool+// runner, and stand-ins for the runner windows and the video decoder.++import fs from "node:fs/promises";+import os from "node:os";+import path from "node:path";+import { Client } from "@modelcontextprotocol/sdk/client/index.js";+import { InMemoryTransport } from "@modelcontextprotocol/sdk/inMemory.js";+import type { CallToolResult } from "@modelcontextprotocol/sdk/types.js";+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";+import type { CursorTelemetryPoint } from "../../src/components/video-editor/types";+import type { CliDoneResult, CliRequest } from "../../src/lib/cliContracts";+import { DocumentService } from "../ai-edition/document-service";+import type { RunnerHandlers } from "../cli/runnerHost";+import { frameTimes, MEDIA_PREAMBLE, type MediaToolDeps, registerMediaTools } from "./media-tools";+import {+ createOpenScreenMcpServer,+ createToolRunner,+ type McpToolDeps,+} from "./openscreen-mcp-server";++/** The runner windows: records what was asked, answers like a runner would. */+class FakeRunners {+ readonly started: CliRequest[] = [];+ recordStarts = true;+ results: Partial<Record<CliRequest["kind"], CliDoneResult>> = {};++ constructor(private readonly videoPath: string) {}++ start(request: CliRequest, handlers: RunnerHandlers = {}) {+ this.started.push(request);+ let finish: (result: CliDoneResult) => void = () => undefined;+ const done = new Promise<CliDoneResult>((resolve) => {+ finish = resolve;+ });+ const window = {+ webContents: {+ send: (channel: string) => {+ if (channel === "cli-stop-recording") {+ finish({ success: true, screenVideoPath: this.videoPath });+ }+ },+ },+ };+ if (request.kind !== "record") {+ finish(this.results[request.kind] ?? { success: true });+ } else if (this.recordStarts) {+ queueMicrotask(() => handlers.onLog?.("info", "Recording started"));+ } else {+ finish({ success: false, error: "Recording did not start within 30s" });+ }+ return { window, done };+ }+}++/** A recorded take with one click at 4 s, its pointer moving until the end. */+function clickTrack(): CursorTelemetryPoint[] {+ const samples: CursorTelemetryPoint[] = [];+ for (let timeMs = 0; timeMs <= 12_000; timeMs += 100) {+ samples.push({ timeMs, cx: 0.7, cy: 0.3, interactionType: "move" });+ }+ samples.push({ timeMs: 4_000, cx: 0.7, cy: 0.3, interactionType: "click" });+ return samples.sort((a, b) => a.timeMs - b.timeMs);+}++function textOf(result: CallToolResult): string {+ const first = result.content[0];+ return first?.type === "text" ? first.text : "";+}++let dir = "";+let documents: DocumentService;+let runners: FakeRunners;+let closed: number;+let client: Client;++beforeEach(async () => {+ dir = await fs.mkdtemp(path.join(os.tmpdir(), "media-tools-"));+ documents = new DocumentService(path.join(dir, "projects"), path.join(dir, "recordings"));+ const videoPath = path.join(dir, "recordings", "take.mp4");+ runners = new FakeRunners(videoPath);+ closed = 0;++ const toolDeps: McpToolDeps = {+ host: { snapshot: async () => null, apply: async () => "no-editor" },+ projects: documents,+ editsAllowed: () => true,+ version: "test",+ };+ const runTool = createToolRunner(toolDeps);+ const server = createOpenScreenMcpServer(toolDeps, runTool, MEDIA_PREAMBLE);+ const deps: MediaToolDeps = {+ runners,+ documents,+ openVideo: async () => ({+ durationSec: 12,+ width: 1920,+ height: 1080,+ capture: async (timeSec) => ({+ mimeType: "image/jpeg",+ data: Buffer.from(`frame@${timeSec}`).toString("base64"),+ }),+ close: () => {+ closed++;+ },+ }),+ readCursor: async (path) => (path === videoPath ? clickTrack() : []),+ runTool,+ now: () => 1_000,+ };+ registerMediaTools(server, deps);++ const [clientSide, serverSide] = InMemoryTransport.createLinkedPair();+ await server.connect(serverSide);+ client = new Client({ name: "test", version: "1" });+ await client.connect(clientSide);+});++afterEach(async () => {+ await client.close();+ await fs.rm(dir, { recursive: true, force: true });+});++const call = (name: string, args: Record<string, unknown> = {}) =>+ client.callTool({ name, arguments: args }) as Promise<CallToolResult>;++async function recordTake(title = "Demo") {+ await call("startRecording", { window: "My App" });+ const stopped = await call("stopRecording", { title });+ return JSON.parse(textOf(stopped)) as {+ projectId: string;+ checkpointId: string;+ videoPath: string;+ durationSec: number;+ autoZooms: number;+ recordingStartedAt: number;+ };+}++describe("frameTimes", () => {+ it("centres `count` frames in equal slices, and clamps asked-for times inside the video", () => {+ expect(frameTimes(12, null, 4)).toEqual([1.5, 4.5, 7.5, 10.5]);+ const [start, end] = frameTimes(10, [0, 99], 6);+ expect(start).toBe(0);+ expect(end).toBeLessThan(10);+ });+});++describe("recording", () => {+ it("records one window, then turns the take into a project the document tools can edit", async () => {+ const started = await call("startRecording", { window: "My App", systemAudio: true });+ expect(started.isError).toBeFalsy();+ expect(runners.started[0]).toMatchObject({+ kind: "record",+ windowTitle: "My App",+ systemAudio: true,+ mic: false,+ cursorMode: "editable-overlay",+ });++ const stopped = await call("stopRecording", { title: "Demo" });+ expect(stopped.isError).toBeFalsy();+ const take = JSON.parse(textOf(stopped));+ expect(take).toMatchObject({ durationSec: 12, recordingStartedAt: 1_000 });+ expect(take.autoZooms).toBeGreaterThan(0);+ expect(closed).toBe(1);++ const document = await documents.getProject(take.projectId);+ expect(document.project.title).toBe("Demo");+ expect(document.assets[0]).toMatchObject({+ originalPath: take.videoPath,+ durationSec: 12,+ video: { width: 1920, height: 1080 },+ });+ expect(document.timeline.clips).toHaveLength(1);+ expect(document.timeline.clips[0]).toMatchObject({ sourceStartSec: 0, sourceEndSec: 12 });+ expect(document.zoomRanges).toHaveLength(take.autoZooms);++ // The checkpoint it returns is the way back for the document tools' edits.+ const read = await call("getCurrentDocument", { projectId: take.projectId });+ expect(read.isError).toBeFalsy();+ const restored = await call("restoreCheckpoint", {+ projectId: take.projectId,+ checkpointId: take.checkpointId,+ });+ expect(restored.isError).toBeFalsy();+ });++ it("says why a recording did not start, and holds no take", async () => {+ runners.recordStarts = false;+ const started = await call("startRecording", { display: 1 });+ expect(started.isError).toBe(true);+ expect(textOf(started)).toContain("did not start within 30s");+ expect((await call("stopRecording")).isError).toBe(true);+ });++ it("runs one take at a time, and no export during it", async () => {+ await call("startRecording", {});+ expect((await call("startRecording", {})).isError).toBe(true);+ const exported = await call("exportVideo", { projectId: "proj_x", path: "out.mp4" });+ expect(exported.isError).toBe(true);+ expect(textOf(exported)).toContain("recording is running");+ });+});++describe("exportVideo", () => {+ it("renders the project's own file to the path asked for", async () => {+ const { projectId } = await recordTake();+ const outPath = path.join(dir, "demo.mp4");+ runners.results.export = {+ success: true,+ outputPath: outPath,+ format: "mp4",+ width: 1920,+ height: 1080,+ };+ const exported = await call("exportVideo", { projectId, path: outPath });+ expect(exported.isError).toBeFalsy();+ expect(JSON.parse(textOf(exported))).toMatchObject({ path: outPath, width: 1920 });+ expect(runners.started.at(-1)).toMatchObject({+ kind: "export",+ projectPath: documents.filePathFor(projectId),+ outPath,+ format: "mp4",+ quality: "good",+ });+ });++ it("refuses a path that is not a video, and reports a failed render", async () => {+ expect((await call("exportVideo", { projectId: "proj_x", path: "out.mov" })).isError).toBe(+ true,+ );+ runners.results.export = { success: false, error: "compositor exploded" };+ const failed = await call("exportVideo", { projectId: "proj_x", path: "out.gif" });+ expect(failed.isError).toBe(true);+ expect(textOf(failed)).toContain("compositor exploded");+ expect(runners.started.at(-1)).toMatchObject({ format: "gif" });+ });+});++describe("getFrames", () => {+ it("returns each frame as an image, labelled with its time", async () => {+ const result = await call("getFrames", { path: "take.mp4", atSec: [1, 5] });+ const images = result.content.filter((part) => part.type === "image");+ expect(images).toHaveLength(2);+ expect(images[1]).toMatchObject({ mimeType: "image/jpeg" });+ expect(result.content.map((part) => (part.type === "text" ? part.text : "img"))).toEqual([+ expect.stringContaining("1920x1080"),+ "1.00s",+ "img",+ "5.00s",+ "img",+ ]);+ expect(closed).toBe(1);+ });+});diff --git a/electron/mcp/media-tools.ts b/electron/mcp/media-tools.tsnew file mode 100644index 000000000..ffdfac1d4--- /dev/null+++ b/electron/mcp/media-tools.ts@@ -0,0 +1,371 @@+// What an agent needs around the editing DSL to make a video on its own: record a+// take, have it become a project, export the project, and look at the result.+// Served by `openscreen mcp` next to the document tools, so every edit between the+// recording and the export goes through the DSL like any other client's.+//+// No `electron` import: the runners, the decoder and the documents are handed in,+// so the tools run under plain vitest with stand-ins.++import path from "node:path";+import type { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";+import type { CallToolResult } from "@modelcontextprotocol/sdk/types.js";+import { z } from "zod";+import type { CursorTelemetryPoint } from "../../src/components/video-editor/types";+import type { AxcutDocument } from "../../src/lib/ai-edition/schema";+import {+ appendAutoZoomSuggestions,+ collectAutoZoomSuggestionsForDocument,+} from "../../src/lib/ai-edition/timeline/apply-auto-zooms";+import type { CliDoneResult, CliRequest } from "../../src/lib/cliContracts";+import { runDocumentTool } from "../ai-edition/deep-agent/service";+import type { DocumentService } from "../ai-edition/document-service";+import type { RunnerHandlers } from "../cli/runnerHost";+import type { OpenedVideo } from "./hidden-video";++export interface MediaToolDeps {+ runners: {+ start(+ request: CliRequest,+ handlers?: RunnerHandlers,+ ): {+ window: { webContents: { send(channel: string): void } };+ done: Promise<CliDoneResult>;+ };+ };+ documents: Pick<DocumentService, "createProject" | "addAsset" | "saveProject" | "filePathFor">;+ openVideo(videoPath: string): Promise<OpenedVideo>;+ /** The pointer samples recorded with a video; empty when there are none. */+ readCursor(videoPath: string): Promise<CursorTelemetryPoint[]>;+ /** The document tools' runner: a fresh project's checkpoint goes through it. */+ runTool(name: string, args: unknown): Promise<CallToolResult>;+ now?(): number;+}++export const MAX_FRAMES = 12;+// Wide enough to read UI text, small enough that a dozen frames stay cheap.+const FRAME_WIDTH = 1280;+// A seek to the exact end shows no frame in Chromium; stop one frame short of it.+const END_MARGIN_SEC = 1 / 60;++/** The times to grab: `atSec` clamped inside the video, else `count` centred in equal slices. */+export function frameTimes(durationSec: number, atSec: number[] | null, count: number): number[] {+ const last = Math.max(0, durationSec - END_MARGIN_SEC);+ if (atSec) return atSec.map((t) => Math.min(t, last));+ return Array.from({ length: count }, (_, i) => Math.min(((i + 0.5) * durationSec) / count, last));+}++function text(body: string, isError = false): CallToolResult {+ return { content: [{ type: "text", text: body }], ...(isError ? { isError: true } : {}) };+}++const READ_ONLY = { readOnlyHint: true, destructiveHint: false, openWorldHint: false } as const;++export const MEDIA_PREAMBLE = [+ "OpenScreen records the screen, turns each take into a project you edit with the document tools, and exports it as a video.",+ "A demo goes: listSources, then startRecording, then drive the app, then stopRecording. It returns the projectId and a checkpointId. Edit with the document tools, always passing that projectId: every edit is saved to the project straight away, and restoreCheckpoint with the checkpointId takes the project back to the raw take in one step. Then exportVideo, and getFrames on the export to check it before you hand it over.",+ "These projects are in the OpenScreen app's project list, so the user can open one in the editor. Do not edit a project while the user has it open there: the editor would keep its own copy.",+ "",+].join("\n");++export function registerMediaTools(server: McpServer, deps: MediaToolDeps): void {+ const now = deps.now ?? Date.now;+ let take: { run: ReturnType<MediaToolDeps["runners"]["start"]>; startedAt: number } | null = null;+ // An export loads its project into the slot a finishing take reads its files from.+ let exporting = false;++ /** The take as a project, laid out as the editor lays out a fresh recording. */+ async function createProjectFromVideo(videoPath: string, title: string) {+ const video = await deps.openVideo(videoPath);+ video.close();+ const { durationSec, width, height } = video;+ if (!Number.isFinite(durationSec) || durationSec <= 0) {+ throw new Error(`Could not read the duration of ${videoPath}`);+ }+ const created = await deps.documents.createProject(title);+ let document: AxcutDocument = await deps.documents.addAsset(created.project.id, {+ path: videoPath,+ });+ const assetId = document.project.primaryAssetId;+ document = {+ ...document,+ assets: document.assets.map((asset) =>+ asset.id === assetId+ ? {+ ...asset,+ durationSec,+ video: {+ codec: asset.video?.codec ?? "unknown",+ width,+ height,+ fps: asset.video?.fps ?? 0,+ },+ }+ : asset,+ ),+ };+ const seeded = await runDocumentTool(+ document,+ "replaceTimeline",+ { intervals: [{ startSec: 0, endSec: durationSec }], reason: "The recorded take" },+ true,+ {},+ );+ if (!seeded.ok || !seeded.document) throw new Error(seeded.resultJson);+ // The zooms a recording made in the app starts with: one on each recorded click.+ const suggestions = await collectAutoZoomSuggestionsForDocument(+ seeded.document,+ deps.readCursor,+ );+ const saved = await deps.documents.saveProject(+ appendAutoZoomSuggestions(seeded.document, suggestions),+ );+ return { document: saved, durationSec, autoZooms: suggestions.length };+ }++ server.registerTool(+ "listSources",+ {+ description:+ "List the displays and windows OpenScreen can record, and the microphones. Pass a display's index or part of a window's title to startRecording.",+ inputSchema: z.object({}),+ annotations: READ_ONLY,+ },+ async () => {+ const result = await deps.runners.start({ kind: "sources" }).done;+ if (!result.success || !result.sources) {+ return text(`Could not list the sources: ${result.error ?? "unknown error"}`, true);+ }+ const { displays, windows, microphones } = result.sources;+ return text(+ JSON.stringify({+ displays: displays.map(({ index, name }) => ({ index, name })),+ windows: windows.map((window) => window.name),+ microphones: microphones.map((mic) => mic.label),+ }),+ );+ },+ );++ server.registerTool(+ "startRecording",+ {+ description:+ "Start recording one window (preferred: nothing else on the desktop is captured) or one display. Returns once capture has started; then drive the app and call stopRecording. The pointer is recorded as data, so the export draws a clean cursor and zooms on the clicks. On Linux the desktop's own picker asks the user which window to record.",+ inputSchema: z.object({+ window: z+ .string()+ .optional()+ .describe("Part of the title of the window to record, ignoring case. Wins over display."),+ display: z+ .number()+ .int()+ .min(0)+ .optional()+ .describe("Index of the display to record, from listSources. Default 0."),+ systemAudio: z.boolean().optional().describe("Also record the computer's sound."),+ microphone: z+ .union([z.boolean(), z.string()])+ .optional()+ .describe("true for the default microphone, or part of a microphone's name."),+ }),+ annotations: { readOnlyHint: false, destructiveHint: false, openWorldHint: false },+ },+ async (args) => {+ if (take) return text("A recording is already running. Call stopRecording first.", true);+ if (exporting) return text("An export is running. Wait for it, then try again.", true);+ let markStarted: () => void = () => undefined;+ const started = new Promise<"started">((resolve) => {+ markStarted = () => resolve("started");+ });+ const run = deps.runners.start(+ {+ kind: "record",+ displayIndex: args.display ?? 0,+ windowTitle: args.window ?? null,+ mic: args.microphone !== undefined && args.microphone !== false,+ micDevice: typeof args.microphone === "string" ? args.microphone : null,+ systemAudio: args.systemAudio === true,+ cursorMode: "editable-overlay",+ durationMs: null,+ projectOut: null,+ },+ {+ onLog: (_level, message) => {+ if (message === "Recording started") markStarted();+ },+ },+ );+ const outcome = await Promise.race([started, run.done]);+ if (outcome !== "started") {+ return text(`The recording did not start: ${outcome.error ?? "unknown error"}`, true);+ }+ take = { run, startedAt: now() };+ return text(+ JSON.stringify({ recording: true, startedAt: take.startedAt }) ++ "\nRecording. Drive the app now, pausing about a second after each result appears, then call stopRecording.",+ );+ },+ );++ server.registerTool(+ "stopRecording",+ {+ description:+ "Stop the recording and turn the take into a new project, laid out as the editor lays out a fresh recording: one clip, plus a zoom on each recorded click. Returns the projectId to edit, a checkpointId that restores the raw take, and the take's videoPath.",+ inputSchema: z.object({+ title: z.string().optional().describe("The project's title in OpenScreen."),+ }),+ annotations: { readOnlyHint: false, destructiveHint: false, openWorldHint: false },+ },+ async (args) => {+ const current = take;+ if (!current) return text("No recording is running.", true);+ try {+ current.run.window.webContents.send("cli-stop-recording");+ } catch {+ // The runner already went away; its result says why.+ }+ const result = await current.run.done;+ take = null;+ if (!result.success || !result.screenVideoPath) {+ return text(`The recording failed: ${result.error ?? "unknown error"}`, true);+ }+ const title =+ args.title?.trim() || `Recording ${new Date(current.startedAt).toLocaleString()}`;+ const project = await createProjectFromVideo(result.screenVideoPath, title);+ const projectId = project.document.project.id;+ const checkpoint = await deps.runTool("createCheckpoint", { projectId });+ const checkpointText =+ checkpoint.content[0]?.type === "text" ? checkpoint.content[0].text : "";+ const checkpointId = checkpoint.isError+ ? null+ : (JSON.parse(checkpointText) as { checkpointId?: string }).checkpointId;+ return text(+ JSON.stringify({+ projectId,+ checkpointId,+ videoPath: result.screenVideoPath,+ durationSec: project.durationSec,+ recordingStartedAt: current.startedAt,+ autoZooms: project.autoZooms,+ ...(result.warnings?.length ? { warnings: result.warnings } : {}),+ }),+ );+ },+ );++ server.registerTool(+ "exportVideo",+ {+ description:+ "Render a project to an MP4 (H.264, 60 fps) or a GIF with OpenScreen's compositor: the background, cursor, zooms, cuts, speed changes and annotations as edited. Returns the file's path; call getFrames on it to check the result.",+ inputSchema: z.object({+ projectId: z.string(),+ path: z+ .string()+ .describe(+ "Where to write the video: an absolute path, or one relative to this server's working directory, ending in .mp4 or .gif.",+ ),+ quality: z+ .enum(["medium", "good", "source"])+ .optional()+ .describe("medium is 720p, good is 1080p (default), source keeps the recording's size."),+ }),+ annotations: { readOnlyHint: false, destructiveHint: false, openWorldHint: false },+ },+ async (args, extra) => {+ if (take) return text("A recording is running. Stop it before exporting.", true);+ const outPath = path.resolve(args.path);+ const extension = path.extname(outPath).toLowerCase();+ if (extension !== ".mp4" && extension !== ".gif") {+ return text(`The path must end in .mp4 or .gif, got "${args.path}".`, true);+ }+ const progressToken = extra._meta?.progressToken;+ exporting = true;+ try {+ const result = await deps.runners.start(+ {+ kind: "export",+ projectPath: deps.documents.filePathFor(args.projectId),+ outPath,+ format: extension === ".gif" ? "gif" : "mp4",+ quality: args.quality ?? "good",+ gifFrameRate: null,+ gifSizePreset: null,+ autoZoom: false,+ audioPath: null,+ audioMode: "mix",+ audioOffsetSec: 0,+ },+ {+ onProgress: (progress) => {+ if (progressToken === undefined) return;+ void extra+ .sendNotification({+ method: "notifications/progress",+ params: { progressToken, progress: progress.percentage, total: 100 },+ })+ .catch(() => undefined);+ },+ },+ ).done;+ if (!result.success) {+ return text(`The export failed: ${result.error ?? "unknown error"}`, true);+ }+ return text(+ JSON.stringify({+ path: result.outputPath,+ format: result.format,+ width: result.width,+ height: result.height,+ }),+ );+ } finally {+ exporting = false;+ }+ },+ );++ server.registerTool(+ "getFrames",+ {+ description: `Look at a video: still frames of a take (videoPath from stopRecording) or of an export, as images. Ask for the moments you need to check, such as each zoom and each callout. At most ${MAX_FRAMES} frames per call.`,+ inputSchema: z.object({+ path: z.string().describe("The video file."),+ atSec: z+ .array(z.number().nonnegative())+ .max(MAX_FRAMES)+ .optional()+ .describe("Times in seconds of that video. A time past the end gives the last frame."),+ count: z+ .number()+ .int()+ .min(1)+ .max(MAX_FRAMES)+ .optional()+ .describe("Frames spread evenly across the video, when atSec is not given. Default 6."),+ }),+ annotations: READ_ONLY,+ },+ async (args) => {+ const video = await deps.openVideo(path.resolve(args.path));+ try {+ const times = frameTimes(video.durationSec, args.atSec ?? null, args.count ?? 6);+ const content: CallToolResult["content"] = [+ {+ type: "text",+ text: `${times.length} frame(s) of a ${video.width}x${video.height} video lasting ${video.durationSec.toFixed(2)}s.`,+ },+ ];+ for (const timeSec of times) {+ const frame = await video.capture(timeSec, FRAME_WIDTH);+ content.push({ type: "text", text: `${timeSec.toFixed(2)}s` });+ content.push({ type: "image", data: frame.data, mimeType: frame.mimeType });+ }+ return { content };+ } finally {+ video.close();+ }+ },+ );+}diff --git a/electron/mcp/openscreen-mcp-server.ts b/electron/mcp/openscreen-mcp-server.tsindex 2feb6e084..acba66f52 100644--- a/electron/mcp/openscreen-mcp-server.ts+++ b/electron/mcp/openscreen-mcp-server.ts@@ -353,10 +353,12 @@ export function createToolRunner(deps: McpToolDeps) {export function createOpenScreenMcpServer(deps: McpToolDeps,runTool: (name: string, args: unknown) => Promise<CallToolResult>,+ /** What the client reads before the editing guidance; the stdio server says what it adds. */+ preamble = MCP_PREAMBLE,): McpServer {const server = new McpServer({ name: MCP_SERVER_NAME, version: deps.version },- { instructions: MCP_PREAMBLE + buildSystemPrompt({ editsAllowed: deps.editsAllowed() }) },+ { instructions: preamble + buildSystemPrompt({ editsAllowed: deps.editsAllowed() }) },);server.registerTool(LIST_PROJECTS_TOOL,diff --git a/electron/mcp/stdio-server.ts b/electron/mcp/stdio-server.tsnew file mode 100644index 000000000..553762b69--- /dev/null+++ b/electron/mcp/stdio-server.ts@@ -0,0 +1,43 @@+// `openscreen mcp`: the MCP tools on stdin/stdout, for the agent that started+// this process. The document tools are the in-app server's own, run by the same+// executor; the media tools around them record, export and look at the result.+//+// No port and no token: the client launched this process itself, with the+// user's rights, as it would launch `openscreen record` or `openscreen export`.+// For the same reason it may edit, where the in-app server waits for the user to+// allow it: there is no other caller to keep out.++import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js";+import type { DocumentService } from "../ai-edition/document-service";+import { openStdin } from "../cli/stdin";+import { MEDIA_PREAMBLE, type MediaToolDeps, registerMediaTools } from "./media-tools";+import {+ createOpenScreenMcpServer,+ createToolRunner,+ type McpToolDeps,+} from "./openscreen-mcp-server";++export async function serveMcpOverStdio(options: {+ mcpToolDeps: McpToolDeps;+ documents: DocumentService;+ readCursorTelemetry: MediaToolDeps["readCursor"];+ runners: MediaToolDeps["runners"];+ openVideo: MediaToolDeps["openVideo"];+ /** The client went away: nothing is left to serve. */+ onClose(): void;+}): Promise<void> {+ const deps: McpToolDeps = { ...options.mcpToolDeps, editsAllowed: () => true };+ const runTool = createToolRunner(deps);+ const server = createOpenScreenMcpServer(deps, runTool, MEDIA_PREAMBLE);+ registerMediaTools(server, {+ runners: options.runners,+ documents: options.documents,+ openVideo: options.openVideo,+ readCursor: options.readCursorTelemetry,+ runTool,+ });+ const stdin = openStdin();+ // The transport only reads stdin; a client that exits just closes it.+ stdin.once("end", options.onClose);+ await server.connect(new StdioServerTransport(stdin, process.stdout));+}diff --git a/src/lib/ai-edition/timeline/zoom-suggestions.ts b/src/lib/ai-edition/timeline/zoom-suggestions.tsindex 8df686f48..9ca9709b5 100644--- a/src/lib/ai-edition/timeline/zoom-suggestions.ts+++ b/src/lib/ai-edition/timeline/zoom-suggestions.ts@@ -12,7 +12,9 @@// automatic zoom.import type { CursorTelemetryPoint, ZoomFocus } from "@/components/video-editor/types";-import { zoomTransitionMs } from "@/lib/zoomMath/constants";+// Relative, not `@/`: the main process plans a fresh MCP take's zooms with this file,+// and its build does not resolve the alias.+import { zoomTransitionMs } from "../../zoomMath/constants";import type { AxcutClip } from "../schema";import { DEFAULT_ZOOM_DEPTH, ZOOM_DEPTH_SCALES, type ZoomDepth } from "./zoom-scale";