Skip to content

Record your screen, ship a demo. Free and open-source, GPU-accelerated, no watermarks, no subscriptions. Windows, macOS, Linux. Actively maintained.

feat(mcp): serve the media tools from the in-app server

EtienneLescotb81cd37601ff
diff --git a/electron/cli/cliMain.ts b/electron/cli/cliMain.ts
index 353787fe5..0a31a5b02 100644
--- a/electron/cli/cliMain.ts
+++ b/electron/cli/cliMain.ts
@@ -5,7 +5,6 @@
import fs from "node:fs/promises";
import path from "node:path";
import readline from "node:readline";
-import { fileURLToPath } from "node:url";
import { app, BrowserWindow, ipcMain, session, systemPreferences } from "electron";
import type { CliProgressEvent, CliSourcesResult } from "../../src/lib/cliContracts";
import { isDiagnosticModeEnabled } from "../diagnostics/main-log-buffer";
@@ -13,17 +12,13 @@ import { getSelectedDesktopSource, registerIpcHandlers } from "../ipc/handlers";
import { openVideoInHiddenWindow } from "../mcp/hidden-video";
import { serveMcpOverStdio } from "../mcp/stdio-server";
import { registerSttIpc } from "../stt";
-import { ASSET_BASE_URL_ARG } from "../windows";
+import { createRunnerWindow } from "../windows";
import { CLI_USAGE, type CliCommand } from "./args";
import { runInfoCommand, runPackCommand } from "./projectCommands";
import { createRunnerHost } from "./runnerHost";
import { isolateCliSessionData } from "./sessionData";
import { openStdin } from "./stdin";
-const __dirname = path.dirname(fileURLToPath(import.meta.url));
-const VITE_DEV_SERVER_URL = process.env["VITE_DEV_SERVER_URL"];
-const RENDERER_DIST = path.join(__dirname, "..", "dist");
-
interface CliOutput {
json: boolean;
event(event: string, data?: Record<string, unknown>): void;
@@ -121,31 +116,6 @@ function createOutput(json: boolean): CliOutput {
};
}
-function loadRunnerWindow(windowType: string): BrowserWindow {
- const win = new BrowserWindow({
- width: 1280,
- height: 720,
- show: false,
- webPreferences: {
- preload: path.join(__dirname, "preload.mjs"),
- additionalArguments: [ASSET_BASE_URL_ARG],
- nodeIntegration: false,
- contextIsolation: true,
- // Same relaxation as the editor window: exporters load recording media
- // via file:// URLs.
- webSecurity: false,
- backgroundThrottling: false,
- },
- });
-
- if (VITE_DEV_SERVER_URL) {
- win.loadURL(`${VITE_DEV_SERVER_URL}?windowType=${windowType}`);
- } else {
- win.loadFile(path.join(RENDERER_DIST, "index.html"), { query: { windowType } });
- }
- return win;
-}
-
function registerAppHandlersForCli(cliWindowRef: () => BrowserWindow | null) {
// The recording/export pipelines are driven through the same IPC surface the
// GUI uses. Window-management callbacks become no-ops; "switch-to-editor"
@@ -402,7 +372,7 @@ export function runCli(command: CliCommand): void {
// The latest runner is the app's "main window" for the recording handlers, as the
// HUD is in the GUI.
const runners = createRunnerHost(ipcMain, (windowType): BrowserWindow => {
- const runnerWindow = loadRunnerWindow(windowType);
+ const runnerWindow = createRunnerWindow(windowType);
cliWindow = runnerWindow;
return runnerWindow;
});
diff --git a/electron/ipc/handlers.ts b/electron/ipc/handlers.ts
index 3f6a949d9..13c48543a 100644
--- a/electron/ipc/handlers.ts
+++ b/electron/ipc/handlers.ts
@@ -24,6 +24,7 @@ import {
isAxcutDocumentFile,
parseDocumentFile,
} from "../../src/lib/ai-edition/schema";
+import { pickRecordingSource } from "../../src/lib/cliSourcePick";
import {
type NativeLinuxRecordingRequest,
portalCursorMode,
@@ -68,6 +69,7 @@ import { DocumentService } from "../ai-edition/document-service";
import { LlmConfigStore } from "../ai-edition/llm-config-store";
import { StylePresetService } from "../ai-edition/style-preset-service";
import { AppSettingsStore } from "../app-settings";
+import { createRunnerHost } from "../cli/runnerHost";
import { isDiagnosticModeEnabled, mainLogBuffer } from "../diagnostics/main-log-buffer";
import { mainT } from "../i18n";
import { getInstallChannel } from "../install-channel";
@@ -77,9 +79,12 @@ import {
RECORDINGS_DIR,
setRecordingsDir,
} from "../main";
+import { createAppRecorder } from "../mcp/app-recorder";
import { EditorDocumentHost } from "../mcp/editor-document-host";
+import { openVideoInHiddenWindow } from "../mcp/hidden-video";
import { McpController } from "../mcp/mcp-controller";
import { McpSettingsStore } from "../mcp/mcp-settings-store";
+import { createMediaTools, type MediaToolDeps, type RecordRequest } from "../mcp/media-tools";
import { createToolRunner, type McpToolDeps } from "../mcp/openscreen-mcp-server";
import { type AudioPeaksResult, getAudioPeaks } from "../media/audioPeaks";
import {
@@ -151,6 +156,7 @@ import {
shouldEnumerateRecordingSources,
shouldPersistSelectedSource,
} from "../recording-source-settings";
+import { createRunnerWindow } from "../windows";
import { getSourcesByType, shouldEnumerateSourceTypesSeparately } from "./desktopSourceTypes";
import { registerNativeBridgeHandlers } from "./nativeBridge";
import { createNativeMacMidCaptureErrorWatch } from "./nativeMacMidCaptureErrorWatch";
@@ -1975,7 +1981,11 @@ export function registerIpcHandlers(
getCountdownOverlayWindow?: () => BrowserWindow | null,
onRecordingStateChange?: (recording: boolean, sourceName: string) => void,
_switchToHud?: () => void,
+ /** The editor holds changes it has not saved (main.ts): an MCP take would close it. */
+ editorHasUnsavedChanges?: () => boolean,
) {
+ /** The HUD is recording a take, the user's or an MCP client's (`set-recording-state`). */
+ let appRecording = false;
const appSettings = new AppSettingsStore(app.getPath("userData"));
const broadcastSelectedSource = (source: SelectedSource | null) => {
for (const window of BrowserWindow.getAllWindows()) {
@@ -2000,7 +2010,9 @@ export function registerIpcHandlers(
clipboard.writeText(text);
});
- ipcMain.handle("get-sources", async (_, opts) => {
+ /** The sources the pickers list, in their order: the in-app MCP server's startRecording
+ * picks from the same list its listSources shows. */
+ async function getSourcesForPickers(opts: Electron.SourcesOptions) {
// desktopCapturer.getSources can never settle where the GL stack cannot be
// reached -- a container, a CI runner, a host whose ANGLE fails to
// initialise. Bounded here rather than per-caller because every caller has
@@ -2084,7 +2096,8 @@ export function registerIpcHandlers(
thumbnail: source.thumbnail ? source.thumbnail.toDataURL() : null,
appIcon: source.appIcon ? source.appIcon.toDataURL() : null,
}));
- });
+ }
+ ipcMain.handle("get-sources", (_, opts) => getSourcesForPickers(opts));
const selectSourceContext: SelectSourceContext<DesktopCapturerSource> = {
generation: selectSourceGeneration,
@@ -2099,37 +2112,37 @@ export function registerIpcHandlers(
},
};
- ipcMain.handle(
- "select-source",
- async (_, source: SelectedSource, options?: { persist?: boolean }) => {
- const next = await selectSourceWithOwnership(
- selectSourceContext,
- { id: source.id, name: source.name, display_id: source.display_id },
- options,
- {
- getSources: () =>
- desktopCapturer.getSources({
- types: ["screen", "window"],
- thumbnailSize: { width: 0, height: 0 },
- fetchWindowIcons: true,
- }),
- persist: (live) => {
- appSettings.setLastSource(
- describeRecordingSource(process.platform, live as Required<SelectedSource>),
- );
- },
- broadcast: broadcastSelectedSource,
- shouldPersist: shouldPersistSelectedSource,
+ async function selectSource(source: SelectedSource, options?: { persist?: boolean }) {
+ const next = await selectSourceWithOwnership(
+ selectSourceContext,
+ { id: source.id, name: source.name, display_id: source.display_id },
+ options,
+ {
+ getSources: () =>
+ desktopCapturer.getSources({
+ types: ["screen", "window"],
+ thumbnailSize: { width: 0, height: 0 },
+ fetchWindowIcons: true,
+ }),
+ persist: (live) => {
+ appSettings.setLastSource(
+ describeRecordingSource(process.platform, live as Required<SelectedSource>),
+ );
},
- );
- if (next) {
- const sourceSelectorWin = getSourceSelectorWindow();
- if (sourceSelectorWin) {
- sourceSelectorWin.close();
- }
+ broadcast: broadcastSelectedSource,
+ shouldPersist: shouldPersistSelectedSource,
+ },
+ );
+ if (next) {
+ const sourceSelectorWin = getSourceSelectorWindow();
+ if (sourceSelectorWin) {
+ sourceSelectorWin.close();
}
- return next;
- },
+ }
+ return next;
+ }
+ ipcMain.handle("select-source", (_, source: SelectedSource, options?: { persist?: boolean }) =>
+ selectSource(source, options),
);
/**
@@ -2282,7 +2295,7 @@ export function registerIpcHandlers(
return selectedSource;
});
- registerRecordingPrefsHandlers(
+ const recordingPrefs = registerRecordingPrefsHandlers(
defaultRecordingPrefs,
getMainWindow,
() => BrowserWindow.getAllWindows(),
@@ -2484,7 +2497,8 @@ export function registerIpcHandlers(
return { success: true };
});
- ipcMain.handle("start-new-recording", () => {
+ /** The editor's Record button: over to the HUD, which starts the take by itself. */
+ function startNewRecording() {
_switchToHud?.();
const hudWindow = getMainWindow();
if (hudWindow && !hudWindow.isDestroyed()) {
@@ -2495,6 +2509,9 @@ export function registerIpcHandlers(
sendAutoStart();
}
}
+ }
+ ipcMain.handle("start-new-recording", () => {
+ startNewRecording();
return { success: true };
});
@@ -4112,6 +4129,8 @@ export function registerIpcHandlers(
}
const source = selectedSource || { name: "Screen" };
+ appRecording = recording;
+ mcpAppRecorder.recordingChanged(recording);
if (onRecordingStateChange) {
onRecordingStateChange(recording, source.name);
}
@@ -4856,6 +4875,7 @@ export function registerIpcHandlers(
setCurrentRecordingSessionState(normalizedSession);
currentVideoPath = normalizedSession?.screenVideoPath ?? null;
currentProjectPath = null;
+ if (normalizedSession) mcpAppRecorder.takeSaved(normalizedSession.screenVideoPath);
return { success: true, session: currentRecordingSession };
});
@@ -5043,7 +5063,109 @@ export function registerIpcHandlers(
// One runner, so one queue and one checkpoint store, for the in-app server and every
// `openscreen mcp` the app answers through its channel (app-channel.ts, started by main.ts).
const mcpRunTool = createToolRunner(mcpToolDeps);
- const mcpController = new McpController(mcpSettings, mcpToolDeps, mcpRunTool);
+
+ // The in-app server's media tools. listSources and exportVideo run the CLI's hidden
+ // runners in this process, their IPC registered on first use: never in the headless
+ // CLI, which registers its own and never starts this server.
+ let mcpRunnerHost: ReturnType<typeof createRunnerHost<BrowserWindow>> | null = null;
+ const mcpRunners: MediaToolDeps["runners"] = {
+ start(request, handlers) {
+ mcpRunnerHost ??= createRunnerHost(ipcMain, createRunnerWindow);
+ return mcpRunnerHost.start(request, handlers);
+ },
+ };
+
+ /**
+ * Points the HUD at a client's take, as Record mode does before it hands over, and
+ * returns what puts back what the user had. Restored once the take is captured, so the
+ * user's own source and settings are what their next take starts from.
+ */
+ async function pointHudAtTake(request: RecordRequest): Promise<() => void> {
+ const before = { source: selectedSource, live: selectedDesktopSource };
+ if (process.platform === "linux" && findPipeWireCursorHelperPath()) {
+ // The portal asks the user once the take starts: nothing here to select.
+ } else if (macPickerOwnsSources()) {
+ // Apple's picker asks for this take: an earlier pick must not stand in for it.
+ selectedSource = null;
+ selectedDesktopSource = null;
+ broadcastSelectedSource(null);
+ } else {
+ // The list listSources shows, in its order: its display indexes are this list's.
+ const sources = await getSourcesForPickers({
+ types: ["screen", "window"],
+ thumbnailSize: { width: 0, height: 0 },
+ });
+ const picked = pickRecordingSource(sources, request);
+ if (!(await selectSource(picked, { persist: false }))) {
+ throw new Error(`"${picked.name}" can no longer be recorded. Call listSources again.`);
+ }
+ }
+ let microphone: Pick<RecordingPrefs, "micDeviceId" | "micDeviceName"> = {
+ micDeviceId: null,
+ micDeviceName: null,
+ };
+ if (request.mic && request.micDevice) {
+ // Its id as well as its name, both of which the HUD records with: a renderer's to read.
+ const run = mcpRunners.start({ kind: "sources" });
+ const listed = await run.done.finally(() => run.close());
+ const needle = request.micDevice.toLowerCase();
+ const match = listed.sources?.microphones.find((mic) =>
+ mic.label.toLowerCase().includes(needle),
+ );
+ if (!match) {
+ throw new Error(
+ `No microphone's name contains "${request.micDevice}". Call listSources for their names.`,
+ );
+ }
+ microphone = { micDeviceId: match.deviceId ?? null, micDeviceName: match.label };
+ }
+ const prefs = recordingPrefs.get();
+ recordingPrefs.set({
+ micEnabled: request.mic,
+ ...microphone,
+ systemAudioEnabled: request.systemAudio,
+ cursorCaptureMode: "editable-overlay",
+ });
+ return () => {
+ recordingPrefs.set({
+ micEnabled: prefs.micEnabled,
+ micDeviceId: prefs.micDeviceId,
+ micDeviceName: prefs.micDeviceName,
+ systemAudioEnabled: prefs.systemAudioEnabled,
+ cursorCaptureMode: prefs.cursorCaptureMode,
+ });
+ selectedSource = before.source;
+ selectedDesktopSource = before.live;
+ broadcastSelectedSource(before.source);
+ };
+ }
+
+ const mcpAppRecorder = createAppRecorder({
+ isRecording: () => appRecording,
+ editorHasUnsavedChanges: () => editorHasUnsavedChanges?.() ?? false,
+ prepare: pointHudAtTake,
+ startTake: startNewRecording,
+ stopTake: () => {
+ const hudWindow = getMainWindow();
+ if (hudWindow && !hudWindow.isDestroyed()) {
+ hudWindow.webContents.send("stop-recording-from-tray");
+ }
+ },
+ host: mcpToolDeps.host,
+ });
+ const mcpMedia = createMediaTools({
+ transport: "http",
+ // Recording and exporting change things too: the same switch as edits.
+ editsAllowed: mcpToolDeps.editsAllowed,
+ recorder: mcpAppRecorder,
+ runners: mcpRunners,
+ documents: aiEditionDocuments,
+ openVideo: openVideoInHiddenWindow,
+ readCursor: async (videoPath) => (await readCursorTelemetryFile(videoPath)).samples,
+ // Never overriding the switch: setPointerTrack is refused while MCP edits is off.
+ runTool: mcpRunTool,
+ });
+ const mcpController = new McpController(mcpSettings, mcpToolDeps, mcpRunTool, mcpMedia);
// The editor says which project it holds, so `openscreen mcp`, a process of its own
// that cannot ask it, refuses to edit that project behind its back (#1138).
diff --git a/electron/ipc/recordingPrefs.ts b/electron/ipc/recordingPrefs.ts
index af1ff2291..49341563c 100644
--- a/electron/ipc/recordingPrefs.ts
+++ b/electron/ipc/recordingPrefs.ts
@@ -2,7 +2,11 @@ import { app, type BrowserWindow, ipcMain } from "electron";
import { AppSettingsStore } from "../app-settings";
import type { RecordingPrefs } from "./handlers";
-/** Shared durable recording preferences. Persist before publishing any new snapshot. */
+/**
+ * Shared durable recording preferences. Persist before publishing any new snapshot.
+ * Returns the same read and write for the main process: the MCP server points the
+ * recording bar at a take with them (handlers.ts).
+ */
export function registerRecordingPrefsHandlers(
defaults: RecordingPrefs,
getMainWindow: () => BrowserWindow | null,
@@ -11,7 +15,7 @@ export function registerRecordingPrefsHandlers(
return mainWindow ? [mainWindow] : [];
},
onChange?: (previous: RecordingPrefs, next: RecordingPrefs) => void,
-): void {
+): { get(): RecordingPrefs; set(prefs: Partial<RecordingPrefs>): RecordingPrefs } {
const userData = app.getPath("userData");
const settings = new AppSettingsStore(userData);
let recordingPrefs = { ...defaults, ...settings.getSnapshot().recording };
@@ -23,8 +27,7 @@ export function registerRecordingPrefsHandlers(
}
};
- ipcMain.handle("get-recording-prefs", () => recordingPrefs);
- ipcMain.handle("set-recording-prefs", (_, prefs: Partial<RecordingPrefs>) => {
+ const set = (prefs: Partial<RecordingPrefs>) => {
// Persist every validated field first. A failed save must leave both the
// durable value and the main-process published snapshot unchanged.
const previous = recordingPrefs;
@@ -32,5 +35,8 @@ export function registerRecordingPrefsHandlers(
onChange?.(previous, recordingPrefs);
publish();
return recordingPrefs;
- });
+ };
+ ipcMain.handle("get-recording-prefs", () => recordingPrefs);
+ ipcMain.handle("set-recording-prefs", (_, prefs: Partial<RecordingPrefs>) => set(prefs));
+ return { get: () => recordingPrefs, set };
}
diff --git a/electron/main.ts b/electron/main.ts
index 953deabad..22807b8d7 100644
--- a/electron/main.ts
+++ b/electron/main.ts
@@ -1412,6 +1412,7 @@ appReady?.then(async () => {
}
},
switchToHudWrapper,
+ () => editorHasUnsavedChanges,
);
// Native STT (whisper.cpp + forced alignment) — single instance per app.
diff --git a/electron/mcp/app-channel.ts b/electron/mcp/app-channel.ts
index eb166a333..5a2a56f72 100644
--- a/electron/mcp/app-channel.ts
+++ b/electron/mcp/app-channel.ts
@@ -129,8 +129,9 @@ export async function serveAppChannel(options: {
input.write(head.subarray(end + 1));
socket.pipe(input);
const session = createOpenScreenMcpServer(deps, runTool);
- // Only here until the in-app server offers the media tools: the edit is computed in
- // this process from what `openscreen mcp` knows of its takes, then run like any edit.
+ // Not the media tools' own setPointerTrack: `openscreen mcp` records its takes itself,
+ // so the call carries their start times. The edit is computed in this process from
+ // them, then run like any edit.
session.registerTool(
POINTER_TRACK_TOOL,
{ inputSchema: FORWARDED_POINTER_TRACK_SCHEMA },
diff --git a/electron/mcp/app-recorder.test.ts b/electron/mcp/app-recorder.test.ts
new file mode 100644
index 000000000..81ea83f1a
--- /dev/null
+++ b/electron/mcp/app-recorder.test.ts
@@ -0,0 +1,188 @@
+// The in-app recorder against stand-ins for the app: the HUD's signals are called by
+// hand, and the editor is a document behind a revision.
+
+import path from "node:path";
+import { describe, expect, it } from "vitest";
+import {
+ type AxcutDocument,
+ createEmptyDocument,
+ documentSchema,
+} from "../../src/lib/ai-edition/schema";
+import { type AppRecorderDeps, createAppRecorder } from "./app-recorder";
+import type { McpApplyResult } from "./openscreen-mcp-server";
+
+const VIDEO = path.resolve("recordings", "take.webm");
+const FAST = { startMs: 200, saveMs: 200, openMs: 400, pollMs: 5, settleMs: 20 };
+const REQUEST = {
+ displayIndex: 0,
+ windowTitle: "My App",
+ mic: false,
+ micDevice: null,
+ systemAudio: false,
+};
+
+/** The editor's project for the take: `clipEndSec` 60 is the placeholder it starts with. */
+function takeDocument(clipEndSec: number, zooms = 0): AxcutDocument {
+ const base = createEmptyDocument({ title: "Recording 1", projectId: "proj_take" });
+ return documentSchema.parse({
+ ...base,
+ project: { ...base.project, primaryAssetId: "asset_1" },
+ assets: [{ id: "asset_1", kind: "video", label: "take", originalPath: VIDEO, durationSec: 12 }],
+ timeline: {
+ ...base.timeline,
+ clips: [
+ {
+ id: "clip_1",
+ assetId: "asset_1",
+ sourceStartSec: 0,
+ sourceEndSec: clipEndSec,
+ timelineStartSec: 0,
+ timelineEndSec: clipEndSec,
+ wordRefs: [],
+ origin: "user",
+ reason: "",
+ },
+ ],
+ },
+ zoomRanges: Array.from({ length: zooms }, (_, i) => ({
+ id: `zoom_${i}`,
+ startMs: 3_000 + i * 4_000,
+ endMs: 5_000 + i * 4_000,
+ clipId: "clip_1",
+ sourceStartSec: 3 + i * 4,
+ sourceEndSec: 5 + i * 4,
+ depth: 3,
+ focus: { cx: 0.5, cy: 0.5 },
+ focusMode: "manual",
+ source: "manual",
+ })),
+ });
+}
+
+function setup(overrides: Partial<AppRecorderDeps> = {}) {
+ const calls: string[] = [];
+ const editor = {
+ document: null as AxcutDocument | null,
+ revision: 0,
+ applied: [] as AxcutDocument[],
+ };
+ const recorder = createAppRecorder({
+ isRecording: () => false,
+ editorHasUnsavedChanges: () => false,
+ prepare: async (request) => {
+ calls.push(`prepare ${request.windowTitle}`);
+ return () => calls.push("restore");
+ },
+ startTake: () => calls.push("startTake"),
+ stopTake: () => calls.push("stopTake"),
+ host: {
+ snapshot: async () =>
+ editor.document ? { document: editor.document, revision: editor.revision } : null,
+ apply: async (document, revision): Promise<McpApplyResult> => {
+ if (revision !== editor.revision) return "conflict";
+ editor.applied.push(document);
+ editor.document = document;
+ editor.revision++;
+ return "applied";
+ },
+ },
+ timing: FAST,
+ ...overrides,
+ });
+ return { recorder, calls, editor };
+}
+
+/** A take started as the app would: the HUD reports it recording once it was handed over. */
+async function started(recorder: ReturnType<typeof setup>["recorder"], calls: string[]) {
+ const handedOver = calls.filter((call) => call === "startTake").length;
+ const starting = recorder.start(REQUEST);
+ while (calls.filter((call) => call === "startTake").length === handedOver) {
+ await new Promise((resolve) => setTimeout(resolve, 1));
+ }
+ recorder.recordingChanged(true);
+ return starting;
+}
+
+describe("an MCP take in the app", () => {
+ it("goes through the recording bar, and hands back the project the editor opened", async () => {
+ const { recorder, calls, editor } = setup();
+ const starting = recorder.start(REQUEST);
+ await new Promise((resolve) => setTimeout(resolve, 5));
+ expect(calls).toEqual(["prepare My App", "startTake"]);
+ recorder.recordingChanged(true);
+ const take = await starting;
+ expect(take.startedAt).toBeGreaterThan(0);
+
+ const stopping = take.stop("Demo");
+ expect(calls.at(-1)).toBe("stopTake");
+ recorder.recordingChanged(false);
+ // The take is captured: the user's settings are back before it saves.
+ expect(calls.at(-1)).toBe("restore");
+ recorder.takeSaved(VIDEO);
+ // The editor lays it out in steps: first the placeholder clip, then the measured one.
+ editor.document = takeDocument(60);
+ await new Promise((resolve) => setTimeout(resolve, 40));
+ editor.document = takeDocument(12, 2);
+ editor.revision = 3;
+
+ expect(await stopping).toEqual({
+ projectId: "proj_take",
+ videoPath: VIDEO,
+ durationSec: 12,
+ autoZooms: 2,
+ });
+ // The title, through the editor like any change to the open project.
+ expect(editor.applied.map((document) => document.project.title)).toEqual(["Demo"]);
+ expect(calls.filter((call) => call === "restore")).toHaveLength(1);
+ });
+
+ it("returns a take the user stopped by hand, without pressing Stop again", async () => {
+ const { recorder, calls, editor } = setup();
+ const take = await started(recorder, calls);
+ recorder.recordingChanged(false);
+ recorder.takeSaved(VIDEO);
+ editor.document = takeDocument(12);
+ const recorded = await take.stop(undefined);
+ expect(recorded.projectId).toBe("proj_take");
+ expect(calls).not.toContain("stopTake");
+ expect(editor.applied).toEqual([]);
+ });
+
+ it("refuses a take while one records, or while the editor holds unsaved changes", async () => {
+ const busy = setup({ isRecording: () => true });
+ await expect(busy.recorder.start(REQUEST)).rejects.toThrow("already recording");
+ const unsaved = setup({ editorHasUnsavedChanges: () => true });
+ await expect(unsaved.recorder.start(REQUEST)).rejects.toThrow("not saved");
+ expect([...busy.calls, ...unsaved.calls]).toEqual([]);
+ });
+
+ it("gives up on a take that never starts, puts the settings back, and leaves a late one to the user", async () => {
+ const { recorder, calls } = setup();
+ await expect(recorder.start(REQUEST)).rejects.toThrow("had not started");
+ expect(calls).toEqual(["prepare My App", "startTake", "restore"]);
+ // The user's own take, started from the bar afterwards: nothing here claims it.
+ recorder.recordingChanged(true);
+ recorder.recordingChanged(false);
+ expect(calls).toHaveLength(3);
+ // And the next client take may start.
+ await expect(started(recorder, calls)).resolves.toMatchObject({
+ startedAt: expect.any(Number),
+ });
+ });
+
+ it("says where the take is when the editor never opens it", async () => {
+ const { recorder, calls } = setup();
+ const take = await started(recorder, calls);
+ recorder.recordingChanged(false);
+ recorder.takeSaved(VIDEO);
+ await expect(take.stop(undefined)).rejects.toThrow(`saved to ${VIDEO}`);
+ });
+
+ it("fails a take the bar never hands over", async () => {
+ const { recorder, calls } = setup();
+ const take = await started(recorder, calls);
+ await expect(take.stop(undefined)).rejects.toThrow("The recording failed");
+ // Released: the next take may start.
+ await expect(started(recorder, calls)).resolves.toBeDefined();
+ });
+});
diff --git a/electron/mcp/app-recorder.ts b/electron/mcp/app-recorder.ts
new file mode 100644
index 000000000..3ce458a71
--- /dev/null
+++ b/electron/mcp/app-recorder.ts
@@ -0,0 +1,210 @@
+// The in-app server's recorder: a take made the way the app makes every take. Not a
+// recorder of its own beside the app's: startRecording points the recording bar at
+// what the client asked for and hands over to it, as the editor's Record button does,
+// and stopRecording presses the bar's Stop, as the tray does. The bar then saves the
+// take and the editor opens it as a new project, laid out and zoomed as any take is,
+// and that project is what the client edits: through the editor, live.
+//
+// No `electron` import: the app's side is handed in (handlers.ts), so this runs under
+// plain vitest with stand-ins.
+
+import path from "node:path";
+import { clipAwaitsProbedDuration } from "../../src/lib/ai-edition/document/timeline";
+import { type AxcutDocument, documentSchema } from "../../src/lib/ai-edition/schema";
+import type { RecordedTake, RecordRequest, RunningTake, TakeRecorder } from "./media-tools";
+import type { McpDocumentHost } from "./openscreen-mcp-server";
+
+// ponytail: fixed deadlines. A take only starts once the countdown, and the system's
+// picker where it asks, are through; a long take takes a while to save.
+const TIMING = {
+ startMs: 120_000,
+ saveMs: 120_000,
+ openMs: 60_000,
+ pollMs: 250,
+ // The editor lays a take out in several saves (the clip, its measured length, the
+ // zooms on its clicks): it is ready once its revision has held this long.
+ // ponytail: a quiet period, not a signal; ask the editor to say "done" if it races.
+ settleMs: 1_500,
+};
+
+export interface AppRecorderDeps {
+ /** The app is recording or saving a take, the user's or a client's. */
+ isRecording(): boolean;
+ /** The editor holds changes it has not saved: closing it for the take would lose them. */
+ editorHasUnsavedChanges(): boolean;
+ /**
+ * Points the recording bar at the request, as Record mode does before it hands over:
+ * its source, microphone, system audio and cursor. Returns what puts back the settings
+ * the user had. Throws when the request matches no source or microphone.
+ */
+ prepare(request: RecordRequest): Promise<() => void>;
+ /** The editor's Record button: hand over to the recording bar, which starts the take. */
+ startTake(): void;
+ /** The recording bar's Stop, as the tray's Stop Recording. */
+ stopTake(): void;
+ /** The editor, where a saved take opens as a new project. */
+ host: Pick<McpDocumentHost, "snapshot" | "apply">;
+ timing?: Partial<typeof TIMING>;
+}
+
+export interface AppRecorder extends TakeRecorder {
+ /** The app's recording state changed (`set-recording-state`). */
+ recordingChanged(recording: boolean): void;
+ /** The recording bar handed a saved take to the editor (`set-current-recording-session`). */
+ takeSaved(videoPath: string): void;
+}
+
+function deferred<T>(): Deferred<T> {
+ let resolve: (value: T) => void = () => undefined;
+ const promise = new Promise<T>((done) => {
+ resolve = done;
+ });
+ return { promise, resolve };
+}
+
+/** `promise`, or `null` once `ms` have passed without it. */
+function within<T>(promise: Promise<T>, ms: number): Promise<T | null> {
+ let timer: ReturnType<typeof setTimeout> | undefined;
+ const expired = new Promise<null>((resolve) => {
+ timer = setTimeout(() => resolve(null), ms);
+ });
+ return Promise.race([promise, expired]).finally(() => clearTimeout(timer));
+}
+
+const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms));
+const seconds = (ms: number) => Math.round(ms / 1000);
+
+/** The take's asset, once the editor has measured it: until then its clip is a placeholder. */
+function takeAsset(document: AxcutDocument, videoPath: string) {
+ const asset = document.assets.find(
+ (candidate) =>
+ !!candidate.originalPath && path.resolve(candidate.originalPath) === path.resolve(videoPath),
+ );
+ if (!asset?.durationSec) return null;
+ if (document.timeline.clips.some((clip) => clipAwaitsProbedDuration(clip, asset.id))) return null;
+ return asset;
+}
+
+interface Deferred<T> {
+ promise: Promise<T>;
+ resolve(value: T): void;
+}
+
+interface ClientTake {
+ phase: "starting" | "recording" | "saving";
+ /** Resolves with when capture started. */
+ started: Deferred<number>;
+ /** Resolves with the saved take's video. */
+ saved: Deferred<string>;
+ restore(): void;
+}
+
+export function createAppRecorder(deps: AppRecorderDeps): AppRecorder {
+ const timing = { ...TIMING, ...deps.timing };
+ /** The client's take, from the moment it is asked for until stopRecording has it. */
+ let current: ClientTake | null = null;
+
+ /** The editor's project for the take, once laid out. */
+ async function openedInEditor(videoPath: string) {
+ const deadline = Date.now() + timing.openMs;
+ let seen = null as { projectId: string; revision: number; since: number } | null;
+ while (Date.now() < deadline) {
+ const snapshot = await deps.host.snapshot();
+ const parsed = snapshot ? documentSchema.safeParse(snapshot.document) : null;
+ const document = parsed?.success ? parsed.data : null;
+ const asset = document ? takeAsset(document, videoPath) : null;
+ if (snapshot && document && asset) {
+ const { revision } = snapshot;
+ if (seen?.projectId !== document.project.id || seen.revision !== revision) {
+ seen = { projectId: document.project.id, revision, since: Date.now() };
+ } else if (Date.now() - seen.since >= timing.settleMs) {
+ return { document, revision, asset };
+ }
+ } else {
+ seen = null;
+ }
+ await sleep(timing.pollMs);
+ }
+ throw new Error(
+ `The take was saved to ${videoPath}, but the OpenScreen editor did not open it within ${seconds(timing.openMs)} s.`,
+ );
+ }
+
+ async function stop(take: ClientTake, title: string | undefined) {
+ try {
+ if (take.phase === "recording") deps.stopTake();
+ const videoPath = await within(take.saved.promise, timing.saveMs);
+ if (videoPath === null) {
+ throw new Error(
+ `The recording failed: OpenScreen did not hand the take to its editor within ${seconds(timing.saveMs)} s. The recording bar says why.`,
+ );
+ }
+ const opened = await openedInEditor(videoPath);
+ const { asset, revision } = opened;
+ let { document } = opened;
+ if (title && title !== document.project.title) {
+ const renamed = { ...document, project: { ...document.project, title } };
+ // A user edit that lands first keeps the editor's title: nothing else depends on it.
+ if ((await deps.host.apply(renamed, revision)) === "applied") document = renamed;
+ }
+ return {
+ projectId: document.project.id,
+ videoPath: asset.originalPath,
+ durationSec: asset.durationSec ?? 0,
+ autoZooms: document.zoomRanges?.length ?? 0,
+ } satisfies RecordedTake;
+ } finally {
+ take.restore();
+ if (current === take) current = null;
+ }
+ }
+
+ return {
+ recordingChanged(recording) {
+ if (!current) return;
+ if (recording && current.phase === "starting") {
+ current.phase = "recording";
+ current.started.resolve(Date.now());
+ } else if (!recording && current.phase === "recording") {
+ // The take is captured: the bar's own settings can come back while it saves.
+ current.phase = "saving";
+ current.restore();
+ }
+ },
+ takeSaved(videoPath) {
+ if (current?.phase === "saving") current.saved.resolve(videoPath);
+ },
+ async start(request): Promise<RunningTake> {
+ if (current || deps.isRecording()) throw new Error("OpenScreen is already recording a take.");
+ if (deps.editorHasUnsavedChanges()) {
+ throw new Error(
+ "the OpenScreen editor has changes it has not saved yet, and the take closes it. Ask the user to save them, then try again.",
+ );
+ }
+ const restoreSettings = await deps.prepare(request);
+ let restored = false;
+ const take: ClientTake = {
+ phase: "starting",
+ started: deferred<number>(),
+ saved: deferred<string>(),
+ restore() {
+ if (restored) return;
+ restored = true;
+ restoreSettings();
+ },
+ };
+ current = take;
+ deps.startTake();
+ const startedAt = await within(take.started.promise, timing.startMs);
+ if (startedAt === null) {
+ // A take that starts after this is the user's: nothing here waits for it.
+ current = null;
+ take.restore();
+ throw new Error(
+ `capture had not started after ${seconds(timing.startMs)} s. OpenScreen's recording bar says why.`,
+ );
+ }
+ return { startedAt, stop: (title) => stop(take, title) };
+ },
+ };
+}
diff --git a/electron/mcp/mcp-controller.ts b/electron/mcp/mcp-controller.ts
index dad7ac156..fe4e3b617 100644
--- a/electron/mcp/mcp-controller.ts
+++ b/electron/mcp/mcp-controller.ts
@@ -6,6 +6,7 @@
import type { AiEditionMcpStatus } from "../../src/native/contracts";
import type { McpSettingsStore } from "./mcp-settings-store";
+import type { MediaTools } from "./media-tools";
import {
MCP_ENDPOINT_PATH,
type McpToolDeps,
@@ -25,6 +26,8 @@ export class McpController {
private readonly deps: McpToolDeps,
/** The app's shared runner, so checkpoints outlive a restart and `openscreen mcp` sees them. */
private readonly runTool?: McpToolRunner,
+ /** Built once, so a take or an export outlives a restart of the server. */
+ private readonly media?: MediaTools,
) {}
private serially<T>(task: () => Promise<T>): Promise<T> {
@@ -52,6 +55,7 @@ export class McpController {
token,
deps: this.deps,
runTool: this.runTool,
+ media: this.media,
});
} catch (error) {
const code = (error as NodeJS.ErrnoException | null)?.code;
diff --git a/electron/mcp/media-tools.test.ts b/electron/mcp/media-tools.test.ts
index 425348959..fa8a391b4 100644
--- a/electron/mcp/media-tools.test.ts
+++ b/electron/mcp/media-tools.test.ts
@@ -14,11 +14,12 @@ import type { CliDoneResult, CliRequest } from "../../src/lib/cliContracts";
import { DocumentService } from "../ai-edition/document-service";
import type { RunnerHandlers } from "../cli/runnerHost";
import {
+ createMediaTools,
frameTimes,
- MEDIA_PREAMBLE,
type MediaToolDeps,
+ mediaPreamble,
pointerSamples,
- registerMediaTools,
+ type TakeRecorder,
} from "./media-tools";
import {
createOpenScreenMcpServer,
@@ -95,6 +96,8 @@ let runners: FakeRunners;
let closed: number;
let openVideoFails: boolean;
let client: Client;
+/** The stdio server's media deps: the in-app tests reuse its stand-ins. */
+let stdioDeps: MediaToolDeps;
/** The pointer recorded with the take, as the sidecar beside the video. */
async function readSidecar(video: string): Promise<CursorTelemetryPoint[]> {
@@ -124,8 +127,9 @@ beforeEach(async () => {
version: "test",
};
const runTool = createToolRunner(toolDeps);
- const server = createOpenScreenMcpServer(toolDeps, runTool, MEDIA_PREAMBLE);
- const deps: MediaToolDeps = {
+ stdioDeps = {
+ transport: "stdio",
+ editsAllowed: () => true,
runners,
documents,
openVideo: async () => {
@@ -147,7 +151,7 @@ beforeEach(async () => {
runTool,
now: () => 1_000,
};
- registerMediaTools(server, deps);
+ const server = createOpenScreenMcpServer(toolDeps, runTool, createMediaTools(stdioDeps));
const [clientSide, serverSide] = InMemoryTransport.createLinkedPair();
await server.connect(serverSide);
@@ -179,7 +183,7 @@ async function recordTake(title = "Demo") {
describe("what `openscreen mcp` tells its client", () => {
it("composes the media block with the checkpoint rule, and nothing true only in the app", () => {
const instructions = client.getInstructions() ?? "";
- expect(instructions.startsWith(MEDIA_PREAMBLE)).toBe(true);
+ expect(instructions.startsWith(mediaPreamble("stdio"))).toBe(true);
expect(instructions).toContain("Before a series of edits, call createCheckpoint");
expect(instructions).toContain("in OpenScreen's memory while the app runs");
expect(instructions).not.toMatch(
@@ -497,6 +501,8 @@ describe("exportVideo", () => {
expect(runners.started.at(-1)).toMatchObject({
kind: "export",
projectPath: documents.filePathFor(projectId),
+ // Read by id: the app's current project and recording session stay as they are.
+ projectId,
outPath,
format: "mp4",
quality: "good",
@@ -610,3 +616,140 @@ describe("getFrames", () => {
expect(closed).toBe(1);
});
});
+
+describe("the media tools in the app", () => {
+ let allowed: boolean;
+ let asked: Parameters<TakeRecorder["start"]>[0][];
+ const appClients: Client[] = [];
+
+ /** The in-app server, its takes made by the app's recorder (a stand-in here). */
+ async function connectApp(): Promise<Client> {
+ const toolDeps: McpToolDeps = {
+ transport: "http",
+ host: { snapshot: async () => null, apply: async () => "no-editor" },
+ projects: documents,
+ editsAllowed: () => allowed,
+ version: "test",
+ };
+ const runTool = createToolRunner(toolDeps);
+ const recorder: TakeRecorder = {
+ async start(request) {
+ asked.push(request);
+ return {
+ startedAt: 5_000,
+ async stop(title) {
+ const created = await documents.createProject(title ?? "Recording");
+ const added = await documents.addAsset(created.project.id, { path: videoPath });
+ // The editor has measured the take by the time the recorder hands it back.
+ await documents.saveProject({
+ ...added,
+ assets: added.assets.map((asset) => ({ ...asset, durationSec: 12 })),
+ });
+ return { projectId: created.project.id, videoPath, durationSec: 12, autoZooms: 1 };
+ },
+ };
+ },
+ };
+ const media = createMediaTools({
+ ...stdioDeps,
+ transport: "http",
+ editsAllowed: () => allowed,
+ recorder,
+ runTool,
+ });
+ const server = createOpenScreenMcpServer(toolDeps, runTool, media);
+ const [clientSide, serverSide] = InMemoryTransport.createLinkedPair();
+ await server.connect(serverSide);
+ const appClient = new Client({ name: "test", version: "1" });
+ await appClient.connect(clientSide);
+ appClients.push(appClient);
+ return appClient;
+ }
+
+ beforeEach(() => {
+ allowed = true;
+ asked = [];
+ });
+ afterEach(async () => {
+ for (const appClient of appClients.splice(0)) await appClient.close();
+ });
+
+ const callOn = (on: Client, name: string, args: Record<string, unknown> = {}) =>
+ on.callTool({ name, arguments: args }) as Promise<CallToolResult>;
+
+ it("records through the app's recorder, never a runner, and hands back the project with its checkpoint", async () => {
+ const app = await connectApp();
+ const started = await callOn(app, "startRecording", { window: "My App", microphone: "USB" });
+ expect(started.isError).toBeFalsy();
+ expect(asked).toEqual([
+ { displayIndex: 0, windowTitle: "My App", mic: true, micDevice: "USB", systemAudio: false },
+ ]);
+ const take = JSON.parse(textOf(await callOn(app, "stopRecording", { title: "Demo" })));
+ expect(take).toMatchObject({
+ videoPath,
+ durationSec: 12,
+ autoZooms: 1,
+ recordingStartedAt: 5_000,
+ });
+ expect(take.checkpointId).toMatch(/^cp_/);
+ expect(runners.started.filter((request) => request.kind === "record")).toEqual([]);
+ expect((await documents.getProject(take.projectId)).project.title).toBe("Demo");
+ // setPointerTrack knows the take's clock, as for a take of `openscreen mcp`.
+ const set = await callOn(app, "setPointerTrack", {
+ projectId: take.projectId,
+ events: [{ at: 5_000 + 4_000, x: 0.5, y: 0.5, click: true }],
+ });
+ expect(set.isError).toBeFalsy();
+ });
+
+ it("records and exports only while MCP edits is on, and says so in its instructions", async () => {
+ allowed = false;
+ const app = await connectApp();
+ expect(app.getInstructions()).toContain(
+ "startRecording and exportVideo are refused too while 'MCP edits'",
+ );
+ const started = await callOn(app, "startRecording", {});
+ expect(started.isError).toBe(true);
+ expect(textOf(started)).toContain("'MCP edits'");
+ expect(asked).toEqual([]);
+ const exported = await callOn(app, "exportVideo", {
+ projectId: "proj_x",
+ path: path.join(dir, "out.mp4"),
+ });
+ expect(textOf(exported)).toContain("'MCP edits'");
+ expect(runners.started).toEqual([]);
+ // Reading stays open: listing what could be recorded changes nothing.
+ runners.results.sources = {
+ success: true,
+ sources: { displays: [], windows: [], microphones: [], microphoneLabelsUnavailable: false },
+ };
+ expect((await callOn(app, "listSources")).isError).toBeFalsy();
+ });
+
+ it("takes only absolute paths: it has no working directory of the client's", async () => {
+ const app = await connectApp();
+ const exported = await callOn(app, "exportVideo", { projectId: "proj_x", path: "out.mp4" });
+ expect(textOf(exported)).toContain("must be absolute");
+ const frames = await callOn(app, "getFrames", { path: "take.mp4" });
+ expect(textOf(frames)).toContain("must be absolute");
+ });
+
+ it("tells its client what is true in the app, and nothing true only of `openscreen mcp`", async () => {
+ const app = await connectApp();
+ const instructions = app.getInstructions() ?? "";
+ expect(instructions.startsWith(mediaPreamble("http"))).toBe(true);
+ expect(instructions).toContain("recording bar");
+ expect(instructions).not.toContain("refused too");
+ const { tools } = await app.listTools();
+ const describeTool = (name: string) =>
+ tools.find((tool) => tool.name === name)?.description ?? "";
+ expect(describeTool("startRecording")).toContain("recording bar");
+ expect(describeTool("startRecording")).not.toContain("On Linux the desktop's own picker");
+ expect(describeTool("stopRecording")).toContain("editor opens the take");
+ expect(describeTool("restoreCheckpoint")).toContain("pointer track");
+ const exportPath = tools.find((tool) => tool.name === "exportVideo")?.inputSchema.properties
+ ?.path as { description?: string };
+ expect(exportPath.description).toContain("absolute path ending in");
+ expect(exportPath.description).not.toContain("working directory, ending");
+ });
+});
diff --git a/electron/mcp/media-tools.ts b/electron/mcp/media-tools.ts
index bdbb13e0a..54b6fe5d3 100644
--- a/electron/mcp/media-tools.ts
+++ b/electron/mcp/media-tools.ts
@@ -1,10 +1,16 @@
// What an agent needs around the editing DSL to make a video on its own: record a
// take, have it become a project, export the project, and look at the result.
-// Served by `openscreen mcp` next to the document tools, so every edit between the
-// recording and the export goes through the DSL like any other client's.
+// Served next to the document tools by both servers, the in-app one and `openscreen
+// mcp`, so every edit between the recording and the export goes through the DSL like
+// any other client's.
//
-// No `electron` import: the runners, the decoder and the documents are handed in,
-// so the tools run under plain vitest with stand-ins.
+// One implementation, two recorders. `openscreen mcp` records in a hidden runner of its
+// own process and lays the take out as a project here (`runnerRecorder`). The app
+// records through its recording bar, as its own Record button does, and its editor opens
+// the take (`app-recorder.ts`): the app never runs a second recorder beside its own.
+//
+// No `electron` import: the runners, the recorder, the decoder and the documents are
+// handed in, so the tools run under plain vitest with stand-ins.
import path from "node:path";
import type { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
@@ -20,16 +26,56 @@ import {
appendAutoZoomSuggestions,
collectAutoZoomSuggestionsForDocument,
} from "../../src/lib/ai-edition/timeline/apply-auto-zooms";
-import type { CliDoneResult, CliRequest } from "../../src/lib/cliContracts";
+import type { CliDoneResult, CliRecordRequest, CliRequest } from "../../src/lib/cliContracts";
import { HIDDEN_WINDOWS_NOTE } from "../../src/lib/cliSourcePick";
import type { CursorRecordingData, CursorRecordingSample } from "../../src/native/contracts";
import { runDocumentTool } from "../ai-edition/deep-agent/service";
import type { DocumentService } from "../ai-edition/document-service";
import type { RunnerHandlers } from "../cli/runnerHost";
import type { OpenedVideo } from "./hidden-video";
-import { type McpToolRunner, type RunnerEdit, textResult as text } from "./openscreen-mcp-server";
+import {
+ MCP_EDITS_SETTING,
+ type McpToolRunner,
+ type McpTransport,
+ type RunnerEdit,
+ textResult as text,
+} from "./openscreen-mcp-server";
+
+/** What startRecording asks a recorder for. */
+export type RecordRequest = Pick<
+ CliRecordRequest,
+ "displayIndex" | "windowTitle" | "mic" | "micDevice" | "systemAudio"
+>;
+
+/** A take turned into a project, ready for the document tools. */
+export interface RecordedTake {
+ projectId: string;
+ /** The take's video, as the project's asset names it: setPointerTrack's key. */
+ videoPath: string;
+ durationSec: number;
+ autoZooms: number;
+ warnings?: string[];
+}
+
+export interface RunningTake {
+ /** Epoch ms when capture started: the clock setPointerTrack's events are on. */
+ startedAt: number;
+ /** Stops the take and hands it back as a project; throws, with the whole story, when it cannot. */
+ stop(title: string | undefined): Promise<RecordedTake>;
+}
+
+export interface TakeRecorder {
+ /** Resolves once capture has started; throws why it did not. */
+ start(request: RecordRequest): Promise<RunningTake>;
+}
export interface MediaToolDeps {
+ /** The server these tools are served by: their texts say what is true on it. */
+ transport: McpTransport;
+ /** Whether a client may record and export: the in-app server's "MCP edits". */
+ editsAllowed(): boolean;
+ /** Records the takes. Default: a hidden record runner, as `openscreen mcp` does. */
+ recorder?: TakeRecorder;
runners: {
start(
request: CliRequest,
@@ -133,14 +179,56 @@ export function pointerSamples(
const READ_ONLY = { readOnlyHint: true, destructiveHint: false, openWorldHint: false } as const;
+/** Where a take is made and where it lands, on each server. */
+const TAKE_NOTES: Record<McpTransport, string> = {
+ http: "Recording is the app's own: startRecording hands over to OpenScreen's recording bar, as the editor's Record button does, so the editor closes for the take, and stopRecording opens the take in the editor, where the user sees each edit land and can undo it. Its zooms on the clicks follow Record mode's Auto-zoom after recording, as for any take.",
+ stdio:
+ "These projects are in the OpenScreen app's project list, so the user can open one in its editor.",
+};
+
/** The media tools' capability block, placed before the projects-and-checkpoints rules
* (`mcpInstructions`) on the server that registers them. */
-export const MEDIA_PREAMBLE = [
- "OpenScreen records the screen, turns each take into a project you edit with the document tools, and exports it as a video.",
- "A demo goes: listSources, then startRecording, then drive the app, then stopRecording. It returns the projectId and a checkpointId. Edit with the document tools, always passing that projectId: every edit is saved to the project straight away, and restoreCheckpoint with the checkpointId takes the project back to the raw take in one step. Then exportVideo, and getFrames with the projectId to check the export before you hand it over: it takes the same ruler seconds as the document tools, so a trim or a speed region never shifts what you look at.",
- "Drive the app with the real mouse when you can: OpenScreen records the pointer, draws a clean cursor and zooms on the clicks as the editor does: clicks close in time and on screen share one zoom, and the first seconds and the end stay on the whole screen. If you drive it from a script instead (Playwright, CDP), the screen pointer never moves: log each action as the script runs it, then hand the log to setPointerTrack after stopRecording.",
- "These projects are in the OpenScreen app's project list, so the user can open one in its editor.",
-].join("\n");
+export function mediaPreamble(transport: McpTransport): string {
+ return [
+ "OpenScreen records the screen, turns each take into a project you edit with the document tools, and exports it as a video.",
+ "A demo goes: listSources, then startRecording, then drive the app, then stopRecording. It returns the projectId and a checkpointId. Edit with the document tools, always passing that projectId: every edit is saved to the project straight away, and restoreCheckpoint with the checkpointId takes the project back to the raw take in one step. Then exportVideo, and getFrames with the projectId to check the export before you hand it over: it takes the same ruler seconds as the document tools, so a trim or a speed region never shifts what you look at.",
+ "Drive the app with the real mouse when you can: OpenScreen records the pointer, draws a clean cursor and zooms on the clicks as the editor does: clicks close in time and on screen share one zoom, and the first seconds and the end stay on the whole screen. If you drive it from a script instead (Playwright, CDP), the screen pointer never moves: log each action as the script runs it, then hand the log to setPointerTrack after stopRecording.",
+ TAKE_NOTES[transport],
+ ].join("\n");
+}
+
+/** What a client reads of the media tools: their preamble, and the tools themselves. */
+export interface MediaTools {
+ preamble: string;
+ /** Adds the tools to a server. Their state (the take, the exports) is shared by every
+ * server they are added to: the in-app server builds one per request. */
+ register(server: McpServer): void;
+}
+
+const START_RECORDING_NOTES: Record<McpTransport, string> = {
+ http: "It goes through OpenScreen's recording bar, as the editor's Record button does: the editor closes for the take, so this is refused while the editor has unsaved changes, and the bar's countdown and camera settings apply. Where OpenScreen leaves the choice to the system's picker (Linux, macOS 15.2 or later), that picker asks the user what to record, whatever window or display says. The bar records with these settings, and gets the user's own back once the take ends.",
+ stdio: "On Linux the desktop's own picker asks the user which window to record.",
+};
+
+const STOP_RECORDING_DESCRIPTIONS: Record<McpTransport, string> = {
+ http: "Stop the recording. OpenScreen's editor opens the take as a new project, as it does after any recording: one clip, plus the zooms planned from the recorded clicks when Record mode's Auto-zoom after recording is on, and autoZooms says how many. Returns the projectId to edit, now open in the editor, a checkpointId that restores the take as it opened, and the take's videoPath.",
+ stdio:
+ "Stop the recording and turn the take into a new project, laid out as the editor lays out a fresh recording: one clip, plus the zooms planned from the recorded clicks. Not one per click: clicks close in time and on screen share one zoom, a click in the first seconds or the one that stopped the take gets none, and autoZooms says how many there are. Returns the projectId to edit, a checkpointId that restores the raw take, and the take's videoPath.",
+};
+
+const EXPORT_PATH_DESCRIPTIONS: Record<McpTransport, string> = {
+ http: "Where to write the video: an absolute path ending in .mp4 or .gif. This server runs inside the app, so it has no working directory of yours to resolve a relative one against.",
+ stdio:
+ "Where to write the video: an absolute path, or one relative to this server's working directory, ending in .mp4 or .gif.",
+};
+
+const GATED_MESSAGE = `Nothing was recorded or exported: OpenScreen lets MCP clients record and export only while the user allows it. Ask the user to turn on ${MCP_EDITS_SETTING}.`;
+
+/** A path a client passed, resolved; `null` for a relative one the in-app server cannot place. */
+function clientPath(transport: McpTransport, value: string): string | null {
+ if (transport === "http" && !path.isAbsolute(value)) return null;
+ return path.resolve(value);
+}
export const POINTER_TRACK_TOOL = "setPointerTrack";
@@ -204,132 +292,68 @@ export async function pointerTrackEdit(
};
}
-export function registerMediaTools(server: McpServer, deps: MediaToolDeps): void {
- const now = deps.now ?? Date.now;
- /** The take being recorded, then saved: held until it is a project. */
- let take: {
- run: ReturnType<MediaToolDeps["runners"]["start"]>;
- startedAt: number;
- stopping: boolean;
- } | null = null;
- // An export loads its project into the slot a finishing take reads its files from.
- let exporting = false;
- /** When each take of this session started, by its video: setPointerTrack's clock. */
- const startedAtByVideo = new Map<string, number>();
- /** Each project's last export of this session, and when it was made: what getFrames reads. */
- const lastExports = new Map<string, { path: string; at: Date; rendered: AxcutDocument }>();
+type Run = ReturnType<MediaToolDeps["runners"]["start"]>;
- /** A runner's result, its hidden window closed: this server outlives every runner. */
- async function finish(run: ReturnType<MediaToolDeps["runners"]["start"]>) {
- try {
- return await run.done;
- } finally {
- run.close();
- }
+/** A runner's result, its hidden window closed: the server outlives every runner. */
+async function finish(run: Run): Promise<CliDoneResult> {
+ try {
+ return await run.done;
+ } finally {
+ run.close();
}
+}
- /** The take as a project, laid out as the editor lays out a fresh recording. */
- async function createProjectFromVideo(videoPath: string, title: string) {
- const video = await deps.openVideo(videoPath);
- video.close();
- const { durationSec, width, height } = video;
- if (!Number.isFinite(durationSec) || durationSec <= 0) {
- throw new Error(`Could not read the duration of ${videoPath}`);
- }
- const created = await deps.documents.createProject(title);
- let document: AxcutDocument = await deps.documents.addAsset(created.project.id, {
- path: videoPath,
- });
- const assetId = document.project.primaryAssetId;
- document = {
- ...document,
- assets: document.assets.map((asset) =>
- asset.id === assetId
- ? {
- ...asset,
- durationSec,
- video: {
- codec: asset.video?.codec ?? "unknown",
- width,
- height,
- fps: asset.video?.fps ?? 0,
- },
- }
- : asset,
- ),
- };
- const seeded = await runDocumentTool(
- document,
- "replaceTimeline",
- { intervals: [{ startSec: 0, endSec: durationSec }], reason: "The recorded take" },
- true,
- {},
- );
- if (!seeded.ok || !seeded.document) throw new Error(seeded.resultJson);
- // The zooms a recording made in the app starts with, planned from its clicks (zoom-suggestions.ts).
- const suggestions = await collectAutoZoomSuggestionsForDocument(
- seeded.document,
- deps.readCursor,
- );
- const saved = await deps.documents.saveProject(
- appendAutoZoomSuggestions(seeded.document, suggestions),
- );
- return { document: saved, durationSec, autoZooms: suggestions.length };
+/** The take as a project, laid out as the editor lays out a fresh recording. */
+async function createProjectFromVideo(deps: MediaToolDeps, videoPath: string, title: string) {
+ const video = await deps.openVideo(videoPath);
+ video.close();
+ const { durationSec, width, height } = video;
+ if (!Number.isFinite(durationSec) || durationSec <= 0) {
+ throw new Error(`Could not read the duration of ${videoPath}`);
}
-
- server.registerTool(
- "listSources",
- {
- description: `List the displays and windows OpenScreen can record, and the microphones. Pass a display's index or part of a window's title to startRecording. ${HIDDEN_WINDOWS_NOTE}`,
- inputSchema: z.object({}),
- annotations: READ_ONLY,
- },
- async () => {
- const result = await finish(deps.runners.start({ kind: "sources" }));
- if (!result.success || !result.sources) {
- return text(`Could not list the sources: ${result.error ?? "unknown error"}`, true);
- }
- const { displays, windows, microphones } = result.sources;
- return text(
- JSON.stringify({
- displays: displays.map(({ index, name }) => ({ index, name })),
- windows: windows.map((window) => window.name),
- microphones: microphones.map((mic) => mic.label),
- }),
- );
- },
+ const created = await deps.documents.createProject(title);
+ let document: AxcutDocument = await deps.documents.addAsset(created.project.id, {
+ path: videoPath,
+ });
+ const assetId = document.project.primaryAssetId;
+ document = {
+ ...document,
+ assets: document.assets.map((asset) =>
+ asset.id === assetId
+ ? {
+ ...asset,
+ durationSec,
+ video: {
+ codec: asset.video?.codec ?? "unknown",
+ width,
+ height,
+ fps: asset.video?.fps ?? 0,
+ },
+ }
+ : asset,
+ ),
+ };
+ const seeded = await runDocumentTool(
+ document,
+ "replaceTimeline",
+ { intervals: [{ startSec: 0, endSec: durationSec }], reason: "The recorded take" },
+ true,
+ {},
);
+ if (!seeded.ok || !seeded.document) throw new Error(seeded.resultJson);
+ // The zooms a recording made in the app starts with, planned from its clicks (zoom-suggestions.ts).
+ const suggestions = await collectAutoZoomSuggestionsForDocument(seeded.document, deps.readCursor);
+ const saved = await deps.documents.saveProject(
+ appendAutoZoomSuggestions(seeded.document, suggestions),
+ );
+ return { document: saved, durationSec, autoZooms: suggestions.length };
+}
- server.registerTool(
- "startRecording",
- {
- description:
- "Start recording one window (preferred: nothing else on the desktop is captured) or one display. Returns once capture has started; then drive the app and call stopRecording. The pointer is recorded as data, so the export draws a clean cursor and the zooms planned from the clicks. On Linux the desktop's own picker asks the user which window to record.",
- inputSchema: z.object({
- window: z
- .string()
- .optional()
- .describe("Part of the title of the window to record, ignoring case. Wins over display."),
- display: z
- .number()
- .int()
- .min(0)
- .optional()
- .describe("Index of the display to record, from listSources. Default 0."),
- systemAudio: z.boolean().optional().describe("Also record the computer's sound."),
- microphone: z
- .union([z.boolean(), z.string()])
- .optional()
- .describe("true for the default microphone, or part of a microphone's name."),
- }),
- annotations: { readOnlyHint: false, destructiveHint: false, openWorldHint: false },
- },
- async (args) => {
- if (take?.stopping) {
- return text("The last take is still being saved. Try again in a moment.", true);
- }
- if (take) return text("A recording is already running. Call stopRecording first.", true);
- if (exporting) return text("An export is running. Wait for it, then try again.", true);
+/** `openscreen mcp`'s recorder: a hidden record runner, and the take laid out as a project here. */
+function runnerRecorder(deps: MediaToolDeps): TakeRecorder {
+ const now = deps.now ?? Date.now;
+ return {
+ async start(request) {
let markStarted: () => void = () => undefined;
const started = new Promise<"started">((resolve) => {
markStarted = () => resolve("started");
@@ -337,11 +361,7 @@ export function registerMediaTools(server: McpServer, deps: MediaToolDeps): void
const run = deps.runners.start(
{
kind: "record",
- displayIndex: args.display ?? 0,
- windowTitle: args.window ?? null,
- mic: args.microphone !== undefined && args.microphone !== false,
- micDevice: typeof args.microphone === "string" ? args.microphone : null,
- systemAudio: args.systemAudio === true,
+ ...request,
cursorMode: "editable-overlay",
durationMs: null,
projectOut: null,
@@ -355,68 +375,78 @@ export function registerMediaTools(server: McpServer, deps: MediaToolDeps): void
const outcome = await Promise.race([started, run.done]);
if (outcome !== "started") {
run.close();
- return text(`The recording did not start: ${outcome.error ?? "unknown error"}`, true);
+ throw new Error(outcome.error ?? "unknown error");
}
- take = { run, startedAt: now(), stopping: false };
- return text(
- JSON.stringify({ recording: true, startedAt: take.startedAt }) +
- "\nRecording. Drive the app now, pausing about a second after each result appears, then call stopRecording.",
- );
+ const startedAt = now();
+ return {
+ startedAt,
+ async stop(title) {
+ try {
+ run.window.webContents.send("cli-stop-recording");
+ } catch {
+ // The runner already went away; its result says why.
+ }
+ const result = await finish(run);
+ if (!result.success || !result.screenVideoPath) {
+ throw new Error(`The recording failed: ${result.error ?? "unknown error"}`);
+ }
+ try {
+ const project = await createProjectFromVideo(
+ deps,
+ result.screenVideoPath,
+ title || `Recording ${new Date(startedAt).toLocaleString()}`,
+ );
+ return {
+ projectId: project.document.project.id,
+ videoPath: result.screenVideoPath,
+ durationSec: project.durationSec,
+ autoZooms: project.autoZooms,
+ ...(result.warnings?.length ? { warnings: result.warnings } : {}),
+ };
+ } catch (error) {
+ // The take is on disk all the same: say where, so it can still be looked at.
+ throw new Error(
+ `The take was saved to ${result.screenVideoPath}, but it could not be turned into a project: ${error instanceof Error ? error.message : String(error)}`,
+ );
+ }
+ },
+ };
},
- );
+ };
+}
- server.registerTool(
- "stopRecording",
- {
- description:
- "Stop the recording and turn the take into a new project, laid out as the editor lays out a fresh recording: one clip, plus the zooms planned from the recorded clicks. Not one per click: clicks close in time and on screen share one zoom, a click in the first seconds or the one that stopped the take gets none, and autoZooms says how many there are. Returns the projectId to edit, a checkpointId that restores the raw take, and the take's videoPath.",
- inputSchema: z.object({
- title: z.string().optional().describe("The project's title in OpenScreen."),
- }),
- annotations: { readOnlyHint: false, destructiveHint: false, openWorldHint: false },
- },
- async (args) => {
- const current = take;
- if (!current) return text("No recording is running.", true);
- if (current.stopping) return text("This recording is already being stopped.", true);
- // Claimed before the first await, so a second call cannot save the same take twice.
- current.stopping = true;
- try {
- return await stopTake(current, args.title);
- } finally {
- take = null;
- }
- },
- );
+/** The take startRecording claimed: `running` once capture has started. */
+interface TakeSlot {
+ running: RunningTake | null;
+ stopping: boolean;
+}
- /** Stops the take, then turns it into a project with its checkpoint. */
+export function createMediaTools(deps: MediaToolDeps): MediaTools {
+ const { transport } = deps;
+ const recorder = deps.recorder ?? runnerRecorder(deps);
+ /** The take being recorded, then saved: held until it is a project. */
+ let take: TakeSlot | null = null;
+ // A take and an export never run together: each is a heavy job of its own, and a
+ // take's capture must not lose frames to an export.
+ let exporting = false;
+ /** When each take of this session started, by its video: setPointerTrack's clock. */
+ const startedAtByVideo = new Map<string, number>();
+ /** Each project's last export of this session, and when it was made: what getFrames reads. */
+ const lastExports = new Map<string, { path: string; at: Date; rendered: AxcutDocument }>();
+
+ /** Stops the take, then hands back its project with a checkpoint of it. */
async function stopTake(
- current: NonNullable<typeof take>,
+ running: RunningTake,
requestedTitle: string | undefined,
): Promise<CallToolResult> {
+ let recorded: RecordedTake;
try {
- current.run.window.webContents.send("cli-stop-recording");
- } catch {
- // The runner already went away; its result says why.
- }
- const result = await finish(current.run);
- if (!result.success || !result.screenVideoPath) {
- return text(`The recording failed: ${result.error ?? "unknown error"}`, true);
- }
- const title =
- requestedTitle?.trim() || `Recording ${new Date(current.startedAt).toLocaleString()}`;
- let project: Awaited<ReturnType<typeof createProjectFromVideo>>;
- try {
- project = await createProjectFromVideo(result.screenVideoPath, title);
+ recorded = await running.stop(requestedTitle?.trim() || undefined);
} catch (error) {
- // The take is on disk all the same: say where, so it can still be looked at.
- return text(
- `The take was saved to ${result.screenVideoPath}, but it could not be turned into a project: ${error instanceof Error ? error.message : String(error)}`,
- true,
- );
+ return text(error instanceof Error ? error.message : String(error), true);
}
- startedAtByVideo.set(result.screenVideoPath, current.startedAt);
- const projectId = project.document.project.id;
+ startedAtByVideo.set(recorded.videoPath, running.startedAt);
+ const { projectId } = recorded;
const checkpoint = await deps.runTool("createCheckpoint", { projectId });
const checkpointText = checkpoint.content[0]?.type === "text" ? checkpoint.content[0].text : "";
const checkpointId = checkpoint.isError
@@ -426,224 +456,346 @@ export function registerMediaTools(server: McpServer, deps: MediaToolDeps): void
JSON.stringify({
projectId,
checkpointId,
- videoPath: result.screenVideoPath,
- durationSec: project.durationSec,
- recordingStartedAt: current.startedAt,
- autoZooms: project.autoZooms,
- ...(result.warnings?.length ? { warnings: result.warnings } : {}),
+ videoPath: recorded.videoPath,
+ durationSec: recorded.durationSec,
+ recordingStartedAt: running.startedAt,
+ autoZooms: recorded.autoZooms,
+ ...(recorded.warnings?.length ? { warnings: recorded.warnings } : {}),
}),
);
}
- server.registerTool(
- "setPointerTrack",
- {
- description: [
- "For a take driven by a script (Playwright, CDP, a browser extension) rather than the real mouse: the screen pointer never moved, so the video has no cursor and no zoom. Give the pointer's path here and OpenScreen draws it, gliding between your actions, and zooms on its clicks as for a real mouse. It replaces the pointer recorded with the take, then plans zooms on the new clicks as stopRecording does, clear of the zooms already there; autoZoomsAdded says how many. Only for a take recorded by this server. Like any edit it needs a checkpoint of the project (stopRecording made one), and restoreCheckpoint undoes it, the pointer track included.",
- "events, one per action: `at` is Date.now() when your script did it (log it as it runs; the take's recordingStartedAt is on the same clock). `x` and `y` are fractions (0 to 1) of the recorded window. For a point (clientX, clientY) of a page shown in that window, from inside the page: border = (outerWidth - innerWidth) / 2; x = (border + clientX) / outerWidth; y = (outerHeight - innerHeight - border + clientY) / outerHeight.",
- ].join("\n"),
- inputSchema: z.object({ projectId: z.string(), events: POINTER_EVENTS }),
- annotations: { readOnlyHint: false, destructiveHint: true, openWorldHint: false },
- },
- // Through the document tools' runner, like any edit: the same queue, the same refusals
- // (a project open in the app's editor, one with no checkpoint) before anything is
- // written, and the sidecar kept by the checkpoints so restoreCheckpoint puts it back.
- // The args carry what the edit needs, so a running app can compute it (app-channel.ts).
- (args) => {
- const takes = Object.fromEntries(startedAtByVideo);
- return deps.runTool(
- POINTER_TRACK_TOOL,
- { projectId: args.projectId, events: args.events, takes },
- { edit: (document) => pointerTrackEdit(document, args.events, takes) },
- );
- },
- );
-
- server.registerTool(
- "exportVideo",
- {
- description:
- "Render a project to an MP4 (H.264, 60 fps) or a GIF with OpenScreen's compositor: the background, cursor, zooms, cuts, speed changes and annotations as edited. Returns the file's path; call getFrames with the projectId to check it at the ruler seconds you edited with. The video's shape follows the project's format (setAspectRatio).",
- inputSchema: z.object({
- projectId: z.string(),
- path: z
- .string()
- .describe(
- "Where to write the video: an absolute path, or one relative to this server's working directory, ending in .mp4 or .gif.",
- ),
- quality: z
- .enum(["medium", "good", "source"])
- .optional()
- .describe(
- "The height: medium is 720 pixels, good is 1080 (default), source keeps the recording's. The width follows the format.",
- ),
- }),
- annotations: { readOnlyHint: false, destructiveHint: false, openWorldHint: false },
- },
- async (args, extra) => {
- if (take) return text("A recording is running. Stop it before exporting.", true);
- if (exporting)
- return text("An export is already running. Wait for it, then try again.", true);
- const outPath = path.resolve(args.path);
- const extension = path.extname(outPath).toLowerCase();
- if (extension !== ".mp4" && extension !== ".gif") {
- return text(`The path must end in .mp4 or .gif, got "${args.path}".`, true);
- }
- const progressToken = extra._meta?.progressToken;
- exporting = true;
- try {
- // The project as the export reads it, and when: getFrames maps ruler seconds through
- // THIS document's cuts and speeds, and calls anything saved after `at` a change the
- // video does not have.
- const at = new Date();
- const rendered = await deps.documents.getProject(args.projectId).catch(() => null);
- if (!rendered) return text(`No project has the id "${args.projectId}".`, true);
- const run = deps.runners.start(
- {
- kind: "export",
- projectPath: deps.documents.filePathFor(args.projectId),
- outPath,
- format: extension === ".gif" ? "gif" : "mp4",
- quality: args.quality ?? "good",
- gifFrameRate: null,
- gifSizePreset: null,
- autoZoom: false,
- audioPath: null,
- audioMode: "mix",
- audioOffsetSec: 0,
- },
- {
- onProgress: (progress) => {
- if (progressToken === undefined) return;
- void extra
- .sendNotification({
- method: "notifications/progress",
- params: { progressToken, progress: progress.percentage, total: 100 },
- })
- .catch(() => undefined);
- },
- },
- );
- const result = await finish(run);
- if (!result.success) {
- return text(`The export failed: ${result.error ?? "unknown error"}`, true);
+ function register(server: McpServer): void {
+ server.registerTool(
+ "listSources",
+ {
+ description: `List the displays and windows OpenScreen can record, and the microphones. Pass a display's index or part of a window's title to startRecording. ${HIDDEN_WINDOWS_NOTE}`,
+ inputSchema: z.object({}),
+ annotations: READ_ONLY,
+ },
+ async () => {
+ const result = await finish(deps.runners.start({ kind: "sources" }));
+ if (!result.success || !result.sources) {
+ return text(`Could not list the sources: ${result.error ?? "unknown error"}`, true);
}
- const written = result.outputPath ?? outPath;
- // The file now holds this video: no other project's record may still point at it.
- for (const [id, record] of lastExports) {
- if (id !== args.projectId && record.path === written) lastExports.delete(id);
+ const { displays, windows, microphones } = result.sources;
+ return text(
+ JSON.stringify({
+ displays: displays.map(({ index, name }) => ({ index, name })),
+ windows: windows.map((window) => window.name),
+ microphones: microphones.map((mic) => mic.label),
+ }),
+ );
+ },
+ );
+
+ server.registerTool(
+ "startRecording",
+ {
+ description: `Start recording one window (preferred: nothing else on the desktop is captured) or one display. Returns once capture has started; then drive the app and call stopRecording. The pointer is recorded as data, so the export draws a clean cursor and the zooms planned from the clicks. ${START_RECORDING_NOTES[transport]}`,
+ inputSchema: z.object({
+ window: z
+ .string()
+ .optional()
+ .describe(
+ "Part of the title of the window to record, ignoring case. Wins over display.",
+ ),
+ display: z
+ .number()
+ .int()
+ .min(0)
+ .optional()
+ .describe("Index of the display to record, from listSources. Default 0."),
+ systemAudio: z.boolean().optional().describe("Also record the computer's sound."),
+ microphone: z
+ .union([z.boolean(), z.string()])
+ .optional()
+ .describe("true for the default microphone, or part of a microphone's name."),
+ }),
+ annotations: { readOnlyHint: false, destructiveHint: false, openWorldHint: false },
+ },
+ async (args) => {
+ if (!deps.editsAllowed()) return text(GATED_MESSAGE, true);
+ if (take?.stopping) {
+ return text("The last take is still being saved. Try again in a moment.", true);
}
- // The runner read the project file itself. Unless nothing was saved since `rendered`
- // was read, it may have rendered another version, so this video gets no record.
- const after = await deps.documents.getProject(args.projectId).catch(() => null);
- if (after?.project.updatedAt !== rendered.project.updatedAt) {
- lastExports.delete(args.projectId);
+ if (take) return text("A recording is already running. Call stopRecording first.", true);
+ if (exporting) return text("An export is running. Wait for it, then try again.", true);
+ // Claimed before the first await, so a second call cannot start a second take.
+ const claimed: TakeSlot = { running: null, stopping: false };
+ take = claimed;
+ try {
+ claimed.running = await recorder.start({
+ displayIndex: args.display ?? 0,
+ windowTitle: args.window ?? null,
+ mic: args.microphone !== undefined && args.microphone !== false,
+ micDevice: typeof args.microphone === "string" ? args.microphone : null,
+ systemAudio: args.systemAudio === true,
+ });
+ } catch (error) {
+ take = null;
return text(
- `${JSON.stringify({ path: result.outputPath, format: result.format, width: result.width, height: result.height })}\nThe project changed while it exported, so this video may not match it: export again before checking it with getFrames.`,
+ `The recording did not start: ${error instanceof Error ? error.message : String(error)}`,
+ true,
);
}
- lastExports.set(args.projectId, { path: written, at, rendered });
return text(
- JSON.stringify({
- path: result.outputPath,
- format: result.format,
- width: result.width,
- height: result.height,
- }),
+ JSON.stringify({ recording: true, startedAt: claimed.running.startedAt }) +
+ "\nRecording. Drive the app now, pausing about a second after each result appears, then call stopRecording.",
);
- } finally {
- exporting = false;
- }
- },
- );
+ },
+ );
- server.registerTool(
- "getFrames",
- {
- description: [
- `Look at a video as still images, at most ${MAX_FRAMES} per call. Ask for the moments you need to check, such as each zoom and each callout.`,
- "To check an edit, pass projectId: you get the project's last export, and atSec are the ruler seconds the document tools use, mapped through the trims and speed regions to the moment the export plays them. With path instead, you get any video file (a take's videoPath, say) and atSec are that file's own seconds.",
- ].join("\n"),
- inputSchema: z.object({
- projectId: z.string().optional().describe("Look at this project's last export."),
- path: z.string().optional().describe("Or look at this video file."),
- atSec: z
- .array(z.number().nonnegative())
- .max(MAX_FRAMES)
- .optional()
- .describe("Times in seconds: ruler seconds with projectId, the file's own with path."),
- count: z
- .number()
- .int()
- .min(1)
- .max(MAX_FRAMES)
- .optional()
- .describe("Frames spread evenly, when atSec is not given. Default 6."),
- }),
- annotations: READ_ONLY,
- },
- async (args) => {
- if ((args.projectId === undefined) === (args.path === undefined)) {
- return text("Pass either projectId or path.", true);
- }
- let videoPath: string;
- /** Ruler seconds → seconds of the exported file, when looking at a project. */
- let toVideoSec = (sec: number) => sec;
- let rulerEndSec: number | null = null;
- let note = "";
- if (args.projectId !== undefined) {
- const exported = lastExports.get(args.projectId);
- if (!exported) {
- return text("This project has no export yet: call exportVideo first.", true);
+ server.registerTool(
+ "stopRecording",
+ {
+ description: STOP_RECORDING_DESCRIPTIONS[transport],
+ inputSchema: z.object({
+ title: z.string().optional().describe("The project's title in OpenScreen."),
+ }),
+ annotations: { readOnlyHint: false, destructiveHint: false, openWorldHint: false },
+ },
+ async (args) => {
+ const current = take;
+ if (!current?.running) {
+ return text(
+ current ? "The recording is still starting." : "No recording is running.",
+ true,
+ );
}
- // Mapped through the document the export rendered, so a cut made since cannot shift
- // which frame of this video is shown; the current one only says whether it is stale.
- const { rendered } = exported;
- videoPath = exported.path;
- toVideoSec = (sec) => programmeSecOf(rendered, sec);
- rulerEndSec = Math.max(0, ...rendered.timeline.clips.map((clip) => clip.timelineEndSec));
- const current = await deps.documents.getProject(args.projectId);
- if (Date.parse(current.project.updatedAt) > exported.at.getTime()) {
- note = " The project changed after this export: export it again to see the changes.";
+ if (current.stopping) return text("This recording is already being stopped.", true);
+ // Claimed before the first await, so a second call cannot save the same take twice.
+ current.stopping = true;
+ try {
+ return await stopTake(current.running, args.title);
+ } finally {
+ take = null;
}
- } else {
- videoPath = path.resolve(args.path ?? "");
- }
+ },
+ );
- const video = await deps.openVideo(videoPath);
- try {
- const asked =
- args.atSec ??
- (rulerEndSec === null ? null : frameTimes(rulerEndSec, null, args.count ?? 6));
- const times = asked
- ? asked.map((sec) => ({
- asked: sec,
- at: frameTimes(video.durationSec, [toVideoSec(sec)], 1)[0],
- }))
- : frameTimes(video.durationSec, null, args.count ?? 6).map((at) => ({ asked: at, at }));
- const content: CallToolResult["content"] = [
- {
- type: "text",
- text: `${times.length} frame(s) of a ${video.width}x${video.height} video lasting ${video.durationSec.toFixed(2)}s.${note}`,
- },
- ];
- for (const { asked: askedSec, at } of times) {
- const frame = await video.capture(at, FRAME_WIDTH);
- content.push({
- type: "text",
- text:
- rulerEndSec === null
- ? `${at.toFixed(2)}s`
- : `${askedSec.toFixed(2)}s on the ruler, ${at.toFixed(2)}s into the video`,
- });
- content.push({ type: "image", data: frame.data, mimeType: frame.mimeType });
+ server.registerTool(
+ "setPointerTrack",
+ {
+ description: [
+ "For a take driven by a script (Playwright, CDP, a browser extension) rather than the real mouse: the screen pointer never moved, so the video has no cursor and no zoom. Give the pointer's path here and OpenScreen draws it, gliding between your actions, and zooms on its clicks as for a real mouse. It replaces the pointer recorded with the take, then plans zooms on the new clicks as stopRecording does, clear of the zooms already there; autoZoomsAdded says how many. Only for a take recorded by this server. It is an edit like any other: on a project that is not open in the editor it needs a checkpoint (stopRecording made one), and restoreCheckpoint undoes it, the pointer track included.",
+ "events, one per action: `at` is Date.now() when your script did it (log it as it runs; the take's recordingStartedAt is on the same clock). `x` and `y` are fractions (0 to 1) of the recorded window. For a point (clientX, clientY) of a page shown in that window, from inside the page: border = (outerWidth - innerWidth) / 2; x = (border + clientX) / outerWidth; y = (outerHeight - innerHeight - border + clientY) / outerHeight.",
+ ].join("\n"),
+ inputSchema: z.object({ projectId: z.string(), events: POINTER_EVENTS }),
+ annotations: { readOnlyHint: false, destructiveHint: true, openWorldHint: false },
+ },
+ // Through the document tools' runner, like any edit: the same queue, the same refusals
+ // (edits turned off, a project open in the app's editor, one with no checkpoint) before
+ // anything is written, and the sidecar kept by the checkpoints so restoreCheckpoint puts
+ // it back. The args carry what the edit needs, so a running app can compute it
+ // (app-channel.ts).
+ (args) => {
+ const takes = Object.fromEntries(startedAtByVideo);
+ return deps.runTool(
+ POINTER_TRACK_TOOL,
+ { projectId: args.projectId, events: args.events, takes },
+ { edit: (document) => pointerTrackEdit(document, args.events, takes) },
+ );
+ },
+ );
+
+ server.registerTool(
+ "exportVideo",
+ {
+ description:
+ "Render a project to an MP4 (H.264, 60 fps) or a GIF with OpenScreen's compositor: the background, cursor, zooms, cuts, speed changes and annotations as edited. Returns the file's path; call getFrames with the projectId to check it at the ruler seconds you edited with. The video's shape follows the project's format (setAspectRatio).",
+ inputSchema: z.object({
+ projectId: z.string(),
+ path: z.string().describe(EXPORT_PATH_DESCRIPTIONS[transport]),
+ quality: z
+ .enum(["medium", "good", "source"])
+ .optional()
+ .describe(
+ "The height: medium is 720 pixels, good is 1080 (default), source keeps the recording's. The width follows the format.",
+ ),
+ }),
+ annotations: { readOnlyHint: false, destructiveHint: false, openWorldHint: false },
+ },
+ async (args, extra) => {
+ if (!deps.editsAllowed()) return text(GATED_MESSAGE, true);
+ if (take) return text("A recording is running. Stop it before exporting.", true);
+ if (exporting)
+ return text("An export is already running. Wait for it, then try again.", true);
+ const outPath = clientPath(transport, args.path);
+ if (!outPath) return text(`The path must be absolute, got "${args.path}".`, true);
+ const extension = path.extname(outPath).toLowerCase();
+ if (extension !== ".mp4" && extension !== ".gif") {
+ return text(`The path must end in .mp4 or .gif, got "${args.path}".`, true);
}
- return { content };
- } finally {
- video.close();
- }
- },
- );
+ const progressToken = extra._meta?.progressToken;
+ exporting = true;
+ try {
+ // The project as the export reads it, and when: getFrames maps ruler seconds through
+ // THIS document's cuts and speeds, and calls anything saved after `at` a change the
+ // video does not have.
+ const at = new Date();
+ const rendered = await deps.documents.getProject(args.projectId).catch(() => null);
+ if (!rendered) return text(`No project has the id "${args.projectId}".`, true);
+ const run = deps.runners.start(
+ {
+ kind: "export",
+ projectPath: deps.documents.filePathFor(args.projectId),
+ // Read by id, as the editor opens a project: nothing of the app's own state moves.
+ projectId: args.projectId,
+ outPath,
+ format: extension === ".gif" ? "gif" : "mp4",
+ quality: args.quality ?? "good",
+ gifFrameRate: null,
+ gifSizePreset: null,
+ autoZoom: false,
+ audioPath: null,
+ audioMode: "mix",
+ audioOffsetSec: 0,
+ },
+ {
+ onProgress: (progress) => {
+ if (progressToken === undefined) return;
+ void extra
+ .sendNotification({
+ method: "notifications/progress",
+ params: { progressToken, progress: progress.percentage, total: 100 },
+ })
+ .catch(() => undefined);
+ },
+ },
+ );
+ const result = await finish(run);
+ if (!result.success) {
+ return text(`The export failed: ${result.error ?? "unknown error"}`, true);
+ }
+ const written = result.outputPath ?? outPath;
+ // The file now holds this video: no other project's record may still point at it.
+ for (const [id, record] of lastExports) {
+ if (id !== args.projectId && record.path === written) lastExports.delete(id);
+ }
+ // The runner read the project file itself. Unless nothing was saved since `rendered`
+ // was read, it may have rendered another version, so this video gets no record.
+ const after = await deps.documents.getProject(args.projectId).catch(() => null);
+ if (after?.project.updatedAt !== rendered.project.updatedAt) {
+ lastExports.delete(args.projectId);
+ return text(
+ `${JSON.stringify({ path: result.outputPath, format: result.format, width: result.width, height: result.height })}\nThe project changed while it exported, so this video may not match it: export again before checking it with getFrames.`,
+ );
+ }
+ lastExports.set(args.projectId, { path: written, at, rendered });
+ return text(
+ JSON.stringify({
+ path: result.outputPath,
+ format: result.format,
+ width: result.width,
+ height: result.height,
+ }),
+ );
+ } finally {
+ exporting = false;
+ }
+ },
+ );
+
+ server.registerTool(
+ "getFrames",
+ {
+ description: [
+ `Look at a video as still images, at most ${MAX_FRAMES} per call. Ask for the moments you need to check, such as each zoom and each callout.`,
+ "To check an edit, pass projectId: you get the project's last export, and atSec are the ruler seconds the document tools use, mapped through the trims and speed regions to the moment the export plays them. With path instead, you get any video file (a take's videoPath, say) and atSec are that file's own seconds.",
+ ].join("\n"),
+ inputSchema: z.object({
+ projectId: z.string().optional().describe("Look at this project's last export."),
+ path: z
+ .string()
+ .optional()
+ .describe(
+ transport === "http"
+ ? "Or look at this video file, by its absolute path."
+ : "Or look at this video file.",
+ ),
+ atSec: z
+ .array(z.number().nonnegative())
+ .max(MAX_FRAMES)
+ .optional()
+ .describe("Times in seconds: ruler seconds with projectId, the file's own with path."),
+ count: z
+ .number()
+ .int()
+ .min(1)
+ .max(MAX_FRAMES)
+ .optional()
+ .describe("Frames spread evenly, when atSec is not given. Default 6."),
+ }),
+ annotations: READ_ONLY,
+ },
+ async (args) => {
+ if ((args.projectId === undefined) === (args.path === undefined)) {
+ return text("Pass either projectId or path.", true);
+ }
+ let videoPath: string;
+ /** Ruler seconds → seconds of the exported file, when looking at a project. */
+ let toVideoSec = (sec: number) => sec;
+ let rulerEndSec: number | null = null;
+ let note = "";
+ if (args.projectId !== undefined) {
+ const exported = lastExports.get(args.projectId);
+ if (!exported) {
+ return text("This project has no export yet: call exportVideo first.", true);
+ }
+ // Mapped through the document the export rendered, so a cut made since cannot shift
+ // which frame of this video is shown; the current one only says whether it is stale.
+ const { rendered } = exported;
+ videoPath = exported.path;
+ toVideoSec = (sec) => programmeSecOf(rendered, sec);
+ rulerEndSec = Math.max(0, ...rendered.timeline.clips.map((clip) => clip.timelineEndSec));
+ const current = await deps.documents.getProject(args.projectId);
+ if (Date.parse(current.project.updatedAt) > exported.at.getTime()) {
+ note = " The project changed after this export: export it again to see the changes.";
+ }
+ } else {
+ const resolved = clientPath(transport, args.path ?? "");
+ if (!resolved) return text(`The path must be absolute, got "${args.path}".`, true);
+ videoPath = resolved;
+ }
+
+ const video = await deps.openVideo(videoPath);
+ try {
+ const asked =
+ args.atSec ??
+ (rulerEndSec === null ? null : frameTimes(rulerEndSec, null, args.count ?? 6));
+ const times = asked
+ ? asked.map((sec) => ({
+ asked: sec,
+ at: frameTimes(video.durationSec, [toVideoSec(sec)], 1)[0],
+ }))
+ : frameTimes(video.durationSec, null, args.count ?? 6).map((at) => ({ asked: at, at }));
+ const content: CallToolResult["content"] = [
+ {
+ type: "text",
+ text: `${times.length} frame(s) of a ${video.width}x${video.height} video lasting ${video.durationSec.toFixed(2)}s.${note}`,
+ },
+ ];
+ for (const { asked: askedSec, at } of times) {
+ const frame = await video.capture(at, FRAME_WIDTH);
+ content.push({
+ type: "text",
+ text:
+ rulerEndSec === null
+ ? `${at.toFixed(2)}s`
+ : `${askedSec.toFixed(2)}s on the ruler, ${at.toFixed(2)}s into the video`,
+ });
+ content.push({ type: "image", data: frame.data, mimeType: frame.mimeType });
+ }
+ return { content };
+ } finally {
+ video.close();
+ }
+ },
+ );
+ }
+
+ return { preamble: mediaPreamble(transport), register };
}
/**
diff --git a/electron/mcp/openscreen-mcp-server.test.ts b/electron/mcp/openscreen-mcp-server.test.ts
index 31aab5a56..edb226ff5 100644
--- a/electron/mcp/openscreen-mcp-server.test.ts
+++ b/electron/mcp/openscreen-mcp-server.test.ts
@@ -18,6 +18,7 @@ import {
import { OPENSCREEN_TOOL_NAMES } from "../ai-edition/agent-tools";
import { TOOL_DESCRIPTIONS } from "../ai-edition/deep-agent/service";
import { DocumentService } from "../ai-edition/document-service";
+import { createMediaTools } from "./media-tools";
import {
LIST_PROJECTS_TOOL,
MCP_ENDPOINT_PATH,
@@ -142,16 +143,32 @@ async function connect(
editor: FakeEditor,
options: { editsAllowed?: boolean; headers?: Record<string, string> } = {},
) {
+ const editsAllowed = () => options.editsAllowed ?? true;
+ const deps = {
+ transport: "http" as const,
+ host: editor,
+ projects,
+ editsAllowed,
+ version: "0.0.0",
+ };
+ const unused = () => {
+ throw new Error("not used by these tests");
+ };
running = await startMcpHttpServer({
port: 0,
token: TOKEN,
- deps: {
+ deps,
+ // As the app serves them; media-tools.test.ts calls them.
+ media: createMediaTools({
transport: "http",
- host: editor,
- projects,
- editsAllowed: () => options.editsAllowed ?? true,
- version: "0.0.0",
- },
+ editsAllowed,
+ recorder: { start: unused },
+ runners: { start: unused },
+ documents: projects,
+ openVideo: unused,
+ readCursor: unused,
+ runTool: unused,
+ }),
});
client = new Client({ name: "test", version: "0.0.0" });
const url = new URL(`http://127.0.0.1:${running.port}${MCP_ENDPOINT_PATH}`);
@@ -169,7 +186,7 @@ function resultText(result: Awaited<ReturnType<Client["callTool"]>>): string {
}
describe("the MCP tool surface", () => {
- it("is listProjects, then exactly the in-app agent's tools with its descriptions, then the checkpoints", async () => {
+ it("is listProjects, then exactly the in-app agent's tools with its descriptions, then the checkpoints and the media tools", async () => {
const mcp = await connect(new FakeEditor());
const { tools } = await mcp.listTools();
expect(tools.map((t) => t.name)).toEqual([
@@ -177,6 +194,13 @@ describe("the MCP tool surface", () => {
...OPENSCREEN_TOOL_NAMES,
"createCheckpoint",
"restoreCheckpoint",
+ // The same as `openscreen mcp` serves (#1174).
+ "listSources",
+ "startRecording",
+ "stopRecording",
+ "setPointerTrack",
+ "exportVideo",
+ "getFrames",
]);
for (const tool of tools.slice(1, 1 + OPENSCREEN_TOOL_NAMES.length)) {
expect(tool.description).toBe(TOOL_DESCRIPTIONS[tool.name]);
@@ -184,8 +208,9 @@ describe("the MCP tool surface", () => {
expect(Object.keys(tool.inputSchema.properties ?? {})).toContain("projectId");
expect(tool.inputSchema.required ?? []).not.toContain("projectId");
}
- for (const tool of tools.slice(-2)) {
- expect(Object.keys(tool.inputSchema.properties ?? {})).toContain("projectId");
+ for (const name of ["createCheckpoint", "restoreCheckpoint"]) {
+ const tool = tools.find((t) => t.name === name);
+ expect(Object.keys(tool?.inputSchema.properties ?? {})).toContain("projectId");
}
// The zod schemas survive the trip to JSON Schema with their fields intact.
const addTrim = tools.find((t) => t.name === "addTrim");
@@ -215,20 +240,21 @@ describe("the MCP tool surface", () => {
expect(mcp.getInstructions()).toContain("Time-bases (do not mix them up)");
});
- it("tells the client what is true here: an editor, its undo, no media, and its own switch", async () => {
+ it("tells the client what is true here: an editor, its undo, its recording bar, and its own switch", async () => {
const mcp = await connect(new FakeEditor(), { editsAllowed: false });
const instructions = mcp.getInstructions() ?? "";
expect(instructions).toContain(
"Without a projectId they act on the project open in the editor",
);
- expect(instructions).toContain("Nothing here records, exports or imports media.");
+ expect(instructions).toContain("startRecording hands over to OpenScreen's recording bar");
+ expect(instructions).toContain("startRecording and exportVideo are refused too");
expect(instructions).toContain("all of them are lost when OpenScreen quits.");
expect(instructions).toContain("turn on 'MCP edits' under MCP server in AI settings");
expect(instructions).not.toContain("Agent edits");
const { tools } = await mcp.listTools();
const restore = tools.find((t) => t.name === "restoreCheckpoint");
expect(restore?.description).toContain("one undo step");
- expect(restore?.description).not.toContain("pointer track");
+ expect(restore?.description).toContain("pointer track");
});
});
@@ -236,17 +262,17 @@ describe("the instructions, composed per transport", () => {
const media = "MEDIA BLOCK";
it("puts the media block first, then the same checkpoint rule on both transports", () => {
const stdio = mcpInstructions({ transport: "stdio", editsAllowed: true, media });
- const http = mcpInstructions({ transport: "http", editsAllowed: true });
- expect(stdio.startsWith(media)).toBe(true);
+ const http = mcpInstructions({ transport: "http", editsAllowed: true, media });
for (const instructions of [stdio, http]) {
+ expect(instructions.startsWith(media)).toBe(true);
expect(instructions).toContain("Before a series of edits, call createCheckpoint");
expect(instructions).toContain("Time-bases (do not mix them up)");
+ // Both export, so exporting is no example of a request without a tool.
+ expect(instructions).not.toContain("exporting)");
}
- // stdio reaches the editor only through the app, and exports.
- expect(stdio).not.toMatch(/Without a projectId they act|Nothing here records/);
+ // stdio reaches the editor only through the app.
+ expect(stdio).not.toContain("Without a projectId they act");
expect(stdio).toContain("While the OpenScreen app runs, this server hands each call to it");
- expect(stdio).not.toContain("exporting)");
- expect(http).toContain("exporting)");
expect(stdio).toContain("in OpenScreen's memory while the app runs");
});
});
diff --git a/electron/mcp/openscreen-mcp-server.ts b/electron/mcp/openscreen-mcp-server.ts
index 2d27d3ace..0a78778ff 100644
--- a/electron/mcp/openscreen-mcp-server.ts
+++ b/electron/mcp/openscreen-mcp-server.ts
@@ -26,7 +26,8 @@
// in-app agent has no need for them: its whole turn is already a single apply.
//
// Two transports serve these tools: the in-app server over HTTP (below) and
-// `openscreen mcp` over stdio (`stdio-server.ts`, which adds the media tools). The
+// `openscreen mcp` over stdio (`stdio-server.ts`), both with the media tools around
+// them (media-tools.ts: record, export, look). The
// runner and the tools are the same; the texts a client reads are composed per
// transport (`mcpInstructions`, the `*_MESSAGES` tables), so each says what is true
// on the server that serves it. While the app runs, `openscreen mcp` forwards its document
@@ -66,6 +67,7 @@ import {
type ProjectSummary,
type ProjectVersion,
} from "../ai-edition/document-service";
+import type { MediaTools } from "./media-tools";
export const MCP_SERVER_NAME = "openscreen";
export const MCP_ENDPOINT_PATH = "/mcp";
@@ -115,8 +117,9 @@ type ProjectRead = Awaited<ReturnType<McpProjectStore["getProjectForUpdate"]>>;
/**
* Which server this is. The tools are the same; what differs is what a call can reach:
- * the in-app server (`http`) has the editor, its undo stack and a switch for edits;
- * `openscreen mcp` (`stdio`) is a process of its own, with no editor and no switch.
+ * the in-app server (`http`) has the editor, its undo stack, the app's recording bar and
+ * a switch for edits; `openscreen mcp` (`stdio`) is a process of its own, with no editor
+ * and no switch, recording in a hidden runner.
* Every text the client reads says what is true on the one it is connected to.
*/
export type McpTransport = "http" | "stdio";
@@ -134,7 +137,7 @@ export interface McpToolDeps {
export const LIST_PROJECTS_TOOL = "listProjects";
/** The in-app server's edit switch, as AI settings label it (`mcpServer.editsLabel`). */
-const MCP_EDITS_SETTING = "'MCP edits' under MCP server in AI settings";
+export const MCP_EDITS_SETTING = "'MCP edits' under MCP server in AI settings";
const LIST_PROJECTS_DESCRIPTION =
"List every OpenScreen project: id, title, last update, asset count, and `open`: whether the OpenScreen editor has it open, in which case an edit to it goes through the editor, shows there live and the user can undo it with Ctrl/Cmd+Z. Pass a project's id as `projectId` to any other tool to read or edit that project.";
@@ -204,17 +207,16 @@ const CORE_PREAMBLES: Record<McpTransport, readonly string[]> = {
],
};
-const NO_MEDIA_NOTE = "Nothing here records, exports or imports media.";
-
/**
- * The server's instructions: the capability block of the media tools when they are served,
- * the projects-and-checkpoints rules of this transport, then the in-app agent's editing
- * guidance, naming this server's edit switch and whether it can export.
+ * The server's instructions: the capability block of the media tools, the
+ * projects-and-checkpoints rules of this transport, then the in-app agent's editing
+ * guidance, naming this server's edit switch, which holds back recording and export too.
*/
export function mcpInstructions(options: {
transport: McpTransport;
editsAllowed: boolean;
- /** The media tools' preamble, when this server registers them. */
+ /** The media tools' preamble. Both servers register them; only the app's channel for
+ * `openscreen mcp` (app-channel.ts), which no agent reads, goes without. */
media?: string;
}): string {
const { transport, editsAllowed, media } = options;
@@ -222,7 +224,9 @@ export function mcpInstructions(options: {
...(media ? [media] : []),
...CORE_PREAMBLES[transport],
CHECKPOINT_LIFETIMES[transport],
- ...(media ? [] : [NO_MEDIA_NOTE]),
+ ...(media && !editsAllowed
+ ? [`startRecording and exportVideo are refused too while ${MCP_EDITS_SETTING} is off.`]
+ : []),
"",
].join("\n");
return (
@@ -547,8 +551,8 @@ export function createToolRunner(deps: McpToolDeps): McpToolRunner {
export function createOpenScreenMcpServer(
deps: McpToolDeps,
runTool: McpToolRunner,
- /** The media tools' preamble, when the caller registers them on this server. */
- media?: string,
+ /** The media tools (media-tools.ts), served by both servers but not the app's channel. */
+ media?: MediaTools,
): McpServer {
const server = new McpServer(
{ name: MCP_SERVER_NAME, version: deps.version },
@@ -556,7 +560,7 @@ export function createOpenScreenMcpServer(
instructions: mcpInstructions({
transport: deps.transport,
editsAllowed: deps.editsAllowed(),
- media,
+ media: media?.preamble,
}),
},
);
@@ -601,6 +605,7 @@ export function createOpenScreenMcpServer(
(args: unknown) => runTool(tool.name, args),
);
}
+ media?.register(server);
return server;
}
@@ -637,6 +642,8 @@ export async function startMcpHttpServer(options: {
deps: McpToolDeps;
/** The app's shared runner (see createToolRunner); a fresh one when omitted. */
runTool?: McpToolRunner;
+ /** The media tools, built once: the take and the exports outlive each request. */
+ media?: MediaTools;
}): Promise<RunningMcpServer> {
const { token, deps } = options;
const shared = options.runTool ?? createToolRunner(deps);
@@ -657,8 +664,9 @@ export async function startMcpHttpServer(options: {
// Stateless: one server + transport per request, the SDK's documented shape
// for a server that keeps no per-session state. Tools read the live editor
- // on every call, so there is nothing a session would need to remember.
- const server = createOpenScreenMcpServer(deps, runTool);
+ // on every call, and what the media tools hold (the take, the exports) lives in
+ // `media`, built once, so there is nothing a session would need to remember.
+ const server = createOpenScreenMcpServer(deps, runTool, options.media);
const transport = new StreamableHTTPServerTransport({
sessionIdGenerator: undefined,
enableJsonResponse: true,
diff --git a/electron/mcp/stdio-server.ts b/electron/mcp/stdio-server.ts
index 9818e474c..54dbfdc63 100644
--- a/electron/mcp/stdio-server.ts
+++ b/electron/mcp/stdio-server.ts
@@ -9,14 +9,14 @@
//
// No port and no token: the client launched this process itself, with the
// user's rights, as it would launch `openscreen record` or `openscreen export`.
-// For the same reason it may edit, where the in-app server waits for the user to
-// allow it: there is no other caller to keep out.
+// For the same reason it may edit, record and export, where the in-app server waits
+// for the user to allow it: there is no other caller to keep out.
import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js";
import type { DocumentService } from "../ai-edition/document-service";
import { openStdin } from "../cli/stdin";
import { routeToApp } from "./app-channel";
-import { MEDIA_PREAMBLE, type MediaToolDeps, registerMediaTools } from "./media-tools";
+import { createMediaTools, type MediaToolDeps } from "./media-tools";
import {
createOpenScreenMcpServer,
createToolRunner,
@@ -46,14 +46,16 @@ export async function serveMcpOverStdio(options: {
version: deps.version,
local: createToolRunner(deps),
});
- const server = createOpenScreenMcpServer(deps, runTool, MEDIA_PREAMBLE);
- registerMediaTools(server, {
+ const media = createMediaTools({
+ transport: "stdio",
+ editsAllowed: () => true,
runners: options.runners,
documents: options.documents,
openVideo: options.openVideo,
readCursor: options.readCursorTelemetry,
runTool,
});
+ const server = createOpenScreenMcpServer(deps, runTool, media);
const stdin = openStdin();
// The transport only reads stdin; a client that exits just closes it. A pipe can
// report that as "end", "close" or "error", depending on how the client went.
diff --git a/electron/windows.ts b/electron/windows.ts
index df71910b1..b4fef84c1 100644
--- a/electron/windows.ts
+++ b/electron/windows.ts
@@ -835,3 +835,30 @@ export function createNotesWindow(): BrowserWindow {
return win;
}
+
+/** A hidden window running one of the CLI runners (src/cli/): `openscreen <command>`, or a
+ * media tool of an MCP server. */
+export function createRunnerWindow(windowType: string): BrowserWindow {
+ const win = new BrowserWindow({
+ width: 1280,
+ height: 720,
+ show: false,
+ webPreferences: {
+ preload: path.join(__dirname, "preload.mjs"),
+ additionalArguments: [ASSET_BASE_URL_ARG],
+ nodeIntegration: false,
+ contextIsolation: true,
+ // Same relaxation as the editor window: exporters load recording media
+ // via file:// URLs.
+ webSecurity: false,
+ backgroundThrottling: false,
+ },
+ });
+
+ if (VITE_DEV_SERVER_URL) {
+ win.loadURL(`${VITE_DEV_SERVER_URL}?windowType=${windowType}`);
+ } else {
+ win.loadFile(path.join(RENDERER_DIST, "index.html"), { query: { windowType } });
+ }
+ return win;
+}
diff --git a/src/cli/CliExportRunner.tsx b/src/cli/CliExportRunner.tsx
index 85b066851..be36f65ad 100644
--- a/src/cli/CliExportRunner.tsx
+++ b/src/cli/CliExportRunner.tsx
@@ -220,21 +220,27 @@ async function loadLegacyProject(project: EditorProjectData): Promise<ExportProj
};
}
-async function runExport(request: CliExportRequest): Promise<CliDoneResult> {
+/** The project to export: by id when the MCP server names one, else the file at its path. */
+async function loadExportProject(request: CliExportRequest): Promise<ExportProject> {
+ const probe = (videoPath: string) => probeVideoDimensions(toFileUrl(videoPath));
+ if (request.projectId) {
+ const stored = await nativeBridgeClient.aiEdition.get(request.projectId);
+ if (!stored.success || !stored.document) {
+ throw new Error(stored.error ?? `No project has the id "${request.projectId}"`);
+ }
+ return loadDocumentProject(stored.document, probe);
+ }
const loaded = await nativeBridgeClient.project.loadProjectFileFromPath(request.projectPath);
if (!loaded.success || loaded.project === undefined) {
throw new Error(loaded.error ?? loaded.message ?? "Failed to load project file");
}
- let project: ExportProject;
- if (isAxcutDocumentFile(loaded.project)) {
- project = await loadDocumentProject(loaded.project, (videoPath) =>
- probeVideoDimensions(toFileUrl(videoPath)),
- );
- } else if (validateProjectData(loaded.project)) {
- project = await loadLegacyProject(loaded.project);
- } else {
- throw new Error("Project file is not a valid .openscreen project");
- }
+ if (isAxcutDocumentFile(loaded.project)) return loadDocumentProject(loaded.project, probe);
+ if (validateProjectData(loaded.project)) return loadLegacyProject(loaded.project);
+ throw new Error("Project file is not a valid .openscreen project");
+}
+
+async function runExport(request: CliExportRequest): Promise<CliDoneResult> {
+ const project = await loadExportProject(request);
const { editor } = project;
let axcutDocument = project.document;
diff --git a/src/cli/CliSourcesRunner.tsx b/src/cli/CliSourcesRunner.tsx
index 0e4bbdd0d..0a5f3dc39 100644
--- a/src/cli/CliSourcesRunner.tsx
+++ b/src/cli/CliSourcesRunner.tsx
@@ -104,7 +104,7 @@ async function enumerateMicrophones(): Promise<{
const labeled = inputs.filter((device) => device.label);
return {
- microphones: labeled.map((device) => ({ label: device.label })),
+ microphones: labeled.map((device) => ({ label: device.label, deviceId: device.deviceId })),
microphoneLabelsUnavailable: inputs.length > 0 && labeled.length === 0,
};
}
diff --git a/src/i18n/locales/ar/editor.json b/src/i18n/locales/ar/editor.json
index 011728781..da053d2d7 100644
--- a/src/i18n/locales/ar/editor.json
+++ b/src/i18n/locales/ar/editor.json
@@ -414,8 +414,8 @@
"enableLabel": "الخادم",
"enable": "تفعيل خادم MCP",
"editsLabel": "تعديلات MCP",
- "allowEdits": "السماح لعملاء MCP بتعديل مشاريعك",
- "editsHint": "معطّل افتراضيًا: يمكن للعملاء قراءة مشاريعك، وتُرفض كل محاولة تعديل. عند التفعيل، يُحفظ كل تعديل فور وصوله، ويمكن التراجع عن تعديل المشروع المفتوح في المحرر من هناك.",
+ "allowEdits": "السماح لعملاء MCP بتعديل مشاريعك والتسجيل والتصدير",
+ "editsHint": "معطّل افتراضيًا: يمكن للعملاء قراءة مشاريعك، وتُرفض كل محاولة تعديل أو تسجيل أو تصدير. عند التفعيل، يُحفظ كل تعديل فور وصوله، ويمكن التراجع عن تعديل المشروع المفتوح في المحرر من هناك.",
"statusRunning": "يعمل",
"statusOff": "متوقف",
"statusError": "لا يعمل",
diff --git a/src/i18n/locales/cs/editor.json b/src/i18n/locales/cs/editor.json
index 20d7c47e7..086d3f74b 100644
--- a/src/i18n/locales/cs/editor.json
+++ b/src/i18n/locales/cs/editor.json
@@ -414,8 +414,8 @@
"enableLabel": "Server",
"enable": "Zapnout MCP server",
"editsLabel": "Úpravy přes MCP",
- "allowEdits": "Povolit MCP klientům úpravy vašich projektů",
- "editsHint": "Ve výchozím stavu vypnuto: klienti mohou vaše projekty číst a každá úprava, o kterou se pokusí, je odmítnuta. Po zapnutí se každá úprava hned uloží; úpravu projektu otevřeného v editoru lze tam vrátit zpět.",
+ "allowEdits": "Povolit MCP klientům upravovat vaše projekty, nahrávat a exportovat",
+ "editsHint": "Ve výchozím stavu vypnuto: klienti mohou vaše projekty číst a každá úprava, nahrávání nebo export, o které se pokusí, je odmítnuta. Po zapnutí se každá úprava hned uloží; úpravu projektu otevřeného v editoru lze tam vrátit zpět.",
"statusRunning": "Běží",
"statusOff": "Vypnuto",
"statusError": "Neběží",
diff --git a/src/i18n/locales/de/editor.json b/src/i18n/locales/de/editor.json
index b5859a41d..eb9a7fdf9 100644
--- a/src/i18n/locales/de/editor.json
+++ b/src/i18n/locales/de/editor.json
@@ -414,8 +414,8 @@
"enableLabel": "Server",
"enable": "MCP-Server aktivieren",
"editsLabel": "MCP-Änderungen",
- "allowEdits": "MCP-Clients erlauben, deine Projekte zu bearbeiten",
- "editsHint": "Standardmäßig aus: Clients können deine Projekte lesen, und jede Bearbeitung wird abgelehnt. Wenn aktiviert, wird jede Bearbeitung sofort gespeichert; eine Bearbeitung des im Editor geöffneten Projekts lässt sich dort rückgängig machen.",
+ "allowEdits": "MCP-Clients erlauben, deine Projekte zu bearbeiten, aufzunehmen und zu exportieren",
+ "editsHint": "Standardmäßig aus: Clients können deine Projekte lesen, und jede Bearbeitung, Aufnahme oder jeder Export wird abgelehnt. Wenn aktiviert, wird jede Bearbeitung sofort gespeichert; eine Bearbeitung des im Editor geöffneten Projekts lässt sich dort rückgängig machen.",
"statusRunning": "Läuft",
"statusOff": "Aus",
"statusError": "Läuft nicht",
diff --git a/src/i18n/locales/en/editor.json b/src/i18n/locales/en/editor.json
index 5113242b8..8e6d253d9 100644
--- a/src/i18n/locales/en/editor.json
+++ b/src/i18n/locales/en/editor.json
@@ -414,8 +414,8 @@
"enableLabel": "Server",
"enable": "Enable MCP server",
"editsLabel": "MCP edits",
- "allowEdits": "Allow MCP clients to edit your projects",
- "editsHint": "Off by default: clients can read your projects, and every edit they try is refused. When on, each edit is saved as it lands; an edit to the project open in the editor can be undone there.",
+ "allowEdits": "Allow MCP clients to edit your projects, record and export",
+ "editsHint": "Off by default: clients can read your projects, and every edit, recording or export they try is refused. When on, each edit is saved as it lands; an edit to the project open in the editor can be undone there.",
"statusRunning": "Running",
"statusOff": "Off",
"statusError": "Not running",
diff --git a/src/i18n/locales/es/editor.json b/src/i18n/locales/es/editor.json
index 05da13ca0..680632caa 100644
--- a/src/i18n/locales/es/editor.json
+++ b/src/i18n/locales/es/editor.json
@@ -414,8 +414,8 @@
"enableLabel": "Servidor",
"enable": "Activar servidor MCP",
"editsLabel": "Ediciones por MCP",
- "allowEdits": "Permitir que los clientes MCP editen tus proyectos",
- "editsHint": "Desactivado por defecto: los clientes pueden leer tus proyectos y se rechaza cada edición que intenten. Si se activa, cada edición se guarda al llegar; una edición del proyecto abierto en el editor se puede deshacer allí.",
+ "allowEdits": "Permitir que los clientes MCP editen tus proyectos, graben y exporten",
+ "editsHint": "Desactivado por defecto: los clientes pueden leer tus proyectos y se rechaza cada edición, grabación o exportación que intenten. Si se activa, cada edición se guarda al llegar; una edición del proyecto abierto en el editor se puede deshacer allí.",
"statusRunning": "En ejecución",
"statusOff": "Desactivado",
"statusError": "Sin ejecutar",
diff --git a/src/i18n/locales/fr/editor.json b/src/i18n/locales/fr/editor.json
index d242f893f..6fd774cd6 100644
--- a/src/i18n/locales/fr/editor.json
+++ b/src/i18n/locales/fr/editor.json
@@ -414,8 +414,8 @@
"enableLabel": "Serveur",
"enable": "Activer le serveur MCP",
"editsLabel": "Modifications par MCP",
- "allowEdits": "Autoriser les clients MCP à modifier vos projets",
- "editsHint": "Désactivé par défaut : les clients peuvent lire vos projets, et chaque modification tentée est refusée. Une fois activé, chaque modification est enregistrée dès son arrivée ; une modification du projet ouvert dans l’éditeur peut y être annulée.",
+ "allowEdits": "Autoriser les clients MCP à modifier vos projets, enregistrer et exporter",
+ "editsHint": "Désactivé par défaut : les clients peuvent lire vos projets, et chaque modification, enregistrement ou export tenté est refusé. Une fois activé, chaque modification est enregistrée dès son arrivée ; une modification du projet ouvert dans l’éditeur peut y être annulée.",
"statusRunning": "En cours",
"statusOff": "Désactivé",
"statusError": "Arrêté",
diff --git a/src/i18n/locales/it/editor.json b/src/i18n/locales/it/editor.json
index c7a25b8e3..27fd5cf6c 100644
--- a/src/i18n/locales/it/editor.json
+++ b/src/i18n/locales/it/editor.json
@@ -414,8 +414,8 @@
"enableLabel": "Server",
"enable": "Attiva server MCP",
"editsLabel": "Modifiche via MCP",
- "allowEdits": "Consenti ai client MCP di modificare i tuoi progetti",
- "editsHint": "Disattivato per impostazione predefinita: i client possono leggere i tuoi progetti e ogni modifica tentata viene rifiutata. Se attivo, ogni modifica viene salvata appena arriva; una modifica al progetto aperto nell’editor si può annullare lì.",
+ "allowEdits": "Consenti ai client MCP di modificare i tuoi progetti, registrare ed esportare",
+ "editsHint": "Disattivato per impostazione predefinita: i client possono leggere i tuoi progetti e ogni modifica, registrazione o esportazione tentata viene rifiutata. Se attivo, ogni modifica viene salvata appena arriva; una modifica al progetto aperto nell’editor si può annullare lì.",
"statusRunning": "In esecuzione",
"statusOff": "Disattivato",
"statusError": "Non in esecuzione",
diff --git a/src/i18n/locales/ja-JP/editor.json b/src/i18n/locales/ja-JP/editor.json
index 1ec988af8..3de6f0a77 100644
--- a/src/i18n/locales/ja-JP/editor.json
+++ b/src/i18n/locales/ja-JP/editor.json
@@ -414,8 +414,8 @@
"enableLabel": "サーバー",
"enable": "MCP サーバーを有効にする",
"editsLabel": "MCP による編集",
- "allowEdits": "MCP クライアントによるプロジェクトの編集を許可",
- "editsHint": "既定ではオフです。クライアントはプロジェクトを読み取れますが、編集はすべて拒否されます。オンにすると、各編集は届いた時点で保存されます。エディターで開いているプロジェクトへの編集は、エディターで元に戻せます。",
+ "allowEdits": "MCP クライアントによるプロジェクトの編集、録画、書き出しを許可",
+ "editsHint": "既定ではオフです。クライアントはプロジェクトを読み取れますが、編集、録画、書き出しはすべて拒否されます。オンにすると、各編集は届いた時点で保存されます。エディターで開いているプロジェクトへの編集は、エディターで元に戻せます。",
"statusRunning": "実行中",
"statusOff": "オフ",
"statusError": "停止中",
diff --git a/src/i18n/locales/ko-KR/editor.json b/src/i18n/locales/ko-KR/editor.json
index fbd59e7ca..77731a1d3 100644
--- a/src/i18n/locales/ko-KR/editor.json
+++ b/src/i18n/locales/ko-KR/editor.json
@@ -414,8 +414,8 @@
"enableLabel": "서버",
"enable": "MCP 서버 사용",
"editsLabel": "MCP 편집",
- "allowEdits": "MCP 클라이언트의 프로젝트 편집 허용",
- "editsHint": "기본값은 꺼짐입니다. 클라이언트는 프로젝트를 읽을 수 있지만 모든 편집 시도는 거부됩니다. 켜면 각 편집이 도착하는 즉시 저장되며, 편집기에 열린 프로젝트의 편집은 편집기에서 실행 취소할 수 있습니다.",
+ "allowEdits": "MCP 클라이언트의 프로젝트 편집, 녹화, 내보내기 허용",
+ "editsHint": "기본값은 꺼짐입니다. 클라이언트는 프로젝트를 읽을 수 있지만 모든 편집, 녹화, 내보내기 시도는 거부됩니다. 켜면 각 편집이 도착하는 즉시 저장되며, 편집기에 열린 프로젝트의 편집은 편집기에서 실행 취소할 수 있습니다.",
"statusRunning": "실행 중",
"statusOff": "꺼짐",
"statusError": "실행되지 않음",
diff --git a/src/i18n/locales/pt-BR/editor.json b/src/i18n/locales/pt-BR/editor.json
index 39af8ad00..55e1fa5bf 100644
--- a/src/i18n/locales/pt-BR/editor.json
+++ b/src/i18n/locales/pt-BR/editor.json
@@ -414,8 +414,8 @@
"enableLabel": "Servidor",
"enable": "Ativar servidor MCP",
"editsLabel": "Edições via MCP",
- "allowEdits": "Permitir que clientes MCP editem seus projetos",
- "editsHint": "Desativado por padrão: os clientes podem ler seus projetos, e toda edição que tentarem é recusada. Quando ativado, cada edição é salva assim que chega; uma edição do projeto aberto no editor pode ser desfeita lá.",
+ "allowEdits": "Permitir que clientes MCP editem seus projetos, gravem e exportem",
+ "editsHint": "Desativado por padrão: os clientes podem ler seus projetos, e toda edição, gravação ou exportação que tentarem é recusada. Quando ativado, cada edição é salva assim que chega; uma edição do projeto aberto no editor pode ser desfeita lá.",
"statusRunning": "Em execução",
"statusOff": "Desligado",
"statusError": "Parado",
diff --git a/src/i18n/locales/ru/editor.json b/src/i18n/locales/ru/editor.json
index 85ddc1908..c89607df1 100644
--- a/src/i18n/locales/ru/editor.json
+++ b/src/i18n/locales/ru/editor.json
@@ -414,8 +414,8 @@
"enableLabel": "Сервер",
"enable": "Включить MCP-сервер",
"editsLabel": "Правки через MCP",
- "allowEdits": "Разрешить MCP-клиентам редактировать ваши проекты",
- "editsHint": "По умолчанию выключено: клиенты могут читать ваши проекты, а любая попытка правки отклоняется. Если включено, каждая правка сохраняется сразу; правку проекта, открытого в редакторе, можно отменить там.",
+ "allowEdits": "Разрешить MCP-клиентам редактировать ваши проекты, записывать и экспортировать",
+ "editsHint": "По умолчанию выключено: клиенты могут читать ваши проекты, а любая попытка правки, записи или экспорта отклоняется. Если включено, каждая правка сохраняется сразу; правку проекта, открытого в редакторе, можно отменить там.",
"statusRunning": "Работает",
"statusOff": "Выключен",
"statusError": "Не работает",
diff --git a/src/i18n/locales/tr/editor.json b/src/i18n/locales/tr/editor.json
index ac764226e..f052f504e 100644
--- a/src/i18n/locales/tr/editor.json
+++ b/src/i18n/locales/tr/editor.json
@@ -414,8 +414,8 @@
"enableLabel": "Sunucu",
"enable": "MCP sunucusunu etkinleştir",
"editsLabel": "MCP düzenlemeleri",
- "allowEdits": "MCP istemcilerinin projelerinizi düzenlemesine izin ver",
- "editsHint": "Varsayılan olarak kapalı: istemciler projelerinizi okuyabilir ve denedikleri her düzenleme reddedilir. Açıkken her düzenleme geldiği anda kaydedilir; düzenleyicide açık olan projedeki bir düzenleme orada geri alınabilir.",
+ "allowEdits": "MCP istemcilerinin projelerinizi düzenlemesine, kayıt yapmasına ve dışa aktarmasına izin ver",
+ "editsHint": "Varsayılan olarak kapalı: istemciler projelerinizi okuyabilir ve denedikleri her düzenleme, kayıt veya dışa aktarma reddedilir. Açıkken her düzenleme geldiği anda kaydedilir; düzenleyicide açık olan projedeki bir düzenleme orada geri alınabilir.",
"statusRunning": "Çalışıyor",
"statusOff": "Kapalı",
"statusError": "Çalışmıyor",
diff --git a/src/i18n/locales/vi/editor.json b/src/i18n/locales/vi/editor.json
index b99882eee..f85a6e755 100644
--- a/src/i18n/locales/vi/editor.json
+++ b/src/i18n/locales/vi/editor.json
@@ -414,8 +414,8 @@
"enableLabel": "Máy chủ",
"enable": "Bật máy chủ MCP",
"editsLabel": "Chỉnh sửa qua MCP",
- "allowEdits": "Cho phép ứng dụng khách MCP chỉnh sửa các dự án của bạn",
- "editsHint": "Tắt theo mặc định: ứng dụng khách có thể đọc các dự án của bạn và mọi thao tác chỉnh sửa đều bị từ chối. Khi bật, mỗi chỉnh sửa được lưu ngay khi đến; chỉnh sửa trên dự án đang mở trong trình chỉnh sửa có thể được hoàn tác tại đó.",
+ "allowEdits": "Cho phép ứng dụng khách MCP chỉnh sửa các dự án của bạn, ghi hình và xuất video",
+ "editsHint": "Tắt theo mặc định: ứng dụng khách có thể đọc các dự án của bạn và mọi thao tác chỉnh sửa, ghi hình hay xuất video đều bị từ chối. Khi bật, mỗi chỉnh sửa được lưu ngay khi đến; chỉnh sửa trên dự án đang mở trong trình chỉnh sửa có thể được hoàn tác tại đó.",
"statusRunning": "Đang chạy",
"statusOff": "Tắt",
"statusError": "Không chạy",
diff --git a/src/i18n/locales/zh-CN/editor.json b/src/i18n/locales/zh-CN/editor.json
index b02212816..940a28963 100644
--- a/src/i18n/locales/zh-CN/editor.json
+++ b/src/i18n/locales/zh-CN/editor.json
@@ -414,8 +414,8 @@
"enableLabel": "服务器",
"enable": "启用 MCP 服务器",
"editsLabel": "MCP 编辑",
- "allowEdits": "允许 MCP 客户端编辑你的项目",
- "editsHint": "默认关闭:客户端可以读取你的项目,但所有编辑都会被拒绝。开启后,每次编辑都会立即保存;对编辑器中打开的项目所做的编辑可以在编辑器中撤销。",
+ "allowEdits": "允许 MCP 客户端编辑你的项目、录制和导出",
+ "editsHint": "默认关闭:客户端可以读取你的项目,但所有编辑、录制和导出都会被拒绝。开启后,每次编辑都会立即保存;对编辑器中打开的项目所做的编辑可以在编辑器中撤销。",
"statusRunning": "运行中",
"statusOff": "已关闭",
"statusError": "未运行",
diff --git a/src/i18n/locales/zh-TW/editor.json b/src/i18n/locales/zh-TW/editor.json
index c7e9c3ad1..54fe4028e 100644
--- a/src/i18n/locales/zh-TW/editor.json
+++ b/src/i18n/locales/zh-TW/editor.json
@@ -414,8 +414,8 @@
"enableLabel": "伺服器",
"enable": "啟用 MCP 伺服器",
"editsLabel": "MCP 編輯",
- "allowEdits": "允許 MCP 用戶端編輯你的專案",
- "editsHint": "預設關閉:用戶端可以讀取你的專案,但所有編輯都會被拒絕。開啟後,每次編輯都會立即儲存;對編輯器中開啟的專案所做的編輯可以在編輯器中復原。",
+ "allowEdits": "允許 MCP 用戶端編輯你的專案、錄製和匯出",
+ "editsHint": "預設關閉:用戶端可以讀取你的專案,但所有編輯、錄製和匯出都會被拒絕。開啟後,每次編輯都會立即儲存;對編輯器中開啟的專案所做的編輯可以在編輯器中復原。",
"statusRunning": "執行中",
"statusOff": "已關閉",
"statusError": "未執行",
diff --git a/src/lib/cliContracts.ts b/src/lib/cliContracts.ts
index 03657887a..bfd57def5 100644
--- a/src/lib/cliContracts.ts
+++ b/src/lib/cliContracts.ts
@@ -10,6 +10,13 @@ export interface CliExportRequest {
kind: "export";
/** Absolute path to the .openscreen project file. */
projectPath: string;
+ /**
+ * Set by the MCP server: the project of the projects folder that `projectPath` is the file
+ * of, read by id as the editor opens a project. Unlike a project file opened by path, that
+ * leaves the main process's current project and recording session alone, which the app's
+ * editor relies on while an export runs beside it.
+ */
+ projectId?: string;
/** Absolute output path; null = derive from projectPath + format. */
outPath: string | null;
/** null = use the format stored in the project. */
@@ -75,7 +82,8 @@ export type CliRequest =
export interface CliSourcesResult {
displays: { index: number; id: string; name: string }[];
windows: { id: string; name: string }[];
- microphones: { label: string }[];
+ /** `deviceId`: the renderer's id for it, as the HUD records with it beside the label. */
+ microphones: { label: string; deviceId?: string }[];
/** True when microphone labels required a permission the user hasn't granted. */
microphoneLabelsUnavailable: boolean;
}