Record your screen, ship a demo. Free and open-source, GPU-accelerated, no watermarks, no subscriptions. Windows, macOS, Linux. Actively maintained.
TypeScript cross-platformelectronffmpegopen-sourceproduct-demorustscreen-capturescreen-recorderscreen-recordingscreen-studioscreencastspeech-to-texttypescriptvideo-editingvideo-editorwaylandwgpuwhisper
feat(mcp): forward openscreen mcp document calls to the running app
EtienneLescot65444df3e807
diff --git a/electron/ai-edition/document-service.ts b/electron/ai-edition/document-service.tsindex 9c14c8c87..8fea6d027 100644--- a/electron/ai-edition/document-service.ts+++ b/electron/ai-edition/document-service.ts@@ -71,11 +71,15 @@ export class DocumentNotFoundError extends Error {}}-/** A write refused because the project is open in an editor, which would save over it. */+/**+ * A file write refused because an editor holds the project and would save over it. Edits+ * to it go through that editor, `openscreen mcp`'s through the app (#1174), so this is+ * only met when the editor or the app does not answer, or the app predates forwarding.+ */export class ProjectOpenInEditorError extends Error {constructor(public readonly projectId: string) {super(- "The edit was NOT applied: this project is open in the OpenScreen editor, which would overwrite it with its own copy on its next save. Ask the user to close the project in OpenScreen and retry, or to turn on the MCP server in OpenScreen's AI settings and connect to that one, which edits the open project through the editor.",+ "The edit was NOT applied: this project is open in the OpenScreen editor, and the editor could not be reached to apply it there (a file edit would be overwritten by its next save). Retry in a moment. If it is refused again, ask the user to close the project in OpenScreen, or to restart or update OpenScreen.",);this.name = "ProjectOpenInEditorError";}diff --git a/electron/cli/cliMain.ts b/electron/cli/cliMain.tsindex 2a02cb285..353787fe5 100644--- a/electron/cli/cliMain.ts+++ b/electron/cli/cliMain.ts@@ -410,6 +410,7 @@ export function runCli(command: CliCommand): void {if (command.kind === "mcp") {await serveMcpOverStdio({...agentServices,+ userDataDir,runners,openVideo: openVideoInHiddenWindow,onClose: () => app.exit(0),diff --git a/electron/ipc/handlers.ts b/electron/ipc/handlers.tsindex 328b4f69b..3f6a949d9 100644--- a/electron/ipc/handlers.ts+++ b/electron/ipc/handlers.ts@@ -80,7 +80,7 @@ import {import { EditorDocumentHost } from "../mcp/editor-document-host";import { McpController } from "../mcp/mcp-controller";import { McpSettingsStore } from "../mcp/mcp-settings-store";-import type { McpToolDeps } from "../mcp/openscreen-mcp-server";+import { createToolRunner, type McpToolDeps } from "../mcp/openscreen-mcp-server";import { type AudioPeaksResult, getAudioPeaks } from "../media/audioPeaks";import {readCursorRecordingFile as readCursorRecordingFileFrom,@@ -5040,7 +5040,10 @@ export function registerIpcHandlers(cursor: agentCursorTelemetryReader,version: app.getVersion(),};- const mcpController = new McpController(mcpSettings, mcpToolDeps);+ // One runner, so one queue and one checkpoint store, for the in-app server and every+ // `openscreen mcp` the app answers through its channel (app-channel.ts, started by main.ts).+ const mcpRunTool = createToolRunner(mcpToolDeps);+ const mcpController = new McpController(mcpSettings, mcpToolDeps, mcpRunTool);// The editor says which project it holds, so `openscreen mcp`, a process of its own// that cannot ask it, refuses to edit that project behind its back (#1138).@@ -5123,6 +5126,7 @@ export function registerIpcHandlers(// `openscreen mcp` serves the same tools on stdio, from the CLI process.return {mcpController,+ mcpAppChannel: { deps: mcpToolDeps, runTool: mcpRunTool },agentServices: {mcpToolDeps,documents: aiEditionDocuments,diff --git a/electron/main.ts b/electron/main.tsindex 7dac8e528..953deabad 100644--- a/electron/main.ts+++ b/electron/main.ts@@ -60,6 +60,7 @@ import {} from "./ipc/handlers";import { isOnlyLingeringOverlay } from "./lingeringOverlay";import { installMainProcessErrorGuards } from "./main-process-errors";+import { serveAppChannel } from "./mcp/app-channel";import { showMessageBoxOver } from "./messageBox";import {registerPermissionsIpc,@@ -1388,7 +1389,7 @@ appReady?.then(async () => {showMainWindow();}- const { mcpController } = registerIpcHandlers(+ const { mcpController, mcpAppChannel } = registerIpcHandlers(createEditorWindowWrapper,createSourceSelectorWindowWrapper,createCountdownOverlayWindowWrapper,@@ -1436,6 +1437,12 @@ appReady?.then(async () => {// Off unless the user turned it on in Settings → AI. Started here rather than// in registerIpcHandlers so neither the headless CLI nor a bench run binds it.void mcpController.startIfEnabled();+ // Always on, unlike the in-app server: `openscreen mcp` forwards its document calls here+ // so the app stays the one writer of its projects (app-channel.ts). Without it, that+ // server works on the files and refuses the project the editor holds.+ serveAppChannel({ userDataDir: app.getPath("userData"), ...mcpAppChannel })+ .then((channel) => app.once("will-quit", () => channel.close()))+ .catch((error) => console.warn("[mcp] could not open the channel for openscreen mcp:", error));void showPermissionsWindowIfNeeded().catch((error) =>console.warn("[permissions] could not read the permissions at launch:", error),);diff --git a/electron/mcp/app-channel.test.ts b/electron/mcp/app-channel.test.tsnew file mode 100644index 000000000..f28d6d218--- /dev/null+++ b/electron/mcp/app-channel.test.ts@@ -0,0 +1,406 @@+// `openscreen mcp` forwarding to the running app (#1174), over the real channel: a+// named pipe on Windows, a Unix socket elsewhere. The "app" is `serveAppChannel` with+// an in-memory editor and a real DocumentService; the "stdio process" is `routeToApp`+// with its own local runner, as stdio-server.ts wires them.++import { spawnSync } from "node:child_process";+import { mkdtempSync, rmSync, statSync, writeFileSync } from "node:fs";+import net from "node:net";+import os from "node:os";+import path from "node:path";+import { Client } from "@modelcontextprotocol/sdk/client/index.js";+import { StreamableHTTPClientTransport } from "@modelcontextprotocol/sdk/client/streamableHttp.js";+import type { CallToolResult } from "@modelcontextprotocol/sdk/types.js";+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";+import {+ type AxcutDocument,+ createEmptyDocument,+ documentSchema,+} from "../../src/lib/ai-edition/schema";+import { DocumentService } from "../ai-edition/document-service";+import { type AppChannel, discoveryFilePath, routeToApp, serveAppChannel } from "./app-channel";+import {+ createToolRunner,+ LIST_PROJECTS_TOOL,+ MCP_ENDPOINT_PATH,+ type McpApplyResult,+ type McpDocumentHost,+ type McpToolDeps,+ type McpToolRunner,+ type RunningMcpServer,+ startMcpHttpServer,+} from "./openscreen-mcp-server";++function fixtureDocument(projectId: string, video = "/tmp/rec.mp4"): AxcutDocument {+ const base = createEmptyDocument({+ title: `Test ${projectId}`,+ projectId,+ createdAt: "2026-01-01T00:00:00.000Z",+ });+ return documentSchema.parse({+ ...base,+ project: { ...base.project, primaryAssetId: "asset_1" },+ assets: [{ id: "asset_1", kind: "video", label: "Rec", originalPath: video, durationSec: 30 }],+ timeline: {+ ...base.timeline,+ clips: [+ {+ id: "clip_1",+ assetId: "asset_1",+ sourceStartSec: 0,+ sourceEndSec: 30,+ timelineStartSec: 0,+ timelineEndSec: 30,+ wordRefs: [],+ origin: "user",+ reason: "",+ },+ ],+ },+ });+}++/** The app's editor window: a document and the revision guarding it. */+class FakeEditor implements McpDocumentHost {+ document: AxcutDocument | null = fixtureDocument("proj_open");+ revision = 1;+ applied = 0;+ /** Runs between the snapshot and the apply, where a user edit would land. */+ onSnapshot?: () => void;+ /** Holds the apply until resolved: an app that stops answering mid-call. */+ hold?: Promise<void>;+ applying = false;++ async snapshot() {+ if (!this.document) return null;+ const snapshot = { document: structuredClone(this.document), revision: this.revision };+ this.onSnapshot?.();+ return snapshot;+ }++ async apply(document: AxcutDocument, expectedRevision: number): Promise<McpApplyResult> {+ this.applying = true;+ await this.hold;+ if (expectedRevision !== this.revision) return "conflict";+ this.document = document;+ this.revision += 1;+ this.applied += 1;+ return "applied";+ }+}++const trimArgs = (projectId?: string) => ({+ assetId: "asset_1",+ startSec: 5,+ endSec: 6,+ ...(projectId ? { projectId } : {}),+});++function text(result: CallToolResult): string {+ return result.content.map((part) => (part.type === "text" ? part.text : "")).join("");+}++let dir: string;+let userData: string;+/** The app's projects, and the CLI's own view of the same folder. */+let appProjects: DocumentService;+let cliProjects: DocumentService;+let editor: FakeEditor;+let appDeps: McpToolDeps;+let appRunTool: McpToolRunner;+let channel: AppChannel | null;+let http: RunningMcpServer | null;+let localCalls: string[];++function cliRouter() {+ const local = createToolRunner({+ transport: "stdio",+ host: { snapshot: async () => null, apply: async () => "no-editor" },+ projects: cliProjects,+ editsAllowed: () => true,+ version: "0.0.0",+ });+ return routeToApp({+ userDataDir: userData,+ version: "0.0.0",+ local: (name, args, call) => {+ localCalls.push(name);+ return local(name, args, call);+ },+ });+}++async function startApp() {+ channel = await serveAppChannel({ userDataDir: userData, deps: appDeps, runTool: appRunTool });+ appProjects.markOpenInEditor("proj_open");+}++beforeEach(async () => {+ dir = mkdtempSync(path.join(os.tmpdir(), "openscreen-app-channel-"));+ userData = path.join(dir, "userData");+ const projectsDir = path.join(userData, "projects");+ appProjects = new DocumentService(projectsDir, dir);+ cliProjects = new DocumentService(projectsDir, dir);+ await appProjects.saveProject(fixtureDocument("proj_open"));+ await appProjects.saveProject(fixtureDocument("proj_closed", path.join(dir, "take.mp4")));+ editor = new FakeEditor();+ // The in-app server's own switch is off: the channel must not depend on it.+ appDeps = {+ transport: "http",+ host: editor,+ projects: appProjects,+ editsAllowed: () => false,+ version: "0.0.0",+ };+ appRunTool = createToolRunner(appDeps);+ channel = null;+ http = null;+ localCalls = [];+});++afterEach(async () => {+ channel?.close();+ await http?.close();+ appProjects.markOpenInEditor(null);+ rmSync(dir, { recursive: true, force: true });+});++describe("discovery", () => {+ it("runs locally when no app announced itself", async () => {+ const route = cliRouter();+ const result = await route(LIST_PROJECTS_TOOL, {});+ expect(result.isError).toBeFalsy();+ expect(localCalls).toEqual([LIST_PROJECTS_TOOL]);+ });++ it("ignores a crashed app's file: a dead pid, or a live pid with no endpoint", async () => {+ const dead = spawnSync(process.execPath, ["-e", ""]).pid;+ const file = discoveryFilePath(userData);+ const route = cliRouter();+ for (const pid of [dead, process.pid]) {+ const endpoint =+ process.platform === "win32"+ ? "\\\\.\\pipe\\openscreen-mcp-gone"+ : path.join(userData, "gone.sock");+ writeFileSync(file, JSON.stringify({ pid, endpoint, token: "x" }));+ expect((await route(LIST_PROJECTS_TOOL, {})).isError).toBeFalsy();+ }+ expect(localCalls).toHaveLength(2);+ });++ it("gives up on an endpoint that accepts and never answers", async () => {+ const endpoint =+ process.platform === "win32"+ ? `\\\\.\\pipe\\openscreen-mcp-mute-${process.pid}`+ : path.join(userData, "mute.sock");+ const mute = net.createServer(() => undefined);+ await new Promise<void>((resolve) => mute.listen(endpoint, resolve));+ try {+ writeFileSync(+ discoveryFilePath(userData),+ JSON.stringify({ pid: process.pid, endpoint, token: "x" }),+ );+ const started = Date.now();+ expect((await cliRouter()(LIST_PROJECTS_TOOL, {})).isError).toBeFalsy();+ expect(localCalls).toEqual([LIST_PROJECTS_TOOL]);+ expect(Date.now() - started).toBeLessThan(10_000);+ } finally {+ mute.close();+ }+ }, 15_000);++ it("finds a running app and removes its announcement when it closes", async () => {+ await startApp();+ const result = await cliRouter()(LIST_PROJECTS_TOOL, {});+ const listed = JSON.parse(text(result)).projects as Array<{ id: string; open: boolean }>;+ expect(listed.find((p) => p.id === "proj_open")?.open).toBe(true);+ expect(localCalls).toEqual([]);+ channel?.close();+ channel = null;+ expect(() => statSync(discoveryFilePath(userData))).toThrow();+ });+});++describe("forwarding to the app", () => {+ it("edits the project open in the editor through the editor, once", async () => {+ await startApp();+ const route = cliRouter();+ const result = await route("addTrim", trimArgs("proj_open"));+ expect(result.isError).toBeFalsy();+ expect(editor.applied).toBe(1);+ expect(editor.document?.timeline.trimRanges).toHaveLength(1);+ // Without a projectId too: the open project is the app's, not this process's.+ expect(+ (await route("addTrim", { ...trimArgs(), startSec: 10, endSec: 11 })).isError,+ ).toBeFalsy();+ expect(editor.applied).toBe(2);+ expect(localCalls).toEqual([]);+ });++ it("keeps the editor's revision guard", async () => {+ await startApp();+ editor.onSnapshot = () => {+ editor.revision += 1;+ };+ const result = await cliRouter()("addTrim", trimArgs("proj_open"));+ expect(result.isError).toBe(true);+ expect(text(result)).toContain("NOT applied");+ expect(editor.applied).toBe(0);+ });++ it("serves several stdio clients at once, one call at a time", async () => {+ await startApp();+ const results = await Promise.all(+ [0, 1, 2].map((i) =>+ cliRouter()("addTrim", { ...trimArgs("proj_open"), startSec: i * 3, endSec: i * 3 + 1 }),+ ),+ );+ expect(results.every((result) => !result.isError)).toBe(true);+ expect(editor.applied).toBe(3);+ expect(editor.document?.timeline.trimRanges).toHaveLength(3);+ });++ it("has the app compute and write setPointerTrack, under its checkpoints", async () => {+ await startApp();+ const route = cliRouter();+ const video = path.join(dir, "take.mp4");+ const created = await route("createCheckpoint", { projectId: "proj_closed" });+ const { checkpointId } = JSON.parse(text(created));+ const result = await route(+ "setPointerTrack",+ {+ projectId: "proj_closed",+ events: [{ at: 6_000, x: 0.5, y: 0.5, click: true }],+ takes: { [video]: 1_000 },+ },+ // Forwarded: this closure never runs, the app computes the edit from the args.+ { edit: () => Promise.reject(new Error("ran locally")) },+ );+ expect(result.isError).toBeFalsy();+ expect(JSON.parse(text(result)).samples).toBeGreaterThan(0);+ expect(statSync(`${video}.cursor.json`).isFile()).toBe(true);+ expect(localCalls).toEqual([]);++ await route("restoreCheckpoint", { projectId: "proj_closed", checkpointId });+ expect(() => statSync(`${video}.cursor.json`)).toThrow();+ });++ it("runs a tool the app does not serve locally", async () => {+ await startApp();+ await cliRouter()("notAnAppTool", {});+ expect(localCalls).toEqual(["notAnAppTool"]);+ });+});++describe("checkpoints", () => {+ it("are one store for the in-app server and forwarded sessions", async () => {+ await startApp();+ http = await startMcpHttpServer({ port: 0, token: "t", deps: appDeps, runTool: appRunTool });+ const client = new Client({ name: "test", version: "0.0.0" });+ await client.connect(+ new StreamableHTTPClientTransport(+ new URL(`http://127.0.0.1:${http.port}${MCP_ENDPOINT_PATH}`),+ { requestInit: { headers: { Authorization: "Bearer t" } } },+ ),+ );+ try {+ const created = (await client.callTool({+ name: "createCheckpoint",+ arguments: { projectId: "proj_closed" },+ })) as CallToolResult;+ const { checkpointId } = JSON.parse(text(created));++ // The closed project needs a checkpoint before an edit: the HTTP one counts.+ const route = cliRouter();+ expect((await route("addTrim", trimArgs("proj_closed"))).isError).toBeFalsy();+ const edited = await appProjects.getProject("proj_closed");+ expect(edited.timeline.trimRanges).toHaveLength(1);++ const restored = await route("restoreCheckpoint", { projectId: "proj_closed", checkpointId });+ expect(restored.isError).toBeFalsy();+ expect((await appProjects.getProject("proj_closed")).timeline.trimRanges).toHaveLength(0);+ expect(localCalls).toEqual([]);+ } finally {+ await client.close();+ }+ });+});++describe("the app going away", () => {+ it("falls back to the files, refusing the project a live app's editor holds", async () => {+ await startApp();+ const route = cliRouter();+ expect((await route("addTrim", trimArgs("proj_open"))).isError).toBeFalsy();+ channel?.close();+ channel = null;+ // The marker stays, as when the app is alive but not answering.+ appProjects.markOpenInEditor("proj_open");+ await vi.waitFor(async () => {+ const result = await route("addTrim", { ...trimArgs("proj_open"), startSec: 20, endSec: 21 });+ expect(localCalls).toContain("addTrim");+ expect(result.isError).toBe(true);+ expect(text(result)).toContain("open in the OpenScreen editor");+ });+ expect(editor.applied).toBe(1);+ });++ it("never runs again here an edit the app received but did not answer", async () => {+ await startApp();+ let release: () => void = () => undefined;+ editor.hold = new Promise((resolve) => {+ release = resolve;+ });+ const route = cliRouter();+ const pending = route("addTrim", trimArgs("proj_open"));+ await vi.waitFor(() => expect(editor.applying).toBe(true));+ channel?.close();+ channel = null;+ const result = await pending;+ expect(result.isError).toBe(true);+ expect(text(result)).toContain("whether it took effect is unknown");+ expect(localCalls).toEqual([]);+ release();+ });++ it("reruns a read locally when its answer is lost", async () => {+ await startApp();+ const route = cliRouter();+ await route(LIST_PROJECTS_TOOL, {});+ // A read that reaches the editor's snapshot, then loses the app.+ const snapshot = editor.snapshot.bind(editor);+ editor.snapshot = async () => {+ channel?.close();+ channel = null;+ await new Promise(() => undefined);+ return snapshot();+ };+ const result = await route("getCurrentDocument", { projectId: "proj_closed" });+ expect(result.isError).toBeFalsy();+ expect(localCalls).toEqual(["getCurrentDocument"]);+ });+});++describe("access", () => {+ it("hangs up on a client without the token", async () => {+ await startApp();+ const socket = net.createConnection(channel?.endpoint ?? "");+ socket.on("error", () => undefined);+ socket.write("wrong-token\n");+ socket.write(+ `${JSON.stringify({ jsonrpc: "2.0", id: 1, method: "tools/list", params: {} })}\n`,+ );+ const received: Buffer[] = [];+ socket.on("data", (chunk) => received.push(chunk));+ await new Promise((resolve) => socket.once("close", resolve));+ expect(Buffer.concat(received).length).toBe(0);+ });++ it.skipIf(process.platform === "win32")(+ "keeps the socket and the token file private",+ async () => {+ await startApp();+ expect(statSync(discoveryFilePath(userData)).mode & 0o077).toBe(0);+ expect(statSync(channel?.endpoint ?? "").mode & 0o077).toBe(0);+ },+ );+});diff --git a/electron/mcp/app-channel.ts b/electron/mcp/app-channel.tsnew file mode 100644index 000000000..8f612466e--- /dev/null+++ b/electron/mcp/app-channel.ts@@ -0,0 +1,304 @@+// How `openscreen mcp` reaches the running app, so the app stays the one writer of+// its projects (#1174). The project open in the editor is then edited through the+// editor, live and one Ctrl/Cmd+Z step per call, and both transports share the+// app's one tool runner: its queue and its checkpoints.+//+// The channel is MCP itself. The app serves the same `createOpenScreenMcpServer` on+// a named pipe (Windows) or a Unix socket, and `openscreen mcp` is its client: each+// document tool call is forwarded as a `tools/call` and its result handed back+// untouched. No second protocol to keep in step with the tools, and the handshake+// says which tools this app has, so a tool an older app lacks runs locally instead.+//+// Discovery: the app writes `<userData>/mcp-app.json` ({ pid, endpoint, token }) once+// it listens, and removes it as it quits. `openscreen mcp` shares that userData, reads+// the file on every call it is not connected for, and ignores it when the pid is gone+// or the endpoint does not answer: a crashed app's file is just stale.+//+// Trust: the endpoint answers only a client that first sends the token from that+// file. Node cannot set a named pipe's DACL, and the default one lets every user open+// it for reading (not writing); the token is what limits the channel to the current+// user, as the file sits in their profile (0600 on Unix, the socket too). Admins and+// SYSTEM can read it, but they can do anything the user can already. Any process of+// this user may connect, which grants nothing new: it could run `openscreen mcp`.+// So calls over this channel edit as `openscreen mcp` does, whatever the in-app+// server's switches say: those guard the HTTP port, which is not ours to bypass.++import { randomBytes, randomUUID, timingSafeEqual } from "node:crypto";+import { chmodSync, renameSync, rmSync, writeFileSync } from "node:fs";+import fs from "node:fs/promises";+import net from "node:net";+import path from "node:path";+import { PassThrough } from "node:stream";+import { Client } from "@modelcontextprotocol/sdk/client/index.js";+import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js";+import type { CallToolResult } from "@modelcontextprotocol/sdk/types.js";+import { isMutatingTool } from "../ai-edition/agent-tools";+import { isProcessAlive } from "../ai-edition/document-service";+import {+ FORWARDED_POINTER_TRACK_SCHEMA,+ POINTER_TRACK_TOOL,+ pointerTrackEdit,+} from "./media-tools";+import {+ createOpenScreenMcpServer,+ LIST_PROJECTS_TOOL,+ type McpToolDeps,+ type McpToolRunner,+} from "./openscreen-mcp-server";++const DISCOVERY_FILE = "mcp-app.json";+/** Connect, token and MCP handshake: the app answers in milliseconds when it is alive. */+const CONNECT_TIMEOUT_MS = 3_000;+// Above the editor's own 30 s for a snapshot plus 30 s for an apply.+const CALL_TIMEOUT_MS = 90_000;+const MAX_TOKEN_LINE_BYTES = 256;++const LOST_MESSAGE =+ "OpenScreen stopped answering during this call, so whether it took effect is unknown. Call getCurrentDocument with the same projectId to check before retrying.";++interface Discovery {+ pid: number;+ endpoint: string;+ token: string;+}++export function discoveryFilePath(userDataDir: string): string {+ return path.join(userDataDir, DISCOVERY_FILE);+}++function newEndpoint(userDataDir: string): string {+ // A fresh pipe name per launch: no other user can create it first to pose as the app.+ if (process.platform === "win32") return `\\\\.\\pipe\\openscreen-mcp-${randomUUID()}`;+ // ponytail: a socket path is capped near 104 bytes; a longer userData fails to listen+ // and `openscreen mcp` keeps working on files. Move it to the runtime dir if that bites.+ return path.join(userDataDir, "mcp.sock");+}++function sameSecret(presented: string, token: string): boolean {+ const a = Buffer.from(presented);+ const b = Buffer.from(token);+ return a.length === b.length && timingSafeEqual(a, b);+}++export interface AppChannel {+ endpoint: string;+ /** Synchronous, so it completes as the app quits. */+ close(): void;+}++/**+ * The app's side: serve the document tools to `openscreen mcp` through `runTool`, the+ * runner the in-app server uses too. Started by the GUI only, under its single-instance+ * lock, so no other process serves this userData.+ */+export async function serveAppChannel(options: {+ userDataDir: string;+ deps: McpToolDeps;+ runTool: McpToolRunner;+}): Promise<AppChannel> {+ const token = randomBytes(32).toString("hex");+ const endpoint = newEndpoint(options.userDataDir);+ const deps: McpToolDeps = { ...options.deps, editsAllowed: () => true };+ const runTool = (name: string, args: unknown) =>+ options.runTool(name, args, { editsAllowed: true });+ const sockets = new Set<net.Socket>();++ const server = net.createServer((socket) => {+ sockets.add(socket);+ socket.once("close", () => sockets.delete(socket));+ socket.on("error", () => socket.destroy());+ // A client that never says the token is cut off.+ socket.setTimeout(CONNECT_TIMEOUT_MS, () => socket.destroy());+ let head = Buffer.alloc(0);+ const onData = (chunk: Buffer) => {+ head = Buffer.concat([head, chunk]);+ const end = head.indexOf(0x0a);+ if (end < 0) {+ if (head.length > MAX_TOKEN_LINE_BYTES) socket.destroy();+ return;+ }+ socket.off("data", onData);+ socket.setTimeout(0);+ if (!sameSecret(head.subarray(0, end).toString("utf8"), token)) {+ socket.destroy();+ return;+ }+ // What followed the token on the wire is already MCP.+ const input = new PassThrough();+ input.write(head.subarray(end + 1));+ socket.pipe(input);+ const session = createOpenScreenMcpServer(deps, runTool);+ // Only here until the in-app server offers the media tools: the edit is computed in+ // this process from what `openscreen mcp` knows of its takes, then run like any edit.+ session.registerTool(+ POINTER_TRACK_TOOL,+ { inputSchema: FORWARDED_POINTER_TRACK_SCHEMA },+ ({ projectId, events, takes }) =>+ options.runTool(+ POINTER_TRACK_TOOL,+ { projectId },+ { editsAllowed: true, edit: (document) => pointerTrackEdit(document, events, takes) },+ ),+ );+ socket.once("close", () => void session.close());+ // A JSON-RPC-lines transport over any stream pair, despite its name.+ session.connect(new StdioServerTransport(input, socket)).catch(() => socket.destroy());+ };+ socket.on("data", onData);+ });++ if (process.platform !== "win32") rmSync(endpoint, { force: true }); // a crashed app's+ await new Promise<void>((resolve, reject) => {+ server.once("error", reject);+ server.listen(endpoint, () => {+ server.off("error", reject);+ resolve();+ });+ });+ if (process.platform !== "win32") chmodSync(endpoint, 0o600);++ const file = discoveryFilePath(options.userDataDir);+ const temp = `${file}.tmp-${process.pid}`;+ writeFileSync(temp, JSON.stringify({ pid: process.pid, endpoint, token } satisfies Discovery), {+ encoding: "utf8",+ mode: 0o600,+ });+ renameSync(temp, file);++ return {+ endpoint,+ close() {+ rmSync(file, { force: true });+ server.close();+ for (const socket of sockets) socket.destroy();+ },+ };+}++async function readDiscovery(userDataDir: string): Promise<Discovery | null> {+ const raw = await fs.readFile(discoveryFilePath(userDataDir), "utf8").catch(() => null);+ if (raw === null) return null;+ try {+ const found = JSON.parse(raw) as Partial<Discovery>;+ if (+ typeof found.pid !== "number" ||+ typeof found.endpoint !== "string" ||+ typeof found.token !== "string"+ ) {+ return null;+ }+ return isProcessAlive(found.pid) ? (found as Discovery) : null;+ } catch {+ return null;+ }+}++interface AppConnection {+ client: Client;+ /** The tools this app serves: an older app may lack one. */+ tools: Set<string>;+ closed: boolean;+ drop(): void;+}++function withTimeout<T>(promise: Promise<T>, ms: number): Promise<T> {+ let timer: ReturnType<typeof setTimeout> | undefined;+ const expired = new Promise<never>((_, reject) => {+ timer = setTimeout(() => reject(new Error("timed out")), ms);+ });+ return Promise.race([promise, expired]).finally(() => clearTimeout(timer));+}++/** The running app, or `null` when there is none or it does not answer. */+export async function connectToApp(+ userDataDir: string,+ version: string,+): Promise<AppConnection | null> {+ const found = await readDiscovery(userDataDir);+ if (!found) return null;+ const socket = net.createConnection(found.endpoint);+ const client = new Client({ name: "openscreen-mcp", version });+ const connection: AppConnection = {+ client,+ tools: new Set(),+ closed: false,+ drop() {+ connection.closed = true;+ socket.destroy();+ },+ };+ socket.on("error", () => socket.destroy());+ // Rejects whatever is in flight at once, rather than at its timeout.+ socket.once("close", () => {+ connection.closed = true;+ void client.close();+ });+ try {+ await withTimeout(+ new Promise<void>((resolve, reject) => {+ socket.once("connect", resolve);+ socket.once("error", reject);+ }),+ CONNECT_TIMEOUT_MS,+ );+ socket.write(`${found.token}\n`);+ await client.connect(new StdioServerTransport(socket, socket), { timeout: CONNECT_TIMEOUT_MS });+ const { tools } = await client.listTools(undefined, { timeout: CONNECT_TIMEOUT_MS });+ for (const tool of tools) connection.tools.add(tool.name);+ return connection;+ } catch {+ connection.drop();+ return null;+ }+}++/** A call that changes nothing, so running it again here after a lost answer is safe. */+function isReadOnly(name: string): boolean {+ if (name === LIST_PROJECTS_TOOL) return true;+ if (name === "createCheckpoint" || name === "restoreCheckpoint") return false;+ return !isMutatingTool(name);+}++/**+ * `openscreen mcp`'s runner: each call goes to the running app when there is one, and+ * to `local` (this process, on files) when there is none. Routed per call, so the app+ * starting or quitting mid-session is picked up on the next call. A call the app+ * received but never answered is not run again here unless it changes nothing: it may+ * have landed, and running it twice is worse than saying so.+ */+export function routeToApp(options: {+ userDataDir: string;+ version: string;+ local: McpToolRunner;+}): McpToolRunner {+ let current: AppConnection | null = null;+ let connecting: Promise<AppConnection | null> | null = null;++ async function app(): Promise<AppConnection | null> {+ if (current && !current.closed) return current;+ connecting ??= connectToApp(options.userDataDir, options.version).finally(() => {+ connecting = null;+ });+ current = await connecting;+ return current;+ }++ return async (name, args, call) => {+ const connection = await app();+ // A runner edit is a closure: it travels only as a tool the app serves (setPointerTrack's+ // args carry what it needs), and `call` is dropped once forwarded.+ if (!connection?.tools.has(name)) return options.local(name, args, call);+ try {+ return (await connection.client.callTool(+ { name, arguments: (args ?? {}) as Record<string, unknown> },+ undefined,+ { timeout: CALL_TIMEOUT_MS },+ )) as CallToolResult;+ } catch {+ // Closed or timed out: a hung app must not hold every later call too.+ connection.drop();+ if (!call?.edit && isReadOnly(name)) return options.local(name, args, call);+ return { content: [{ type: "text", text: LOST_MESSAGE }], isError: true };+ }+ };+}diff --git a/electron/mcp/mcp-controller.ts b/electron/mcp/mcp-controller.tsindex 49eb1e538..dad7ac156 100644--- a/electron/mcp/mcp-controller.ts+++ b/electron/mcp/mcp-controller.ts@@ -9,6 +9,7 @@ import type { McpSettingsStore } from "./mcp-settings-store";import {MCP_ENDPOINT_PATH,type McpToolDeps,+ type McpToolRunner,type RunningMcpServer,startMcpHttpServer,} from "./openscreen-mcp-server";@@ -22,6 +23,8 @@ export class McpController {constructor(private readonly store: McpSettingsStore,private readonly deps: McpToolDeps,+ /** The app's shared runner, so checkpoints outlive a restart and `openscreen mcp` sees them. */+ private readonly runTool?: McpToolRunner,) {}private serially<T>(task: () => Promise<T>): Promise<T> {@@ -44,7 +47,12 @@ export class McpController {if (!enabled) return;try {const token = await this.store.getToken();- this.running = await startMcpHttpServer({ port, token, deps: this.deps });+ this.running = await startMcpHttpServer({+ port,+ token,+ deps: this.deps,+ runTool: this.runTool,+ });} catch (error) {const code = (error as NodeJS.ErrnoException | null)?.code;this.error =diff --git a/electron/mcp/media-tools.test.ts b/electron/mcp/media-tools.test.tsindex ef69bfd07..425348959 100644--- a/electron/mcp/media-tools.test.ts+++ b/electron/mcp/media-tools.test.ts@@ -181,7 +181,7 @@ describe("what `openscreen mcp` tells its client", () => {const instructions = client.getInstructions() ?? "";expect(instructions.startsWith(MEDIA_PREAMBLE)).toBe(true);expect(instructions).toContain("Before a series of edits, call createCheckpoint");- expect(instructions).toContain("lost when it exits");+ expect(instructions).toContain("in OpenScreen's memory while the app runs");expect(instructions).not.toMatch(/project open in the editor|Project edits|Agent edits|Nothing here records|undo step|exporting\)/,);@@ -193,15 +193,16 @@ describe("what `openscreen mcp` tells its client", () => {const projectId = byName.get("addTrim")?.inputSchema.properties?.projectId as {description?: string;};- expect(projectId.description).toContain("Pass it on every call");+ expect(projectId.description).toContain("only while the app runs");const restore = byName.get("restoreCheckpoint")?.description ?? "";expect(restore).toContain("pointer track");- expect(restore).not.toContain("undo step");+ // Through the app's editor while it runs (app-channel.ts).+ expect(restore).toContain("one undo step");});- it("answers in its own terms: no editor to fall back on, no Ctrl/Cmd+Z, its own lifetime", async () => {+ it("answers in its own terms when no app answers: no editor, no Ctrl/Cmd+Z, its own lifetime", async () => {const noProject = await call("getCurrentDocument");- expect(textOf(noProject)).toContain("this server has no editor");+ expect(textOf(noProject)).toContain("OpenScreen is not running");expect(textOf(noProject)).not.toContain("No project is open");const other = await documents.createProject("Other");@@ -215,7 +216,7 @@ describe("what `openscreen mcp` tells its client", () => {projectId: other.project.id,checkpointId: "cp_nope",});- expect(textOf(unknown)).toContain("lost when it exits");+ expect(textOf(unknown)).toContain("this server when the client disconnects");});});diff --git a/electron/mcp/media-tools.ts b/electron/mcp/media-tools.tsindex 1e332e902..bdbb13e0a 100644--- a/electron/mcp/media-tools.ts+++ b/electron/mcp/media-tools.ts@@ -142,6 +142,68 @@ export const MEDIA_PREAMBLE = ["These projects are in the OpenScreen app's project list, so the user can open one in its editor.",].join("\n");+export const POINTER_TRACK_TOOL = "setPointerTrack";++const POINTER_EVENTS = z+ .array(+ z.object({+ at: z.number().describe("Epoch milliseconds, from Date.now()."),+ x: z.number().min(0).max(1),+ y: z.number().min(0).max(1),+ click: z.boolean().optional().describe("The action clicked here."),+ }),+ )+ .min(1);++/** setPointerTrack as `openscreen mcp` forwards it to the app: with its takes' start times. */+export const FORWARDED_POINTER_TRACK_SCHEMA = z.object({+ projectId: z.string(),+ events: POINTER_EVENTS,+ /** When each take of the session started, by its video path. */+ takes: z.record(z.string(), z.number()),+});++/**+ * setPointerTrack's change: the new sidecar, and the zooms planned on its clicks. Computed+ * by the runner's process (the app's, when it runs), and written by that runner.+ */+export async function pointerTrackEdit(+ document: AxcutDocument,+ events: PointerEvent[],+ takes: Readonly<Record<string, number>>,+): ReturnType<RunnerEdit> {+ const asset = document.assets.find((a) => a.id === document.project.primaryAssetId);+ const startedAt = asset ? takes[asset.originalPath] : undefined;+ if (!asset || startedAt === undefined || !asset.durationSec) {+ return {+ ok: false,+ resultJson:+ "This project's take was not recorded by this server, so its start time is unknown. setPointerTrack only works on a take from stopRecording.",+ };+ }+ const samples = pointerSamples(events, startedAt, asset.durationSec * 1000);+ const sidecar: CursorRecordingData = { version: 2, provider: "native", samples, assets: [] };+ // Planned from the new track before it is on disk, as readCursor would read it back.+ const suggestions = await collectAutoZoomSuggestionsForDocument(+ document,+ async () =>+ samples.map(({ timeMs, cx, cy, visible, interactionType }) => ({+ timeMs,+ cx,+ cy,+ visible,+ interactionType,+ })),+ (candidate) => candidate.id === asset.id,+ );+ return {+ ok: true,+ resultJson: JSON.stringify({ samples: samples.length, autoZoomsAdded: suggestions.length }),+ document: appendAutoZoomSuggestions(document, suggestions),+ sidecars: [{ path: `${asset.originalPath}.cursor.json`, content: JSON.stringify(sidecar) }],+ };+}+export function registerMediaTools(server: McpServer, deps: MediaToolDeps): void {const now = deps.now ?? Date.now;/** The take being recorded, then saved: held until it is a project. */@@ -380,67 +442,23 @@ export function registerMediaTools(server: McpServer, deps: MediaToolDeps): void"For a take driven by a script (Playwright, CDP, a browser extension) rather than the real mouse: the screen pointer never moved, so the video has no cursor and no zoom. Give the pointer's path here and OpenScreen draws it, gliding between your actions, and zooms on its clicks as for a real mouse. It replaces the pointer recorded with the take, then plans zooms on the new clicks as stopRecording does, clear of the zooms already there; autoZoomsAdded says how many. Only for a take recorded by this server. Like any edit it needs a checkpoint of the project (stopRecording made one), and restoreCheckpoint undoes it, the pointer track included.","events, one per action: `at` is Date.now() when your script did it (log it as it runs; the take's recordingStartedAt is on the same clock). `x` and `y` are fractions (0 to 1) of the recorded window. For a point (clientX, clientY) of a page shown in that window, from inside the page: border = (outerWidth - innerWidth) / 2; x = (border + clientX) / outerWidth; y = (outerHeight - innerHeight - border + clientY) / outerHeight.",].join("\n"),- inputSchema: z.object({- projectId: z.string(),- events: z- .array(- z.object({- at: z.number().describe("Epoch milliseconds, from Date.now()."),- x: z.number().min(0).max(1),- y: z.number().min(0).max(1),- click: z.boolean().optional().describe("The action clicked here."),- }),- )- .min(1),- }),+ inputSchema: z.object({ projectId: z.string(), events: POINTER_EVENTS }),annotations: { readOnlyHint: false, destructiveHint: true, openWorldHint: false },},// Through the document tools' runner, like any edit: the same queue, the same refusals// (a project open in the app's editor, one with no checkpoint) before anything is// written, and the sidecar kept by the checkpoints so restoreCheckpoint puts it back.- (args) =>- deps.runTool("setPointerTrack", { projectId: args.projectId }, (document) =>- pointerTrackEdit(document, args.events),- ),+ // The args carry what the edit needs, so a running app can compute it (app-channel.ts).+ (args) => {+ const takes = Object.fromEntries(startedAtByVideo);+ return deps.runTool(+ POINTER_TRACK_TOOL,+ { projectId: args.projectId, events: args.events, takes },+ { edit: (document) => pointerTrackEdit(document, args.events, takes) },+ );+ },);- /** The new sidecar, and the zooms planned on its clicks: computed here, written by the runner. */- async function pointerTrackEdit(- document: AxcutDocument,- events: PointerEvent[],- ): ReturnType<RunnerEdit> {- const asset = document.assets.find((a) => a.id === document.project.primaryAssetId);- const startedAt = asset ? startedAtByVideo.get(asset.originalPath) : undefined;- if (!asset || startedAt === undefined || !asset.durationSec) {- return {- ok: false,- resultJson:- "This project's take was not recorded by this server, so its start time is unknown. setPointerTrack only works on a take from stopRecording.",- };- }- const samples = pointerSamples(events, startedAt, asset.durationSec * 1000);- const sidecar: CursorRecordingData = { version: 2, provider: "native", samples, assets: [] };- // Planned from the new track before it is on disk, as readCursor would read it back.- const suggestions = await collectAutoZoomSuggestionsForDocument(- document,- async () =>- samples.map(({ timeMs, cx, cy, visible, interactionType }) => ({- timeMs,- cx,- cy,- visible,- interactionType,- })),- (candidate) => candidate.id === asset.id,- );- return {- ok: true,- resultJson: JSON.stringify({ samples: samples.length, autoZoomsAdded: suggestions.length }),- document: appendAutoZoomSuggestions(document, suggestions),- sidecars: [{ path: `${asset.originalPath}.cursor.json`, content: JSON.stringify(sidecar) }],- };- }-server.registerTool("exportVideo",{diff --git a/electron/mcp/openscreen-mcp-server.test.ts b/electron/mcp/openscreen-mcp-server.test.tsindex 942b8eac4..31aab5a56 100644--- a/electron/mcp/openscreen-mcp-server.test.ts+++ b/electron/mcp/openscreen-mcp-server.test.ts@@ -222,7 +222,7 @@ describe("the MCP tool surface", () => {"Without a projectId they act on the project open in the editor",);expect(instructions).toContain("Nothing here records, exports or imports media.");- expect(instructions).toContain("OpenScreen quits or its MCP server restarts");+ expect(instructions).toContain("all of them are lost when OpenScreen quits.");expect(instructions).toContain("turn on 'MCP edits' under MCP server in AI settings");expect(instructions).not.toContain("Agent edits");const { tools } = await mcp.listTools();@@ -242,11 +242,12 @@ describe("the instructions, composed per transport", () => {expect(instructions).toContain("Before a series of edits, call createCheckpoint");expect(instructions).toContain("Time-bases (do not mix them up)");}- // Only the in-app server has an editor, its undo stack, and no export.- expect(stdio).not.toMatch(/project open in the editor|undo step|Nothing here records/);+ // stdio reaches the editor only through the app, and exports.+ expect(stdio).not.toMatch(/Without a projectId they act|Nothing here records/);+ expect(stdio).toContain("While the OpenScreen app runs, this server hands each call to it");expect(stdio).not.toContain("exporting)");expect(http).toContain("exporting)");- expect(stdio).toContain("lost when it exits");+ expect(stdio).toContain("in OpenScreen's memory while the app runs");});});@@ -418,7 +419,7 @@ describe("checkpoints", () => {arguments: { checkpointId: "cp_nope" },});expect(result.isError).toBe(true);- expect(resultText(result)).toContain("its port or token changed");+ expect(resultText(result)).toContain("lost when OpenScreen quits");expect(editor.applied).toHaveLength(0);});@@ -690,8 +691,9 @@ describe("projects other than the open one", () => {});});-// `openscreen mcp` beside the app (#1138): this server's own editor holds nothing,-// the app's editor holds proj_2, and only the marker the app leaves says so.+// `openscreen mcp` beside an app that does not answer it (#1138; forwarding to one that+// does is app-channel.test.ts): this server's own editor holds nothing, the app's editor+// holds proj_2, and only the marker the app leaves says so.describe("a project open in another process's editor", () => {const addTrimArgs = { assetId: "asset_1", startSec: 5, endSec: 6, projectId: "proj_2" };@@ -725,7 +727,7 @@ describe("a project open in another process's editor", () => {const result = await mcp.callTool({ name: "addTrim", arguments: addTrimArgs });expect(result.isError).toBe(true);expect(resultText(result)).toContain("open in the OpenScreen editor");- expect(resultText(result)).toContain("MCP server");+ expect(resultText(result)).toContain("close the project in OpenScreen");expect((await projects.getProject("proj_2")).timeline.trimRanges).toHaveLength(0);});diff --git a/electron/mcp/openscreen-mcp-server.ts b/electron/mcp/openscreen-mcp-server.tsindex f9cef6cce..2d27d3ace 100644--- a/electron/mcp/openscreen-mcp-server.ts+++ b/electron/mcp/openscreen-mcp-server.ts@@ -29,7 +29,8 @@// `openscreen mcp` over stdio (`stdio-server.ts`, which adds the media tools). The// runner and the tools are the same; the texts a client reads are composed per// transport (`mcpInstructions`, the `*_MESSAGES` tables), so each says what is true-// on the server that serves it. A media tool that edits a project+// on the server that serves it. While the app runs, `openscreen mcp` forwards its document+// calls to it (`app-channel.ts`), so the app's one runner serves both. A media tool that edits a project// (setPointerTrack) hands its change to the runner too, so it meets the same// refusals and checkpoints, sidecar included.//@@ -136,18 +137,19 @@ export const LIST_PROJECTS_TOOL = "listProjects";const MCP_EDITS_SETTING = "'MCP edits' under MCP server in AI settings";const LIST_PROJECTS_DESCRIPTION =- "List every OpenScreen project: id, title, last update, asset count, and whether it is the one open in the editor. Pass a project's id as `projectId` to any other tool to read or edit that project.";+ "List every OpenScreen project: id, title, last update, asset count, and `open`: whether the OpenScreen editor has it open, in which case an edit to it goes through the editor, shows there live and the user can undo it with Ctrl/Cmd+Z. Pass a project's id as `projectId` to any other tool to read or edit that project.";const PROJECT_ID_DESCRIPTIONS: Record<McpTransport, string> = {http: "The project to act on, from listProjects. Omit it for the project open in the editor.",stdio:- "The project to act on, from listProjects. Pass it on every call: this server has no editor, so there is no open project to fall back on.",+ "The project to act on, from listProjects. Omit it only for the project open in the OpenScreen editor, which works only while the app runs.",};+// stdio's are only met on its own, with no app to forward to (app-channel.ts).const NO_PROJECT_MESSAGES: Record<McpTransport, string> = {http: "No project is open in the OpenScreen editor. Call listProjects and pass a projectId, or ask the user to open one in OpenScreen.",stdio:- "Pass a projectId: this server has no editor, so there is no open project to act on. Call listProjects for the ids.",+ "Pass a projectId: OpenScreen is not running (or not answering), so there is no project open in its editor to act on. Call listProjects for the ids.",};// The editor would overwrite a file edit with its own copy on its next save, so a@@ -158,16 +160,17 @@ const PROJECT_CHANGED_MESSAGE =const CHECKPOINT_REQUIRED_MESSAGES: Record<McpTransport, string> = {http: "The edit was NOT applied: this project is not open in the editor, so the user cannot undo an edit to it with Ctrl/Cmd+Z. Call createCheckpoint with this projectId first, so restoreCheckpoint can take your edits back in one step, then retry.",stdio:- "The edit was NOT applied: this server saves each edit straight to the project's file, where Ctrl/Cmd+Z cannot reach it. Call createCheckpoint with this projectId first, so restoreCheckpoint can take your edits back in one step, then retry.",+ "The edit was NOT applied: OpenScreen is not running (or not answering), so this server saves each edit straight to the project's file, where Ctrl/Cmd+Z cannot reach it. Call createCheckpoint with this projectId first, so restoreCheckpoint can take your edits back in one step, then retry.",};-// ponytail: kept in memory, last 20 only; persist them per project if agents need+// ponytail: kept in memory, last 20 only. In the app one store serves the in-app server+// and every `openscreen mcp` session it answers; persist them per project if agents need// them across sessions.const MAX_CHECKPOINTS = 20;const CHECKPOINT_LIFETIMES: Record<McpTransport, string> = {- http: `Checkpoints live in OpenScreen's memory: only the ${MAX_CHECKPOINTS} most recent are kept, and all of them are lost when OpenScreen quits or its MCP server restarts (turned off and on, or its port or token changed).`,- stdio: `Checkpoints live in this server's memory: only the ${MAX_CHECKPOINTS} most recent are kept, and all of them are lost when it exits, which it does when the client disconnects.`,+ http: `Checkpoints live in OpenScreen's memory: only the ${MAX_CHECKPOINTS} most recent are kept, and all of them are lost when OpenScreen quits.`,+ stdio: `Checkpoints live in OpenScreen's memory while the app runs, and in this server's when it does not: only the ${MAX_CHECKPOINTS} most recent are kept, they are lost when the one holding them exits (OpenScreen quitting, or this server when the client disconnects), and neither knows the other's.`,};const APPLY_FAILURE_MESSAGES: Record<Exclude<McpApplyResult, "applied">, string> = {@@ -196,8 +199,8 @@ const CORE_PREAMBLES: Record<McpTransport, readonly string[]> = {"Before a series of edits, call createCheckpoint with the same projectId. Undo is one step per call, so if the result is not what the user wanted, restoreCheckpoint takes the project back in one step instead of many. A project that is not open in the editor cannot be edited until it has a checkpoint, since restoreCheckpoint is its only undo.",],stdio: [- "These tools act on the user's OpenScreen projects, read from and saved to their files: call listProjects for their ids, and pass one as projectId on every call. Every edit is saved straight away, and Ctrl/Cmd+Z cannot undo it. An edit to a project the user has open in the OpenScreen app is refused, since its editor would save over it: ask them to close it there.",- "Before a series of edits, call createCheckpoint with the same projectId: restoreCheckpoint then takes the project back in one step, and it is the only way back. A project cannot be edited until it has a checkpoint.",+ "These tools act on the user's OpenScreen projects: call listProjects for their ids, and pass one as projectId. Every edit is saved straight away. While the OpenScreen app runs, this server hands each call to it: an edit to the project open in its editor appears there, where the user can undo it with Ctrl/Cmd+Z, and any other project is saved to its file. When the app is not running, every project is read from and saved to its file, and Ctrl/Cmd+Z cannot reach the edit.",+ "Before a series of edits, call createCheckpoint with the same projectId. Undo is one step per call, so if the result is not what the user wanted, restoreCheckpoint takes the project back in one step instead of many. A project that is not open in the editor cannot be edited until it has a checkpoint, since restoreCheckpoint is its only undo.",],};@@ -228,15 +231,12 @@ export function mcpInstructions(options: {);}-function checkpointTools(transport: McpTransport, media: boolean) {+function checkpointTools(media: boolean) {const restore = ["Put the project back exactly as it was when createCheckpoint returned this checkpointId, discarding every edit made since, including the user's. Pass the same projectId as to createCheckpoint.",...(media ? ["It also puts back the pointer track setPointerTrack replaced."] : []),- ...(transport === "http"- ? [- "On the project open in the editor it lands as one undo step, so the user can undo the restore itself.",- ]- : []),+ // Over stdio too: while the app runs, the restore goes through its editor.+ "On the project open in the editor it lands as one undo step, so the user can undo the restore itself.",].join(" ");return [{@@ -275,7 +275,21 @@ export type RunnerEdit = (document: AxcutDocument,) => Promise<AgentToolExecution & { sidecars?: SidecarWrite[] }>;-export type McpToolRunner = ReturnType<typeof createToolRunner>;+export interface CallOptions {+ /** Computes the call's change in place of the executor. */+ edit?: RunnerEdit;+ editsAllowed: boolean;+}++/**+ * Runs one tool call. `editsAllowed` overrides `deps.editsAllowed()` for this call: the+ * app's channel for `openscreen mcp` edits whatever the in-app server's switch says.+ */+export type McpToolRunner = (+ name: string,+ args: unknown,+ options?: Partial<CallOptions>,+) => Promise<CallToolResult>;async function readSidecar(file: string): Promise<string | null> {try {@@ -305,9 +319,10 @@ function splitProjectId(rawArgs: unknown): { projectId: string | undefined; args/*** Runs tool calls one at a time. Each call is snapshot → execute → apply, and* two of those interleaving would make the second apply against a revision the- * first just moved — a spurious conflict at best.+ * first just moved — a spurious conflict at best. The app builds one and shares+ * it, queue and checkpoints, between the in-app server and `openscreen mcp`.*/-export function createToolRunner(deps: McpToolDeps) {+export function createToolRunner(deps: McpToolDeps): McpToolRunner {let queue: Promise<unknown> = Promise.resolve();/*** Each checkpoint's document, and what each sidecar an edit rewrote since held before@@ -334,8 +349,9 @@ export function createToolRunner(deps: McpToolDeps) {function restoreCheckpoint(document: AxcutDocument,args: unknown,+ editsAllowed: boolean,): AgentToolExecution & { sidecars?: SidecarWrite[] } {- if (!deps.editsAllowed()) {+ if (!editsAllowed) {return {ok: false,resultJson: `Edits by MCP clients are turned off in OpenScreen, so the checkpoint was NOT restored. Ask the user to turn on ${MCP_EDITS_SETTING}, or to undo the edits themselves.`,@@ -417,13 +433,13 @@ export function createToolRunner(deps: McpToolDeps) {snapshot: McpDocumentSnapshot,name: string,args: unknown,- edit: RunnerEdit | undefined,+ call: CallOptions,): Promise<CallToolResult> {const parsed = documentSchema.safeParse(snapshot.document);if (!parsed.success) {return textResult("The project open in the editor could not be read.", true);}- const execution = await execute(parsed.data, name, args, edit);+ const execution = await execute(parsed.data, name, args, call);if (execution.document) {const applied = await deps.host.apply(execution.document, snapshot.revision);if (applied !== "applied") return textResult(APPLY_FAILURE_MESSAGES[applied], true);@@ -439,9 +455,9 @@ export function createToolRunner(deps: McpToolDeps) {read: ProjectRead,name: string,args: unknown,- edit: RunnerEdit | undefined,+ call: CallOptions,): Promise<CallToolResult> {- const execution = await execute(read.document, name, args, edit);+ const execution = await execute(read.document, name, args, call);if (execution.document) {// An editor this server cannot reach (the app, beside `openscreen mcp`) holds// it and would save over this edit. Checked again just before the write.@@ -469,29 +485,29 @@ export function createToolRunner(deps: McpToolDeps) {document: AxcutDocument,name: string,args: unknown,- edit: RunnerEdit | undefined,+ { edit, editsAllowed }: CallOptions,): Promise<AgentToolExecution & { sidecars?: SidecarWrite[] }> {if (name === "createCheckpoint") return createCheckpoint(document);- if (name === "restoreCheckpoint") return restoreCheckpoint(document, args);+ if (name === "restoreCheckpoint") return restoreCheckpoint(document, args, editsAllowed);// This server's own switch, so the refusal names it rather than the in-app agent's.- if ((edit || isMutatingTool(name)) && !deps.editsAllowed()) {+ if ((edit || isMutatingTool(name)) && !editsAllowed) {return consentRequired(name, args, MCP_EDITS_SETTING);}if (edit) return edit(document);const availableByAssetId = await probeCursorTelemetry(document, deps.cursor);- return runDocumentTool(document, name, args, deps.editsAllowed(), {+ return runDocumentTool(document, name, args, editsAllowed, {cursor: deps.cursor,availableByAssetId,});}- async function run(name: string, rawArgs: unknown, edit?: RunnerEdit): Promise<CallToolResult> {+ async function run(name: string, rawArgs: unknown, call: CallOptions): Promise<CallToolResult> {if (name === LIST_PROJECTS_TOOL) return listProjects();const { projectId, args } = splitProjectId(rawArgs);if (projectId === undefined) {const snapshot = await deps.host.snapshot();return snapshot- ? runOnEditor(snapshot, name, args, edit)+ ? runOnEditor(snapshot, name, args, call): textResult(NO_PROJECT_MESSAGES[deps.transport], true);}// The file first, the editor second. Reads of a project queue behind each other,@@ -504,7 +520,7 @@ export function createToolRunner(deps: McpToolDeps) {});const snapshot = await deps.host.snapshot();if (snapshot && openProjectId(snapshot) === projectId) {- return runOnEditor(snapshot, name, args, edit);+ return runOnEditor(snapshot, name, args, call);}if (!read) {return textResult(@@ -512,15 +528,17 @@ export function createToolRunner(deps: McpToolDeps) {true,);}- return runOnFile(projectId, read, name, args, edit);+ return runOnFile(projectId, read, name, args, call);}- /** Runs one call; `edit`, when given, computes the call's change in place of the executor. */- return (name: string, args: unknown, edit?: RunnerEdit): Promise<CallToolResult> => {- const next = queue.then(- () => run(name, args, edit),- () => run(name, args, edit),- );+ return (name, args, options = {}) => {+ // The setting is read when the call's turn comes: it may change while it waits.+ const runNow = () =>+ run(name, args, {+ edit: options.edit,+ editsAllowed: options.editsAllowed ?? deps.editsAllowed(),+ });+ const next = queue.then(runNow, runNow);queue = next.catch(() => undefined);return next.catch((error) => textResult(`Tool failed: ${errorMessage(error)}`, true));};@@ -568,7 +586,7 @@ export function createOpenScreenMcpServer((args: unknown) => runTool(name, args),);}- for (const tool of checkpointTools(deps.transport, media !== undefined)) {+ for (const tool of checkpointTools(media !== undefined)) {server.registerTool(tool.name,{@@ -617,9 +635,13 @@ export async function startMcpHttpServer(options: {port: number;token: string;deps: McpToolDeps;+ /** The app's shared runner (see createToolRunner); a fresh one when omitted. */+ runTool?: McpToolRunner;}): Promise<RunningMcpServer> {const { token, deps } = options;- const runTool = createToolRunner(deps);+ const shared = options.runTool ?? createToolRunner(deps);+ // This server obeys its own switch: never pass an override through.+ const runTool = (name: string, args: unknown) => shared(name, args);let boundPort = options.port;const handle = async (req: IncomingMessage, res: ServerResponse) => {diff --git a/electron/mcp/stdio-server.ts b/electron/mcp/stdio-server.tsindex 0393d5f9b..9818e474c 100644--- a/electron/mcp/stdio-server.ts+++ b/electron/mcp/stdio-server.ts@@ -2,6 +2,11 @@// this process. The document tools are the in-app server's own, run by the same// executor; the media tools around them record, export and look at the result.//+// When the OpenScreen app runs, it is the authority: each document tool call is+// forwarded to it (app-channel.ts), so the project open in its editor is edited+// there, live, and checkpoints live in one place. With no app, the calls run here+// on the project files, where no editor can hold one.+//// No port and no token: the client launched this process itself, with the// user's rights, as it would launch `openscreen record` or `openscreen export`.// For the same reason it may edit, where the in-app server waits for the user to@@ -10,6 +15,7 @@import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js";import type { DocumentService } from "../ai-edition/document-service";import { openStdin } from "../cli/stdin";+import { routeToApp } from "./app-channel";import { MEDIA_PREAMBLE, type MediaToolDeps, registerMediaTools } from "./media-tools";import {createOpenScreenMcpServer,@@ -20,6 +26,8 @@ import {export async function serveMcpOverStdio(options: {mcpToolDeps: McpToolDeps;documents: DocumentService;+ /** Shared with the app: where its channel is announced. */+ userDataDir: string;readCursorTelemetry: MediaToolDeps["readCursor"];runners: MediaToolDeps["runners"];openVideo: MediaToolDeps["openVideo"];@@ -31,7 +39,13 @@ export async function serveMcpOverStdio(options: {transport: "stdio",editsAllowed: () => true,};- const runTool = createToolRunner(deps);+ // The media tools' own document calls (stopRecording's checkpoint, setPointerTrack)+ // take the same route.+ const runTool = routeToApp({+ userDataDir: options.userDataDir,+ version: deps.version,+ local: createToolRunner(deps),+ });const server = createOpenScreenMcpServer(deps, runTool, MEDIA_PREAMBLE);registerMediaTools(server, {runners: options.runners,diff --git a/skills/openscreen-demo/SKILL.md b/skills/openscreen-demo/SKILL.mdindex 94362b8f5..20c9ce43d 100644--- a/skills/openscreen-demo/SKILL.md+++ b/skills/openscreen-demo/SKILL.md@@ -83,7 +83,7 @@ Use the document tools, always with the `projectId`. Read the project first with- **Zoom on the action:** clicks close in time and on screen share one zoom, and the first 2.5 s stay wide. Check them, then add or move zooms with `addZooms` and `setZoom` where a result deserves its own. Hold a zoom at least 1.5 s.- **Callouts:** one `addAnnotation` per beat, short, placed clear of the action. Text sits on the output frame and stays put when the picture zooms. Use its look for hierarchy: a large title with `animation: "typewriter"`, step callouts with `animation: "pop"`. `setAnnotation` moves or restyles one.- **Undo:** `restoreCheckpoint` with the `checkpointId` takes the project back to the raw take in one step. It also puts back the take's own pointer track, undoing any `setPointerTrack` made since.-- **Open in the app:** while the user has this project open in the OpenScreen editor, every edit is refused, since the editor would save its own copy over yours. Reads still work. Ask them to close the project there, then retry.+- **Open in the app:** while OpenScreen is running, your edits go through it, so a project the user has open in its editor changes there live and they can undo each edit. If an edit is refused as "open in the OpenScreen editor", the app did not answer: retry once, then ask the user to close the project there or restart OpenScreen.Keep the rest as it is: the background, padding, shadow and cursor style are chosen to look good.diff --git a/technical-documentation/architecture/mcp-server.md b/technical-documentation/architecture/mcp-server.mdindex 1a97895e0..5ff70a4b6 100644--- a/technical-documentation/architecture/mcp-server.md+++ b/technical-documentation/architecture/mcp-server.md@@ -10,12 +10,12 @@ The same tools are served over two transports, by two different processes:| Setup | **AI settings → MCP server**: turn it on, copy the `claude mcp add` / `codex mcp add` command (it carries the token) | `claude mcp add openscreen -- <executable> mcp` ([executables](../../website/docs/cli.md#running-the-cli)) || Auth | Bearer token, `127.0.0.1` only, Host/Origin check | None: the client launched the process with the user's rights || Edits | Off until the user turns on **MCP edits** | Always allowed |-| Project open in the editor | Edited live through the editor, one undo step per call | Read only: an edit to it is refused ([the marker](#the-open-in-editor-marker)) |+| Project open in the editor | Edited live through the editor, one undo step per call | The same while the app runs: the call is forwarded to it. With no app answering, refused ([the marker](#the-open-in-editor-marker)) || Other projects | Read from and saved to their file, a checkpoint first | Same || Tools | `listProjects`, the agent's tools, the checkpoint tools | The same, plus the media tools: `listSources`, `startRecording`, `stopRecording`, `setPointerTrack`, `exportVideo`, `getFrames` |-| Checkpoints | In memory, lost when the app quits or the server restarts | In memory, lost when the process exits, which it does when the client disconnects |+| Checkpoints | In the app's memory, shared with forwarded `openscreen mcp` calls, lost when the app quits | In the app's while it runs; otherwise in this process's, lost when it exits, which it does when the client disconnects |-Neither server knows about the other. Two processes may therefore write the same project folder; the [open-in-editor marker](#the-open-in-editor-marker) and the byte compare of `saveProjectIfUnchanged` are what keep them from overwriting each other's work.+**When the app runs, it is the authority**: `openscreen mcp` forwards its document calls to it ([below](#openscreen-mcp-and-the-running-app)), so one process writes the projects. Only when no app answers do two processes write the same project folder; the [open-in-editor marker](#the-open-in-editor-marker) and the byte compare of `saveProjectIfUnchanged` are then what keep them from overwriting each other's work.| File | Role ||---|---|@@ -23,6 +23,7 @@ Neither server knows about the other. Two processes may therefore write the same| [`electron/mcp/stdio-server.ts`](../../electron/mcp/stdio-server.ts) | `openscreen mcp`: the same server and runner on stdin/stdout, with the media tools. || [`electron/mcp/media-tools.ts`](../../electron/mcp/media-tools.ts) | The media tools and their capability block (`MEDIA_PREAMBLE`). || [`electron/mcp/editor-document-host.ts`](../../electron/mcp/editor-document-host.ts) | Main-process side of the live document: asks the editor window for a snapshot and hands edits back. |+| [`electron/mcp/app-channel.ts`](../../electron/mcp/app-channel.ts) | The local channel `openscreen mcp` forwards its document calls through to the running app. See [`openscreen mcp` and the running app](#openscreen-mcp-and-the-running-app). || [`electron/mcp/mcp-controller.ts`](../../electron/mcp/mcp-controller.ts) | In-app server lifecycle: start when enabled, restart on a port or token change, status for the settings UI. || [`electron/mcp/mcp-settings-store.ts`](../../electron/mcp/mcp-settings-store.ts) | `mcp-server.json` (enabled, port, allowEdits) and `mcp-token.enc` (bearer token, `safeStorage`). || [`src/lib/ai-edition/store/mcpDocumentHost.ts`](../../src/lib/ai-edition/store/mcpDocumentHost.ts) | Renderer side: answers snapshot / apply requests from the project store. Mounted by `NewEditorShell`. |@@ -43,21 +44,21 @@ So a tool added to the agent appears over MCP with no further work, and the MCPEvery text a client reads must be true on the server that serves it, so the texts are composed rather than shared verbatim. `McpToolDeps.transport` (`"http"` or `"stdio"`) selects them:-- **Instructions** (`mcpInstructions`): the media tools' capability block when they are registered, then the projects-and-checkpoints rules of the transport, then how long checkpoints live there, then `buildSystemPrompt`. The checkpoint rule is in both. Over HTTP the rules speak of the editor and its undo stack and add "Nothing here records, exports or imports media"; over stdio they say a `projectId` is needed on every call, that `Ctrl/Cmd+Z` cannot undo an edit, and that a project open in the app is refused.+- **Instructions** (`mcpInstructions`): the media tools' capability block when they are registered, then the projects-and-checkpoints rules of the transport, then how long checkpoints live there, then `buildSystemPrompt`. The checkpoint rule is in both. Over HTTP the rules speak of the editor and its undo stack and add "Nothing here records, exports or imports media"; over stdio they say that calls go through the app while it runs (the open project live, with `Ctrl/Cmd+Z`), and to the files when it does not.- **`buildSystemPrompt`** takes the transport's facts as options: `editsSetting` names the switch its consent block points to (the MCP server's **MCP edits**, not the in-app agent's **Agent edits**), and `canExport` drops "exporting" from its examples of a request with no tool when `exportVideo` is served.-- **Tool descriptions**: `projectId` is "omit it for the open project" over HTTP and "pass it on every call" over stdio; `restoreCheckpoint` mentions the undo step only over HTTP, and the pointer track only where `setPointerTrack` is served.-- **Messages**: no project to fall back on, a checkpoint required, an unknown checkpoint, edits turned off. Each has a version per transport, or exists only on the one that can produce it.+- **Tool descriptions**: `projectId` is "omit it for the open project" over HTTP and "omit it only for the open project, while the app runs" over stdio; `restoreCheckpoint` mentions the pointer track only where `setPointerTrack` is served.+- **Messages**: no project to fall back on, a checkpoint required, an unknown checkpoint, edits turned off. Each has a version per transport, or exists only on the one that can produce it. The stdio versions are only met with no app answering, and say so.The in-app chat agent keeps `buildSystemPrompt`'s defaults: its own switch, no export.-**Writes on the in-app server are a second opt-in.** MCP clients have their own **MCP edits** switch, `allowEdits` in `mcp-server.json`, **off by default** and independent of the in-app agent's **Agent edits** (`allowAgentEdits`). Turning the server on therefore grants read access only. The value is read on every call; while it is off every mutating tool and `setPointerTrack` are refused with the executor's `consent_required` payload naming **MCP edits**, `restoreCheckpoint` with its own refusal text naming it too, and the consent block of the system prompt is in the instructions. It is independent because `allowAgentEdits` defaults to allowed and lives in the provider form, so a user with no provider configured could never have turned MCP writes off. Toggling it does not restart the server. `openscreen mcp` has no such switch: the client started it itself, and there is no other caller to keep out.+**Writes on the in-app server are a second opt-in.** MCP clients have their own **MCP edits** switch, `allowEdits` in `mcp-server.json`, **off by default** and independent of the in-app agent's **Agent edits** (`allowAgentEdits`). Turning the server on therefore grants read access only. The value is read on every call; while it is off every mutating tool and `setPointerTrack` are refused with the executor's `consent_required` payload naming **MCP edits**, `restoreCheckpoint` with its own refusal text naming it too, and the consent block of the system prompt is in the instructions. It is independent because `allowAgentEdits` defaults to allowed and lives in the provider form, so a user with no provider configured could never have turned MCP writes off. Toggling it does not restart the server. `openscreen mcp` has no such switch: the client started it itself, and there is no other caller to keep out. Its calls forwarded to the app keep that: the runner takes `editsAllowed` per call.## Which project a call acts on| `projectId` | In-app server | `openscreen mcp` ||---|---|---|-| omitted | The project open in the editor, through the editor. With none open the call fails and points at `listProjects`. | Fails: there is no editor to fall back on. The message asks for a `projectId`. |-| the id of the project open in this process's editor | The same: through the editor, never its file. The file is read first all the same (see below). | Cannot happen: this process has no editor. |+| omitted | The project open in the editor, through the editor. With none open the call fails and points at `listProjects`. | While the app runs, forwarded: the same as the in-app server. Otherwise fails, asking for a `projectId`. |+| the id of the project open in the app's editor | The same: through the editor, never its file. The file is read first all the same (see below). | While the app runs, forwarded: the same. Otherwise refused ([the marker](#the-open-in-editor-marker)). || any other id | That project's file, through the process's one `DocumentService` (`getProjectForUpdate` → tool → `saveProjectIfUnchanged`). Works with no editor window at all. A write needs a checkpoint of that project first, and is refused while another process's editor holds it. | Same. |The open project is never written to disk from here because the editor holds it in memory and saves it as a whole: a file edit under it would be overwritten by the editor's next save, and the editor would never show it.@@ -76,7 +77,7 @@ A user edit that lands between 1 and 3 moves the revision and the apply is refusOnly the webContents that registered on `ai-edition.mcp-host` is asked, and only its replies count. An editor that unmounts or is destroyed stops being asked. A request unanswered for 30 s resolves to "no project" (reads) or `timeout` (writes, reported as "did not confirm", not as a failure, since the save may have landed).-`openscreen mcp` builds the same `EditorDocumentHost`, but no editor window ever exists in that process, so its snapshot is always `null`.+`openscreen mcp` builds the same `EditorDocumentHost`, but no editor window ever exists in that process, so its snapshot is always `null`: it reaches the app's editor by forwarding the call to the app.### Any other project@@ -96,9 +97,9 @@ The check runs inside the project's queue, so nothing in the process reads or sa### The open-in-editor marker-`openscreen mcp` cannot ask the app's editor anything, so the app says which project its editor holds: the editor sends the project id on `AI_EDITION_OPEN_PROJECT_CHANNEL`, and `DocumentService.markOpenInEditor` writes `.open-in-editor.json` (`{pid, projectId}`, temp file then rename) in the projects folder. Only the window that set it clears it, on closing the project or on being destroyed. `projectOpenInEditor()` reads it back and ignores a marker whose `pid` is no longer running, so an app that crashed blocks nothing. The runner checks it just before any write to a file; `listProjects` reports the project as `open`.+When no app answers its channel, `openscreen mcp` cannot ask the app's editor anything, so the app says which project its editor holds: the editor sends the project id on `AI_EDITION_OPEN_PROJECT_CHANNEL`, and `DocumentService.markOpenInEditor` writes `.open-in-editor.json` (`{pid, projectId}`, temp file then rename) in the projects folder. Only the window that set it clears it, on closing the project or on being destroyed. `projectOpenInEditor()` reads it back and ignores a marker whose `pid` is no longer running, so an app that crashed blocks nothing. The runner checks it just before any write to a file; `listProjects` reports the project as `open`.-An edit refused this way is answered with `ProjectOpenInEditorError`'s message: close the project in the app, or connect to the in-app server, which edits the open project through the editor.+An edit refused this way is answered with `ProjectOpenInEditorError`'s message: the editor could not be reached, so retry, and if it is refused again, close the project in the app, or restart or update it. It takes an app that is alive but not answering, or one that predates forwarding.## Checkpoints@@ -109,16 +110,26 @@ A client chains several edits in one turn, and each is its own undo step, so rev- **They cover the cursor sidecar.** `setPointerTrack` rewrites `<video>.cursor.json`. Before the runner writes a sidecar, it keeps the file's previous content (or its absence) in every checkpoint of that project that has none for it yet; a restore writes those back, and deletes a sidecar the take never had. A restore is itself such a write, so each checkpoint gets back its own state whatever order they are restored in.- A restore is a write: refused while **MCP edits** is off, and refused when the checkpoint belongs to another project than the one the call acts on. That message names the checkpoint's `projectId`: restoring a closed project's checkpoint takes it explicitly.- It discards every edit since the checkpoint, the user's included. The tool description says so.-- **Lifetime.** Checkpoints live in the runner's memory, the 20 most recent. On the in-app server they are lost when the app quits or the server restarts: turned off and on, its port changed, or its token regenerated, since each of those starts a new server with a new runner. Over stdio they are lost when the process exits, which it does when the client disconnects. The unknown-checkpoint message says which. The in-app agent does not need them: its whole turn is a single apply.+- **Lifetime.** Checkpoints live in the runner's memory, the 20 most recent. In the app that is one runner (`createToolRunner`, built in `registerIpcHandlers`), shared by the in-app server and every forwarded `openscreen mcp` call; a restart of the in-app server keeps them, and they are lost when the app quits. An `openscreen mcp` with no app answering keeps its own, lost when the process exits, which it does when the client disconnects. The unknown-checkpoint message says which. The in-app agent does not need them: its whole turn is a single apply.## The media tools (stdio)-`registerMediaTools` adds the tools that make a video around the edits. They share the runner the document tools use (`createToolRunner` in `stdio-server.ts`):+`registerMediaTools` adds the tools that make a video around the edits. They share the runner the document tools use, the routed one of `stdio-server.ts`, so their document calls go to the app while it runs:-- `stopRecording` turns the take into a project and creates its checkpoint through the runner.-- `setPointerTrack` gives a script-driven take a pointer track and plans zooms on its clicks. It computes the new sidecar and the zooms from the project the runner read, writes nothing itself, and hands both to the runner as a `RunnerEdit`. The runner then applies the same refusals as any edit (another process's editor, no checkpoint, a project changed mid-call) before writing anything, saves the project, and only then writes the sidecar, keeping its previous content for the checkpoints. Every refusal comes back as the tool's own text result. A sidecar write that fails after the save is reported as a partial edit: the project was saved, the pointer track was not.+- `stopRecording` turns the take into a project and creates its checkpoint through the runner. The new project itself is written by the CLI process: a fresh id no editor can hold.+- `setPointerTrack` gives a script-driven take a pointer track and plans zooms on its clicks. `pointerTrackEdit` computes the new sidecar and the zooms from the project the runner read, writes nothing itself, and hands both to the runner as a `RunnerEdit`. The runner then applies the same refusals as any edit (another process's editor, no checkpoint, a project changed mid-call) before writing anything, saves the project, and only then writes the sidecar, keeping its previous content for the checkpoints. Every refusal comes back as the tool's own text result. A sidecar write that fails after the save is reported as a partial edit: the project was saved, the pointer track was not. A `RunnerEdit` is a closure and cannot be forwarded, so the call carries what the edit needs (`events`, and `takes`, each take's start time by video): the app's channel serves `setPointerTrack` with those args and runs `pointerTrackEdit` in its own runner.- `exportVideo` and `getFrames` read projects; they write none.+## `openscreen mcp` and the running app++The stdio server (`openscreen mcp`, [`stdio-server.ts`](../../electron/mcp/stdio-server.ts)) is a process apart from the app. **When the app runs, it is the authority** (#1174): `listProjects`, every document tool, both checkpoint tools and `setPointerTrack` are forwarded to it, and the app runs them through the same runner as the in-app server. The project open in the editor is then edited through the editor, live and one undo step per call, and checkpoints live in one place. The other media tools (record, export, `getFrames`, `listSources`) stay in the CLI process.++- **Channel.** MCP itself, over a Windows named pipe (a fresh random name per launch) or a Unix socket (`<userData>/mcp.sock`, mode 0600). The app serves `createOpenScreenMcpServer` on it, `openscreen mcp` is its client and forwards each `tools/call` unchanged. The handshake lists the app's tools, so a tool an older app lacks runs locally.+- **Discovery.** Once listening, the app writes `<userData>/mcp-app.json` (`{ pid, endpoint, token }`, mode 0600) and removes it as it quits. `openscreen mcp` shares that userData and reads the file on each call it is not connected for: the app starting or quitting mid-session is picked up on the next call. A file whose pid is dead, or whose endpoint does not answer within 3 s, is a crashed app's and is ignored.+- **Trust.** Every connection must first send the token from that file, or it is dropped. Node cannot set a named pipe's DACL, and the default one lets any user open it read-only; the token, in a file inside the user's profile, is what limits the channel to the current user. Any process of that user can connect, which grants nothing new: it could run `openscreen mcp` itself. So calls over the channel edit as `openscreen mcp` always did, whatever the in-app server's **MCP server** and **MCP edits** switches say: those guard the HTTP port, a door a web page or another tool could knock on.+- **Fallback.** With no app, or one that does not answer, the calls run in the CLI process on the files, as before. The project a live app's editor holds is then still refused (#1138): the app's `.open-in-editor.json` marker says so, and the refusal asks to retry, close the project, or restart or update OpenScreen.+- **Never twice.** A call the app received but did not answer within 90 s, or whose connection dropped, is not run again locally unless it is a read: the client is told the outcome is unknown and to re-read. The connection is dropped, so a hung app does not hold later calls.+## Transport and security**In-app server (HTTP)**@@ -131,11 +142,11 @@ A client chains several edits in one turn, and each is its own undo step, so rev**`openscreen mcp` (stdio)**- One `McpServer` on the process's stdin/stdout for the whole session. It exits when stdin ends, closes or errors, whichever the client's pipe reports.-- No port and no token: only the client that launched the process can talk to it, with the user's rights, as it would run `openscreen record` or `openscreen export`.+- No port and no token: only the client that launched the process can talk to it, with the user's rights, as it would run `openscreen record` or `openscreen export`. Its own channel to the app is described [above](#openscreen-mcp-and-the-running-app).## Lifecycle-`registerIpcHandlers` builds the `McpController` (it is where the agent's own dependencies — the cursor reader, the LLM config — live) and returns it; **`main.ts` starts it**. The headless CLI shares `registerIpcHandlers` and must never bind the port a running app is listening on, and a bench run has no use for it, so neither calls `startIfEnabled`. `openscreen mcp` reuses the same dependencies (`mcpToolDeps`) with `transport: "stdio"` and edits always allowed.+`registerIpcHandlers` builds the `McpController` (it is where the agent's own dependencies — the cursor reader, the LLM config — live) and returns it; **`main.ts` starts it**. The headless CLI shares `registerIpcHandlers` and must never bind the port a running app is listening on, and a bench run has no use for it, so neither calls `startIfEnabled`. `main.ts` also opens the channel for `openscreen mcp` (`serveAppChannel`), always, and closes it on `will-quit`. `openscreen mcp` reuses the same dependencies (`mcpToolDeps`) with `transport: "stdio"` and edits always allowed.## Connecting a client@@ -157,7 +168,7 @@ codex mcp add openscreen -- "<executable>" mcp## Known gaps-- **Two servers that ignore each other.** No connection gives an agent the whole loop: the in-app server edits the open project live but cannot record or export; `openscreen mcp` records and exports but refuses the project open in the app. Checkpoints are per process and never shared. Tracked in #1174.+- **The in-app server cannot record or export.** `openscreen mcp` now covers the whole loop, editing the open project through the app; the in-app server gets the media tools later in #1174.- **No Ctrl+Z for edits to closed projects.** Their undo is the checkpoint the write requires, which lives in memory: it is gone once its server goes away or 20 newer ones push it out, and the edits stay.- **An editor that stops answering looks closed.** A snapshot unanswered for 30 s reads as "no project open", so an edit naming the project that editor holds would go to its file, and the editor's next save would drop it. It takes a renderer hung for 30 s.- **No project management.** Projects can be listed, read and edited, and `openscreen mcp` creates one per take; none can be renamed or deleted, and nothing imports media.diff --git a/technical-documentation/testing/manual-e2e-checklist.md b/technical-documentation/testing/manual-e2e-checklist.mdindex c65cdaa74..980a3adc5 100644--- a/technical-documentation/testing/manual-e2e-checklist.md+++ b/technical-documentation/testing/manual-e2e-checklist.md@@ -577,7 +577,7 @@ Two servers. The **in-app** one is HTTP, in *AI settings* → *MCP server*, off- [ ] Confirm `"<exe>" help` lists `mcp`, register the stdio server in an MCP client, and confirm `listSources` lists displays, windows and microphones.- [ ] Run `startRecording` on a window, click inside it a few times, `stopRecording`, and confirm the returned project is in the editor's project list with one clip and the zooms its `autoZooms` count announces. Clicks close in time (3 s or less) and on screen share one zoom, and a click in the first seconds or the stopping click gets none, so there can be fewer zooms than clicks. On macOS 15.2+, confirm Apple's picker never appears; on Linux, confirm the portal asks.- [ ] Edit that project while it is closed, `restoreCheckpoint` with the returned checkpoint, and confirm the project is back to the raw take. Confirm an edit to a closed project with no checkpoint is refused.-- [ ] Open the project in the editor, then send an MCP edit to it as a closed project, and confirm it is refused rather than overwriting the editor's copy. Do it through the **stdio** server, a process apart from the app (#1138): `listProjects` reports the project `open: true`, and an edit, even after `createCheckpoint`, is refused with a message naming the in-app MCP server. Open another project in the editor, or quit OpenScreen, and confirm the same edit then lands.+- [ ] Open the project in the editor, with the in-app MCP server turned off, then send an edit to it through the **stdio** server, a process apart from the app (#1174): `listProjects` reports the project `open: true`, the edit shows in the editor at once, Ctrl+Z takes it back in one step, and a later change in the editor (its autosave) keeps it. `restoreCheckpoint` with the checkpoint from `stopRecording` also lands in the editor as one undo step. Quit OpenScreen mid-session: the next stdio edit to that project, now closed, asks for a checkpoint (the app's left with it), and after `createCheckpoint` lands in its file. Relaunch OpenScreen and open the project: the next stdio edit goes through the editor again, with no restart of the MCP client.- [ ] `exportVideo`, then `getFrames` at a few ruler seconds after a trim, and confirm the frames show the moments asked for; edit the project again and confirm `getFrames` warns that the export is stale.- [ ] Drive a take from a script instead of the mouse, hand its action log to `setPointerTrack`, and confirm the export draws a cursor and zooms on the logged clicks.diff --git a/website/docs/ai-editing.md b/website/docs/ai-editing.mdindex 1882aa709..d30acc0b0 100644--- a/website/docs/ai-editing.md+++ b/website/docs/ai-editing.md@@ -95,6 +95,6 @@ The executable on each system is listed in [Running the CLI](./cli.md#running-th- **No port, no token, no setting.** The agent starts the process itself, with your rights, as it would run `openscreen record`. So it can record your screen and edit without the **MCP edits** switch.- **Each take becomes a new project** in your project list, zoomed on its clicks as a recording made in the app is: clicks close in time and on screen share one zoom, and the first seconds and the end stay on the whole screen. Every edit the agent makes is saved to the project's file. Its undo is the checkpoint the agent takes, which lasts until the agent's session ends. Only the 20 most recent checkpoints are kept, so an older one can be gone sooner.-- **A project open in the OpenScreen editor is off limits.** The agent can read it, but its edits are refused until you close the project there, since the editor would save its own copy over them.+- **While OpenScreen is running, the agent works through it.** An edit to the project open in the editor appears there at once, and `Ctrl/Cmd + Z` undoes it. The agent's checkpoints then last until OpenScreen quits. If the app does not answer, edits to that project are refused, since the editor would save its own copy over them.- **A take driven by a script** (Playwright, for example) never moves the screen pointer. The agent hands OpenScreen its own log of the actions with `setPointerTrack`, and the video gets a cursor and its zooms all the same.- **The `openscreen-demo` skill** teaches an agent the whole job, from the storyboard to the finished video. It then attaches the video to a pull request, so reviewers watch the change in place, or drafts a post on X that it publishes only once you say yes. Add it with `npx skills add getopenscreen/openscreen --skill openscreen-demo`. The recipe is [`skills/openscreen-demo/SKILL.md`](https://github.com/getopenscreen/openscreen/blob/main/skills/openscreen-demo/SKILL.md).diff --git a/website/i18n/de/docusaurus-plugin-content-docs/current/ai-editing.md b/website/i18n/de/docusaurus-plugin-content-docs/current/ai-editing.mdindex d71a93389..1002aa56e 100644--- a/website/i18n/de/docusaurus-plugin-content-docs/current/ai-editing.md+++ b/website/i18n/de/docusaurus-plugin-content-docs/current/ai-editing.md@@ -95,6 +95,6 @@ Die ausführbare Datei für jedes System steht unter [Die CLI ausführen](./cli.- **Kein Port, kein Token, keine Einstellung.** Der Agent startet den Prozess selbst, mit deinen Rechten, so wie er `openscreen record` ausführen würde. Deshalb kann er deinen Bildschirm aufnehmen und bearbeiten, ohne den Schalter **MCP-Änderungen**.- **Jeder Take wird zu einem neuen Projekt** in deiner Projektliste, mit Zooms auf seine Klicks, wie eine in der App gemachte Aufnahme: Zeitlich und räumlich nah beieinanderliegende Klicks teilen sich einen Zoom, und die ersten Sekunden und das Ende bleiben auf dem ganzen Bildschirm. Jede Änderung des Agenten wird in der Projektdatei gespeichert. Rückgängig macht sie der Wiederherstellungspunkt, den der Agent anlegt und der bis zum Ende seiner Sitzung besteht. Nur die 20 neuesten Wiederherstellungspunkte werden behalten, ein älterer kann also früher verschwinden.-- **Ein im OpenScreen-Editor geöffnetes Projekt ist tabu.** Der Agent kann es lesen, aber seine Änderungen werden abgelehnt, bis du das Projekt dort schließt, denn der Editor würde seine eigene Kopie darüber speichern.+- **Läuft OpenScreen, arbeitet der Agent über die App.** Eine Änderung am im Editor geöffneten Projekt erscheint dort sofort, und `Ctrl/Cmd + Z` macht sie rückgängig. Die Wiederherstellungspunkte des Agenten gelten dann, bis OpenScreen beendet wird. Antwortet die App nicht, werden Änderungen an diesem Projekt abgelehnt, denn der Editor würde seine eigene Kopie darüber speichern.- **Ein per Skript gesteuerter Take** (zum Beispiel mit Playwright) bewegt nie den Zeiger auf dem Bildschirm. Der Agent übergibt OpenScreen mit `setPointerTrack` sein eigenes Protokoll der Aktionen, und das Video bekommt trotzdem einen Cursor und dessen Zooms.- **Der Skill `openscreen-demo`** bringt einem Agenten die ganze Arbeit bei, vom Storyboard bis zum fertigen Video. Danach hängt er das Video an einen Pull Request, damit die Reviewer die Änderung direkt dort sehen, oder entwirft einen Post auf X, den er erst veröffentlicht, wenn du Ja sagst. Füge ihn mit `npx skills add getopenscreen/openscreen --skill openscreen-demo` hinzu. Das Rezept ist [`skills/openscreen-demo/SKILL.md`](https://github.com/getopenscreen/openscreen/blob/main/skills/openscreen-demo/SKILL.md).diff --git a/website/i18n/es/docusaurus-plugin-content-docs/current/ai-editing.md b/website/i18n/es/docusaurus-plugin-content-docs/current/ai-editing.mdindex 826252ba0..41d3cf334 100644--- a/website/i18n/es/docusaurus-plugin-content-docs/current/ai-editing.md+++ b/website/i18n/es/docusaurus-plugin-content-docs/current/ai-editing.md@@ -95,6 +95,6 @@ El ejecutable de cada sistema aparece en [Ejecutar la CLI](./cli.md#running-the-- **Sin puerto, sin token, sin ajuste.** El agente inicia el proceso él mismo, con tus permisos, como ejecutaría `openscreen record`. Así puede grabar tu pantalla y editar sin el interruptor **Ediciones por MCP**.- **Cada toma se convierte en un proyecto nuevo** en tu lista de proyectos, con zooms en sus clics, como una grabación hecha en la app: los clics cercanos en el tiempo y en la pantalla comparten un zoom, y los primeros segundos y el final se quedan en la pantalla completa. Cada edición del agente se guarda en el archivo del proyecto. Se deshace con el punto de restauración que toma el agente, que dura hasta que termina su sesión. Solo se conservan los 20 puntos de restauración más recientes, así que uno más antiguo puede desaparecer antes.-- **Un proyecto abierto en el editor de OpenScreen queda fuera de su alcance.** El agente puede leerlo, pero sus ediciones se rechazan hasta que cierres el proyecto ahí, porque el editor guardaría su propia copia encima.+- **Mientras OpenScreen está abierto, el agente trabaja a través de la app.** Una edición del proyecto abierto en el editor aparece ahí al instante, y `Ctrl/Cmd + Z` la deshace. Los puntos de restauración del agente duran entonces hasta que cierres OpenScreen. Si la app no responde, las ediciones de ese proyecto se rechazan, porque el editor guardaría su propia copia encima.- **Una toma controlada por un script** (Playwright, por ejemplo) nunca mueve el puntero de la pantalla. El agente le pasa a OpenScreen su propio registro de las acciones con `setPointerTrack`, y el video obtiene igualmente un cursor y sus zooms.- **La skill `openscreen-demo`** le enseña a un agente todo el trabajo, desde el storyboard hasta el video terminado. Después adjunta el video a un pull request, para que los revisores vean el cambio ahí mismo, o redacta un post en X que solo publica cuando dices que sí. Agrégala con `npx skills add getopenscreen/openscreen --skill openscreen-demo`. La receta es [`skills/openscreen-demo/SKILL.md`](https://github.com/getopenscreen/openscreen/blob/main/skills/openscreen-demo/SKILL.md).diff --git a/website/i18n/fr/docusaurus-plugin-content-docs/current/ai-editing.md b/website/i18n/fr/docusaurus-plugin-content-docs/current/ai-editing.mdindex bbb57d319..d0eebb869 100644--- a/website/i18n/fr/docusaurus-plugin-content-docs/current/ai-editing.md+++ b/website/i18n/fr/docusaurus-plugin-content-docs/current/ai-editing.md@@ -95,6 +95,6 @@ L'exécutable de chaque système est indiqué dans [Lancer la CLI](./cli.md#runn- **Ni port, ni jeton, ni réglage.** L'agent lance lui-même le processus, avec vos droits, comme il lancerait `openscreen record`. Il peut donc enregistrer votre écran et faire des modifications sans l'interrupteur **Modifications par MCP**.- **Chaque prise devient un nouveau projet** dans votre liste de projets, zoomé sur ses clics comme un enregistrement fait dans l'application : des clics rapprochés dans le temps et à l'écran partagent un zoom, et les premières secondes comme la fin restent sur l'écran entier. Chaque modification de l'agent est enregistrée dans le fichier du projet. Elle s'annule avec le point de restauration que prend l'agent, qui dure jusqu'à la fin de sa session. Seuls les 20 points de restauration les plus récents sont gardés : un plus ancien peut disparaître avant.-- **Un projet ouvert dans l'éditeur d'OpenScreen est hors de portée.** L'agent peut le lire, mais ses modifications sont refusées tant que vous ne l'avez pas fermé dans l'éditeur, qui enregistrerait sinon sa propre copie par-dessus.+- **Quand OpenScreen est lancé, l'agent passe par l'application.** Une modification du projet ouvert dans l'éditeur y apparaît aussitôt, et `Ctrl/Cmd + Z` l'annule. Les points de restauration de l'agent durent alors jusqu'à ce que vous quittiez OpenScreen. Si l'application ne répond pas, les modifications de ce projet sont refusées, car l'éditeur enregistrerait sa propre copie par-dessus.- **Une prise pilotée par un script** (Playwright, par exemple) ne déplace jamais le pointeur de l'écran. L'agent transmet à OpenScreen son propre journal des actions avec `setPointerTrack`, et la vidéo a malgré tout un curseur et ses zooms.- **La skill `openscreen-demo`** enseigne à un agent tout le travail, du storyboard jusqu'à la vidéo finie. Il joint ensuite la vidéo à une pull request, pour que les relecteurs voient le changement sur place, ou rédige un post sur X qu'il ne publie qu'une fois que vous avez dit oui. Ajoutez-la avec `npx skills add getopenscreen/openscreen --skill openscreen-demo`. La recette est [`skills/openscreen-demo/SKILL.md`](https://github.com/getopenscreen/openscreen/blob/main/skills/openscreen-demo/SKILL.md).diff --git a/website/i18n/ja/docusaurus-plugin-content-docs/current/ai-editing.md b/website/i18n/ja/docusaurus-plugin-content-docs/current/ai-editing.mdindex f02fd367b..30e0fb684 100644--- a/website/i18n/ja/docusaurus-plugin-content-docs/current/ai-editing.md+++ b/website/i18n/ja/docusaurus-plugin-content-docs/current/ai-editing.md@@ -95,6 +95,6 @@ codex mcp add openscreen -- "/Applications/Openscreen.app/Contents/MacOS/Openscr- **ポートも、トークンも、設定も不要。** エージェントは、`openscreen record` を実行するときと同じように、あなたの権限でプロセスを自分で起動します。そのため、**MCP による編集**スイッチなしで、画面を録画し、編集できます。- **テイクはそれぞれ新しいプロジェクトになり**、プロジェクト一覧に表示されます。アプリで作った録画と同じく、記録されたクリックにズームが付きます。時間的にも画面上でも近いクリックは1つのズームにまとめられ、最初の数秒と最後は画面全体のままです。エージェントによる編集はすべてプロジェクトのファイルに保存されます。元に戻すには、エージェントが作成するチェックポイントを使います。チェックポイントはエージェントのセッションが終わるまで有効です。保持されるのは直近 20 個のチェックポイントだけなので、古いものはそれより前に消えることがあります。-- **OpenScreen のエディターで開いているプロジェクトは編集できません。** エージェントは読み取れますが、そこでプロジェクトを閉じるまで編集は拒否されます。エディターが自分のコピーで上書きしてしまうためです。+- **OpenScreen の起動中は、エージェントはアプリを通して作業します。** エディターで開いているプロジェクトへの編集はすぐにそこに反映され、`Ctrl/Cmd + Z` で元に戻せます。このときエージェントのチェックポイントは OpenScreen を終了するまで有効です。アプリが応答しない場合、そのプロジェクトへの編集は拒否されます。エディターが自分のコピーで上書きしてしまうためです。- **スクリプトで操作するテイク**(たとえば Playwright)では、画面のポインターはまったく動きません。エージェントが `setPointerTrack` で自分の操作ログを OpenScreen に渡せば、動画には同じようにカーソルとそのズームが付きます。- **`openscreen-demo` スキル**は、絵コンテから完成した動画まで、作業全体をエージェントに教えます。そのあとエージェントは、レビュアーがその場で変更を確認できるよう動画をプルリクエストに添付するか、X への投稿を下書きします。投稿は、あなたが了承してはじめて公開されます。`npx skills add getopenscreen/openscreen --skill openscreen-demo` で追加します。手順は [`skills/openscreen-demo/SKILL.md`](https://github.com/getopenscreen/openscreen/blob/main/skills/openscreen-demo/SKILL.md) にあります。diff --git a/website/i18n/pt-BR/docusaurus-plugin-content-docs/current/ai-editing.md b/website/i18n/pt-BR/docusaurus-plugin-content-docs/current/ai-editing.mdindex db8782cb1..03fd609a8 100644--- a/website/i18n/pt-BR/docusaurus-plugin-content-docs/current/ai-editing.md+++ b/website/i18n/pt-BR/docusaurus-plugin-content-docs/current/ai-editing.md@@ -95,6 +95,6 @@ O executável de cada sistema está listado em [Como executar a CLI](./cli.md#ru- **Sem porta, sem token, sem configuração.** O agente inicia o processo por conta própria, com as suas permissões, como faria ao rodar `openscreen record`. Por isso ele pode gravar a sua tela e editar sem o interruptor **Edições via MCP**.- **Cada tomada vira um novo projeto** na sua lista de projetos, com zooms nos seus cliques, como uma gravação feita no app: cliques próximos no tempo e na tela dividem um zoom, e os primeiros segundos e o final ficam na tela inteira. Cada edição do agente é salva no arquivo do projeto. Ela se desfaz com o ponto de restauração que o agente cria, que dura até o fim da sessão dele. Só os 20 pontos de restauração mais recentes são mantidos, então um mais antigo pode sumir antes.-- **Um projeto aberto no editor do OpenScreen fica fora do alcance.** O agente pode lê-lo, mas as edições dele são recusadas até você fechar o projeto ali, porque o editor salvaria a própria cópia por cima.+- **Enquanto o OpenScreen está aberto, o agente trabalha por meio do app.** Uma edição do projeto aberto no editor aparece ali na hora, e `Ctrl/Cmd + Z` a desfaz. Os pontos de restauração do agente duram então até você fechar o OpenScreen. Se o app não responder, as edições desse projeto são recusadas, porque o editor salvaria a própria cópia por cima.- **Uma tomada conduzida por um script** (o Playwright, por exemplo) nunca move o ponteiro da tela. O agente passa ao OpenScreen o próprio registro das ações com `setPointerTrack`, e o vídeo ganha um cursor e seus zooms do mesmo jeito.- **A skill `openscreen-demo`** ensina a um agente o trabalho inteiro, do storyboard até o vídeo pronto. Depois ele anexa o vídeo a um pull request, para que os revisores vejam a mudança ali mesmo, ou redige um post no X que só publica depois que você disser sim. Adicione-a com `npx skills add getopenscreen/openscreen --skill openscreen-demo`. A receita é [`skills/openscreen-demo/SKILL.md`](https://github.com/getopenscreen/openscreen/blob/main/skills/openscreen-demo/SKILL.md).diff --git a/website/i18n/zh-CN/docusaurus-plugin-content-docs/current/ai-editing.md b/website/i18n/zh-CN/docusaurus-plugin-content-docs/current/ai-editing.mdindex 5feedb032..40c882244 100644--- a/website/i18n/zh-CN/docusaurus-plugin-content-docs/current/ai-editing.md+++ b/website/i18n/zh-CN/docusaurus-plugin-content-docs/current/ai-editing.md@@ -95,6 +95,6 @@ codex mcp add openscreen -- "/Applications/Openscreen.app/Contents/MacOS/Openscr- **无需端口、令牌或设置**。AI 代理以你的权限自行启动进程,就像它运行 `openscreen record` 一样。因此,它无需**MCP 编辑**开关,就能录制你的屏幕并进行编辑。- **每次录制都会成为一个新项目**,出现在你的项目列表中,并且与在应用中录制的内容一样,在录制下来的点击处添加缩放:时间和位置都相近的点击共用一个缩放,开头几秒和结尾保持全屏。AI 代理的每项编辑都会保存到项目文件中。撤销它们要靠 AI 代理创建的检查点,检查点在代理的会话结束前一直有效。只保留最近的 20 个检查点,因此较早的检查点可能会更早失效。-- **在 OpenScreen 编辑器中打开的项目无法编辑。** AI 代理可以读取它,但在你于编辑器中关闭该项目之前,它的编辑都会被拒绝,因为编辑器会用自己的副本覆盖这些编辑。+- **OpenScreen 运行时,AI 代理通过应用进行编辑。** 对编辑器中打开的项目所做的编辑会立即显示在那里,并可用 `Ctrl/Cmd + Z` 撤销。此时 AI 代理的检查点会一直保留到 OpenScreen 退出。如果应用没有响应,对该项目的编辑会被拒绝,因为编辑器会用自己的副本覆盖这些编辑。- **由脚本驱动的录制**(例如 Playwright)永远不会移动屏幕指针。AI 代理通过 `setPointerTrack` 把它自己的操作日志交给 OpenScreen,视频照样会有光标及其缩放。- **`openscreen-demo` 技能**会教 AI 代理完成整个工作,从分镜脚本到成品视频。之后,它会把视频附加到拉取请求中,让审阅者直接在那里看到改动;或者起草一条 X 帖子,而且只有在你同意之后,它才会发布这条帖子。使用 `npx skills add getopenscreen/openscreen --skill openscreen-demo` 添加它。具体步骤见 [`skills/openscreen-demo/SKILL.md`](https://github.com/getopenscreen/openscreen/blob/main/skills/openscreen-demo/SKILL.md)。diff --git a/website/i18n/zh-TW/docusaurus-plugin-content-docs/current/ai-editing.md b/website/i18n/zh-TW/docusaurus-plugin-content-docs/current/ai-editing.mdindex 3ac582023..46c1a2df8 100644--- a/website/i18n/zh-TW/docusaurus-plugin-content-docs/current/ai-editing.md+++ b/website/i18n/zh-TW/docusaurus-plugin-content-docs/current/ai-editing.md@@ -95,6 +95,6 @@ codex mcp add openscreen -- "/Applications/Openscreen.app/Contents/MacOS/Openscr- **不需要連接埠、權杖或設定。** 代理會以你的權限自行啟動程序,就像它執行 `openscreen record` 一樣。因此,它不需要**MCP 編輯**開關,就能錄製你的螢幕並進行剪輯。- **每段錄影都會成為一個新專案**,出現在你的專案清單中,並和在應用程式中錄製的影片一樣,在錄下的點擊處加上縮放:時間與位置都相近的點擊共用一個縮放,開頭幾秒與結尾保持全螢幕。代理的每項編輯都會儲存到專案檔案中。要復原,就靠代理建立的檢查點,檢查點在代理的工作階段結束前都有效。只會保留最近的 20 個檢查點,因此較早的檢查點可能會提早失效。-- **在 OpenScreen 編輯器中開啟的專案無法編輯。** 代理可以讀取它,但在你於編輯器中關閉該專案之前,它的編輯都會被拒絕,因為編輯器會用自己的副本覆寫這些編輯。+- **OpenScreen 執行時,代理會透過應用程式進行編輯。** 對編輯器中開啟的專案所做的編輯會立即顯示在那裡,並可用 `Ctrl/Cmd + Z` 復原。此時代理的檢查點會保留到 OpenScreen 結束為止。如果應用程式沒有回應,對該專案的編輯會被拒絕,因為編輯器會用自己的副本覆寫這些編輯。- **由腳本驅動的錄影**(例如 Playwright)永遠不會移動螢幕上的指標。代理會用 `setPointerTrack` 把它自己的操作記錄交給 OpenScreen,影片照樣會有游標與其縮放。- **`openscreen-demo` 技能**會教代理完成整個工作,從分鏡腳本到完成的影片。之後,它會把影片附加到拉取請求中,讓審閱者直接在那裡看到變更;或者草擬一則 X 貼文,而且只有在你同意之後,它才會發布這則貼文。使用 `npx skills add getopenscreen/openscreen --skill openscreen-demo` 加入它。完整步驟請見 [`skills/openscreen-demo/SKILL.md`](https://github.com/getopenscreen/openscreen/blob/main/skills/openscreen-demo/SKILL.md)。