Record your screen, ship a demo. Free and open-source, GPU-accelerated, no watermarks, no subscriptions. Windows, macOS, Linux. Actively maintained.
TypeScript cross-platformelectronffmpegopen-sourceproduct-demorustscreen-capturescreen-recorderscreen-recordingscreen-studioscreencastspeech-to-texttypescriptvideo-editingvideo-editorwaylandwgpuwhisper
test: verify shared layout contracts over MCP
EtienneLescot3bd8f6414a1d
diff --git a/electron/ai-edition/agent-tools.ts b/electron/ai-edition/agent-tools.tsindex 128d0b910..3cbccdb01 100644--- a/electron/ai-edition/agent-tools.ts+++ b/electron/ai-edition/agent-tools.ts@@ -1402,7 +1402,7 @@ export function documentSnapshotForModel(preset: r.member.preset,})),layoutNote:- "Layouts form a continuous partition. A change lasts until the next one. Removing a change extends the preceding layout. Camera presets require linked webcam footage (assets[].hasCameraTrack).",+ "layoutSegments use RAW document-timeline seconds, matching addLayoutChange.timeSec and moveLayoutBoundary.timeSec. Layouts form a continuous partition. A change lasts until the next one. Removing a change extends the preceding layout. Camera presets require linked webcam footage (assets[].hasCameraTrack).",primaryAssetId: document.project.primaryAssetId ?? document.assets[0]?.id ?? null,autoFocusAll,hasAnyCamera: hasAnyClipWithCamera(document.assets, document.timeline.clips),diff --git a/electron/mcp/openscreen-mcp-server.test.ts b/electron/mcp/openscreen-mcp-server.test.tsindex 5e5cec129..c33d4ce77 100644--- a/electron/mcp/openscreen-mcp-server.test.ts+++ b/electron/mcp/openscreen-mcp-server.test.ts@@ -10,11 +10,20 @@ import path from "node:path";import { Client } from "@modelcontextprotocol/sdk/client/index.js";import { StreamableHTTPClientTransport } from "@modelcontextprotocol/sdk/client/streamableHttp.js";import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";+import {+ addLayoutChange,+ layoutRuns,+ moveLayoutBoundary,+ removeLayoutChange,+ setLayoutPreset,+} from "../../src/lib/ai-edition/document/layout";import {type AxcutDocument,createEmptyDocument,documentSchema,} from "../../src/lib/ai-edition/schema";+import { LAYOUT_PRESETS } from "../../src/lib/layoutTypes";+import { buildSceneDescription } from "../../src/native/sceneDescription";import { OPENSCREEN_TOOL_NAMES } from "../ai-edition/agent-tools";import { TOOL_DESCRIPTIONS } from "../ai-edition/deep-agent/service";import { DocumentService } from "../ai-edition/document-service";@@ -194,31 +203,109 @@ function resultText(result: Awaited<ReturnType<Client["callTool"]>>): string {describe("the MCP tool surface", () => {it("edits layout passages through the same document operations and exposes their result", async () => {const editor = new FakeEditor();+ editor.document = documentSchema.parse({+ ...editor.document,+ assets: editor.document?.assets.map((asset) => ({+ ...asset,+ cameraTrack: { sourcePath: "/tmp/camera.mp4", startMs: 0, offsetMs: 0, visible: true },+ })),+ });+ let expected = editor.document;+ const sameDocumentAndScene = () => {+ const actual = editor.document;+ if (!actual) throw new Error("The editor lost its document");+ const passages = (doc: AxcutDocument) =>+ layoutRuns(doc).map((r) => ({+ start: r.start,+ end: r.end,+ preset: r.member.preset,+ }));+ expect(passages(actual)).toEqual(passages(expected));+ const anchors = (doc: AxcutDocument) =>+ (doc.layoutRegions ?? []).map(({ id: _id, ...region }) => region);+ expect(anchors(actual)).toEqual(anchors(expected));+ expect(actual.legacyEditor?.cameraFullscreenRegions).toBeUndefined();+ expect(buildSceneDescription(actual).layoutRegions).toEqual(+ buildSceneDescription(expected).layoutRegions,+ );+ };const mcp = await connect(editor);const changed = await mcp.callTool({name: "addLayoutChange",- arguments: { timeSec: 5, preset: "no-webcam" },+ arguments: { timeSec: 5, preset: "dual-frame" },});expect(changed.isError).not.toBe(true);expect(editor.applied).toHaveLength(1);- expect(editor.document?.layoutRegions?.some((r) => r.preset === "no-webcam")).toBe(true);+ expected = addLayoutChange(expected, 5, "dual-frame");+ sameDocumentAndScene();+ const layoutId = JSON.parse(resultText(changed)).layoutSegments[1].id as string;+ const moved = await mcp.callTool({+ name: "moveLayoutBoundary",+ arguments: { layoutId, timeSec: 8 },+ });+ expect(moved.isError).not.toBe(true);+ expected = moveLayoutBoundary(expected, layoutRuns(expected)[1].ids[0], 8);+ sameDocumentAndScene();+ const restyled = await mcp.callTool({+ name: "setLayoutPreset",+ arguments: { layoutId, preset: "vertical-stack" },+ });+ expect(restyled.isError).not.toBe(true);+ expected = setLayoutPreset(expected, layoutRuns(expected)[1].ids[0], "vertical-stack");+ sameDocumentAndScene();+ const noOp = await mcp.callTool({+ name: "setLayoutPreset",+ arguments: { layoutId, preset: "vertical-stack" },+ });+ expect(noOp.isError).toBe(true);+ expect(editor.applied).toHaveLength(3);const snapshot = await mcp.callTool({ name: "getCurrentDocument", arguments: {} });const segments = JSON.parse(resultText(snapshot)).layoutSegments;expect(segments.map((r: { startSec: number; preset: string }) => [r.startSec, r.preset]),).toEqual([[0, "picture-in-picture"],- [5, "no-webcam"],+ [8, "vertical-stack"],]);const removed = await mcp.callTool({name: "removeModifier",arguments: { id: segments[1].id },});expect(removed.isError).not.toBe(true);+ expected = removeLayoutChange(expected, layoutRuns(expected)[1].ids[0]);+ sameDocumentAndScene();expect(editor.document?.layoutRegions).toEqual([]);const { tools } = await mcp.listTools();expect(tools.map((t) => t.name)).toContain("moveLayoutBoundary");expect(tools.map((t) => t.name)).not.toContain("addCameraFullscreen");+ expect(tools.map((t) => t.name)).not.toContain("setCameraFullscreen");+ });++ it("advertises the shared presets and marks all layout tools as mutations", async () => {+ const mcp = await connect(new FakeEditor());+ const { tools } = await mcp.listTools();+ for (const name of ["addLayoutChange", "setLayoutPreset", "moveLayoutBoundary"]) {+ const tool = tools.find((t) => t.name === name);+ expect(tool?.description).toBe(TOOL_DESCRIPTIONS[name]);+ expect(tool?.annotations?.readOnlyHint).toBe(false);+ if (name !== "moveLayoutBoundary")+ expect(tool?.inputSchema.properties?.preset).toMatchObject({ enum: [...LAYOUT_PRESETS] });+ }+ });++ it.each([+ { name: "addLayoutChange", args: { timeSec: 5, preset: "no-webcam" } },+ { name: "setLayoutPreset", args: { layoutId: "layout", preset: "no-webcam" } },+ { name: "moveLayoutBoundary", args: { layoutId: "layout", timeSec: 8 } },+ ])("guards $name with MCP edit consent", async ({ name, args }) => {+ const editor = new FakeEditor();+ const before = editor.document;+ const mcp = await connect(editor, { editsAllowed: false });+ const result = await mcp.callTool({ name, arguments: args });+ expect(result.isError).toBe(true);+ expect(resultText(result)).toMatch(/consent_required/);+ expect(editor.document).toBe(before);+ expect(editor.applied).toHaveLength(0);});it("is listProjects, then exactly the in-app agent's tools with its descriptions, then the checkpoints and the media tools", async () => {diff --git a/technical-documentation/architecture/ai-agent.md b/technical-documentation/architecture/ai-agent.mdindex 17fddf104..a4489c235 100644--- a/technical-documentation/architecture/ai-agent.md+++ b/technical-documentation/architecture/ai-agent.md@@ -41,7 +41,7 @@ The agent is built with LangChain's `createAgent`, not `deepagents`' `createDeep## Tool schema-The model never free-writes the project document. It can only call the fixed set of tools built by `deep-agent/service.ts#buildTools` (`OPENSCREEN_TOOL_NAMES`), described by `TOOL_DESCRIPTIONS` in the same file and validated against the Zod argument schemas in `agent-tools.ts`; the executor parses JSON, validates arguments, and returns either a new schema-valid snapshot or an error. Those three surfaces (descriptions, built tools, executor cases) are pinned to one another by `deep-agent/service.test.ts` — an earlier fourth surface, `AGENT_TOOL_SPECS`, described the tools in JSON Schema for a provider it had stopped reaching, and drifted. The tools operate on the same [document model](document-model.md) as manual editing.+The model never free-writes the project document. `deep-agent/service.ts#buildTools` builds the tools from `TOOL_ARG_SCHEMAS`, the registry also used by both MCP transports. `TOOL_DESCRIPTIONS` supplies their descriptions, and the Zod argument schemas come from `agent-tools.ts`. The executor parses JSON, validates arguments, and returns either a new schema-valid snapshot or an error. These surfaces and the public `OPENSCREEN_TOOL_NAMES` list are pinned to one another by `deep-agent/service.test.ts`. The tools operate on the same [document model](document-model.md) and layout mutations as manual editing.| Tool | What it does | What it mutates ||---|---|---|@@ -62,17 +62,25 @@ The model never free-writes the project document. It can only call the fixed set| `setSpeed` | Moves, resizes, or changes an existing speed pill. | The corresponding `legacyEditor.speedRegions` fragments. || `addAnnotation` | Adds a text, arrow (`figure`) or privacy `blur` annotation over virtual timeline time. An arrow or a blur is the editor's text switched through `convertAnnotationKind`, then given its box and its `figureData` / `blurData` (validated by `figureDataSchema` / `blurDataSchema`). Images are left out: they need a file. A freehand blur is refused, as the inspector no longer offers it. | `annotations`. || `setAnnotation` | Moves, resizes, retypes or restyles an annotation. A type change goes through `convertAnnotationKind` with the footage under each fragment (`annotationFootageRect`), as the inspector's type switch does; `figureData` / `blurData` merge only the keys sent. Arguments of another kind (width on a text, blurData on an arrow) are refused. | The corresponding clip-anchored `annotations` fragments. |-| `addCameraFullscreen` | Adds a camera-fullscreen region over virtual timeline time; refused when no clip under the span comes from an asset with a linked `cameraTrack`, since such a region can only render nothing. | `legacyEditor.cameraFullscreenRegions`. |-| `setCameraFullscreen` | Moves or resizes a camera-fullscreen pill, under the same camera requirement as `addCameraFullscreen`. | The corresponding `legacyEditor.cameraFullscreenRegions` fragments. |+| `addLayoutChange` | Changes the layout from a RAW document-timeline time until the next change. Camera presets require linked camera footage. | The shared `layoutRegions` partition, or the global preset when uniform. |+| `setLayoutPreset` | Changes one passage by an ID from `getCurrentDocument.layoutSegments`; equal neighbours merge. An unchanged preset is refused. | The same layout partition. |+| `moveLayoutBoundary` | Moves a passage's start in RAW seconds, updating both neighbours atomically. The first boundary is protected. | The same layout partition. || `addEffect` | Writes a shader effect (WGSL, [docs/shader-effects.md](../../docs/shader-effects.md)) and poses it over virtual timeline time, or poses one of the project's effects again by `effectId`. The compositor checks the code first (`checkShaderEffect`); code that does not compile is refused with the error at the agent's line. The result gives the cost tier and, when the tier runs long, a recommendation, never a refusal. One effect runs at a time: two cannot overlap. | `shaderEffects` (a new definition) and `shaderEffectRanges`. || `setEffect` | Moves, re-tunes (`values`) or retargets an effect pill. | The pill's `shaderEffectRanges` fragments. || `setEffectDefinition` | Changes an effect's name, description, code or parameters, and so every region using it; new code is checked first. A removed parameter's values drop. | The `shaderEffects` entry and its regions' `values`. || `getEffect` | Reads one effect in full, code included; `getCurrentDocument` lists effects without their code. | Read only. || `removeTrim` | Deletes a trim so its source span plays and exports again. | `timeline.trimRanges`. |-| `removeModifier` | Resolves and deletes a zoom, speed, annotation, camera-fullscreen, audio, or effect pill by ID; `removedIds` reports every touching storage row deleted with that pill. | The matching modifier collection. |+| `removeModifier` | Resolves and deletes a zoom, speed, annotation, layout change, audio, or effect by ID; `removedIds` reports the removed fragments. Removing a layout change extends the preceding passage; the first passage is protected. | The matching modifier collection. || `removeClip` | Deletes a placed clip, closes the gap, and drops effects anchored only to it. The result names the modifiers and trims it took with it. | Timeline clips and affected anchored modifiers. |-Clips and trims use source time. Zoom, speed, annotation, camera-fullscreen, and effect tools use virtual edited-timeline time; the executor converts these spans to the clip-anchored millisecond representation used by the document.+Clips and trims use source time. Zoom, speed, annotation and effect tools use virtual edited-timeline time; the executor converts these spans to the clip-anchored millisecond representation used by the document.++Layout point tools use RAW document-timeline seconds. `getCurrentDocument.layoutSegments`+exposes their logical IDs, bounds and presets. The document stores clip/source anchors;+`buildSceneDescription` projects them into source time for native preview and export.+Legacy Full Camera executors remain compatibility aliases over this partition and are+not advertised to the model or MCP clients. Migration-only camera envelopes are derived+from layout metadata rather than maintained as a second mutable collection.## Checkpoints and undodiff --git a/technical-documentation/architecture/mcp-server.md b/technical-documentation/architecture/mcp-server.mdindex 8e8aac3b4..446212da4 100644--- a/technical-documentation/architecture/mcp-server.md+++ b/technical-documentation/architecture/mcp-server.md@@ -250,5 +250,8 @@ camera footage. A temporary passage needs a change at its start and another at itools for a local edit. Compatibility Full Camera executor aliases are not advertised.HTTP and stdio obtain schemas and descriptions from `TOOL_ARG_SCHEMAS` and-`TOOL_DESCRIPTIONS`; mutation consent still goes through `executeAgentTool`. The MCP-HTTP tests exercise a real client/server create/read/remove cycle on the shared document.+`TOOL_DESCRIPTIONS`. Both call `runDocumentTool` and `executeAgentTool`; the shared MCP+runner checks mutation consent with `isMutatingTool` before executing a write. The MCP+HTTP tests exercise a real client/server add/move/set/read/remove cycle, compare the+document and native scene with the UI's shared mutations, verify the preset enum and+mutation annotations, and refuse all three layout writes when MCP edits are disabled.