Skip to content

Record your screen, ship a demo. Free and open-source, GPU-accelerated, no watermarks, no subscriptions. Windows, macOS, Linux. Actively maintained.

test: verify shared layout contracts over MCP

EtienneLescot3bd8f6414a1d
diff --git a/electron/ai-edition/agent-tools.ts b/electron/ai-edition/agent-tools.ts
index 128d0b910..3cbccdb01 100644
--- a/electron/ai-edition/agent-tools.ts
+++ b/electron/ai-edition/agent-tools.ts
@@ -1402,7 +1402,7 @@ export function documentSnapshotForModel(
preset: r.member.preset,
})),
layoutNote:
- "Layouts form a continuous partition. A change lasts until the next one. Removing a change extends the preceding layout. Camera presets require linked webcam footage (assets[].hasCameraTrack).",
+ "layoutSegments use RAW document-timeline seconds, matching addLayoutChange.timeSec and moveLayoutBoundary.timeSec. Layouts form a continuous partition. A change lasts until the next one. Removing a change extends the preceding layout. Camera presets require linked webcam footage (assets[].hasCameraTrack).",
primaryAssetId: document.project.primaryAssetId ?? document.assets[0]?.id ?? null,
autoFocusAll,
hasAnyCamera: hasAnyClipWithCamera(document.assets, document.timeline.clips),
diff --git a/electron/mcp/openscreen-mcp-server.test.ts b/electron/mcp/openscreen-mcp-server.test.ts
index 5e5cec129..c33d4ce77 100644
--- a/electron/mcp/openscreen-mcp-server.test.ts
+++ b/electron/mcp/openscreen-mcp-server.test.ts
@@ -10,11 +10,20 @@ import path from "node:path";
import { Client } from "@modelcontextprotocol/sdk/client/index.js";
import { StreamableHTTPClientTransport } from "@modelcontextprotocol/sdk/client/streamableHttp.js";
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
+import {
+ addLayoutChange,
+ layoutRuns,
+ moveLayoutBoundary,
+ removeLayoutChange,
+ setLayoutPreset,
+} from "../../src/lib/ai-edition/document/layout";
import {
type AxcutDocument,
createEmptyDocument,
documentSchema,
} from "../../src/lib/ai-edition/schema";
+import { LAYOUT_PRESETS } from "../../src/lib/layoutTypes";
+import { buildSceneDescription } from "../../src/native/sceneDescription";
import { OPENSCREEN_TOOL_NAMES } from "../ai-edition/agent-tools";
import { TOOL_DESCRIPTIONS } from "../ai-edition/deep-agent/service";
import { DocumentService } from "../ai-edition/document-service";
@@ -194,31 +203,109 @@ function resultText(result: Awaited<ReturnType<Client["callTool"]>>): string {
describe("the MCP tool surface", () => {
it("edits layout passages through the same document operations and exposes their result", async () => {
const editor = new FakeEditor();
+ editor.document = documentSchema.parse({
+ ...editor.document,
+ assets: editor.document?.assets.map((asset) => ({
+ ...asset,
+ cameraTrack: { sourcePath: "/tmp/camera.mp4", startMs: 0, offsetMs: 0, visible: true },
+ })),
+ });
+ let expected = editor.document;
+ const sameDocumentAndScene = () => {
+ const actual = editor.document;
+ if (!actual) throw new Error("The editor lost its document");
+ const passages = (doc: AxcutDocument) =>
+ layoutRuns(doc).map((r) => ({
+ start: r.start,
+ end: r.end,
+ preset: r.member.preset,
+ }));
+ expect(passages(actual)).toEqual(passages(expected));
+ const anchors = (doc: AxcutDocument) =>
+ (doc.layoutRegions ?? []).map(({ id: _id, ...region }) => region);
+ expect(anchors(actual)).toEqual(anchors(expected));
+ expect(actual.legacyEditor?.cameraFullscreenRegions).toBeUndefined();
+ expect(buildSceneDescription(actual).layoutRegions).toEqual(
+ buildSceneDescription(expected).layoutRegions,
+ );
+ };
const mcp = await connect(editor);
const changed = await mcp.callTool({
name: "addLayoutChange",
- arguments: { timeSec: 5, preset: "no-webcam" },
+ arguments: { timeSec: 5, preset: "dual-frame" },
});
expect(changed.isError).not.toBe(true);
expect(editor.applied).toHaveLength(1);
- expect(editor.document?.layoutRegions?.some((r) => r.preset === "no-webcam")).toBe(true);
+ expected = addLayoutChange(expected, 5, "dual-frame");
+ sameDocumentAndScene();
+ const layoutId = JSON.parse(resultText(changed)).layoutSegments[1].id as string;
+ const moved = await mcp.callTool({
+ name: "moveLayoutBoundary",
+ arguments: { layoutId, timeSec: 8 },
+ });
+ expect(moved.isError).not.toBe(true);
+ expected = moveLayoutBoundary(expected, layoutRuns(expected)[1].ids[0], 8);
+ sameDocumentAndScene();
+ const restyled = await mcp.callTool({
+ name: "setLayoutPreset",
+ arguments: { layoutId, preset: "vertical-stack" },
+ });
+ expect(restyled.isError).not.toBe(true);
+ expected = setLayoutPreset(expected, layoutRuns(expected)[1].ids[0], "vertical-stack");
+ sameDocumentAndScene();
+ const noOp = await mcp.callTool({
+ name: "setLayoutPreset",
+ arguments: { layoutId, preset: "vertical-stack" },
+ });
+ expect(noOp.isError).toBe(true);
+ expect(editor.applied).toHaveLength(3);
const snapshot = await mcp.callTool({ name: "getCurrentDocument", arguments: {} });
const segments = JSON.parse(resultText(snapshot)).layoutSegments;
expect(
segments.map((r: { startSec: number; preset: string }) => [r.startSec, r.preset]),
).toEqual([
[0, "picture-in-picture"],
- [5, "no-webcam"],
+ [8, "vertical-stack"],
]);
const removed = await mcp.callTool({
name: "removeModifier",
arguments: { id: segments[1].id },
});
expect(removed.isError).not.toBe(true);
+ expected = removeLayoutChange(expected, layoutRuns(expected)[1].ids[0]);
+ sameDocumentAndScene();
expect(editor.document?.layoutRegions).toEqual([]);
const { tools } = await mcp.listTools();
expect(tools.map((t) => t.name)).toContain("moveLayoutBoundary");
expect(tools.map((t) => t.name)).not.toContain("addCameraFullscreen");
+ expect(tools.map((t) => t.name)).not.toContain("setCameraFullscreen");
+ });
+
+ it("advertises the shared presets and marks all layout tools as mutations", async () => {
+ const mcp = await connect(new FakeEditor());
+ const { tools } = await mcp.listTools();
+ for (const name of ["addLayoutChange", "setLayoutPreset", "moveLayoutBoundary"]) {
+ const tool = tools.find((t) => t.name === name);
+ expect(tool?.description).toBe(TOOL_DESCRIPTIONS[name]);
+ expect(tool?.annotations?.readOnlyHint).toBe(false);
+ if (name !== "moveLayoutBoundary")
+ expect(tool?.inputSchema.properties?.preset).toMatchObject({ enum: [...LAYOUT_PRESETS] });
+ }
+ });
+
+ it.each([
+ { name: "addLayoutChange", args: { timeSec: 5, preset: "no-webcam" } },
+ { name: "setLayoutPreset", args: { layoutId: "layout", preset: "no-webcam" } },
+ { name: "moveLayoutBoundary", args: { layoutId: "layout", timeSec: 8 } },
+ ])("guards $name with MCP edit consent", async ({ name, args }) => {
+ const editor = new FakeEditor();
+ const before = editor.document;
+ const mcp = await connect(editor, { editsAllowed: false });
+ const result = await mcp.callTool({ name, arguments: args });
+ expect(result.isError).toBe(true);
+ expect(resultText(result)).toMatch(/consent_required/);
+ expect(editor.document).toBe(before);
+ expect(editor.applied).toHaveLength(0);
});
it("is listProjects, then exactly the in-app agent's tools with its descriptions, then the checkpoints and the media tools", async () => {
diff --git a/technical-documentation/architecture/ai-agent.md b/technical-documentation/architecture/ai-agent.md
index 17fddf104..a4489c235 100644
--- a/technical-documentation/architecture/ai-agent.md
+++ b/technical-documentation/architecture/ai-agent.md
@@ -41,7 +41,7 @@ The agent is built with LangChain's `createAgent`, not `deepagents`' `createDeep
## Tool schema
-The model never free-writes the project document. It can only call the fixed set of tools built by `deep-agent/service.ts#buildTools` (`OPENSCREEN_TOOL_NAMES`), described by `TOOL_DESCRIPTIONS` in the same file and validated against the Zod argument schemas in `agent-tools.ts`; the executor parses JSON, validates arguments, and returns either a new schema-valid snapshot or an error. Those three surfaces (descriptions, built tools, executor cases) are pinned to one another by `deep-agent/service.test.ts` — an earlier fourth surface, `AGENT_TOOL_SPECS`, described the tools in JSON Schema for a provider it had stopped reaching, and drifted. The tools operate on the same [document model](document-model.md) as manual editing.
+The model never free-writes the project document. `deep-agent/service.ts#buildTools` builds the tools from `TOOL_ARG_SCHEMAS`, the registry also used by both MCP transports. `TOOL_DESCRIPTIONS` supplies their descriptions, and the Zod argument schemas come from `agent-tools.ts`. The executor parses JSON, validates arguments, and returns either a new schema-valid snapshot or an error. These surfaces and the public `OPENSCREEN_TOOL_NAMES` list are pinned to one another by `deep-agent/service.test.ts`. The tools operate on the same [document model](document-model.md) and layout mutations as manual editing.
| Tool | What it does | What it mutates |
|---|---|---|
@@ -62,17 +62,25 @@ The model never free-writes the project document. It can only call the fixed set
| `setSpeed` | Moves, resizes, or changes an existing speed pill. | The corresponding `legacyEditor.speedRegions` fragments. |
| `addAnnotation` | Adds a text, arrow (`figure`) or privacy `blur` annotation over virtual timeline time. An arrow or a blur is the editor's text switched through `convertAnnotationKind`, then given its box and its `figureData` / `blurData` (validated by `figureDataSchema` / `blurDataSchema`). Images are left out: they need a file. A freehand blur is refused, as the inspector no longer offers it. | `annotations`. |
| `setAnnotation` | Moves, resizes, retypes or restyles an annotation. A type change goes through `convertAnnotationKind` with the footage under each fragment (`annotationFootageRect`), as the inspector's type switch does; `figureData` / `blurData` merge only the keys sent. Arguments of another kind (width on a text, blurData on an arrow) are refused. | The corresponding clip-anchored `annotations` fragments. |
-| `addCameraFullscreen` | Adds a camera-fullscreen region over virtual timeline time; refused when no clip under the span comes from an asset with a linked `cameraTrack`, since such a region can only render nothing. | `legacyEditor.cameraFullscreenRegions`. |
-| `setCameraFullscreen` | Moves or resizes a camera-fullscreen pill, under the same camera requirement as `addCameraFullscreen`. | The corresponding `legacyEditor.cameraFullscreenRegions` fragments. |
+| `addLayoutChange` | Changes the layout from a RAW document-timeline time until the next change. Camera presets require linked camera footage. | The shared `layoutRegions` partition, or the global preset when uniform. |
+| `setLayoutPreset` | Changes one passage by an ID from `getCurrentDocument.layoutSegments`; equal neighbours merge. An unchanged preset is refused. | The same layout partition. |
+| `moveLayoutBoundary` | Moves a passage's start in RAW seconds, updating both neighbours atomically. The first boundary is protected. | The same layout partition. |
| `addEffect` | Writes a shader effect (WGSL, [docs/shader-effects.md](../../docs/shader-effects.md)) and poses it over virtual timeline time, or poses one of the project's effects again by `effectId`. The compositor checks the code first (`checkShaderEffect`); code that does not compile is refused with the error at the agent's line. The result gives the cost tier and, when the tier runs long, a recommendation, never a refusal. One effect runs at a time: two cannot overlap. | `shaderEffects` (a new definition) and `shaderEffectRanges`. |
| `setEffect` | Moves, re-tunes (`values`) or retargets an effect pill. | The pill's `shaderEffectRanges` fragments. |
| `setEffectDefinition` | Changes an effect's name, description, code or parameters, and so every region using it; new code is checked first. A removed parameter's values drop. | The `shaderEffects` entry and its regions' `values`. |
| `getEffect` | Reads one effect in full, code included; `getCurrentDocument` lists effects without their code. | Read only. |
| `removeTrim` | Deletes a trim so its source span plays and exports again. | `timeline.trimRanges`. |
-| `removeModifier` | Resolves and deletes a zoom, speed, annotation, camera-fullscreen, audio, or effect pill by ID; `removedIds` reports every touching storage row deleted with that pill. | The matching modifier collection. |
+| `removeModifier` | Resolves and deletes a zoom, speed, annotation, layout change, audio, or effect by ID; `removedIds` reports the removed fragments. Removing a layout change extends the preceding passage; the first passage is protected. | The matching modifier collection. |
| `removeClip` | Deletes a placed clip, closes the gap, and drops effects anchored only to it. The result names the modifiers and trims it took with it. | Timeline clips and affected anchored modifiers. |
-Clips and trims use source time. Zoom, speed, annotation, camera-fullscreen, and effect tools use virtual edited-timeline time; the executor converts these spans to the clip-anchored millisecond representation used by the document.
+Clips and trims use source time. Zoom, speed, annotation and effect tools use virtual edited-timeline time; the executor converts these spans to the clip-anchored millisecond representation used by the document.
+
+Layout point tools use RAW document-timeline seconds. `getCurrentDocument.layoutSegments`
+exposes their logical IDs, bounds and presets. The document stores clip/source anchors;
+`buildSceneDescription` projects them into source time for native preview and export.
+Legacy Full Camera executors remain compatibility aliases over this partition and are
+not advertised to the model or MCP clients. Migration-only camera envelopes are derived
+from layout metadata rather than maintained as a second mutable collection.
## Checkpoints and undo
diff --git a/technical-documentation/architecture/mcp-server.md b/technical-documentation/architecture/mcp-server.md
index 8e8aac3b4..446212da4 100644
--- a/technical-documentation/architecture/mcp-server.md
+++ b/technical-documentation/architecture/mcp-server.md
@@ -250,5 +250,8 @@ camera footage. A temporary passage needs a change at its start and another at i
tools for a local edit. Compatibility Full Camera executor aliases are not advertised.
HTTP and stdio obtain schemas and descriptions from `TOOL_ARG_SCHEMAS` and
-`TOOL_DESCRIPTIONS`; mutation consent still goes through `executeAgentTool`. The MCP
-HTTP tests exercise a real client/server create/read/remove cycle on the shared document.
+`TOOL_DESCRIPTIONS`. Both call `runDocumentTool` and `executeAgentTool`; the shared MCP
+runner checks mutation consent with `isMutatingTool` before executing a write. The MCP
+HTTP tests exercise a real client/server add/move/set/read/remove cycle, compare the
+document and native scene with the UI's shared mutations, verify the preset enum and
+mutation annotations, and refuse all three layout writes when MCP edits are disabled.