Skip to content

Free open-source image editor for the browser

Expose editing commands to browser agents as WebMCP tools (#54)

Rodrigo Belfiore321baa842c18
diff --git a/API.md b/API.md
index dc9f943..1f0bcee 100644
--- a/API.md
+++ b/API.md
@@ -171,3 +171,20 @@ On a fresh load the start screen shows a notice in the viewport's corner with **
`getState()` returns a detached snapshot containing `file`, `failure`, `documentId`, `scene`, `selectedLayerId`, `size`, `frame`, `adjustments`, `whiteBalance`, `details`, `toneCurve`, `colorMixer`, `vignette`, `preview`, and `history`. `scene` contains `frame` and the layer tree. The top-level adjustment, tone-curve, and white-balance values come from `scene.layers[0]`; details, color-mixer, and vignette values describe the first matching root layer from the bottom, or its defaults. `colorMixer` contains eight-value `hue`, `saturation`, and `luminance` arrays in the color order above, defaulting to zero. `whiteBalance` contains absolute temperature/tint for RAW sources and is undefined for other sources. `file` is the open document's filename, or without one the file loading or failed, `failure` is the last file that failed to open and its error until another opens or the notice is dismissed, and `history` contains `undoCount`, `redoCount`, and `editing`, which is true while a group is open.
Without a document, `documentId`, `size`, `frame`, and `preview` are undefined. Adjustments, details, and the tone curve use their defaults, and history counts are zero. Mutating the snapshot does not edit the document.
+
+## WebMCP
+
+In browsers with [WebMCP](https://webmachinelearning.github.io/webmcp/), OpenLight also registers tools on `document.modelContext`, so a browser agent can edit the open photo. Chrome offers WebMCP from version 149 through an origin trial or `chrome://flags/#enable-webmcp-testing`. Tools validate their JSON input against schemas generated from the same models as the commands above, make the same undoable edits, and return errors as text, so the agent can correct its input.
+
+| Tool | Behavior |
+| --- | --- |
+| `get_state` | Returns the file, source size, frame, layers, comparison, and history; brush strokes and healing patches are counted rather than listed. |
+| `set_adjustments`, `set_tone_curve` | Call `setAdjustments` and `setToneCurve`; `layerId` addresses a mask instead of the image. |
+| `set_white_balance`, `set_color_mixer`, `set_details`, `set_vignette` | Call the matching command on the image or the first root effect. |
+| `add_mask` | Adds a mask layer with the given mask on top of the stack as one edit and returns its `layerId`. |
+| `delete_layer` | Calls `deleteLayer`. |
+| `set_crop` | Replaces the frame with the largest centered crop of the source at `aspectRatio`, straightened by −45 to 45 degrees. |
+| `undo`, `redo` | Call the matching command and return the history. |
+| `set_preview` | Sets the preview `comparison`. |
+
+WebMCP passes only JSON, so the tools neither open nor export files: the agent edits the photo the user opened.
diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md
index 148cf75..d8e5942 100644
--- a/ARCHITECTURE.md
+++ b/ARCHITECTURE.md
@@ -12,6 +12,7 @@ Where the code lives and the rules that keep it in place. Keep this file true: w
| `app/loaders` | Image, scene file, and Camera Raw XMP loaders. |
| `app/draft` | The draft autosaved in the browser. |
| `app/controls.ts` | `window.openlight`, documented in [API.md](API.md). |
+| `app/webmcp.ts` | WebMCP tools for browser agents, built on the controls; see [API.md](API.md#webmcp). |
| `app/editor/index.tsx` | The editor with a document: header, then a tool's canvas and controls in the layout. A tool's `View` replaces both. |
| `app/editor/empty.tsx` | The editor before a document opens: welcome or loading status, and the placeholder sidebar. |
| `app/editor/sidebar.tsx` | Sidebar sections in order: `EditorSidebar` with a document, `PlaceholderSidebar` without one. |
diff --git a/playwright.config.ts b/playwright.config.ts
index cf14fb4..b96ad57 100644
--- a/playwright.config.ts
+++ b/playwright.config.ts
@@ -1,17 +1,16 @@
import { defineConfig } from "@playwright/test";
const args = ["--enable-unsafe-webgpu"];
+const features = ["WebMCP"];
if (process.platform !== "darwin") {
args.push("--use-webgpu-adapter=swiftshader");
}
if (process.platform === "linux") {
// Vulkan enables image transfers; ANGLE's SwiftShader path presents canvases in headless Chromium.
- args.push(
- "--enable-features=Vulkan",
- "--use-angle=swiftshader",
- "--use-vulkan=swiftshader",
- );
+ features.push("Vulkan");
+ args.push("--use-angle=swiftshader", "--use-vulkan=swiftshader");
}
+args.push(`--enable-features=${features.join(",")}`);
export default defineConfig({
testDir: "./tests",
diff --git a/src/app/webmcp.ts b/src/app/webmcp.ts
new file mode 100644
index 0000000..3b96db4
--- /dev/null
+++ b/src/app/webmcp.ts
@@ -0,0 +1,205 @@
+import { z } from "zod/mini";
+import type { EditorDocument } from "@/core/document";
+import { imageFrame } from "@/core/image/frame";
+import { adjustmentsSchema } from "@/features/adjustments/model";
+import { mixerChange, mixerColor } from "@/features/color-mixer/model";
+import { fitRatio, rotate } from "@/features/crop/geometry";
+import { detailsSchema } from "@/features/details/model";
+import { addLayer } from "@/features/layers/edits";
+import { maskSchema } from "@/features/layers/model";
+import { curveSchema } from "@/features/tone-curves/curve";
+import { vignetteSchema } from "@/features/vignette/model";
+import { change, parse, range } from "@/lib/parse";
+import type { createControls } from "./controls";
+import { createMask } from "./editor/layers";
+import type { Workspace } from "./workspace";
+
+type Controls = ReturnType<typeof createControls>;
+
+type ModelContextTool = {
+ name: string;
+ description: string;
+ inputSchema: object;
+ annotations?: { readOnlyHint?: boolean };
+ execute: (input: unknown) => Promise<unknown>;
+};
+
+type RegisteredTool = { name: string; description: string };
+
+declare global {
+ interface Document {
+ readonly modelContext?: {
+ registerTool(
+ tool: ModelContextTool,
+ options?: { signal?: AbortSignal },
+ ): Promise<void>;
+ getTools(): Promise<RegisteredTool[]>;
+ executeTool(tool: RegisteredTool, input: string): Promise<string>;
+ };
+ }
+}
+
+function defineTool<S extends z.ZodMiniType>(
+ name: string,
+ description: string,
+ input: S,
+ run: (input: z.output<S>) => unknown,
+ annotations?: ModelContextTool["annotations"],
+): ModelContextTool {
+ return {
+ name,
+ description,
+ inputSchema: z.toJSONSchema(input, { io: "input" }),
+ annotations,
+ async execute(value) {
+ try {
+ return run(parse(input, value, "Invalid input")) ?? "Done.";
+ } catch (error) {
+ // Returned, since browsers pass a thrown error to the agent without its message.
+ return String(error);
+ }
+ },
+ };
+}
+
+function sourceSize(document: EditorDocument) {
+ const [image] = document.scene.getState().layers;
+ return document.resources.get(image.source).image.size;
+}
+
+/** The open photo as an agent reads it; brush strokes and healing patches are counted, not listed. */
+function describe(workspace: Workspace, controls: Controls) {
+ const { file, failure, scene, selectedLayerId, preview, history } =
+ controls.getState();
+ const document = workspace.state.getState().document;
+ const state = {
+ file,
+ failure,
+ sourceSize: document && sourceSize(document),
+ frame: scene?.frame,
+ layers: scene?.layers,
+ selectedLayerId,
+ comparison: preview?.comparison,
+ history,
+ };
+ return JSON.stringify(state, (key, value) =>
+ Array.isArray(value) && (key === "strokes" || key === "patches")
+ ? `${value.length} omitted`
+ : value,
+ );
+}
+
+const none = z.strictObject({});
+const layerId = z.optional(z.string());
+
+/** Registers the editor's commands as WebMCP tools for browser agents until `signal` aborts. */
+export function registerTools(
+ workspace: Workspace,
+ controls: Controls,
+ signal: AbortSignal,
+) {
+ const context = document.modelContext;
+ if (!context) {
+ return;
+ }
+ const tools = [
+ defineTool(
+ "get_state",
+ "Describes the open photo: its file, source size in pixels, crop frame, layers from bottom to top with their IDs and settings, and undo history. Color mixer arrays list red, orange, yellow, green, aqua, blue, purple, and magenta. Call it first; when no photo is open, ask the user to open one.",
+ none,
+ () => describe(workspace, controls),
+ { readOnlyHint: true },
+ ),
+ defineTool(
+ "set_adjustments",
+ "Sets basic adjustments, keeping the ones omitted. Exposure is in stops from -5 to 5; the others go from -100 to 100, where 0 is neutral. Without layerId they tone the whole photo; with a mask's layerId they apply inside that mask.",
+ z.extend(change(adjustmentsSchema), { layerId }),
+ ({ layerId, ...adjustments }) =>
+ controls.setAdjustments(adjustments, layerId),
+ ),
+ defineTool(
+ "set_tone_curve",
+ "Replaces the tone curve: points map input x to output y, from shadows at x 0 to highlights at x 1, ordered by x. Omit points to reset it. Without layerId it tones the whole photo after its adjustments; with a mask's layerId it applies inside that mask.",
+ z.strictObject({ points: z.optional(curveSchema), layerId }),
+ ({ points, layerId }) => controls.setToneCurve(points, layerId),
+ ),
+ defineTool(
+ "set_white_balance",
+ "Sets a RAW photo's white balance: temperature in Kelvin from 2000 to 25000 and tint from -150 to 150. Other photos have no white balance; warm or cool them with set_adjustments' incrementalTemperature and incrementalTint.",
+ z.partial(z.strictObject({ temperature: z.number(), tint: z.number() })),
+ (change) => controls.setWhiteBalance(change),
+ ),
+ defineTool(
+ "set_color_mixer",
+ "Shifts one color range's hue, saturation, or luminance from -100 to 100, keeping other colors and omitted values. Neutral grays are unaffected.",
+ z.extend(mixerChange, { color: mixerColor }),
+ ({ color, ...change }) => controls.setColorMixer(color, change),
+ ),
+ defineTool(
+ "set_details",
+ "Sets clarity (local contrast, -100 to 100), sharpening (0 to 150), and sharpenRadius (0.5 to 3 pixels), keeping omitted values.",
+ change(detailsSchema),
+ (change) => controls.setDetails(change),
+ ),
+ defineTool(
+ "set_vignette",
+ "Darkens the photo's edges: intensity from 0 (off) to 100 and softness from 0 to 100, keeping omitted values.",
+ change(vignetteSchema),
+ (change) => controls.setVignette(change),
+ ),
+ defineTool(
+ "add_mask",
+ "Adds a mask layer and returns its layerId; give it adjustments or a tone curve with that layerId. Coordinates are source pixels (see sourceSize), unaffected by crop. A linear mask covers fully at start and fades out at end; a radial mask covers an ellipse around center with radius [x, y], angle in degrees, and feather from 0 to 1.",
+ z.strictObject({ mask: maskSchema }),
+ ({ mask }) => ({
+ layerId: addLayer(workspace.getDocument(), createMask(mask)),
+ }),
+ ),
+ defineTool(
+ "delete_layer",
+ "Removes a layer and the layers inside it.",
+ z.strictObject({ layerId: z.string() }),
+ ({ layerId }) => controls.deleteLayer(layerId),
+ ),
+ defineTool(
+ "set_crop",
+ "Replaces the crop, rotation, and flips with the largest centered crop of the photo. aspectRatio is width / height and defaults to the photo's; straighten rotates by -45 to 45 degrees. Omit both to remove the crop.",
+ z.strictObject({
+ aspectRatio: z.optional(z.number().check(z.positive())),
+ straighten: z.optional(range(-45, 45)),
+ }),
+ ({ aspectRatio, straighten = 0 }) => {
+ const size = sourceSize(workspace.getDocument());
+ const frame = rotate(imageFrame(size), straighten, size);
+ controls.setFrame(aspectRatio ? fitRatio(frame, aspectRatio) : frame);
+ },
+ ),
+ defineTool(
+ "undo",
+ "Undoes the last edit and returns the history.",
+ none,
+ () => {
+ controls.undo();
+ return controls.getState().history;
+ },
+ ),
+ defineTool(
+ "redo",
+ "Redoes the last undone edit and returns the history.",
+ none,
+ () => {
+ controls.redo();
+ return controls.getState().history;
+ },
+ ),
+ defineTool(
+ "set_preview",
+ "Shows the edited photo, the original, or a split comparison of both, without editing the photo.",
+ z.strictObject({ comparison: z.enum(["edited", "original", "split"]) }),
+ ({ comparison }) => controls.setPreview({ comparison }),
+ ),
+ ];
+ for (const tool of tools) {
+ void context.registerTool(tool, { signal });
+ }
+}
diff --git a/src/app/workspace/use-workspace.ts b/src/app/workspace/use-workspace.ts
index a8ed6e3..ff2d325 100644
--- a/src/app/workspace/use-workspace.ts
+++ b/src/app/workspace/use-workspace.ts
@@ -4,10 +4,11 @@ import { createControls } from "@/app/controls";
import { createAutosave } from "@/app/draft/autosave";
import { createDraftSession } from "@/app/draft/session";
import { createDraftStore } from "@/app/draft/store";
+import { registerTools } from "@/app/webmcp";
import { createWorkspace } from ".";
/**
- * Connects an imperative workspace, its browser commands, and draft autosave to the app lifetime.
+ * Connects an imperative workspace, its browser commands and agent tools, and draft autosave to the app lifetime.
* A startup URL opens while the session is created, so the first paint is already loading; otherwise a stored draft is offered.
*/
export function useWorkspace(startup?: string) {
@@ -28,7 +29,10 @@ export function useWorkspace(startup?: string) {
const { workspace, store, controls, drafts } = session;
const autosave = createAutosave(workspace, store, drafts.report);
window.openlight = controls;
+ const tools = new AbortController();
+ registerTools(workspace, controls, tools.signal);
return () => {
+ tools.abort();
Reflect.deleteProperty(window, "openlight");
autosave.dispose();
workspace.dispose();
diff --git a/tests/webmcp.e2e.ts b/tests/webmcp.e2e.ts
new file mode 100644
index 0000000..f5a6738
--- /dev/null
+++ b/tests/webmcp.e2e.ts
@@ -0,0 +1,76 @@
+import type { Page } from "@playwright/test";
+import { expect, test } from "./fixtures";
+import { readImage } from "./images";
+
+/** Calls a tool as a browser agent does, with its input as JSON. */
+function callTool(page: Page, name: string, input: object = {}) {
+ return page.evaluate(
+ async ([name, input]) => {
+ const context = document.modelContext;
+ if (!context) throw new Error("WebMCP is unavailable.");
+ const tools = await context.getTools();
+ const tool = tools.find((tool) => tool.name === name);
+ if (!tool) throw new Error(`There is no ${name} tool.`);
+ return context.executeTool(tool, JSON.stringify(input));
+ },
+ [name, input] as const,
+ );
+}
+
+test("a browser agent edits, masks, crops, and undoes through WebMCP tools", async ({
+ page,
+}) => {
+ await page.goto("/");
+ await page.waitForFunction(
+ async () => (await document.modelContext?.getTools())?.length,
+ );
+ expect(await callTool(page, "set_adjustments", { exposure: 1 })).toBe(
+ "Error: Load an image before editing.",
+ );
+ await page.locator('input[type="file"]').setInputFiles({
+ name: "gray.svg",
+ mimeType: "image/svg+xml",
+ buffer: Buffer.from(
+ '<svg xmlns="http://www.w3.org/2000/svg" width="128" height="64"><rect width="128" height="64" fill="#808080"/></svg>',
+ ),
+ });
+ await expect
+ .poll(async () => JSON.parse(await callTool(page, "get_state")).sourceSize)
+ .toEqual([128, 64]);
+ const gray = await readImage(page);
+
+ expect(await callTool(page, "set_adjustments", { exposure: 9 })).toMatch(
+ /exposure/,
+ );
+ expect(await callTool(page, "set_adjustments", { exposure: 1 })).toBe(
+ "Done.",
+ );
+ const brighter = await readImage(page);
+ expect(brighter.center[0]).toBeGreaterThan(gray.center[0]);
+
+ const { layerId } = JSON.parse(
+ await callTool(page, "add_mask", {
+ mask: { kind: "linear", start: [0, 0], end: [0, 32] },
+ }),
+ );
+ await callTool(page, "set_adjustments", { layerId, exposure: -2 });
+ const masked = await readImage(page, undefined, [
+ [64, 2],
+ [64, 60],
+ ]);
+ expect(masked.samples?.[0][0]).toBeLessThan(gray.center[0]);
+ expect(masked.samples?.[1]).toEqual(brighter.center);
+
+ await callTool(page, "set_crop", { aspectRatio: 1 });
+ expect((await readImage(page)).size).toEqual([64, 64]);
+
+ expect(JSON.parse(await callTool(page, "undo"))).toMatchObject({
+ undoCount: 3,
+ redoCount: 1,
+ });
+ await callTool(page, "undo");
+ await callTool(page, "undo");
+ expect(await readImage(page)).toEqual(brighter);
+ await callTool(page, "undo");
+ expect(await readImage(page)).toEqual(gray);
+});