Skip to content

Free open-source image editor for the browser

Add an experimental assistant that edits the photo through commands (#56)

Rodrigo Belfiore61abe30f1724
diff --git a/AGENTS.md b/AGENTS.md
index 743fffb..ad1a429 100644
--- a/AGENTS.md
+++ b/AGENTS.md
@@ -14,6 +14,8 @@ bunx --no-install playwright install chromium
The last command installs the Chromium build that the project's Playwright version expects; it is needed once per Playwright version for browser tests and scripted screenshots. `bun run test:browser` starts Vite itself and runs WebGPU on SwiftShader; reuse the launch flags in [playwright.config.ts](playwright.config.ts) for any other scripted browser.
+The experimental assistant shows only at `/?assistant` and answers through the [Vercel AI Gateway](https://vercel.com/docs/ai-gateway). `bun dev` serves the app alone; to try the assistant, run `vercel dev` with `AI_GATEWAY_API_KEY` in `.env.local`.
+
| Command | Checks |
| --- | --- |
| `bun run check` | Format and lint; writes fixes. |
diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md
index fc305a4..f509c8e 100644
--- a/ARCHITECTURE.md
+++ b/ARCHITECTURE.md
@@ -14,6 +14,8 @@ Where the code lives and the rules that keep it in place. Keep this file true: w
| `app/controls.ts` | `window.openlight`, documented in [API.md](API.md). |
| `app/commands.ts` | The commands: serializable, validated edits that `run`, WebMCP, and other callers share; see [API.md](API.md#commands). |
| `app/webmcp.ts` | WebMCP tools for browser agents, one per command; see [API.md](API.md#webmcp). |
+| `app/assistant` | Experimental chat that edits the photo, mounted only at `/?assistant`. `index.tsx` sends a message with a summary of the photo and runs the commands that come back; `jev.ts` asks the Jev evaluation model and turns its answers into commands. |
+| `/api`, at the repository root | Vercel functions: `assistant.ts` answers the chat through `jev.ts`; `vercel dev` serves them locally. |
| `app/editor/index.tsx` | The editor with a document: header, then a tool's canvas and controls in the layout. A tool's `View` replaces both. |
| `app/editor/empty.tsx` | The editor before a document opens: welcome or loading status, and the placeholder sidebar. |
| `app/editor/sidebar.tsx` | Sidebar sections in order: `EditorSidebar` with a document, `PlaceholderSidebar` without one. |
@@ -58,6 +60,11 @@ Dependencies point downward. Features do not import each other; `app/` connects
- UI controls, `window.openlight`, and commands call the same edits. A slider or curve gesture is one edit; cancelling restores the previous scene. Preview settings and navigation stay outside history.
- The engine owns GPU resources, rendering, and derived data such as histogram bins; frame data stays out of React state. Every resource owner disposes what it creates.
+## Assistant
+
+- The model, how it is asked, and its key stay on the server. The browser sends a message and a summary of the photo, and receives commands and a message, so another model can answer without changing the client.
+- `protocol.ts` validates both directions. Vercel runs `api/assistant.ts` unbundled in Node, so the modules it reaches import app modules only as types, and its own imports name their `.js` output instead of using `@/`.
+
## Rendering
- `core/renderer` defines passes as nodes (`node`, `pipeline`, `split`, `merge`); the graph owns intermediate textures and timing. Features declare passes; `app/editor/renderer.ts` connects them.
diff --git a/api/assistant.ts b/api/assistant.ts
new file mode 100644
index 0000000..a41dc4b
--- /dev/null
+++ b/api/assistant.ts
@@ -0,0 +1,35 @@
+import { createGateway } from "ai";
+// Vercel runs functions unbundled in Node, so these name their .js output instead of using `@/`.
+import { ask, interpret } from "../src/app/assistant/jev.js";
+import { assistantRequest } from "../src/app/assistant/protocol.js";
+
+const model = createGateway().evaluationModel("typesafe-ai/jev");
+
+/**
+ * Answers an `AssistantRequest` with an `AssistantResponse`. The model and how it is asked stay on
+ * the server; the gateway reads `AI_GATEWAY_API_KEY` or Vercel's OIDC token.
+ */
+export async function POST(request: Request) {
+ const body = assistantRequest.safeParse(
+ await request.json().catch(() => undefined),
+ );
+ if (!body.success) {
+ return Response.json({ error: "Invalid request." }, { status: 400 });
+ }
+ const { state, questions } = ask(body.data);
+ // A stalled model would otherwise hold the chat until the function times out.
+ const abortSignal = AbortSignal.timeout(15_000);
+ // The model directly: the SDK's evaluate rejects answers whose top options tie.
+ const evaluate = async () =>
+ model.doEvaluate({ state, questions, abortSignal });
+ try {
+ const { answers } = await evaluate().catch(evaluate);
+ return Response.json(interpret(body.data, answers));
+ } catch (error) {
+ console.error(error);
+ return Response.json(
+ { error: "The assistant is unavailable right now." },
+ { status: 502 },
+ );
+ }
+}
diff --git a/biome.json b/biome.json
index 3286a5a..a433d4f 100644
--- a/biome.json
+++ b/biome.json
@@ -28,5 +28,13 @@
}
}
}
- }
+ },
+ "overrides": [
+ {
+ "includes": ["api/**"],
+ "linter": {
+ "rules": { "style": { "noRestrictedImports": "off" } }
+ }
+ }
+ ]
}
diff --git a/bun.lock b/bun.lock
index 7d2529c..b89139b 100644
--- a/bun.lock
+++ b/bun.lock
@@ -12,6 +12,7 @@
"@tailwindcss/vite": "^4.3.3",
"@vercel/analytics": "^2.0.1",
"@vercel/speed-insights": "^2.0.0",
+ "ai": "^7.0.119",
"class-variance-authority": "^0.7.1",
"cn": "^0.4.0",
"idb": "^8.0.3",
@@ -39,6 +40,12 @@
},
},
"packages": {
+ "@ai-sdk/gateway": ["@ai-sdk/[email protected]", "", { "dependencies": { "@ai-sdk/provider": "4.0.19", "@ai-sdk/provider-utils": "5.0.50", "@vercel/oidc": "3.2.0" }, "peerDependencies": { "zod": "^3.25.76 || ^4.1.8" } }, "sha512-YFS9McgEDHXPwXOWlu1JCylkFyDsgPNLwH9jXgMxdiaur3k6SaIUPKCRSciQQ5LbEZOSb4JAaPYYO7B5Cvq7ow=="],
+
+ "@ai-sdk/provider": ["@ai-sdk/[email protected]", "", { "dependencies": { "json-schema": "^0.4.0" } }, "sha512-ryVZSSfmpEsr4bM8rbt9cSBWal+KGYRrm57C+k/hiPQHMiMahaqg2bPF0MpxSUWY8mGpt7s/ajM6PAqNTMKqPQ=="],
+
+ "@ai-sdk/provider-utils": ["@ai-sdk/[email protected]", "", { "dependencies": { "@ai-sdk/provider": "4.0.19", "@standard-schema/spec": "^1.1.0", "@workflow/serde": "4.1.0", "eventsource-parser": "^3.0.8", "undici": "^7.29.0" }, "peerDependencies": { "zod": "^3.25.76 || ^4.1.8" } }, "sha512-qg2LutM5RlUT/tuuFf+VkhFGNZ2sEJHL0oDIrFN/sMoVbDBANglqtGCJUOnBqGMoFBH8IfAXMrFZvbFWx4dMOg=="],
+
"@babel/runtime": ["@babel/[email protected]", "", {}, "sha512-Nq8OhGWiZIZGV6hLHoyAKLLcJihP/xFeBMGJoUrxTX2psI8dCifzLhZISFb+VWS3wFMRDmCGw5R+dOySCqPLhw=="],
"@base-ui/react": ["@base-ui/[email protected]", "", { "dependencies": { "@babel/runtime": "^7.29.7", "@base-ui/utils": "0.4.0", "@floating-ui/react-dom": "^2.1.9", "@floating-ui/utils": "^0.2.12", "use-sync-external-store": "^1.6.0" }, "peerDependencies": { "@date-fns/tz": "^1.2.0", "@types/react": "^17 || ^18 || ^19", "date-fns": "^4.0.0", "react": "^17 || ^18 || ^19", "react-dom": "^17 || ^18 || ^19" }, "optionalPeers": ["@date-fns/tz", "@types/react", "date-fns"] }, "sha512-P0/1sxo6SBVZOklKMIedvTWqw2s2IQzi9x5bIVsXu980cuSOD4NeuRSs+/L7LZQfDkZP/uRZyGPyfFl/B1oH+Q=="],
@@ -131,6 +138,8 @@
"@roprgm/ui": ["@roprgm/[email protected]", "", { "dependencies": { "@base-ui/react": "^1.8.0", "class-variance-authority": "^0.7.1", "cn": "^0.4.0" }, "peerDependencies": { "react": ">=19" } }, "sha512-UoR9Vf0VdzBpdQvEydw4b8Ew0nCP3Webu+dhFXb+3mKvDGJFt0MqbGRnJqhsGP121Xo8z5R8IPmh/ID8wTmNOg=="],
+ "@standard-schema/spec": ["@standard-schema/[email protected]", "", {}, "sha512-l2aFy5jALhniG5HgqrD6jXLi/rUWrKvqN/qJx6yoJsgKhblVd+iqqU4RCXavm/jPityDo5TCvKMnpjKnOriy0w=="],
+
"@tailwindcss/node": ["@tailwindcss/[email protected]", "", { "dependencies": { "@jridgewell/remapping": "^2.3.5", "enhanced-resolve": "^5.24.1", "jiti": "^2.7.0", "lightningcss": "1.32.0", "magic-string": "^0.30.21", "source-map-js": "^1.2.1", "tailwindcss": "4.3.3" } }, "sha512-/T8IKEsf9VTU6tLjgC7+sv2mOPtQxzE2jMw7u4Tt40Tx+QSZxpzh95/H6cMKoja9XuW7iMdLJYBB0o9G1CaAgg=="],
"@tailwindcss/oxide": ["@tailwindcss/[email protected]", "", { "optionalDependencies": { "@tailwindcss/oxide-android-arm64": "4.3.3", "@tailwindcss/oxide-darwin-arm64": "4.3.3", "@tailwindcss/oxide-darwin-x64": "4.3.3", "@tailwindcss/oxide-freebsd-x64": "4.3.3", "@tailwindcss/oxide-linux-arm-gnueabihf": "4.3.3", "@tailwindcss/oxide-linux-arm64-gnu": "4.3.3", "@tailwindcss/oxide-linux-arm64-musl": "4.3.3", "@tailwindcss/oxide-linux-x64-gnu": "4.3.3", "@tailwindcss/oxide-linux-x64-musl": "4.3.3", "@tailwindcss/oxide-wasm32-wasi": "4.3.3", "@tailwindcss/oxide-win32-arm64-msvc": "4.3.3", "@tailwindcss/oxide-win32-x64-msvc": "4.3.3" } }, "sha512-krXjAikiaFSPaK/FkAQT5UTx3VormQaiZ5hBFlJZ9UFQGB/rwg1MZIhHAG9smMQRTdyJxP6Qt5MwMtdyU5FWrA=="],
@@ -211,6 +220,8 @@
"@vercel/analytics": ["@vercel/[email protected]", "", { "peerDependencies": { "@remix-run/react": "^2", "@sveltejs/kit": "^1 || ^2", "next": ">= 13", "nuxt": ">= 3", "react": "^18 || ^19 || ^19.0.0-rc", "svelte": ">= 4", "vue": "^3", "vue-router": "^4" }, "optionalPeers": ["@remix-run/react", "@sveltejs/kit", "next", "nuxt", "react", "svelte", "vue", "vue-router"] }, "sha512-MTQG6V9qQrt1tsDeF+2Uoo5aPjqbVPys1xvnIftXSJYG2SrwXRHnqEvVoYID7BTruDz4lCd2Z7rM1BdkUehk2g=="],
+ "@vercel/oidc": ["@vercel/[email protected]", "", {}, "sha512-UycprH3T6n3jH0k44NHMa7pnFHGu/N05MjojYr+Mc6I7obkoLIJujSWwin1pCvdy/eOxrI/l3uDLQsmcrOb4ug=="],
+
"@vercel/speed-insights": ["@vercel/[email protected]", "", { "peerDependencies": { "@sveltejs/kit": "^1 || ^2", "next": ">= 13", "nuxt": ">= 3", "react": "^18 || ^19 || ^19.0.0-rc", "svelte": ">= 4", "vue": "^3", "vue-router": "^4" }, "optionalPeers": ["@sveltejs/kit", "next", "nuxt", "react", "svelte", "vue", "vue-router"] }, "sha512-jwkNcrTeafWxjmWq4AHBaptSqZiJkYU5adLC9QBSqeim0GcqDMgN5Ievh8OG1rJ6W3A4l1oiP7qr9CWxGuzu3w=="],
"@vgpu/adapter-mock": ["@vgpu/[email protected]", "", { "dependencies": { "@vgpu/core": "0.3.1" } }, "sha512-uZxTFPHLYd1nOxGxluvni50tsjnUTQ2eOhjJ/YwWynO7k906AD0byJxN0V6BbYb1MK/xM/EF+/cA8tKCrvY0yA=="],
@@ -227,8 +238,12 @@
"@webgpu/types": ["@webgpu/[email protected]", "", {}, "sha512-0cF7RFM2edNoiIS1ODJp0/Gzv4/xSXhwoR0YCza+OWpJWtn4wmo9DvK91aLlH9+uUnwIriP7ZiC3WitmyhuzBw=="],
+ "@workflow/serde": ["@workflow/[email protected]", "", {}, "sha512-pav4F2BoirECWR7Nf1TKt+2eETcBj7jj4cBefQ8VXQCA6NPkaKeLfj/zMgi+3zYV5ZIBT4GuUiphsj0/b9hPQQ=="],
+
"@xmldom/xmldom": ["@xmldom/[email protected]", "", {}, "sha512-5AXjrcMClTryPe9LgZrygpB1lj7s0S9E0+W+AHaVKAVyHanafK86iPSvG5xHVSp/jC+VH1UXu0TAEmY279xH7A=="],
+ "ai": ["[email protected]", "", { "dependencies": { "@ai-sdk/gateway": "4.0.97", "@ai-sdk/provider": "4.0.19", "@ai-sdk/provider-utils": "5.0.50" }, "peerDependencies": { "zod": "^3.25.76 || ^4.1.8" } }, "sha512-KlHbkKMmdb2g0DXdkhBXzSvLSkdoAg0MdShhTTOyuoHKJetEUlgjn2juZ05lHY97EqCwBuSK8dgiEeB0ty5xZw=="],
+
"bun-types": ["[email protected]", "", { "dependencies": { "@types/node": "*" } }, "sha512-loKuVrAFZKfEv+JvWkHRS9GW5IqLuLRjVXN9p+vZvBN86O5hf/pBZQ5hSoyipsrMmWObZBDvWnlmKvjKTM0PdA=="],
"class-variance-authority": ["[email protected]", "", { "dependencies": { "clsx": "^2.1.1" } }, "sha512-Ka+9Trutv7G8M6WT6SeiRWz792K5qEqIGEGzXKhAE6xOWAY6pPH8U+9IY3oCMv6kqTmLsv7Xh/2w2RigkePMsg=="],
@@ -245,6 +260,8 @@
"enhanced-resolve": ["[email protected]", "", { "dependencies": { "graceful-fs": "^4.2.4", "tapable": "^2.3.3" } }, "sha512-L1l8TNvomm6UVW5B253AGxQagSQr+vGwhMlrrfRS2qmhx46AMpMVJKQYLvWYbysTMY8VoicOvzHzoHMbyzB+4A=="],
+ "eventsource-parser": ["[email protected]", "", {}, "sha512-EKN1vKAMcZ8MlYMpaNuxN6R9yakzH6uajHcHVTqWJzvu5pWw9DyhbP35HH8MVBQ+dZjAfDxk+A8NiR9KWaXiyQ=="],
+
"fdir": ["[email protected]", "", { "peerDependencies": { "picomatch": "^3 || ^4" }, "optionalPeers": ["picomatch"] }, "sha512-tIbYtZbucOs0BRGqPJkshJUYdL+SDH7dVM8gjy+ERp3WAUjLEFJE+02kanyHtwjWOnwrKYBiwAmM0p4kLJAnXg=="],
"fsevents": ["[email protected]", "", { "os": "darwin" }, "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw=="],
@@ -255,6 +272,8 @@
"jiti": ["[email protected]", "", { "bin": { "jiti": "lib/jiti-cli.mjs" } }, "sha512-AC/7JofJvZGrrneWNaEnJeOLUx+JlGt7tNa0wZiRPT4MY1wmfKjt2+6O2p2uz2+skll8OZZmJMNqeke7kKbNgQ=="],
+ "json-schema": ["[email protected]", "", {}, "sha512-es94M3nTIfsEPisRafak+HDLfHXnKBhV3vU5eqPcS3flIWqcxJWgXHXiey3YrpaNsanY5ei1VoYEbOzijuq9BA=="],
+
"lightningcss": ["[email protected]", "", { "dependencies": { "detect-libc": "^2.0.3" }, "optionalDependencies": { "lightningcss-android-arm64": "1.33.0", "lightningcss-darwin-arm64": "1.33.0", "lightningcss-darwin-x64": "1.33.0", "lightningcss-freebsd-x64": "1.33.0", "lightningcss-linux-arm-gnueabihf": "1.33.0", "lightningcss-linux-arm64-gnu": "1.33.0", "lightningcss-linux-arm64-musl": "1.33.0", "lightningcss-linux-x64-gnu": "1.33.0", "lightningcss-linux-x64-musl": "1.33.0", "lightningcss-win32-arm64-msvc": "1.33.0", "lightningcss-win32-x64-msvc": "1.33.0" } }, "sha512-WkUDrojuJs0xkgGf2udWxa3yGBRxPtxUkB79i6aCZLRgc7PM8fZe9TosfPDcvEpQZbuFASnHYmRLBLUbmLOIIA=="],
"lightningcss-android-arm64": ["[email protected]", "", { "os": "android", "cpu": "arm64" }, "sha512-gEpRTalKdosp4Bb8qWtc2iOgE5SeIHlpS1up9bFq2wAyYhl1UdTObYiHe98zEM9SQvSoqQZ1IQD0JNpg3Ml5pg=="],
@@ -323,6 +342,8 @@
"typescript": ["[email protected]", "", { "optionalDependencies": { "@typescript/typescript-aix-ppc64": "7.0.2", "@typescript/typescript-darwin-arm64": "7.0.2", "@typescript/typescript-darwin-x64": "7.0.2", "@typescript/typescript-freebsd-arm64": "7.0.2", "@typescript/typescript-freebsd-x64": "7.0.2", "@typescript/typescript-linux-arm": "7.0.2", "@typescript/typescript-linux-arm64": "7.0.2", "@typescript/typescript-linux-loong64": "7.0.2", "@typescript/typescript-linux-mips64el": "7.0.2", "@typescript/typescript-linux-ppc64": "7.0.2", "@typescript/typescript-linux-riscv64": "7.0.2", "@typescript/typescript-linux-s390x": "7.0.2", "@typescript/typescript-linux-x64": "7.0.2", "@typescript/typescript-netbsd-arm64": "7.0.2", "@typescript/typescript-netbsd-x64": "7.0.2", "@typescript/typescript-openbsd-arm64": "7.0.2", "@typescript/typescript-openbsd-x64": "7.0.2", "@typescript/typescript-sunos-x64": "7.0.2", "@typescript/typescript-win32-arm64": "7.0.2", "@typescript/typescript-win32-x64": "7.0.2" }, "bin": { "tsc": "bin/tsc" } }, "sha512-8FYau96o3NKOhbjKi/qNvG/W5jhzxkbdm5sj9AbZ/5T5sWqn3hJgLfGx27sRKZWTvyzCP8dLRBTf5tBTSRVUNA=="],
+ "undici": ["[email protected]", "", {}, "sha512-dkrQXeHSaoamnItlYbmzG0wFYrM0ZwDxCIg0A7aKjTyyhh9svRzCNFEzV+Vm05/yehjCzjDZ31KXfGEjYSztDQ=="],
+
"undici-types": ["[email protected]", "", {}, "sha512-j375ScV60dom+YkPFIfTLcOiPxkN/buHz5GobjLhixFuANaNs3C9l4GmrWqejgXWJ7BbJcFYpTEUkS1Ge8bpZQ=="],
"use-sync-external-store": ["[email protected]", "", { "peerDependencies": { "react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0" } }, "sha512-6L+EeigHMQhdaIPNIFUKwfWJSwWFQ8gJbJ2DLOs5sDIegTwR9fRxvnM3uciHKjIZhFz+KAv2emhWMRvDmMcY8A=="],
diff --git a/package.json b/package.json
index bc1a6d1..2edca77 100644
--- a/package.json
+++ b/package.json
@@ -19,6 +19,7 @@
"@tailwindcss/vite": "^4.3.3",
"@vercel/analytics": "^2.0.1",
"@vercel/speed-insights": "^2.0.0",
+ "ai": "^7.0.119",
"class-variance-authority": "^0.7.1",
"cn": "^0.4.0",
"idb": "^8.0.3",
diff --git a/src/app/assistant/index.tsx b/src/app/assistant/index.tsx
new file mode 100644
index 0000000..533ae69
--- /dev/null
+++ b/src/app/assistant/index.tsx
@@ -0,0 +1,248 @@
+import { IconButton } from "@roprgm/ui/icon-button";
+import { Input } from "@roprgm/ui/input";
+import { ScrollArea } from "@roprgm/ui/scroll-area";
+import { Section, SectionAction, sections } from "@roprgm/ui/section";
+import { Tooltip, TooltipContent, TooltipTrigger } from "@roprgm/ui/tooltip";
+import { cn } from "cn";
+import { type FormEvent, useEffect, useRef, useState } from "react";
+import { flushSync } from "react-dom";
+import { z } from "zod/mini";
+import { runCommand } from "@/app/commands";
+import type { createControls } from "@/app/controls";
+import type { Workspace } from "@/app/workspace";
+import { CloseIcon } from "@/components/icons/close";
+import { InfoIcon } from "@/components/icons/info";
+import { SparklesIcon } from "@/components/icons/sparkles";
+import { parse } from "@/lib/parse";
+import {
+ type AssistantRequest,
+ assistantResponse,
+ type Photo,
+} from "./protocol";
+
+type Controls = ReturnType<typeof createControls>;
+type Message = { from: "user" | "assistant"; text: string };
+
+const unavailable = "The assistant is unavailable right now.";
+const limited =
+ "You've reached the assistant's usage limit. Try again in a little while.";
+const replaced =
+ "Another photo opened before the answer came, so I left it unedited.";
+const failure = z.object({ error: z.string() });
+
+function describePhoto(workspace: Workspace, controls: Controls): Photo {
+ const { adjustments, vignette, details, toneCurve, preview } =
+ controls.getState();
+ const document = workspace.getDocument();
+ const [image] = document.scene.getState().layers;
+ const [width, height] = document.resources.get(image.source).image.size;
+ return {
+ adjustments,
+ vignette: vignette.intensity,
+ clarity: details.clarity,
+ sharpening: details.sharpening,
+ toneCurve: [...toneCurve],
+ comparison: preview?.comparison ?? "edited",
+ sourceSize: [width, height],
+ };
+}
+
+async function ask(request: AssistantRequest) {
+ const response = await fetch("/api/assistant", {
+ method: "POST",
+ headers: { "content-type": "application/json" },
+ body: JSON.stringify(request),
+ });
+ // Vercel's firewall answers 429 once an address sends too many messages.
+ if (response.status === 429) {
+ throw Error(limited);
+ }
+ const body = await response.json().catch(() => undefined);
+ if (!response.ok) {
+ const error = failure.safeParse(body);
+ throw Error(error.success ? error.data.error : unavailable);
+ }
+ return parse(assistantResponse, body, "Invalid assistant response");
+}
+
+// Shown at once so focus can land, and hidden only once faded out.
+const appear =
+ "visible opacity-100 [transition:opacity_200ms_100ms,visibility_0s]";
+const vanish =
+ "invisible opacity-0 [transition:opacity_150ms,visibility_0s_150ms]";
+
+/** A chat that edits the open photo: the server turns each message into commands, and the last layer they touch is selected. */
+export function Assistant({
+ workspace,
+ controls,
+}: {
+ workspace: Workspace;
+ controls: Controls;
+}) {
+ const [open, setOpen] = useState(false);
+ const [messages, setMessages] = useState<Message[]>([]);
+ const [busy, setBusy] = useState(false);
+ const toggle = useRef<HTMLButtonElement>(null);
+ const input = useRef<HTMLInputElement>(null);
+ const newest = useRef<HTMLLIElement>(null);
+ const content = useRef<HTMLDivElement>(null);
+ const [height, setHeight] = useState(0);
+ // The card's height follows its content, so it grows smoothly as messages arrive too.
+ useEffect(() => {
+ const element = content.current;
+ if (!element) {
+ return;
+ }
+ const observer = new ResizeObserver(([entry]) =>
+ setHeight(entry.borderBoxSize[0].blockSize),
+ );
+ observer.observe(element);
+ return () => observer.disconnect();
+ }, []);
+ useEffect(() => {
+ if (messages.length > 0) {
+ newest.current?.scrollIntoView({ block: "nearest" });
+ }
+ }, [messages.length]);
+ const say = (from: Message["from"], text: string) =>
+ setMessages((messages) => [...messages, { from, text }]);
+
+ /** Opens or closes, moving focus to what shows once the state applies. */
+ function show(next: boolean) {
+ flushSync(() => setOpen(next));
+ (next ? input : toggle).current?.focus();
+ }
+
+ async function submit(event: FormEvent<HTMLFormElement>) {
+ event.preventDefault();
+ const form = event.currentTarget;
+ const message = new FormData(form).get("message")?.toString().trim();
+ if (!message || busy) {
+ return;
+ }
+ form.reset();
+ const earlier = messages
+ .filter(({ from }) => from === "user")
+ .slice(-3)
+ .map(({ text }) => text);
+ say("user", message);
+ setBusy(true);
+ try {
+ const document = workspace.getDocument();
+ const answer = await ask({
+ message,
+ earlier,
+ photo: describePhoto(workspace, controls),
+ });
+ // The answer describes the photo that was open when the message was sent.
+ if (workspace.getDocument() !== document) {
+ throw Error(replaced);
+ }
+ const results = answer.commands.map((command) =>
+ runCommand(workspace, command),
+ );
+ const layerId = results.findLast((result) => result.layerId)?.layerId;
+ if (layerId) {
+ controls.selectLayer(layerId);
+ }
+ say("assistant", answer.message);
+ } catch (error) {
+ say("assistant", error instanceof Error ? error.message : unavailable);
+ }
+ setBusy(false);
+ }
+
+ // One shape grows from the button into the card, over content pinned to its bottom-left
+ // corner, so nothing inside moves. It clips rather than hides overflow, so scrolling a message
+ // into view cannot scroll the shape.
+ return (
+ <div
+ style={open ? { height } : undefined}
+ className={cn(
+ "fixed bottom-3 left-14 z-50 overflow-clip surface-panel bg-level-4/95 backdrop-blur-sm transition-[width,height,border-radius] duration-300 ease-[cubic-bezier(0.2,0,0,1)] motion-reduce:transition-none max-md:hidden",
+ open ? "w-80 rounded-xl" : "size-10 rounded-[1.25rem]",
+ )}
+ >
+ <Tooltip>
+ <TooltipTrigger
+ render={
+ <button
+ ref={toggle}
+ type="button"
+ aria-label="Assistant"
+ onClick={() => show(true)}
+ className={cn(
+ "absolute bottom-0 left-0 grid size-10 place-items-center text-muted transition-colors hover:text-foreground",
+ open ? vanish : appear,
+ )}
+ >
+ <SparklesIcon className="size-4" />
+ </button>
+ }
+ />
+ <TooltipContent>Assistant</TooltipContent>
+ </Tooltip>
+ <div
+ ref={content}
+ className={cn(
+ sections({ lines: true }),
+ "absolute bottom-0 left-0 w-80",
+ open ? appear : vanish,
+ )}
+ >
+ <Section className="flex-row items-center">
+ <div className="flex flex-1 items-center gap-1">
+ <h2 className="font-medium">Assistant</h2>
+ <IconButton
+ label="The assistant is in beta, and its use is limited."
+ size="icon-sm"
+ >
+ <InfoIcon className="size-3.5" />
+ </IconButton>
+ </div>
+ <SectionAction>
+ <IconButton
+ label="Close"
+ shortcut="Esc"
+ size="icon"
+ onClick={() => show(false)}
+ >
+ <CloseIcon className="size-4" />
+ </IconButton>
+ </SectionAction>
+ </Section>
+ {messages.length > 0 && (
+ <ScrollArea fade className="*:max-h-60">
+ <ol className="flex flex-col gap-1 px-3.5 py-2.5">
+ {messages.map((message, index) => (
+ <li
+ key={index}
+ ref={index === messages.length - 1 ? newest : undefined}
+ className={
+ message.from === "user" ? "font-medium" : "text-muted"
+ }
+ >
+ {message.text}
+ </li>
+ ))}
+ </ol>
+ </ScrollArea>
+ )}
+ <Section>
+ <form aria-label="Assistant" onSubmit={submit}>
+ <Input
+ ref={input}
+ name="message"
+ aria-label="Message"
+ placeholder={busy ? "Editing…" : "Ask for an edit"}
+ autoComplete="off"
+ maxLength={500}
+ readOnly={busy}
+ onKeyDown={(event) => event.key === "Escape" && show(false)}
+ />
+ </form>
+ </Section>
+ </div>
+ </div>
+ );
+}
diff --git a/src/app/assistant/jev.ts b/src/app/assistant/jev.ts
new file mode 100644
index 0000000..0a65527
--- /dev/null
+++ b/src/app/assistant/jev.ts
@@ -0,0 +1,458 @@
+import type { Experimental_EvaluationQuestion as Question } from "ai";
+import { z } from "zod/mini";
+import type { Command } from "@/app/commands";
+import type { Adjustments } from "@/core/document";
+import type { AssistantRequest, AssistantResponse, Photo } from "./protocol";
+
+/**
+ * How the assistant asks Jev, an evaluation model: one call answers typed questions about the
+ * message in parallel, each by choosing among options, and the answers become commands.
+ */
+
+/** A control Jev sets by choosing one of its steps; `about` describes it among the commands and in its value question. */
+type Slider = {
+ about: string;
+ min: number;
+ max: number;
+ step: number;
+ read: (photo: Photo) => number;
+};
+
+/** An adjustment, from -100 to 100 in steps of 10 unless given. */
+function adjustment(
+ key: keyof Adjustments,
+ about: string,
+ limit = 100,
+ step = 10,
+): Slider & { key: keyof Adjustments } {
+ const read = (photo: Photo) => photo.adjustments[key];
+ return { key, about, min: -limit, max: limit, step, read };
+}
+
+/** Controls a mask can hold, so they can change one area. */
+const adjustments = {
+ exposure: adjustment(
+ "exposure",
+ "Exposure: make the whole image lighter or darker, in stops.",
+ 5,
+ 0.5,
+ ),
+ contrast: adjustment(
+ "contrast",
+ "Contrast: more or less difference between light and dark.",
+ ),
+ highlights: adjustment(
+ "highlights",
+ "Highlights: recover or brighten the bright areas.",
+ ),
+ shadows: adjustment("shadows", "Shadows: lift or deepen the dark areas."),
+ whites: adjustment("whites", "Whites: move the white point."),
+ blacks: adjustment("blacks", "Blacks: move the black point."),
+ temperature: adjustment(
+ "incrementalTemperature",
+ "Temperature: warmer (yellow) or cooler (blue).",
+ ),
+ tint: adjustment("incrementalTint", "Tint: greener or more magenta."),
+ vibrance: adjustment(
+ "vibrance",
+ "Vibrance: more or less color, gently, protecting vivid colors and skin. The usual choice for “more color”.",
+ ),
+ saturation: adjustment(
+ "saturation",
+ "Saturation: more or less color everywhere, down to black and white.",
+ ),
+};
+
+/** Controls on effect layers, which change the whole photo. */
+const effects = {
+ vignette: {
+ about: "Vignette: darker edges.",
+ min: 0,
+ max: 100,
+ step: 5,
+ read: (photo) => photo.vignette,
+ },
+ clarity: {
+ about: "Clarity: local contrast, punchier or softer texture.",
+ min: -100,
+ max: 100,
+ step: 10,
+ read: (photo) => photo.clarity,
+ },
+ sharpening: {
+ about: "Sharpening: crisper detail.",
+ min: 0,
+ max: 150,
+ step: 10,
+ read: (photo) => photo.sharpening,
+ },
+} satisfies Record<string, Slider>;
+
+const sliders: Record<string, Slider> = { ...adjustments, ...effects };
+
+type Tool = { about: string; commands: Command[]; message: string };
+
+const inverted = [
+ { x: 0, y: 1 },
+ { x: 1, y: 0 },
+];
+
+const crops = {
+ "1:1": "a square",
+ "4:5": "4:5 portrait, as for an Instagram post",
+ "3:2": "3:2 landscape",
+ "16:9": "16:9 widescreen",
+ "9:16": "9:16 vertical, as for a story",
+};
+
+/** Commands beside the sliders; only inverting depends on the photo, which it toggles. */
+function tools({ toneCurve }: Photo): Record<string, Tool> {
+ const isInverted =
+ toneCurve.length === 2 &&
+ toneCurve.every(
+ ({ x, y }, i) => x === inverted[i].x && y === inverted[i].y,
+ );
+ const cropTo = Object.entries(crops).map(([ratio, about]) => {
+ const [width, height] = ratio.split(":").map(Number);
+ const tool: Tool = {
+ about: `Crop to ${about}.`,
+ commands: [{ type: "set-crop", aspectRatio: width / height }],
+ message: `Cropped to ${ratio}`,
+ };
+ return [`crop-${ratio}`, tool];
+ });
+ return {
+ invert: {
+ about: "Invert the colors into a negative, or back.",
+ commands: [
+ isInverted
+ ? { type: "set-tone-curve" }
+ : { type: "set-tone-curve", points: inverted },
+ ],
+ message: isInverted ? "Colors restored" : "Colors inverted",
+ },
+ reset: {
+ about:
+ "Reset every edit at once, returning the whole photo to how it was opened. Resetting one control, such as “reset the contrast”, sets that control instead.",
+ commands: [{ type: "reset" }],
+ message: "Reset all edits",
+ },
+ undo: {
+ about: "Undo the last change.",
+ commands: [{ type: "undo" }],
+ message: "Undone",
+ },
+ redo: {
+ about: "Redo the last undone change.",
+ commands: [{ type: "redo" }],
+ message: "Redone",
+ },
+ original: {
+ about: "Show the original photo for comparison, keeping the edits.",
+ commands: [{ type: "set-preview", comparison: "original" }],
+ message: "Showing the original. Say “show my edits” to go back.",
+ },
+ split: {
+ about: "Compare before and after side by side.",
+ commands: [{ type: "set-preview", comparison: "split" }],
+ message: "Showing before and after",
+ },
+ edited: {
+ about: "Show the edited photo again after showing the original.",
+ commands: [{ type: "set-preview", comparison: "edited" }],
+ message: "Showing your edits",
+ },
+ ...Object.fromEntries(cropTo),
+ "crop-none": {
+ about: "Remove the crop.",
+ commands: [{ type: "set-crop" }],
+ message: "Crop removed",
+ },
+ };
+}
+
+const reply =
+ "Answer with a message instead of editing: a greeting, thanks, a question, an exact number, something the editor cannot do, or anything unrelated to this photo.";
+
+/** Canned answers: conversation first, then why a request cannot be done. */
+const replies: Record<string, string> = {
+ greeting:
+ "Hi! I can edit this photo for you. Try “make it warmer” or “darken the sky”.",
+ thanks: "You're welcome! Anything else you'd like to change?",
+ approval: "Great! Tell me if you'd like to change anything else.",
+ goodbye: "Bye! Your edits stay saved in this browser.",
+ capabilities:
+ "I can change exposure, contrast, highlights, shadows, whites, blacks, temperature, tint, vibrance, saturation, vignette, clarity, and sharpening, in the whole photo or the sky, ground, sides, or center. I can also crop, invert colors, compare before and after, reset, and undo.",
+ "how-to":
+ "Tell me what to change in plain words, like “brighter and warmer”, “more contrast in the sky”, or “crop it for Instagram”.",
+ identity:
+ "I'm Jev, a quick assistant that edits this photo from short requests.",
+ opinion:
+ "I can't judge photos, but I can change whatever you'd like to improve.",
+ export: "To save your photo, use Export in the top right.",
+ open: "To edit another photo, use the folder button next to the file name.",
+ "off-topic": "I can only help with editing this photo.",
+ "unsupported-edit":
+ "I can't do that edit yet. I can change light, color, detail, and vignette, crop, and invert colors.",
+ "exact-value":
+ "I can't set exact numbers yet; I move controls in steps. Try “a bit brighter” or “much more contrast”.",
+ "out-of-range": "That's beyond what this control allows.",
+ "several-areas":
+ "I can change one area at a time. Ask for the sky and the ground separately.",
+ unclear:
+ "I'm not sure what to change. Try something like “brighter”, “warmer”, or “more contrast”.",
+};
+
+const areas = {
+ photo: "The whole photo.",
+ top: "The top of the photo, such as the sky.",
+ bottom: "The bottom of the photo, such as the ground or foreground.",
+ left: "The left side of the photo.",
+ right: "The right side of the photo.",
+ center: "The center of the photo, such as the main subject.",
+};
+
+const counts: Record<string, number> = { one: 1, two: 2, three: 3 };
+
+function round(value: number) {
+ return Math.round(value * 100) / 100;
+}
+
+/** Words for a change's size, so a plain "brighter" lands on a moderate step. */
+function magnitude(change: number, { min, max }: Slider) {
+ const share = Math.abs(change) / (max - min);
+ if (share <= 0.05) {
+ return "slightly";
+ }
+ if (share <= 0.15) {
+ return "moderately";
+ }
+ return share <= 0.3 ? "strongly" : "extremely";
+}
+
+/** Every step from min to max, each described by its change from the current value. */
+function valueQuestion(slider: Slider, current: number): Question {
+ const { min, max, step } = slider;
+ const count = Math.round((max - min) / step);
+ const options = Array.from({ length: count + 1 }, (_, i) => {
+ const value = round(min + i * step);
+ const change = round(value - current);
+ const direction = change > 0 ? "more" : "less";
+ const notes = [
+ change
+ ? `${change > 0 ? "+" : ""}${change}, ${magnitude(change, slider)} ${direction}`
+ : "the current value",
+ value === 0 && "the default",
+ value === min && "the minimum",
+ value === max && "the maximum",
+ ].filter(Boolean);
+ // A fixed decimal keeps keys from sorting as integers, so options stay in order.
+ return [value.toFixed(1), notes.join(", ")];
+ });
+ return {
+ type: "choice",
+ instructions: `The value the request asks for. ${slider.about} It is now ${current}; each option notes its change from now. Without a stated amount, prefer a moderate change.`,
+ criteria: Object.fromEntries(options),
+ };
+}
+
+/**
+ * The state and questions for one message: how many changes it names, the command that fits best
+ * with the others ranked behind it, the area, a reply, and each slider's value in case it is chosen.
+ */
+export function ask({ message, earlier, photo }: AssistantRequest) {
+ const current = Object.fromEntries(
+ Object.entries(sliders).map(([id, slider]) => [id, slider.read(photo)]),
+ );
+ const commands = Object.fromEntries([
+ ...Object.entries({ ...sliders, ...tools(photo) }).map(
+ ([id, { about }]) => [id, about],
+ ),
+ ["reply", reply],
+ ]);
+ const values = Object.entries(sliders).map(([id, slider]) => [
+ `value-${id}`,
+ valueQuestion(slider, slider.read(photo)),
+ ]);
+ const questions: Record<string, Question> = {
+ count: {
+ type: "choice",
+ instructions: "How many different changes the request names.",
+ criteria: {
+ one: "One change, even if described with several words.",
+ two: "Two different changes, such as “warmer and darker”.",
+ three: "Three different changes.",
+ },
+ },
+ command: {
+ type: "choice",
+ instructions:
+ "The command that best fulfills the request; when it names several changes, each of them.",
+ criteria: commands,
+ },
+ area: {
+ type: "choice",
+ instructions: "Which part of the photo the change applies to.",
+ criteria: areas,
+ },
+ reply: {
+ type: "choice",
+ instructions: "The message that best answers the request.",
+ criteria: replies,
+ },
+ ...Object.fromEntries(values),
+ };
+ return {
+ state: { request: message, earlierRequests: earlier, current },
+ questions,
+ };
+}
+
+const answersSchema = z.record(
+ z.string(),
+ z.object({
+ choice: z.optional(z.string()),
+ probabilities: z.optional(z.record(z.string(), z.number())),
+ }),
+);
+type Answers = z.output<typeof answersSchema>;
+
+function choice(answers: Answers, id: string) {
+ const choice = answers[id]?.choice;
+ if (choice === undefined) {
+ throw Error(`Jev did not answer ${id}.`);
+ }
+ return choice;
+}
+
+/** The chosen command, then as many more sliders as the message names changes, by probability. */
+function chosen(answers: Answers) {
+ const first = choice(answers, "command");
+ const count = counts[choice(answers, "count")] ?? 1;
+ const others = Object.entries(answers.command?.probabilities ?? {})
+ .filter(([id]) => id !== first && id in sliders)
+ .sort((a, b) => b[1] - a[1])
+ .slice(0, count - 1)
+ .map(([id]) => id);
+ return [first, ...others];
+}
+
+type MaskInput = Extract<Command, { type: "add-mask" }>["mask"];
+
+function areaMask(area: string, size: readonly number[]): MaskInput {
+ // Whole source pixels, as a person would place the gradient.
+ const [width, height] = size.map(Math.round);
+ const at = (share: number, length: number) => Math.round(share * length);
+ const middle: [number, number] = [at(0.5, width), at(0.5, height)];
+ switch (area) {
+ case "top":
+ return {
+ kind: "linear",
+ start: [middle[0], 0],
+ end: [middle[0], at(0.55, height)],
+ };
+ case "bottom":
+ return {
+ kind: "linear",
+ start: [middle[0], height],
+ end: [middle[0], at(0.45, height)],
+ };
+ case "left":
+ return {
+ kind: "linear",
+ start: [0, middle[1]],
+ end: [at(0.55, width), middle[1]],
+ };
+ case "right":
+ return {
+ kind: "linear",
+ start: [width, middle[1]],
+ end: [at(0.45, width), middle[1]],
+ };
+ default:
+ return {
+ kind: "radial",
+ center: middle,
+ radius: [at(0.3, width), at(0.35, height)],
+ angle: 0,
+ feather: 0.5,
+ };
+ }
+}
+
+/** Commands that set the chosen sliders: adjustments in one edit, in a new mask for an area. */
+function edit(answers: Answers, photo: Photo, ids: string[]) {
+ const area = choice(answers, "area");
+ const changes = ids.map((id) => {
+ const current = sliders[id].read(photo);
+ const value = Number(choice(answers, `value-${id}`));
+ return { id, current, value };
+ });
+ const changed = new Map(changes.map(({ id, value }) => [id, value]));
+ const adjustmentChange: Partial<Adjustments> = {};
+ for (const [id, { key, read }] of Object.entries(adjustments)) {
+ const value = changed.get(id);
+ if (value !== undefined) {
+ // A mask starts neutral, so it takes the change rather than the value.
+ adjustmentChange[key] = area === "photo" ? value : value - read(photo);
+ }
+ }
+ const commands: Command[] = [];
+ if (Object.keys(adjustmentChange).length > 0) {
+ commands.push(
+ area === "photo"
+ ? { type: "set-adjustments", ...adjustmentChange }
+ : {
+ type: "add-mask",
+ mask: areaMask(area, photo.sourceSize),
+ adjustments: adjustmentChange,
+ },
+ );
+ }
+ const vignette = changed.get("vignette");
+ if (vignette !== undefined) {
+ commands.push({ type: "set-vignette", intensity: vignette });
+ }
+ const clarity = changed.get("clarity");
+ const sharpening = changed.get("sharpening");
+ if (clarity !== undefined || sharpening !== undefined) {
+ commands.push({ type: "set-details", clarity, sharpening });
+ }
+ const local = area !== "photo";
+ const summary = changes
+ .map(({ id, current, value }) => {
+ if (local && id in adjustments) {
+ const change = round(value - current);
+ return `${area} ${id} ${change > 0 ? "+" : ""}${change}`;
+ }
+ return `${id} ${current} → ${value}`;
+ })
+ .join(", ");
+ return { commands, message: summary };
+}
+
+/** Turns Jev's answers into commands and a message for the user. */
+export function interpret(
+ { photo }: AssistantRequest,
+ response: unknown,
+): AssistantResponse {
+ const answers = answersSchema.parse(response);
+ const [first, ...others] = chosen(answers);
+ if (first === "reply") {
+ return {
+ commands: [],
+ message: replies[choice(answers, "reply")] ?? replies.unclear,
+ };
+ }
+ const tool = tools(photo)[first];
+ const moved = [first, ...others].filter((id) => id in sliders);
+ const edits = moved.length > 0 ? edit(answers, photo, moved) : undefined;
+ const commands = [...(tool?.commands ?? []), ...(edits?.commands ?? [])];
+ const message = [tool?.message, edits?.message].filter(Boolean).join(" · ");
+ const editing = commands.some((command) => command.type !== "set-preview");
+ // An edit made while the original shows would look like it did nothing.
+ if (editing && photo.comparison === "original") {
+ commands.unshift({ type: "set-preview", comparison: "edited" });
+ }
+ return { commands, message: message || replies.unclear };
+}
diff --git a/src/app/assistant/protocol.ts b/src/app/assistant/protocol.ts
new file mode 100644
index 0000000..b443451
--- /dev/null
+++ b/src/app/assistant/protocol.ts
@@ -0,0 +1,46 @@
+import { z } from "zod/mini";
+import type { Adjustments } from "@/core/document";
+
+// Runs in the browser and in the serverless function, so it imports no app module at runtime.
+
+const value = z.number().check(z.refine(Number.isFinite, "Not finite"));
+
+/** The parts of the open photo the assistant reads. */
+const photo = z.object({
+ adjustments: z.object({
+ exposure: value,
+ contrast: value,
+ highlights: value,
+ shadows: value,
+ whites: value,
+ blacks: value,
+ incrementalTemperature: value,
+ incrementalTint: value,
+ vibrance: value,
+ saturation: value,
+ }) satisfies z.ZodMiniType<Adjustments>,
+ vignette: value,
+ clarity: value,
+ sharpening: value,
+ toneCurve: z.array(z.object({ x: value, y: value })).check(z.maxLength(64)),
+ comparison: z.enum(["edited", "original", "split"]),
+ sourceSize: z.tuple([value, value]),
+});
+
+const text = z.string().check(z.trim(), z.minLength(1), z.maxLength(500));
+
+/** A message to the assistant, with the user's earlier messages for context. */
+export const assistantRequest = z.object({
+ message: text,
+ earlier: z.array(text).check(z.maxLength(3)),
+ photo,
+});
+export type AssistantRequest = z.output<typeof assistantRequest>;
+export type Photo = AssistantRequest["photo"];
+
+/** The assistant's answer: commands to run in order, and a message for the user. */
+export const assistantResponse = z.object({
+ commands: z.array(z.unknown()),
+ message: z.string(),
+});
+export type AssistantResponse = z.output<typeof assistantResponse>;
diff --git a/src/app/index.tsx b/src/app/index.tsx
index ddcb3df..31f5f43 100644
--- a/src/app/index.tsx
+++ b/src/app/index.tsx
@@ -1,4 +1,5 @@
import { useStore } from "zustand";
+import { Assistant } from "@/app/assistant";
import { DraftNotice } from "@/app/draft/notice";
import { Editor } from "@/app/editor";
import { useWorkspace } from "@/app/workspace/use-workspace";
@@ -7,6 +8,9 @@ import { useFileDrop } from "@/hooks/use-file-drop";
/** Visiting /demo opens a bundled photo instead of the empty state. */
const startup = location.pathname === "/demo" ? "/images/demo.jpg" : undefined;
+/** The experimental assistant mounts only when the URL asks for it, as in `/?assistant`. */
+const assistant = new URLSearchParams(location.search).has("assistant");
+
export function App() {
const { workspace, controls, drafts } = useWorkspace(startup);
useFileDrop(controls.openFiles);
@@ -24,6 +28,9 @@ export function App() {
draft={recovery}
/>
{error && <DraftNotice error={error} onDismiss={drafts.dismiss} />}
+ {assistant && state.document && (
+ <Assistant workspace={workspace} controls={controls} />
+ )}
</>
);
}
diff --git a/src/components/icons/info.tsx b/src/components/icons/info.tsx
new file mode 100644
index 0000000..a13ca8d
--- /dev/null
+++ b/src/components/icons/info.tsx
@@ -0,0 +1,11 @@
+import { Icon, type IconProps } from "./icon";
+
+export function InfoIcon(props: IconProps) {
+ return (
+ <Icon {...props}>
+ <circle cx="12" cy="12" r="9" />
+ <path d="M12 11v5" />
+ <circle cx="12" cy="7.75" r="0.75" fill="currentColor" stroke="none" />
+ </Icon>
+ );
+}
diff --git a/src/components/icons/sparkles.tsx b/src/components/icons/sparkles.tsx
new file mode 100644
index 0000000..7c23b5d
--- /dev/null
+++ b/src/components/icons/sparkles.tsx
@@ -0,0 +1,11 @@
+import { Icon, type IconProps } from "./icon";
+
+export function SparklesIcon(props: IconProps) {
+ return (
+ <Icon {...props}>
+ <path d="M10 4c.7 3.4 2.6 5.3 6 6-3.4.7-5.3 2.6-6 6-.7-3.4-2.6-5.3-6-6 3.4-.7 5.3-2.6 6-6Z" />
+ <path d="M18 2.5c.3 1.3 1.2 2.2 2.5 2.5-1.3.3-2.2 1.2-2.5 2.5-.3-1.3-1.2-2.2-2.5-2.5 1.3-.3 2.2-1.2 2.5-2.5Z" />
+ <path d="M18 16c.2 1 .9 1.8 2 2-1.1.2-1.8 1-2 2-.2-1-.9-1.8-2-2 1.1-.2 1.8-1 2-2Z" />
+ </Icon>
+ );
+}
diff --git a/tests/assistant.e2e.ts b/tests/assistant.e2e.ts
new file mode 100644
index 0000000..ed489c3
--- /dev/null
+++ b/tests/assistant.e2e.ts
@@ -0,0 +1,106 @@
+import { expect, test } from "./fixtures";
+
+test("the assistant shows only at /?assistant, opens, runs the commands a message returns on the photo it was sent for, selects the layer they edit, replies, and closes", async ({
+ page,
+}) => {
+ const requests: unknown[] = [];
+ let release = () => {};
+ const released = new Promise<void>((resolve) => {
+ release = resolve;
+ });
+ await page.route("**/api/assistant", async (route) => {
+ requests.push(route.request().postDataJSON());
+ if (requests.length === 3) {
+ await released;
+ await route.fulfill({
+ json: {
+ commands: [{ type: "set-adjustments", exposure: 2 }],
+ message: "exposure 0 → 2",
+ },
+ });
+ return;
+ }
+ if (requests.length === 1) {
+ await route.fulfill({
+ json: {
+ commands: [
+ { type: "set-adjustments", exposure: -1 },
+ { type: "set-vignette", intensity: 40 },
+ ],
+ message: "exposure 0 → -1, vignette 0 → 40",
+ },
+ });
+ return;
+ }
+ await route.fulfill({
+ json: { commands: [], message: "You're welcome!" },
+ });
+ });
+ const open = async () => {
+ await page
+ .locator('input[type="file"]')
+ .setInputFiles("tests/fixtures/photo.svg");
+ await expect(
+ page.getByRole("button", { name: "Open an image or scene" }),
+ ).toBeVisible();
+ };
+ const button = page.getByRole("button", { name: "Assistant", exact: true });
+ await page.goto("/");
+ await open();
+ await expect(button).toBeHidden();
+
+ await page.goto("/?assistant");
+ await open();
+ await button.click();
+ const message = page.getByRole("textbox", { name: "Message", exact: true });
+ await expect(message).toBeFocused();
+
+ await message.fill("darker, with a vignette");
+ await message.press("Enter");
+ await expect(
+ page.getByText("exposure 0 → -1, vignette 0 → 40"),
+ ).toBeVisible();
+ expect(requests[0]).toMatchObject({
+ message: "darker, with a vignette",
+ earlier: [],
+ photo: { sourceSize: [1200, 800], comparison: "edited" },
+ });
+ const state = await page.evaluate(() => window.openlight.getState());
+ expect(state.adjustments.exposure).toBe(-1);
+ const vignette = state.scene?.layers.find(
+ (layer) => layer.kind === "vignette",
+ );
+ expect(vignette).toMatchObject({ vignette: { intensity: 40 } });
+ expect(state.selectedLayerId).toBe(vignette?.id);
+
+ await message.fill("thanks");
+ await message.press("Enter");
+ await expect(page.getByText("You're welcome!")).toBeVisible();
+ expect(requests[1]).toMatchObject({ earlier: ["darker, with a vignette"] });
+
+ // An answer that arrives after another photo opens leaves the new photo alone.
+ await message.fill("brighter");
+ await message.press("Enter");
+ await page
+ .locator('input[type="file"]')
+ .setInputFiles("tests/fixtures/tones.png");
+ await expect
+ .poll(() => page.evaluate(() => window.openlight.getState().file))
+ .toBe("tones.png");
+ release();
+ await expect(
+ page.getByText(
+ "Another photo opened before the answer came, so I left it unedited.",
+ ),
+ ).toBeVisible();
+ expect(
+ (await page.evaluate(() => window.openlight.getState())).adjustments
+ .exposure,
+ ).toBe(0);
+
+ await message.press("Escape");
+ await expect(message).toBeHidden();
+ await expect(
+ page.getByRole("button", { name: "Assistant", exact: true }),
+ ).toBeFocused();
+});
diff --git a/tests/assistant.test.ts b/tests/assistant.test.ts
new file mode 100644
index 0000000..e92cda8
--- /dev/null
+++ b/tests/assistant.test.ts
@@ -0,0 +1,126 @@
+import { expect, test } from "bun:test";
+import { ask, interpret } from "@/app/assistant/jev";
+import type { AssistantRequest } from "@/app/assistant/protocol";
+
+const request: AssistantRequest = {
+ message: "warmer and darker",
+ earlier: [],
+ photo: {
+ adjustments: {
+ exposure: 0.5,
+ contrast: 0,
+ highlights: 0,
+ shadows: 0,
+ whites: 0,
+ blacks: 0,
+ incrementalTemperature: 0,
+ incrementalTint: 0,
+ vibrance: 0,
+ saturation: 0,
+ },
+ vignette: 0,
+ clarity: 0,
+ sharpening: 0,
+ toneCurve: [
+ { x: 0, y: 0 },
+ { x: 1, y: 1 },
+ ],
+ comparison: "edited",
+ sourceSize: [300, 200],
+ },
+};
+
+/** Answers as Jev returns them: a choice per question, with the command's distribution. */
+function answers(
+ command: Record<string, number>,
+ choices: Record<string, string>,
+) {
+ const [first] = Object.entries(command).sort((a, b) => b[1] - a[1]);
+ return {
+ command: { type: "choice", choice: first[0], probabilities: command },
+ ...Object.fromEntries(
+ Object.entries(choices).map(([id, choice]) => [
+ id,
+ { type: "choice", choice },
+ ]),
+ ),
+ };
+}
+
+test("the assistant asks one question per decision and turns the answers into commands", () => {
+ const { state, questions } = ask(request);
+ expect(state.current.exposure).toBe(0.5);
+ expect(Object.keys(questions)).toContain("value-temperature");
+ expect(questions.command).toMatchObject({
+ type: "choice",
+ criteria: { reset: expect.any(String), reply: expect.any(String) },
+ });
+
+ expect(
+ interpret(
+ request,
+ answers(
+ { temperature: 0.6, reply: 0.2, exposure: 0.15, vignette: 0.05 },
+ {
+ count: "two",
+ area: "photo",
+ "value-temperature": "30.0",
+ "value-exposure": "-1.0",
+ },
+ ),
+ ),
+ ).toEqual({
+ commands: [
+ { type: "set-adjustments", exposure: -1, incrementalTemperature: 30 },
+ ],
+ message: "temperature 0 → 30, exposure 0.5 → -1",
+ });
+
+ const sky = interpret(
+ { ...request, photo: { ...request.photo, comparison: "original" } },
+ answers(
+ { exposure: 0.9, vignette: 0.1 },
+ { count: "one", area: "top", "value-exposure": "-0.5" },
+ ),
+ );
+ expect(sky.commands).toEqual([
+ { type: "set-preview", comparison: "edited" },
+ {
+ type: "add-mask",
+ mask: { kind: "linear", start: [150, 0], end: [150, 110] },
+ adjustments: { exposure: -1 },
+ },
+ ]);
+ expect(sky.message).toBe("top exposure -1");
+
+ const inverted = {
+ ...request,
+ photo: {
+ ...request.photo,
+ toneCurve: [
+ { x: 0, y: 1 },
+ { x: 1, y: 0 },
+ ],
+ },
+ };
+ expect(interpret(inverted, answers({ invert: 1 }, { count: "one" }))).toEqual(
+ {
+ commands: [{ type: "set-tone-curve" }],
+ message: "Colors restored",
+ },
+ );
+
+ expect(
+ interpret(
+ request,
+ answers(
+ { reply: 0.9, exposure: 0.1 },
+ { count: "one", reply: "greeting" },
+ ),
+ ),
+ ).toEqual({
+ commands: [],
+ message:
+ "Hi! I can edit this photo for you. Try “make it warmer” or “darken the sky”.",
+ });
+});
diff --git a/tsconfig.json b/tsconfig.json
index 816ca13..6393540 100644
--- a/tsconfig.json
+++ b/tsconfig.json
@@ -14,5 +14,5 @@
"@/*": ["./src/*"]
}
},
- "include": ["src", "tests"]
+ "include": ["src", "tests", "api"]
}