Free open-source image editor for the browser
Add an experimental assistant that edits the photo through commands (#56)
Rodrigo Belfiore61abe30f1724
diff --git a/AGENTS.md b/AGENTS.mdindex 743fffb..ad1a429 100644--- a/AGENTS.md+++ b/AGENTS.md@@ -14,6 +14,8 @@ bunx --no-install playwright install chromiumThe last command installs the Chromium build that the project's Playwright version expects; it is needed once per Playwright version for browser tests and scripted screenshots. `bun run test:browser` starts Vite itself and runs WebGPU on SwiftShader; reuse the launch flags in [playwright.config.ts](playwright.config.ts) for any other scripted browser.+The experimental assistant shows only at `/?assistant` and answers through the [Vercel AI Gateway](https://vercel.com/docs/ai-gateway). `bun dev` serves the app alone; to try the assistant, run `vercel dev` with `AI_GATEWAY_API_KEY` in `.env.local`.+| Command | Checks || --- | --- || `bun run check` | Format and lint; writes fixes. |diff --git a/ARCHITECTURE.md b/ARCHITECTURE.mdindex fc305a4..f509c8e 100644--- a/ARCHITECTURE.md+++ b/ARCHITECTURE.md@@ -14,6 +14,8 @@ Where the code lives and the rules that keep it in place. Keep this file true: w| `app/controls.ts` | `window.openlight`, documented in [API.md](API.md). || `app/commands.ts` | The commands: serializable, validated edits that `run`, WebMCP, and other callers share; see [API.md](API.md#commands). || `app/webmcp.ts` | WebMCP tools for browser agents, one per command; see [API.md](API.md#webmcp). |+| `app/assistant` | Experimental chat that edits the photo, mounted only at `/?assistant`. `index.tsx` sends a message with a summary of the photo and runs the commands that come back; `jev.ts` asks the Jev evaluation model and turns its answers into commands. |+| `/api`, at the repository root | Vercel functions: `assistant.ts` answers the chat through `jev.ts`; `vercel dev` serves them locally. || `app/editor/index.tsx` | The editor with a document: header, then a tool's canvas and controls in the layout. A tool's `View` replaces both. || `app/editor/empty.tsx` | The editor before a document opens: welcome or loading status, and the placeholder sidebar. || `app/editor/sidebar.tsx` | Sidebar sections in order: `EditorSidebar` with a document, `PlaceholderSidebar` without one. |@@ -58,6 +60,11 @@ Dependencies point downward. Features do not import each other; `app/` connects- UI controls, `window.openlight`, and commands call the same edits. A slider or curve gesture is one edit; cancelling restores the previous scene. Preview settings and navigation stay outside history.- The engine owns GPU resources, rendering, and derived data such as histogram bins; frame data stays out of React state. Every resource owner disposes what it creates.+## Assistant++- The model, how it is asked, and its key stay on the server. The browser sends a message and a summary of the photo, and receives commands and a message, so another model can answer without changing the client.+- `protocol.ts` validates both directions. Vercel runs `api/assistant.ts` unbundled in Node, so the modules it reaches import app modules only as types, and its own imports name their `.js` output instead of using `@/`.+## Rendering- `core/renderer` defines passes as nodes (`node`, `pipeline`, `split`, `merge`); the graph owns intermediate textures and timing. Features declare passes; `app/editor/renderer.ts` connects them.diff --git a/api/assistant.ts b/api/assistant.tsnew file mode 100644index 0000000..a41dc4b--- /dev/null+++ b/api/assistant.ts@@ -0,0 +1,35 @@+import { createGateway } from "ai";+// Vercel runs functions unbundled in Node, so these name their .js output instead of using `@/`.+import { ask, interpret } from "../src/app/assistant/jev.js";+import { assistantRequest } from "../src/app/assistant/protocol.js";++const model = createGateway().evaluationModel("typesafe-ai/jev");++/**+ * Answers an `AssistantRequest` with an `AssistantResponse`. The model and how it is asked stay on+ * the server; the gateway reads `AI_GATEWAY_API_KEY` or Vercel's OIDC token.+ */+export async function POST(request: Request) {+ const body = assistantRequest.safeParse(+ await request.json().catch(() => undefined),+ );+ if (!body.success) {+ return Response.json({ error: "Invalid request." }, { status: 400 });+ }+ const { state, questions } = ask(body.data);+ // A stalled model would otherwise hold the chat until the function times out.+ const abortSignal = AbortSignal.timeout(15_000);+ // The model directly: the SDK's evaluate rejects answers whose top options tie.+ const evaluate = async () =>+ model.doEvaluate({ state, questions, abortSignal });+ try {+ const { answers } = await evaluate().catch(evaluate);+ return Response.json(interpret(body.data, answers));+ } catch (error) {+ console.error(error);+ return Response.json(+ { error: "The assistant is unavailable right now." },+ { status: 502 },+ );+ }+}diff --git a/biome.json b/biome.jsonindex 3286a5a..a433d4f 100644--- a/biome.json+++ b/biome.json@@ -28,5 +28,13 @@}}}- }+ },+ "overrides": [+ {+ "includes": ["api/**"],+ "linter": {+ "rules": { "style": { "noRestrictedImports": "off" } }+ }+ }+ ]}diff --git a/bun.lock b/bun.lockindex 7d2529c..b89139b 100644--- a/bun.lock+++ b/bun.lock@@ -12,6 +12,7 @@"@tailwindcss/vite": "^4.3.3","@vercel/analytics": "^2.0.1","@vercel/speed-insights": "^2.0.0",+ "ai": "^7.0.119","class-variance-authority": "^0.7.1","cn": "^0.4.0","idb": "^8.0.3",@@ -39,6 +40,12 @@},},"packages": {+ "@ai-sdk/gateway": ["@ai-sdk/[email protected]", "", { "dependencies": { "@ai-sdk/provider": "4.0.19", "@ai-sdk/provider-utils": "5.0.50", "@vercel/oidc": "3.2.0" }, "peerDependencies": { "zod": "^3.25.76 || ^4.1.8" } }, "sha512-YFS9McgEDHXPwXOWlu1JCylkFyDsgPNLwH9jXgMxdiaur3k6SaIUPKCRSciQQ5LbEZOSb4JAaPYYO7B5Cvq7ow=="],++ "@ai-sdk/provider": ["@ai-sdk/[email protected]", "", { "dependencies": { "json-schema": "^0.4.0" } }, "sha512-ryVZSSfmpEsr4bM8rbt9cSBWal+KGYRrm57C+k/hiPQHMiMahaqg2bPF0MpxSUWY8mGpt7s/ajM6PAqNTMKqPQ=="],++ "@ai-sdk/provider-utils": ["@ai-sdk/[email protected]", "", { "dependencies": { "@ai-sdk/provider": "4.0.19", "@standard-schema/spec": "^1.1.0", "@workflow/serde": "4.1.0", "eventsource-parser": "^3.0.8", "undici": "^7.29.0" }, "peerDependencies": { "zod": "^3.25.76 || ^4.1.8" } }, "sha512-qg2LutM5RlUT/tuuFf+VkhFGNZ2sEJHL0oDIrFN/sMoVbDBANglqtGCJUOnBqGMoFBH8IfAXMrFZvbFWx4dMOg=="],+"@babel/runtime": ["@babel/[email protected]", "", {}, "sha512-Nq8OhGWiZIZGV6hLHoyAKLLcJihP/xFeBMGJoUrxTX2psI8dCifzLhZISFb+VWS3wFMRDmCGw5R+dOySCqPLhw=="],"@base-ui/react": ["@base-ui/[email protected]", "", { "dependencies": { "@babel/runtime": "^7.29.7", "@base-ui/utils": "0.4.0", "@floating-ui/react-dom": "^2.1.9", "@floating-ui/utils": "^0.2.12", "use-sync-external-store": "^1.6.0" }, "peerDependencies": { "@date-fns/tz": "^1.2.0", "@types/react": "^17 || ^18 || ^19", "date-fns": "^4.0.0", "react": "^17 || ^18 || ^19", "react-dom": "^17 || ^18 || ^19" }, "optionalPeers": ["@date-fns/tz", "@types/react", "date-fns"] }, "sha512-P0/1sxo6SBVZOklKMIedvTWqw2s2IQzi9x5bIVsXu980cuSOD4NeuRSs+/L7LZQfDkZP/uRZyGPyfFl/B1oH+Q=="],@@ -131,6 +138,8 @@"@roprgm/ui": ["@roprgm/[email protected]", "", { "dependencies": { "@base-ui/react": "^1.8.0", "class-variance-authority": "^0.7.1", "cn": "^0.4.0" }, "peerDependencies": { "react": ">=19" } }, "sha512-UoR9Vf0VdzBpdQvEydw4b8Ew0nCP3Webu+dhFXb+3mKvDGJFt0MqbGRnJqhsGP121Xo8z5R8IPmh/ID8wTmNOg=="],+ "@standard-schema/spec": ["@standard-schema/[email protected]", "", {}, "sha512-l2aFy5jALhniG5HgqrD6jXLi/rUWrKvqN/qJx6yoJsgKhblVd+iqqU4RCXavm/jPityDo5TCvKMnpjKnOriy0w=="],+"@tailwindcss/node": ["@tailwindcss/[email protected]", "", { "dependencies": { "@jridgewell/remapping": "^2.3.5", "enhanced-resolve": "^5.24.1", "jiti": "^2.7.0", "lightningcss": "1.32.0", "magic-string": "^0.30.21", "source-map-js": "^1.2.1", "tailwindcss": "4.3.3" } }, "sha512-/T8IKEsf9VTU6tLjgC7+sv2mOPtQxzE2jMw7u4Tt40Tx+QSZxpzh95/H6cMKoja9XuW7iMdLJYBB0o9G1CaAgg=="],"@tailwindcss/oxide": ["@tailwindcss/[email protected]", "", { "optionalDependencies": { "@tailwindcss/oxide-android-arm64": "4.3.3", "@tailwindcss/oxide-darwin-arm64": "4.3.3", "@tailwindcss/oxide-darwin-x64": "4.3.3", "@tailwindcss/oxide-freebsd-x64": "4.3.3", "@tailwindcss/oxide-linux-arm-gnueabihf": "4.3.3", "@tailwindcss/oxide-linux-arm64-gnu": "4.3.3", "@tailwindcss/oxide-linux-arm64-musl": "4.3.3", "@tailwindcss/oxide-linux-x64-gnu": "4.3.3", "@tailwindcss/oxide-linux-x64-musl": "4.3.3", "@tailwindcss/oxide-wasm32-wasi": "4.3.3", "@tailwindcss/oxide-win32-arm64-msvc": "4.3.3", "@tailwindcss/oxide-win32-x64-msvc": "4.3.3" } }, "sha512-krXjAikiaFSPaK/FkAQT5UTx3VormQaiZ5hBFlJZ9UFQGB/rwg1MZIhHAG9smMQRTdyJxP6Qt5MwMtdyU5FWrA=="],@@ -211,6 +220,8 @@"@vercel/analytics": ["@vercel/[email protected]", "", { "peerDependencies": { "@remix-run/react": "^2", "@sveltejs/kit": "^1 || ^2", "next": ">= 13", "nuxt": ">= 3", "react": "^18 || ^19 || ^19.0.0-rc", "svelte": ">= 4", "vue": "^3", "vue-router": "^4" }, "optionalPeers": ["@remix-run/react", "@sveltejs/kit", "next", "nuxt", "react", "svelte", "vue", "vue-router"] }, "sha512-MTQG6V9qQrt1tsDeF+2Uoo5aPjqbVPys1xvnIftXSJYG2SrwXRHnqEvVoYID7BTruDz4lCd2Z7rM1BdkUehk2g=="],+ "@vercel/oidc": ["@vercel/[email protected]", "", {}, "sha512-UycprH3T6n3jH0k44NHMa7pnFHGu/N05MjojYr+Mc6I7obkoLIJujSWwin1pCvdy/eOxrI/l3uDLQsmcrOb4ug=="],+"@vercel/speed-insights": ["@vercel/[email protected]", "", { "peerDependencies": { "@sveltejs/kit": "^1 || ^2", "next": ">= 13", "nuxt": ">= 3", "react": "^18 || ^19 || ^19.0.0-rc", "svelte": ">= 4", "vue": "^3", "vue-router": "^4" }, "optionalPeers": ["@sveltejs/kit", "next", "nuxt", "react", "svelte", "vue", "vue-router"] }, "sha512-jwkNcrTeafWxjmWq4AHBaptSqZiJkYU5adLC9QBSqeim0GcqDMgN5Ievh8OG1rJ6W3A4l1oiP7qr9CWxGuzu3w=="],"@vgpu/adapter-mock": ["@vgpu/[email protected]", "", { "dependencies": { "@vgpu/core": "0.3.1" } }, "sha512-uZxTFPHLYd1nOxGxluvni50tsjnUTQ2eOhjJ/YwWynO7k906AD0byJxN0V6BbYb1MK/xM/EF+/cA8tKCrvY0yA=="],@@ -227,8 +238,12 @@"@webgpu/types": ["@webgpu/[email protected]", "", {}, "sha512-0cF7RFM2edNoiIS1ODJp0/Gzv4/xSXhwoR0YCza+OWpJWtn4wmo9DvK91aLlH9+uUnwIriP7ZiC3WitmyhuzBw=="],+ "@workflow/serde": ["@workflow/[email protected]", "", {}, "sha512-pav4F2BoirECWR7Nf1TKt+2eETcBj7jj4cBefQ8VXQCA6NPkaKeLfj/zMgi+3zYV5ZIBT4GuUiphsj0/b9hPQQ=="],+"@xmldom/xmldom": ["@xmldom/[email protected]", "", {}, "sha512-5AXjrcMClTryPe9LgZrygpB1lj7s0S9E0+W+AHaVKAVyHanafK86iPSvG5xHVSp/jC+VH1UXu0TAEmY279xH7A=="],+ "ai": ["[email protected]", "", { "dependencies": { "@ai-sdk/gateway": "4.0.97", "@ai-sdk/provider": "4.0.19", "@ai-sdk/provider-utils": "5.0.50" }, "peerDependencies": { "zod": "^3.25.76 || ^4.1.8" } }, "sha512-KlHbkKMmdb2g0DXdkhBXzSvLSkdoAg0MdShhTTOyuoHKJetEUlgjn2juZ05lHY97EqCwBuSK8dgiEeB0ty5xZw=="],+"bun-types": ["[email protected]", "", { "dependencies": { "@types/node": "*" } }, "sha512-loKuVrAFZKfEv+JvWkHRS9GW5IqLuLRjVXN9p+vZvBN86O5hf/pBZQ5hSoyipsrMmWObZBDvWnlmKvjKTM0PdA=="],"class-variance-authority": ["[email protected]", "", { "dependencies": { "clsx": "^2.1.1" } }, "sha512-Ka+9Trutv7G8M6WT6SeiRWz792K5qEqIGEGzXKhAE6xOWAY6pPH8U+9IY3oCMv6kqTmLsv7Xh/2w2RigkePMsg=="],@@ -245,6 +260,8 @@"enhanced-resolve": ["[email protected]", "", { "dependencies": { "graceful-fs": "^4.2.4", "tapable": "^2.3.3" } }, "sha512-L1l8TNvomm6UVW5B253AGxQagSQr+vGwhMlrrfRS2qmhx46AMpMVJKQYLvWYbysTMY8VoicOvzHzoHMbyzB+4A=="],+ "eventsource-parser": ["[email protected]", "", {}, "sha512-EKN1vKAMcZ8MlYMpaNuxN6R9yakzH6uajHcHVTqWJzvu5pWw9DyhbP35HH8MVBQ+dZjAfDxk+A8NiR9KWaXiyQ=="],+"fdir": ["[email protected]", "", { "peerDependencies": { "picomatch": "^3 || ^4" }, "optionalPeers": ["picomatch"] }, "sha512-tIbYtZbucOs0BRGqPJkshJUYdL+SDH7dVM8gjy+ERp3WAUjLEFJE+02kanyHtwjWOnwrKYBiwAmM0p4kLJAnXg=="],"fsevents": ["[email protected]", "", { "os": "darwin" }, "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw=="],@@ -255,6 +272,8 @@"jiti": ["[email protected]", "", { "bin": { "jiti": "lib/jiti-cli.mjs" } }, "sha512-AC/7JofJvZGrrneWNaEnJeOLUx+JlGt7tNa0wZiRPT4MY1wmfKjt2+6O2p2uz2+skll8OZZmJMNqeke7kKbNgQ=="],+ "json-schema": ["[email protected]", "", {}, "sha512-es94M3nTIfsEPisRafak+HDLfHXnKBhV3vU5eqPcS3flIWqcxJWgXHXiey3YrpaNsanY5ei1VoYEbOzijuq9BA=="],+"lightningcss": ["[email protected]", "", { "dependencies": { "detect-libc": "^2.0.3" }, "optionalDependencies": { "lightningcss-android-arm64": "1.33.0", "lightningcss-darwin-arm64": "1.33.0", "lightningcss-darwin-x64": "1.33.0", "lightningcss-freebsd-x64": "1.33.0", "lightningcss-linux-arm-gnueabihf": "1.33.0", "lightningcss-linux-arm64-gnu": "1.33.0", "lightningcss-linux-arm64-musl": "1.33.0", "lightningcss-linux-x64-gnu": "1.33.0", "lightningcss-linux-x64-musl": "1.33.0", "lightningcss-win32-arm64-msvc": "1.33.0", "lightningcss-win32-x64-msvc": "1.33.0" } }, "sha512-WkUDrojuJs0xkgGf2udWxa3yGBRxPtxUkB79i6aCZLRgc7PM8fZe9TosfPDcvEpQZbuFASnHYmRLBLUbmLOIIA=="],"lightningcss-android-arm64": ["[email protected]", "", { "os": "android", "cpu": "arm64" }, "sha512-gEpRTalKdosp4Bb8qWtc2iOgE5SeIHlpS1up9bFq2wAyYhl1UdTObYiHe98zEM9SQvSoqQZ1IQD0JNpg3Ml5pg=="],@@ -323,6 +342,8 @@"typescript": ["[email protected]", "", { "optionalDependencies": { "@typescript/typescript-aix-ppc64": "7.0.2", "@typescript/typescript-darwin-arm64": "7.0.2", "@typescript/typescript-darwin-x64": "7.0.2", "@typescript/typescript-freebsd-arm64": "7.0.2", "@typescript/typescript-freebsd-x64": "7.0.2", "@typescript/typescript-linux-arm": "7.0.2", "@typescript/typescript-linux-arm64": "7.0.2", "@typescript/typescript-linux-loong64": "7.0.2", "@typescript/typescript-linux-mips64el": "7.0.2", "@typescript/typescript-linux-ppc64": "7.0.2", "@typescript/typescript-linux-riscv64": "7.0.2", "@typescript/typescript-linux-s390x": "7.0.2", "@typescript/typescript-linux-x64": "7.0.2", "@typescript/typescript-netbsd-arm64": "7.0.2", "@typescript/typescript-netbsd-x64": "7.0.2", "@typescript/typescript-openbsd-arm64": "7.0.2", "@typescript/typescript-openbsd-x64": "7.0.2", "@typescript/typescript-sunos-x64": "7.0.2", "@typescript/typescript-win32-arm64": "7.0.2", "@typescript/typescript-win32-x64": "7.0.2" }, "bin": { "tsc": "bin/tsc" } }, "sha512-8FYau96o3NKOhbjKi/qNvG/W5jhzxkbdm5sj9AbZ/5T5sWqn3hJgLfGx27sRKZWTvyzCP8dLRBTf5tBTSRVUNA=="],+ "undici": ["[email protected]", "", {}, "sha512-dkrQXeHSaoamnItlYbmzG0wFYrM0ZwDxCIg0A7aKjTyyhh9svRzCNFEzV+Vm05/yehjCzjDZ31KXfGEjYSztDQ=="],+"undici-types": ["[email protected]", "", {}, "sha512-j375ScV60dom+YkPFIfTLcOiPxkN/buHz5GobjLhixFuANaNs3C9l4GmrWqejgXWJ7BbJcFYpTEUkS1Ge8bpZQ=="],"use-sync-external-store": ["[email protected]", "", { "peerDependencies": { "react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0" } }, "sha512-6L+EeigHMQhdaIPNIFUKwfWJSwWFQ8gJbJ2DLOs5sDIegTwR9fRxvnM3uciHKjIZhFz+KAv2emhWMRvDmMcY8A=="],diff --git a/package.json b/package.jsonindex bc1a6d1..2edca77 100644--- a/package.json+++ b/package.json@@ -19,6 +19,7 @@"@tailwindcss/vite": "^4.3.3","@vercel/analytics": "^2.0.1","@vercel/speed-insights": "^2.0.0",+ "ai": "^7.0.119","class-variance-authority": "^0.7.1","cn": "^0.4.0","idb": "^8.0.3",diff --git a/src/app/assistant/index.tsx b/src/app/assistant/index.tsxnew file mode 100644index 0000000..533ae69--- /dev/null+++ b/src/app/assistant/index.tsx@@ -0,0 +1,248 @@+import { IconButton } from "@roprgm/ui/icon-button";+import { Input } from "@roprgm/ui/input";+import { ScrollArea } from "@roprgm/ui/scroll-area";+import { Section, SectionAction, sections } from "@roprgm/ui/section";+import { Tooltip, TooltipContent, TooltipTrigger } from "@roprgm/ui/tooltip";+import { cn } from "cn";+import { type FormEvent, useEffect, useRef, useState } from "react";+import { flushSync } from "react-dom";+import { z } from "zod/mini";+import { runCommand } from "@/app/commands";+import type { createControls } from "@/app/controls";+import type { Workspace } from "@/app/workspace";+import { CloseIcon } from "@/components/icons/close";+import { InfoIcon } from "@/components/icons/info";+import { SparklesIcon } from "@/components/icons/sparkles";+import { parse } from "@/lib/parse";+import {+ type AssistantRequest,+ assistantResponse,+ type Photo,+} from "./protocol";++type Controls = ReturnType<typeof createControls>;+type Message = { from: "user" | "assistant"; text: string };++const unavailable = "The assistant is unavailable right now.";+const limited =+ "You've reached the assistant's usage limit. Try again in a little while.";+const replaced =+ "Another photo opened before the answer came, so I left it unedited.";+const failure = z.object({ error: z.string() });++function describePhoto(workspace: Workspace, controls: Controls): Photo {+ const { adjustments, vignette, details, toneCurve, preview } =+ controls.getState();+ const document = workspace.getDocument();+ const [image] = document.scene.getState().layers;+ const [width, height] = document.resources.get(image.source).image.size;+ return {+ adjustments,+ vignette: vignette.intensity,+ clarity: details.clarity,+ sharpening: details.sharpening,+ toneCurve: [...toneCurve],+ comparison: preview?.comparison ?? "edited",+ sourceSize: [width, height],+ };+}++async function ask(request: AssistantRequest) {+ const response = await fetch("/api/assistant", {+ method: "POST",+ headers: { "content-type": "application/json" },+ body: JSON.stringify(request),+ });+ // Vercel's firewall answers 429 once an address sends too many messages.+ if (response.status === 429) {+ throw Error(limited);+ }+ const body = await response.json().catch(() => undefined);+ if (!response.ok) {+ const error = failure.safeParse(body);+ throw Error(error.success ? error.data.error : unavailable);+ }+ return parse(assistantResponse, body, "Invalid assistant response");+}++// Shown at once so focus can land, and hidden only once faded out.+const appear =+ "visible opacity-100 [transition:opacity_200ms_100ms,visibility_0s]";+const vanish =+ "invisible opacity-0 [transition:opacity_150ms,visibility_0s_150ms]";++/** A chat that edits the open photo: the server turns each message into commands, and the last layer they touch is selected. */+export function Assistant({+ workspace,+ controls,+}: {+ workspace: Workspace;+ controls: Controls;+}) {+ const [open, setOpen] = useState(false);+ const [messages, setMessages] = useState<Message[]>([]);+ const [busy, setBusy] = useState(false);+ const toggle = useRef<HTMLButtonElement>(null);+ const input = useRef<HTMLInputElement>(null);+ const newest = useRef<HTMLLIElement>(null);+ const content = useRef<HTMLDivElement>(null);+ const [height, setHeight] = useState(0);+ // The card's height follows its content, so it grows smoothly as messages arrive too.+ useEffect(() => {+ const element = content.current;+ if (!element) {+ return;+ }+ const observer = new ResizeObserver(([entry]) =>+ setHeight(entry.borderBoxSize[0].blockSize),+ );+ observer.observe(element);+ return () => observer.disconnect();+ }, []);+ useEffect(() => {+ if (messages.length > 0) {+ newest.current?.scrollIntoView({ block: "nearest" });+ }+ }, [messages.length]);+ const say = (from: Message["from"], text: string) =>+ setMessages((messages) => [...messages, { from, text }]);++ /** Opens or closes, moving focus to what shows once the state applies. */+ function show(next: boolean) {+ flushSync(() => setOpen(next));+ (next ? input : toggle).current?.focus();+ }++ async function submit(event: FormEvent<HTMLFormElement>) {+ event.preventDefault();+ const form = event.currentTarget;+ const message = new FormData(form).get("message")?.toString().trim();+ if (!message || busy) {+ return;+ }+ form.reset();+ const earlier = messages+ .filter(({ from }) => from === "user")+ .slice(-3)+ .map(({ text }) => text);+ say("user", message);+ setBusy(true);+ try {+ const document = workspace.getDocument();+ const answer = await ask({+ message,+ earlier,+ photo: describePhoto(workspace, controls),+ });+ // The answer describes the photo that was open when the message was sent.+ if (workspace.getDocument() !== document) {+ throw Error(replaced);+ }+ const results = answer.commands.map((command) =>+ runCommand(workspace, command),+ );+ const layerId = results.findLast((result) => result.layerId)?.layerId;+ if (layerId) {+ controls.selectLayer(layerId);+ }+ say("assistant", answer.message);+ } catch (error) {+ say("assistant", error instanceof Error ? error.message : unavailable);+ }+ setBusy(false);+ }++ // One shape grows from the button into the card, over content pinned to its bottom-left+ // corner, so nothing inside moves. It clips rather than hides overflow, so scrolling a message+ // into view cannot scroll the shape.+ return (+ <div+ style={open ? { height } : undefined}+ className={cn(+ "fixed bottom-3 left-14 z-50 overflow-clip surface-panel bg-level-4/95 backdrop-blur-sm transition-[width,height,border-radius] duration-300 ease-[cubic-bezier(0.2,0,0,1)] motion-reduce:transition-none max-md:hidden",+ open ? "w-80 rounded-xl" : "size-10 rounded-[1.25rem]",+ )}+ >+ <Tooltip>+ <TooltipTrigger+ render={+ <button+ ref={toggle}+ type="button"+ aria-label="Assistant"+ onClick={() => show(true)}+ className={cn(+ "absolute bottom-0 left-0 grid size-10 place-items-center text-muted transition-colors hover:text-foreground",+ open ? vanish : appear,+ )}+ >+ <SparklesIcon className="size-4" />+ </button>+ }+ />+ <TooltipContent>Assistant</TooltipContent>+ </Tooltip>+ <div+ ref={content}+ className={cn(+ sections({ lines: true }),+ "absolute bottom-0 left-0 w-80",+ open ? appear : vanish,+ )}+ >+ <Section className="flex-row items-center">+ <div className="flex flex-1 items-center gap-1">+ <h2 className="font-medium">Assistant</h2>+ <IconButton+ label="The assistant is in beta, and its use is limited."+ size="icon-sm"+ >+ <InfoIcon className="size-3.5" />+ </IconButton>+ </div>+ <SectionAction>+ <IconButton+ label="Close"+ shortcut="Esc"+ size="icon"+ onClick={() => show(false)}+ >+ <CloseIcon className="size-4" />+ </IconButton>+ </SectionAction>+ </Section>+ {messages.length > 0 && (+ <ScrollArea fade className="*:max-h-60">+ <ol className="flex flex-col gap-1 px-3.5 py-2.5">+ {messages.map((message, index) => (+ <li+ key={index}+ ref={index === messages.length - 1 ? newest : undefined}+ className={+ message.from === "user" ? "font-medium" : "text-muted"+ }+ >+ {message.text}+ </li>+ ))}+ </ol>+ </ScrollArea>+ )}+ <Section>+ <form aria-label="Assistant" onSubmit={submit}>+ <Input+ ref={input}+ name="message"+ aria-label="Message"+ placeholder={busy ? "Editing…" : "Ask for an edit"}+ autoComplete="off"+ maxLength={500}+ readOnly={busy}+ onKeyDown={(event) => event.key === "Escape" && show(false)}+ />+ </form>+ </Section>+ </div>+ </div>+ );+}diff --git a/src/app/assistant/jev.ts b/src/app/assistant/jev.tsnew file mode 100644index 0000000..0a65527--- /dev/null+++ b/src/app/assistant/jev.ts@@ -0,0 +1,458 @@+import type { Experimental_EvaluationQuestion as Question } from "ai";+import { z } from "zod/mini";+import type { Command } from "@/app/commands";+import type { Adjustments } from "@/core/document";+import type { AssistantRequest, AssistantResponse, Photo } from "./protocol";++/**+ * How the assistant asks Jev, an evaluation model: one call answers typed questions about the+ * message in parallel, each by choosing among options, and the answers become commands.+ */++/** A control Jev sets by choosing one of its steps; `about` describes it among the commands and in its value question. */+type Slider = {+ about: string;+ min: number;+ max: number;+ step: number;+ read: (photo: Photo) => number;+};++/** An adjustment, from -100 to 100 in steps of 10 unless given. */+function adjustment(+ key: keyof Adjustments,+ about: string,+ limit = 100,+ step = 10,+): Slider & { key: keyof Adjustments } {+ const read = (photo: Photo) => photo.adjustments[key];+ return { key, about, min: -limit, max: limit, step, read };+}++/** Controls a mask can hold, so they can change one area. */+const adjustments = {+ exposure: adjustment(+ "exposure",+ "Exposure: make the whole image lighter or darker, in stops.",+ 5,+ 0.5,+ ),+ contrast: adjustment(+ "contrast",+ "Contrast: more or less difference between light and dark.",+ ),+ highlights: adjustment(+ "highlights",+ "Highlights: recover or brighten the bright areas.",+ ),+ shadows: adjustment("shadows", "Shadows: lift or deepen the dark areas."),+ whites: adjustment("whites", "Whites: move the white point."),+ blacks: adjustment("blacks", "Blacks: move the black point."),+ temperature: adjustment(+ "incrementalTemperature",+ "Temperature: warmer (yellow) or cooler (blue).",+ ),+ tint: adjustment("incrementalTint", "Tint: greener or more magenta."),+ vibrance: adjustment(+ "vibrance",+ "Vibrance: more or less color, gently, protecting vivid colors and skin. The usual choice for “more color”.",+ ),+ saturation: adjustment(+ "saturation",+ "Saturation: more or less color everywhere, down to black and white.",+ ),+};++/** Controls on effect layers, which change the whole photo. */+const effects = {+ vignette: {+ about: "Vignette: darker edges.",+ min: 0,+ max: 100,+ step: 5,+ read: (photo) => photo.vignette,+ },+ clarity: {+ about: "Clarity: local contrast, punchier or softer texture.",+ min: -100,+ max: 100,+ step: 10,+ read: (photo) => photo.clarity,+ },+ sharpening: {+ about: "Sharpening: crisper detail.",+ min: 0,+ max: 150,+ step: 10,+ read: (photo) => photo.sharpening,+ },+} satisfies Record<string, Slider>;++const sliders: Record<string, Slider> = { ...adjustments, ...effects };++type Tool = { about: string; commands: Command[]; message: string };++const inverted = [+ { x: 0, y: 1 },+ { x: 1, y: 0 },+];++const crops = {+ "1:1": "a square",+ "4:5": "4:5 portrait, as for an Instagram post",+ "3:2": "3:2 landscape",+ "16:9": "16:9 widescreen",+ "9:16": "9:16 vertical, as for a story",+};++/** Commands beside the sliders; only inverting depends on the photo, which it toggles. */+function tools({ toneCurve }: Photo): Record<string, Tool> {+ const isInverted =+ toneCurve.length === 2 &&+ toneCurve.every(+ ({ x, y }, i) => x === inverted[i].x && y === inverted[i].y,+ );+ const cropTo = Object.entries(crops).map(([ratio, about]) => {+ const [width, height] = ratio.split(":").map(Number);+ const tool: Tool = {+ about: `Crop to ${about}.`,+ commands: [{ type: "set-crop", aspectRatio: width / height }],+ message: `Cropped to ${ratio}`,+ };+ return [`crop-${ratio}`, tool];+ });+ return {+ invert: {+ about: "Invert the colors into a negative, or back.",+ commands: [+ isInverted+ ? { type: "set-tone-curve" }+ : { type: "set-tone-curve", points: inverted },+ ],+ message: isInverted ? "Colors restored" : "Colors inverted",+ },+ reset: {+ about:+ "Reset every edit at once, returning the whole photo to how it was opened. Resetting one control, such as “reset the contrast”, sets that control instead.",+ commands: [{ type: "reset" }],+ message: "Reset all edits",+ },+ undo: {+ about: "Undo the last change.",+ commands: [{ type: "undo" }],+ message: "Undone",+ },+ redo: {+ about: "Redo the last undone change.",+ commands: [{ type: "redo" }],+ message: "Redone",+ },+ original: {+ about: "Show the original photo for comparison, keeping the edits.",+ commands: [{ type: "set-preview", comparison: "original" }],+ message: "Showing the original. Say “show my edits” to go back.",+ },+ split: {+ about: "Compare before and after side by side.",+ commands: [{ type: "set-preview", comparison: "split" }],+ message: "Showing before and after",+ },+ edited: {+ about: "Show the edited photo again after showing the original.",+ commands: [{ type: "set-preview", comparison: "edited" }],+ message: "Showing your edits",+ },+ ...Object.fromEntries(cropTo),+ "crop-none": {+ about: "Remove the crop.",+ commands: [{ type: "set-crop" }],+ message: "Crop removed",+ },+ };+}++const reply =+ "Answer with a message instead of editing: a greeting, thanks, a question, an exact number, something the editor cannot do, or anything unrelated to this photo.";++/** Canned answers: conversation first, then why a request cannot be done. */+const replies: Record<string, string> = {+ greeting:+ "Hi! I can edit this photo for you. Try “make it warmer” or “darken the sky”.",+ thanks: "You're welcome! Anything else you'd like to change?",+ approval: "Great! Tell me if you'd like to change anything else.",+ goodbye: "Bye! Your edits stay saved in this browser.",+ capabilities:+ "I can change exposure, contrast, highlights, shadows, whites, blacks, temperature, tint, vibrance, saturation, vignette, clarity, and sharpening, in the whole photo or the sky, ground, sides, or center. I can also crop, invert colors, compare before and after, reset, and undo.",+ "how-to":+ "Tell me what to change in plain words, like “brighter and warmer”, “more contrast in the sky”, or “crop it for Instagram”.",+ identity:+ "I'm Jev, a quick assistant that edits this photo from short requests.",+ opinion:+ "I can't judge photos, but I can change whatever you'd like to improve.",+ export: "To save your photo, use Export in the top right.",+ open: "To edit another photo, use the folder button next to the file name.",+ "off-topic": "I can only help with editing this photo.",+ "unsupported-edit":+ "I can't do that edit yet. I can change light, color, detail, and vignette, crop, and invert colors.",+ "exact-value":+ "I can't set exact numbers yet; I move controls in steps. Try “a bit brighter” or “much more contrast”.",+ "out-of-range": "That's beyond what this control allows.",+ "several-areas":+ "I can change one area at a time. Ask for the sky and the ground separately.",+ unclear:+ "I'm not sure what to change. Try something like “brighter”, “warmer”, or “more contrast”.",+};++const areas = {+ photo: "The whole photo.",+ top: "The top of the photo, such as the sky.",+ bottom: "The bottom of the photo, such as the ground or foreground.",+ left: "The left side of the photo.",+ right: "The right side of the photo.",+ center: "The center of the photo, such as the main subject.",+};++const counts: Record<string, number> = { one: 1, two: 2, three: 3 };++function round(value: number) {+ return Math.round(value * 100) / 100;+}++/** Words for a change's size, so a plain "brighter" lands on a moderate step. */+function magnitude(change: number, { min, max }: Slider) {+ const share = Math.abs(change) / (max - min);+ if (share <= 0.05) {+ return "slightly";+ }+ if (share <= 0.15) {+ return "moderately";+ }+ return share <= 0.3 ? "strongly" : "extremely";+}++/** Every step from min to max, each described by its change from the current value. */+function valueQuestion(slider: Slider, current: number): Question {+ const { min, max, step } = slider;+ const count = Math.round((max - min) / step);+ const options = Array.from({ length: count + 1 }, (_, i) => {+ const value = round(min + i * step);+ const change = round(value - current);+ const direction = change > 0 ? "more" : "less";+ const notes = [+ change+ ? `${change > 0 ? "+" : ""}${change}, ${magnitude(change, slider)} ${direction}`+ : "the current value",+ value === 0 && "the default",+ value === min && "the minimum",+ value === max && "the maximum",+ ].filter(Boolean);+ // A fixed decimal keeps keys from sorting as integers, so options stay in order.+ return [value.toFixed(1), notes.join(", ")];+ });+ return {+ type: "choice",+ instructions: `The value the request asks for. ${slider.about} It is now ${current}; each option notes its change from now. Without a stated amount, prefer a moderate change.`,+ criteria: Object.fromEntries(options),+ };+}++/**+ * The state and questions for one message: how many changes it names, the command that fits best+ * with the others ranked behind it, the area, a reply, and each slider's value in case it is chosen.+ */+export function ask({ message, earlier, photo }: AssistantRequest) {+ const current = Object.fromEntries(+ Object.entries(sliders).map(([id, slider]) => [id, slider.read(photo)]),+ );+ const commands = Object.fromEntries([+ ...Object.entries({ ...sliders, ...tools(photo) }).map(+ ([id, { about }]) => [id, about],+ ),+ ["reply", reply],+ ]);+ const values = Object.entries(sliders).map(([id, slider]) => [+ `value-${id}`,+ valueQuestion(slider, slider.read(photo)),+ ]);+ const questions: Record<string, Question> = {+ count: {+ type: "choice",+ instructions: "How many different changes the request names.",+ criteria: {+ one: "One change, even if described with several words.",+ two: "Two different changes, such as “warmer and darker”.",+ three: "Three different changes.",+ },+ },+ command: {+ type: "choice",+ instructions:+ "The command that best fulfills the request; when it names several changes, each of them.",+ criteria: commands,+ },+ area: {+ type: "choice",+ instructions: "Which part of the photo the change applies to.",+ criteria: areas,+ },+ reply: {+ type: "choice",+ instructions: "The message that best answers the request.",+ criteria: replies,+ },+ ...Object.fromEntries(values),+ };+ return {+ state: { request: message, earlierRequests: earlier, current },+ questions,+ };+}++const answersSchema = z.record(+ z.string(),+ z.object({+ choice: z.optional(z.string()),+ probabilities: z.optional(z.record(z.string(), z.number())),+ }),+);+type Answers = z.output<typeof answersSchema>;++function choice(answers: Answers, id: string) {+ const choice = answers[id]?.choice;+ if (choice === undefined) {+ throw Error(`Jev did not answer ${id}.`);+ }+ return choice;+}++/** The chosen command, then as many more sliders as the message names changes, by probability. */+function chosen(answers: Answers) {+ const first = choice(answers, "command");+ const count = counts[choice(answers, "count")] ?? 1;+ const others = Object.entries(answers.command?.probabilities ?? {})+ .filter(([id]) => id !== first && id in sliders)+ .sort((a, b) => b[1] - a[1])+ .slice(0, count - 1)+ .map(([id]) => id);+ return [first, ...others];+}++type MaskInput = Extract<Command, { type: "add-mask" }>["mask"];++function areaMask(area: string, size: readonly number[]): MaskInput {+ // Whole source pixels, as a person would place the gradient.+ const [width, height] = size.map(Math.round);+ const at = (share: number, length: number) => Math.round(share * length);+ const middle: [number, number] = [at(0.5, width), at(0.5, height)];+ switch (area) {+ case "top":+ return {+ kind: "linear",+ start: [middle[0], 0],+ end: [middle[0], at(0.55, height)],+ };+ case "bottom":+ return {+ kind: "linear",+ start: [middle[0], height],+ end: [middle[0], at(0.45, height)],+ };+ case "left":+ return {+ kind: "linear",+ start: [0, middle[1]],+ end: [at(0.55, width), middle[1]],+ };+ case "right":+ return {+ kind: "linear",+ start: [width, middle[1]],+ end: [at(0.45, width), middle[1]],+ };+ default:+ return {+ kind: "radial",+ center: middle,+ radius: [at(0.3, width), at(0.35, height)],+ angle: 0,+ feather: 0.5,+ };+ }+}++/** Commands that set the chosen sliders: adjustments in one edit, in a new mask for an area. */+function edit(answers: Answers, photo: Photo, ids: string[]) {+ const area = choice(answers, "area");+ const changes = ids.map((id) => {+ const current = sliders[id].read(photo);+ const value = Number(choice(answers, `value-${id}`));+ return { id, current, value };+ });+ const changed = new Map(changes.map(({ id, value }) => [id, value]));+ const adjustmentChange: Partial<Adjustments> = {};+ for (const [id, { key, read }] of Object.entries(adjustments)) {+ const value = changed.get(id);+ if (value !== undefined) {+ // A mask starts neutral, so it takes the change rather than the value.+ adjustmentChange[key] = area === "photo" ? value : value - read(photo);+ }+ }+ const commands: Command[] = [];+ if (Object.keys(adjustmentChange).length > 0) {+ commands.push(+ area === "photo"+ ? { type: "set-adjustments", ...adjustmentChange }+ : {+ type: "add-mask",+ mask: areaMask(area, photo.sourceSize),+ adjustments: adjustmentChange,+ },+ );+ }+ const vignette = changed.get("vignette");+ if (vignette !== undefined) {+ commands.push({ type: "set-vignette", intensity: vignette });+ }+ const clarity = changed.get("clarity");+ const sharpening = changed.get("sharpening");+ if (clarity !== undefined || sharpening !== undefined) {+ commands.push({ type: "set-details", clarity, sharpening });+ }+ const local = area !== "photo";+ const summary = changes+ .map(({ id, current, value }) => {+ if (local && id in adjustments) {+ const change = round(value - current);+ return `${area} ${id} ${change > 0 ? "+" : ""}${change}`;+ }+ return `${id} ${current} → ${value}`;+ })+ .join(", ");+ return { commands, message: summary };+}++/** Turns Jev's answers into commands and a message for the user. */+export function interpret(+ { photo }: AssistantRequest,+ response: unknown,+): AssistantResponse {+ const answers = answersSchema.parse(response);+ const [first, ...others] = chosen(answers);+ if (first === "reply") {+ return {+ commands: [],+ message: replies[choice(answers, "reply")] ?? replies.unclear,+ };+ }+ const tool = tools(photo)[first];+ const moved = [first, ...others].filter((id) => id in sliders);+ const edits = moved.length > 0 ? edit(answers, photo, moved) : undefined;+ const commands = [...(tool?.commands ?? []), ...(edits?.commands ?? [])];+ const message = [tool?.message, edits?.message].filter(Boolean).join(" · ");+ const editing = commands.some((command) => command.type !== "set-preview");+ // An edit made while the original shows would look like it did nothing.+ if (editing && photo.comparison === "original") {+ commands.unshift({ type: "set-preview", comparison: "edited" });+ }+ return { commands, message: message || replies.unclear };+}diff --git a/src/app/assistant/protocol.ts b/src/app/assistant/protocol.tsnew file mode 100644index 0000000..b443451--- /dev/null+++ b/src/app/assistant/protocol.ts@@ -0,0 +1,46 @@+import { z } from "zod/mini";+import type { Adjustments } from "@/core/document";++// Runs in the browser and in the serverless function, so it imports no app module at runtime.++const value = z.number().check(z.refine(Number.isFinite, "Not finite"));++/** The parts of the open photo the assistant reads. */+const photo = z.object({+ adjustments: z.object({+ exposure: value,+ contrast: value,+ highlights: value,+ shadows: value,+ whites: value,+ blacks: value,+ incrementalTemperature: value,+ incrementalTint: value,+ vibrance: value,+ saturation: value,+ }) satisfies z.ZodMiniType<Adjustments>,+ vignette: value,+ clarity: value,+ sharpening: value,+ toneCurve: z.array(z.object({ x: value, y: value })).check(z.maxLength(64)),+ comparison: z.enum(["edited", "original", "split"]),+ sourceSize: z.tuple([value, value]),+});++const text = z.string().check(z.trim(), z.minLength(1), z.maxLength(500));++/** A message to the assistant, with the user's earlier messages for context. */+export const assistantRequest = z.object({+ message: text,+ earlier: z.array(text).check(z.maxLength(3)),+ photo,+});+export type AssistantRequest = z.output<typeof assistantRequest>;+export type Photo = AssistantRequest["photo"];++/** The assistant's answer: commands to run in order, and a message for the user. */+export const assistantResponse = z.object({+ commands: z.array(z.unknown()),+ message: z.string(),+});+export type AssistantResponse = z.output<typeof assistantResponse>;diff --git a/src/app/index.tsx b/src/app/index.tsxindex ddcb3df..31f5f43 100644--- a/src/app/index.tsx+++ b/src/app/index.tsx@@ -1,4 +1,5 @@import { useStore } from "zustand";+import { Assistant } from "@/app/assistant";import { DraftNotice } from "@/app/draft/notice";import { Editor } from "@/app/editor";import { useWorkspace } from "@/app/workspace/use-workspace";@@ -7,6 +8,9 @@ import { useFileDrop } from "@/hooks/use-file-drop";/** Visiting /demo opens a bundled photo instead of the empty state. */const startup = location.pathname === "/demo" ? "/images/demo.jpg" : undefined;+/** The experimental assistant mounts only when the URL asks for it, as in `/?assistant`. */+const assistant = new URLSearchParams(location.search).has("assistant");+export function App() {const { workspace, controls, drafts } = useWorkspace(startup);useFileDrop(controls.openFiles);@@ -24,6 +28,9 @@ export function App() {draft={recovery}/>{error && <DraftNotice error={error} onDismiss={drafts.dismiss} />}+ {assistant && state.document && (+ <Assistant workspace={workspace} controls={controls} />+ )}</>);}diff --git a/src/components/icons/info.tsx b/src/components/icons/info.tsxnew file mode 100644index 0000000..a13ca8d--- /dev/null+++ b/src/components/icons/info.tsx@@ -0,0 +1,11 @@+import { Icon, type IconProps } from "./icon";++export function InfoIcon(props: IconProps) {+ return (+ <Icon {...props}>+ <circle cx="12" cy="12" r="9" />+ <path d="M12 11v5" />+ <circle cx="12" cy="7.75" r="0.75" fill="currentColor" stroke="none" />+ </Icon>+ );+}diff --git a/src/components/icons/sparkles.tsx b/src/components/icons/sparkles.tsxnew file mode 100644index 0000000..7c23b5d--- /dev/null+++ b/src/components/icons/sparkles.tsx@@ -0,0 +1,11 @@+import { Icon, type IconProps } from "./icon";++export function SparklesIcon(props: IconProps) {+ return (+ <Icon {...props}>+ <path d="M10 4c.7 3.4 2.6 5.3 6 6-3.4.7-5.3 2.6-6 6-.7-3.4-2.6-5.3-6-6 3.4-.7 5.3-2.6 6-6Z" />+ <path d="M18 2.5c.3 1.3 1.2 2.2 2.5 2.5-1.3.3-2.2 1.2-2.5 2.5-.3-1.3-1.2-2.2-2.5-2.5 1.3-.3 2.2-1.2 2.5-2.5Z" />+ <path d="M18 16c.2 1 .9 1.8 2 2-1.1.2-1.8 1-2 2-.2-1-.9-1.8-2-2 1.1-.2 1.8-1 2-2Z" />+ </Icon>+ );+}diff --git a/tests/assistant.e2e.ts b/tests/assistant.e2e.tsnew file mode 100644index 0000000..ed489c3--- /dev/null+++ b/tests/assistant.e2e.ts@@ -0,0 +1,106 @@+import { expect, test } from "./fixtures";++test("the assistant shows only at /?assistant, opens, runs the commands a message returns on the photo it was sent for, selects the layer they edit, replies, and closes", async ({+ page,+}) => {+ const requests: unknown[] = [];+ let release = () => {};+ const released = new Promise<void>((resolve) => {+ release = resolve;+ });+ await page.route("**/api/assistant", async (route) => {+ requests.push(route.request().postDataJSON());+ if (requests.length === 3) {+ await released;+ await route.fulfill({+ json: {+ commands: [{ type: "set-adjustments", exposure: 2 }],+ message: "exposure 0 → 2",+ },+ });+ return;+ }+ if (requests.length === 1) {+ await route.fulfill({+ json: {+ commands: [+ { type: "set-adjustments", exposure: -1 },+ { type: "set-vignette", intensity: 40 },+ ],+ message: "exposure 0 → -1, vignette 0 → 40",+ },+ });+ return;+ }+ await route.fulfill({+ json: { commands: [], message: "You're welcome!" },+ });+ });+ const open = async () => {+ await page+ .locator('input[type="file"]')+ .setInputFiles("tests/fixtures/photo.svg");+ await expect(+ page.getByRole("button", { name: "Open an image or scene" }),+ ).toBeVisible();+ };+ const button = page.getByRole("button", { name: "Assistant", exact: true });+ await page.goto("/");+ await open();+ await expect(button).toBeHidden();++ await page.goto("/?assistant");+ await open();+ await button.click();+ const message = page.getByRole("textbox", { name: "Message", exact: true });+ await expect(message).toBeFocused();++ await message.fill("darker, with a vignette");+ await message.press("Enter");+ await expect(+ page.getByText("exposure 0 → -1, vignette 0 → 40"),+ ).toBeVisible();+ expect(requests[0]).toMatchObject({+ message: "darker, with a vignette",+ earlier: [],+ photo: { sourceSize: [1200, 800], comparison: "edited" },+ });+ const state = await page.evaluate(() => window.openlight.getState());+ expect(state.adjustments.exposure).toBe(-1);+ const vignette = state.scene?.layers.find(+ (layer) => layer.kind === "vignette",+ );+ expect(vignette).toMatchObject({ vignette: { intensity: 40 } });+ expect(state.selectedLayerId).toBe(vignette?.id);++ await message.fill("thanks");+ await message.press("Enter");+ await expect(page.getByText("You're welcome!")).toBeVisible();+ expect(requests[1]).toMatchObject({ earlier: ["darker, with a vignette"] });++ // An answer that arrives after another photo opens leaves the new photo alone.+ await message.fill("brighter");+ await message.press("Enter");+ await page+ .locator('input[type="file"]')+ .setInputFiles("tests/fixtures/tones.png");+ await expect+ .poll(() => page.evaluate(() => window.openlight.getState().file))+ .toBe("tones.png");+ release();+ await expect(+ page.getByText(+ "Another photo opened before the answer came, so I left it unedited.",+ ),+ ).toBeVisible();+ expect(+ (await page.evaluate(() => window.openlight.getState())).adjustments+ .exposure,+ ).toBe(0);++ await message.press("Escape");+ await expect(message).toBeHidden();+ await expect(+ page.getByRole("button", { name: "Assistant", exact: true }),+ ).toBeFocused();+});diff --git a/tests/assistant.test.ts b/tests/assistant.test.tsnew file mode 100644index 0000000..e92cda8--- /dev/null+++ b/tests/assistant.test.ts@@ -0,0 +1,126 @@+import { expect, test } from "bun:test";+import { ask, interpret } from "@/app/assistant/jev";+import type { AssistantRequest } from "@/app/assistant/protocol";++const request: AssistantRequest = {+ message: "warmer and darker",+ earlier: [],+ photo: {+ adjustments: {+ exposure: 0.5,+ contrast: 0,+ highlights: 0,+ shadows: 0,+ whites: 0,+ blacks: 0,+ incrementalTemperature: 0,+ incrementalTint: 0,+ vibrance: 0,+ saturation: 0,+ },+ vignette: 0,+ clarity: 0,+ sharpening: 0,+ toneCurve: [+ { x: 0, y: 0 },+ { x: 1, y: 1 },+ ],+ comparison: "edited",+ sourceSize: [300, 200],+ },+};++/** Answers as Jev returns them: a choice per question, with the command's distribution. */+function answers(+ command: Record<string, number>,+ choices: Record<string, string>,+) {+ const [first] = Object.entries(command).sort((a, b) => b[1] - a[1]);+ return {+ command: { type: "choice", choice: first[0], probabilities: command },+ ...Object.fromEntries(+ Object.entries(choices).map(([id, choice]) => [+ id,+ { type: "choice", choice },+ ]),+ ),+ };+}++test("the assistant asks one question per decision and turns the answers into commands", () => {+ const { state, questions } = ask(request);+ expect(state.current.exposure).toBe(0.5);+ expect(Object.keys(questions)).toContain("value-temperature");+ expect(questions.command).toMatchObject({+ type: "choice",+ criteria: { reset: expect.any(String), reply: expect.any(String) },+ });++ expect(+ interpret(+ request,+ answers(+ { temperature: 0.6, reply: 0.2, exposure: 0.15, vignette: 0.05 },+ {+ count: "two",+ area: "photo",+ "value-temperature": "30.0",+ "value-exposure": "-1.0",+ },+ ),+ ),+ ).toEqual({+ commands: [+ { type: "set-adjustments", exposure: -1, incrementalTemperature: 30 },+ ],+ message: "temperature 0 → 30, exposure 0.5 → -1",+ });++ const sky = interpret(+ { ...request, photo: { ...request.photo, comparison: "original" } },+ answers(+ { exposure: 0.9, vignette: 0.1 },+ { count: "one", area: "top", "value-exposure": "-0.5" },+ ),+ );+ expect(sky.commands).toEqual([+ { type: "set-preview", comparison: "edited" },+ {+ type: "add-mask",+ mask: { kind: "linear", start: [150, 0], end: [150, 110] },+ adjustments: { exposure: -1 },+ },+ ]);+ expect(sky.message).toBe("top exposure -1");++ const inverted = {+ ...request,+ photo: {+ ...request.photo,+ toneCurve: [+ { x: 0, y: 1 },+ { x: 1, y: 0 },+ ],+ },+ };+ expect(interpret(inverted, answers({ invert: 1 }, { count: "one" }))).toEqual(+ {+ commands: [{ type: "set-tone-curve" }],+ message: "Colors restored",+ },+ );++ expect(+ interpret(+ request,+ answers(+ { reply: 0.9, exposure: 0.1 },+ { count: "one", reply: "greeting" },+ ),+ ),+ ).toEqual({+ commands: [],+ message:+ "Hi! I can edit this photo for you. Try “make it warmer” or “darken the sky”.",+ });+});diff --git a/tsconfig.json b/tsconfig.jsonindex 816ca13..6393540 100644--- a/tsconfig.json+++ b/tsconfig.json@@ -14,5 +14,5 @@"@/*": ["./src/*"]}},- "include": ["src", "tests"]+ "include": ["src", "tests", "api"]}