Skip to content

Free open-source image editor for the browser

15.6 KB
Raw Blame
import type { Experimental_EvaluationQuestion as Question } from "ai";
import { z } from "zod/mini";
import type { Command } from "@/app/commands";
import type { Adjustments } from "@/core/document";
import type { AssistantRequest, AssistantResponse, Photo } from "./protocol";

/**
 * How the assistant asks Jev, an evaluation model: one call answers typed questions about the
 * message in parallel, each by choosing among options, and the answers become commands.
 */

/** A control Jev sets by choosing one of its steps; `about` describes it among the commands and in its value question. */
type Slider = {
  about: string;
  min: number;
  max: number;
  step: number;
  read: (photo: Photo) => number;
};

/** An adjustment, from -100 to 100 in steps of 10 unless given. */
function adjustment(
  key: keyof Adjustments,
  about: string,
  limit = 100,
  step = 10,
): Slider & { key: keyof Adjustments } {
  const read = (photo: Photo) => photo.adjustments[key];
  return { key, about, min: -limit, max: limit, step, read };
}

/** Controls a mask can hold, so they can change one area. */
const adjustments = {
  exposure: adjustment(
    "exposure",
    "Exposure: make the whole image lighter or darker, in stops.",
    5,
    0.5,
  ),
  contrast: adjustment(
    "contrast",
    "Contrast: more or less difference between light and dark.",
  ),
  highlights: adjustment(
    "highlights",
    "Highlights: recover or brighten the bright areas.",
  ),
  shadows: adjustment("shadows", "Shadows: lift or deepen the dark areas."),
  whites: adjustment("whites", "Whites: move the white point."),
  blacks: adjustment("blacks", "Blacks: move the black point."),
  temperature: adjustment(
    "incrementalTemperature",
    "Temperature: warmer (yellow) or cooler (blue).",
  ),
  tint: adjustment("incrementalTint", "Tint: greener or more magenta."),
  vibrance: adjustment(
    "vibrance",
    "Vibrance: more or less color, gently, protecting vivid colors and skin. The usual choice for “more color”.",
  ),
  saturation: adjustment(
    "saturation",
    "Saturation: more or less color everywhere, down to black and white.",
  ),
};

/** Controls on effect layers, which change the whole photo. */
const effects = {
  vignette: {
    about: "Vignette: darker edges.",
    min: 0,
    max: 100,
    step: 5,
    read: (photo) => photo.vignette,
  },
  clarity: {
    about: "Clarity: local contrast, punchier or softer texture.",
    min: -100,
    max: 100,
    step: 10,
    read: (photo) => photo.clarity,
  },
  sharpening: {
    about: "Sharpening: crisper detail.",
    min: 0,
    max: 150,
    step: 10,
    read: (photo) => photo.sharpening,
  },
} satisfies Record<string, Slider>;

const sliders: Record<string, Slider> = { ...adjustments, ...effects };

type Tool = { about: string; commands: Command[]; message: string };

const inverted = [
  { x: 0, y: 1 },
  { x: 1, y: 0 },
];

const crops = {
  "1:1": "a square",
  "4:5": "4:5 portrait, as for an Instagram post",
  "3:2": "3:2 landscape",
  "16:9": "16:9 widescreen",
  "9:16": "9:16 vertical, as for a story",
};

/** Commands beside the sliders; only inverting depends on the photo, which it toggles. */
function tools({ toneCurve }: Photo): Record<string, Tool> {
  const isInverted =
    toneCurve.length === 2 &&
    toneCurve.every(
      ({ x, y }, i) => x === inverted[i].x && y === inverted[i].y,
    );
  const cropTo = Object.entries(crops).map(([ratio, about]) => {
    const [width, height] = ratio.split(":").map(Number);
    const tool: Tool = {
      about: `Crop to ${about}.`,
      commands: [{ type: "set-crop", aspectRatio: width / height }],
      message: `Cropped to ${ratio}`,
    };
    return [`crop-${ratio}`, tool];
  });
  return {
    invert: {
      about: "Invert the colors into a negative, or back.",
      commands: [
        isInverted
          ? { type: "set-tone-curve" }
          : { type: "set-tone-curve", points: inverted },
      ],
      message: isInverted ? "Colors restored" : "Colors inverted",
    },
    reset: {
      about:
        "Reset every edit at once, returning the whole photo to how it was opened. Resetting one control, such as “reset the contrast”, sets that control instead.",
      commands: [{ type: "reset" }],
      message: "Reset all edits",
    },
    undo: {
      about: "Undo the last change.",
      commands: [{ type: "undo" }],
      message: "Undone",
    },
    redo: {
      about: "Redo the last undone change.",
      commands: [{ type: "redo" }],
      message: "Redone",
    },
    original: {
      about: "Show the original photo for comparison, keeping the edits.",
      commands: [{ type: "set-preview", comparison: "original" }],
      message: "Showing the original. Say “show my edits” to go back.",
    },
    split: {
      about: "Compare before and after side by side.",
      commands: [{ type: "set-preview", comparison: "split" }],
      message: "Showing before and after",
    },
    edited: {
      about: "Show the edited photo again after showing the original.",
      commands: [{ type: "set-preview", comparison: "edited" }],
      message: "Showing your edits",
    },
    ...Object.fromEntries(cropTo),
    "crop-none": {
      about: "Remove the crop.",
      commands: [{ type: "set-crop" }],
      message: "Crop removed",
    },
  };
}

const reply =
  "Answer with a message instead of editing: a greeting, thanks, a question, an exact number, something the editor cannot do, or anything unrelated to this photo.";

/** Canned answers: conversation first, then why a request cannot be done. */
const replies: Record<string, string> = {
  greeting:
    "Hi! I can edit this photo for you. Try “make it warmer” or “darken the sky”.",
  thanks: "You're welcome! Anything else you'd like to change?",
  approval: "Great! Tell me if you'd like to change anything else.",
  goodbye: "Bye! Your edits stay saved in this browser.",
  capabilities:
    "I can change exposure, contrast, highlights, shadows, whites, blacks, temperature, tint, vibrance, saturation, vignette, clarity, and sharpening, in the whole photo or the sky, ground, sides, or center. I can also crop, invert colors, compare before and after, reset, and undo.",
  "how-to":
    "Tell me what to change in plain words, like “brighter and warmer”, “more contrast in the sky”, or “crop it for Instagram”.",
  identity:
    "I'm Jev, a quick assistant that edits this photo from short requests.",
  opinion:
    "I can't judge photos, but I can change whatever you'd like to improve.",
  export: "To save your photo, use Export in the top right.",
  open: "To edit another photo, use the folder button next to the file name.",
  "off-topic": "I can only help with editing this photo.",
  "unsupported-edit":
    "I can't do that edit yet. I can change light, color, detail, and vignette, crop, and invert colors.",
  "exact-value":
    "I can't set exact numbers yet; I move controls in steps. Try “a bit brighter” or “much more contrast”.",
  "out-of-range": "That's beyond what this control allows.",
  "several-areas":
    "I can change one area at a time. Ask for the sky and the ground separately.",
  unclear:
    "I'm not sure what to change. Try something like “brighter”, “warmer”, or “more contrast”.",
};

const areas = {
  photo: "The whole photo.",
  top: "The top of the photo, such as the sky.",
  bottom: "The bottom of the photo, such as the ground or foreground.",
  left: "The left side of the photo.",
  right: "The right side of the photo.",
  center: "The center of the photo, such as the main subject.",
};

const counts: Record<string, number> = { one: 1, two: 2, three: 3 };

function round(value: number) {
  return Math.round(value * 100) / 100;
}

/** Words for a change's size, so a plain "brighter" lands on a moderate step. */
function magnitude(change: number, { min, max }: Slider) {
  const share = Math.abs(change) / (max - min);
  if (share <= 0.05) {
    return "slightly";
  }
  if (share <= 0.15) {
    return "moderately";
  }
  return share <= 0.3 ? "strongly" : "extremely";
}

/** Every step from min to max, each described by its change from the current value. */
function valueQuestion(slider: Slider, current: number): Question {
  const { min, max, step } = slider;
  const count = Math.round((max - min) / step);
  const options = Array.from({ length: count + 1 }, (_, i) => {
    const value = round(min + i * step);
    const change = round(value - current);
    const direction = change > 0 ? "more" : "less";
    const notes = [
      change
        ? `${change > 0 ? "+" : ""}${change}, ${magnitude(change, slider)} ${direction}`
        : "the current value",
      value === 0 && "the default",
      value === min && "the minimum",
      value === max && "the maximum",
    ].filter(Boolean);
    // A fixed decimal keeps keys from sorting as integers, so options stay in order.
    return [value.toFixed(1), notes.join(", ")];
  });
  return {
    type: "choice",
    instructions: `The value the request asks for. ${slider.about} It is now ${current}; each option notes its change from now. Without a stated amount, prefer a moderate change.`,
    criteria: Object.fromEntries(options),
  };
}

/**
 * The state and questions for one message: how many changes it names, the command that fits best
 * with the others ranked behind it, the area, a reply, and each slider's value in case it is chosen.
 */
export function ask({ message, earlier, photo }: AssistantRequest) {
  const current = Object.fromEntries(
    Object.entries(sliders).map(([id, slider]) => [id, slider.read(photo)]),
  );
  const commands = Object.fromEntries([
    ...Object.entries({ ...sliders, ...tools(photo) }).map(
      ([id, { about }]) => [id, about],
    ),
    ["reply", reply],
  ]);
  const values = Object.entries(sliders).map(([id, slider]) => [
    `value-${id}`,
    valueQuestion(slider, slider.read(photo)),
  ]);
  const questions: Record<string, Question> = {
    count: {
      type: "choice",
      instructions: "How many different changes the request names.",
      criteria: {
        one: "One change, even if described with several words.",
        two: "Two different changes, such as “warmer and darker”.",
        three: "Three different changes.",
      },
    },
    command: {
      type: "choice",
      instructions:
        "The command that best fulfills the request; when it names several changes, each of them.",
      criteria: commands,
    },
    area: {
      type: "choice",
      instructions: "Which part of the photo the change applies to.",
      criteria: areas,
    },
    reply: {
      type: "choice",
      instructions: "The message that best answers the request.",
      criteria: replies,
    },
    ...Object.fromEntries(values),
  };
  return {
    state: { request: message, earlierRequests: earlier, current },
    questions,
  };
}

const answersSchema = z.record(
  z.string(),
  z.object({
    choice: z.optional(z.string()),
    probabilities: z.optional(z.record(z.string(), z.number())),
  }),
);
type Answers = z.output<typeof answersSchema>;

function choice(answers: Answers, id: string) {
  const choice = answers[id]?.choice;
  if (choice === undefined) {
    throw Error(`Jev did not answer ${id}.`);
  }
  return choice;
}

/** The chosen command, then as many more sliders as the message names changes, by probability. */
function chosen(answers: Answers) {
  const first = choice(answers, "command");
  const count = counts[choice(answers, "count")] ?? 1;
  const others = Object.entries(answers.command?.probabilities ?? {})
    .filter(([id]) => id !== first && id in sliders)
    .sort((a, b) => b[1] - a[1])
    .slice(0, count - 1)
    .map(([id]) => id);
  return [first, ...others];
}

type MaskInput = Extract<Command, { type: "add-mask" }>["mask"];

function areaMask(area: string, size: readonly number[]): MaskInput {
  // Whole source pixels, as a person would place the gradient.
  const [width, height] = size.map(Math.round);
  const at = (share: number, length: number) => Math.round(share * length);
  const middle: [number, number] = [at(0.5, width), at(0.5, height)];
  switch (area) {
    case "top":
      return {
        kind: "linear",
        start: [middle[0], 0],
        end: [middle[0], at(0.55, height)],
      };
    case "bottom":
      return {
        kind: "linear",
        start: [middle[0], height],
        end: [middle[0], at(0.45, height)],
      };
    case "left":
      return {
        kind: "linear",
        start: [0, middle[1]],
        end: [at(0.55, width), middle[1]],
      };
    case "right":
      return {
        kind: "linear",
        start: [width, middle[1]],
        end: [at(0.45, width), middle[1]],
      };
    default:
      return {
        kind: "radial",
        center: middle,
        radius: [at(0.3, width), at(0.35, height)],
        angle: 0,
        feather: 0.5,
      };
  }
}

/** Commands that set the chosen sliders: adjustments in one edit, in a new mask for an area. */
function edit(answers: Answers, photo: Photo, ids: string[]) {
  const area = choice(answers, "area");
  const changes = ids.map((id) => {
    const current = sliders[id].read(photo);
    const value = Number(choice(answers, `value-${id}`));
    return { id, current, value };
  });
  const changed = new Map(changes.map(({ id, value }) => [id, value]));
  const adjustmentChange: Partial<Adjustments> = {};
  for (const [id, { key, read }] of Object.entries(adjustments)) {
    const value = changed.get(id);
    if (value !== undefined) {
      // A mask starts neutral, so it takes the change rather than the value.
      adjustmentChange[key] = area === "photo" ? value : value - read(photo);
    }
  }
  const commands: Command[] = [];
  if (Object.keys(adjustmentChange).length > 0) {
    commands.push(
      area === "photo"
        ? { type: "set-adjustments", ...adjustmentChange }
        : {
            type: "add-mask",
            mask: areaMask(area, photo.sourceSize),
            adjustments: adjustmentChange,
          },
    );
  }
  const vignette = changed.get("vignette");
  if (vignette !== undefined) {
    commands.push({ type: "set-vignette", intensity: vignette });
  }
  const clarity = changed.get("clarity");
  const sharpening = changed.get("sharpening");
  if (clarity !== undefined || sharpening !== undefined) {
    commands.push({ type: "set-details", clarity, sharpening });
  }
  const local = area !== "photo";
  const summary = changes
    .map(({ id, current, value }) => {
      if (local && id in adjustments) {
        const change = round(value - current);
        return `${area} ${id} ${change > 0 ? "+" : ""}${change}`;
      }
      return `${id} ${current} → ${value}`;
    })
    .join(", ");
  return { commands, message: summary };
}

/** Turns Jev's answers into commands and a message for the user. */
export function interpret(
  { photo }: AssistantRequest,
  response: unknown,
): AssistantResponse {
  const answers = answersSchema.parse(response);
  const [first, ...others] = chosen(answers);
  if (first === "reply") {
    return {
      commands: [],
      message: replies[choice(answers, "reply")] ?? replies.unclear,
    };
  }
  const tool = tools(photo)[first];
  const moved = [first, ...others].filter((id) => id in sliders);
  const edits = moved.length > 0 ? edit(answers, photo, moved) : undefined;
  const commands = [...(tool?.commands ?? []), ...(edits?.commands ?? [])];
  const message = [tool?.message, edits?.message].filter(Boolean).join(" · ");
  const editing = commands.some((command) => command.type !== "set-preview");
  // An edit made while the original shows would look like it did nothing.
  if (editing && photo.comparison === "original") {
    commands.unshift({ type: "set-preview", comparison: "edited" });
  }
  return { commands, message: message || replies.unclear };
}