Skip to content

Free open-source image editor for the browser

21.3 KB
Raw Blame
import type { Gpu, Target, Timer } from "vgpu";
import {
  type BrushStroke,
  hasPaint,
  type MaskLayer,
  type PaintLayer,
  paintingOf,
  type RemoveField,
  type RemovePatch,
  removePatches,
  type Scene,
  walkLayers,
} from "@/core/document";
import type { ImageSource, WhiteBalance } from "@/core/image";
import type { Point } from "@/core/image/frame";
import { createFieldStore, type FieldLattice } from "./fields";
import { createRenderGraph } from "./graph";
import { createMaskCoverage } from "./mask";
import { createPatchRaster, type PatchInput } from "./mask/patches";
import { input, type RenderImage } from "./node";
import {
  type AcceptPainting,
  createPaintRaster,
  type PaintInput,
} from "./paint";
import { createProxy } from "./proxy";
import { createStrokes } from "./strokes";

export {
  curveInput,
  gradientModifiers,
  gradientParams,
  isGradient,
  mixAdjustment,
  modifierData,
} from "./blend";
export {
  type Clipping,
  type CoverageRegion,
  createDisplay,
  type MaskOverlay,
  renderBitmap,
  renderCoverage,
  type View,
} from "./display";
export type { FieldLattice } from "./fields";
export type { PatchInput } from "./mask/patches";
export {
  input,
  merge,
  type NodeDefinition,
  node,
  pipeline,
  type RenderImage,
  type RenderInput,
  type RenderNode,
  type RenderStep,
  type Scale,
  sourceSize,
  split,
} from "./node";
export type { PaintInput } from "./paint";
export { transformImage } from "./transform";
export { createRenderGraph };

export type Composition = {
  inputId?: string;
  /** The mask whose ranges' image, the one below it, to keep for picking colors from it. */
  rangeSourceId?: string;
  /** Keep one stable composition instance and its render-graph resources reusable. */
  retain: (id: string) => void;
  /**
   * A mask's coverage over `below`, the image it applies to, when a brush or range takes part;
   * gradients alone leave it to the mix pass.
   */
  coverage: (layer: MaskLayer, below: RenderImage) => RenderImage | undefined;
  /** Rasterized paint of a paint layer, prepared before composition. */
  paint: (layer: PaintLayer) => PaintInput | undefined;
  /** Rasterized coverage of an effect's own stroke, such as a Healing patch. */
  patch: (id: string, strokes: readonly BrushStroke[]) => PatchInput;
  /** The Remove field a patch shows, when the renderer holds it. */
  field: (patch: RemovePatch) => PatchField | undefined;
  /** Keeps a Remove field this render synthesizes for a patch, for the renders after it. */
  keepField: (
    patch: RemovePatch,
    lattice: FieldLattice,
    texels: RenderImage,
  ) => void;
};

/** A Remove field a render shows. */
export type PatchField = { texels: RenderImage; lattice: FieldLattice };

/** App composition describes requested outputs; the engine owns their storage. */
export type SceneProcessing = (
  source: RenderImage,
  scene: Scene,
  composition: Composition,
) => {
  full: RenderImage;
  output: RenderImage;
  input?: RenderImage;
  rangeSource?: RenderImage;
};

type RenderRequest = {
  scene: Scene;
  inputId?: string;
  rangeSourceId?: string;
  /** Device pixels shown per source pixel, which the render reduces the source to about. */
  density: number;
  /** Source pixels per texel of the composition's source; above 1 renders a reduced proxy. */
  factor: number;
  /** An open gesture, which shows a Remove patch unfilled rather than synthesizing its field. */
  interactive: boolean;
};

/** A Remove field a render synthesized, to keep. */
type Synthesis = { id: string; lattice: FieldLattice; texels: RenderImage };

/** The largest whole reduction that still gives every device pixel a texel. */
function reductionFor(density: number) {
  if (!Number.isFinite(density) || density <= 0) {
    return 1;
  }
  return Math.max(1, Math.floor(1 / density));
}

/** How many reductions keep their proxy and intermediates, each a quarter of the photo at most. */
const keptReductions = 4;

function sameBalance(a: WhiteBalance | undefined, b: WhiteBalance | undefined) {
  return a?.temperature === b?.temperature && a?.tint === b?.tint;
}

/** A renderer's timer, and where it reads what the document stores outside the scene, by the IDs the scene names. */
export type RendererOptions = {
  timer?: Timer;
  /** The settled pixels a paint layer's or brush mask's `raster` names. */
  paintPixels?: (id: string) => Blob;
  /** The Remove field the document saved under an ID, once synthesized. */
  field?: (id: string) => RemoveField | undefined;
  /** Receives each Remove field the renderer synthesizes, read back; without it nothing is read back. */
  saveField?: (id: string, field: RemoveField) => void;
};

/**
 * Owns scene passes, mask and paint rasters, Remove fields, the proxy, and intermediate textures for
 * one decoded source.
 */
export function createRenderer(
  gpu: Gpu,
  resource: ImageSource,
  compose: SceneProcessing,
  {
    timer,
    paintPixels = () => {
      throw Error("This renderer has no settled paint.");
    },
    field: savedField = () => undefined,
    saveField,
  }: RendererOptions = {},
) {
  const source = resource.image;
  const graph = createRenderGraph(gpu, timer);
  const strokes = createStrokes(gpu);
  const brushes = createPaintRaster(gpu, strokes, "r8unorm");
  const masks = createMaskCoverage(brushes);
  const patches = createPatchRaster(gpu, strokes);
  const paints = createPaintRaster(gpu, strokes, "rgba8unorm");
  const fields = createFieldStore(gpu);
  const proxy = createProxy(gpu);
  const release = resource.retain();
  const raw = resource.raw?.createPass();
  let full = source;
  const listeners = new Set<() => void>();
  let rendered = false;
  let output = source;
  let inspected: { id: string; image: Target } | undefined;
  let kept: { id: string; image: Target } | undefined;
  /** Mask coverage the graph rendered for the overlay and thumbnails, by layer ID. */
  let shown = new Map<string, Target>();
  let balance = resource.raw?.asShot;
  /** Counts developments, so the proxy follows white-balance changes. */
  let version = 0;
  let last: RenderRequest | undefined;
  /** Whether the last render, a gesture's proxy, showed a Remove patch unfilled. */
  let lacked = false;
  /** The passes the last render's synthesis ran apart, before its own. */
  let synthesized: string[] = [];
  /**
   * The reductions rendered at last, latest first, whose proxies and intermediates stay: a zoom
   * returns to them, and Safari penalizes textures made and freed. The photo's own size never stays.
   */
  let reductions: number[] = [];
  let next: RenderRequest | undefined;
  let pending: Promise<void> | undefined;
  /** A raster being read back, a painting to settle or a Remove field to save; nothing renders meanwhile. */
  let settling: Promise<unknown> | undefined;
  /** Readbacks of synthesized fields in flight, by ID, which finish even once the renderer closes. */
  const capturing = new Map<string, Promise<RemoveField | undefined>>();
  let disposed = false;
  let instances = new Set<string>();
  /**
   * Composes the scene over `image`, at one reduction: what the composition yields, the Remove fields
   * it synthesizes, the brush coverage to show, the instances it keeps, and whether it lacked a field.
   */
  function composeAt(
    { scene, inputId, rangeSourceId }: RenderRequest,
    image: RenderImage,
    developed: Target,
  ) {
    const active = new Set<string>();
    const covered = new Map<string, RenderImage>();
    // Read brush views after every raster has updated, including inactive submasks.
    for (const { layer } of walkLayers(scene.layers)) {
      if (
        layer.kind === "mask" &&
        layer.mask.kind === "brush" &&
        hasPaint(layer.mask)
      ) {
        const coverage = brushes.coverage(layer.id);
        if (coverage) covered.set(layer.id, coverage);
      }
    }
    const syntheses: Synthesis[] = [];
    let lacking = false;
    const images = compose(image, scene, {
      inputId,
      rangeSourceId,
      retain: (id) => active.add(id),
      coverage: (layer, below) =>
        masks.coverage(layer, below, {
          retain: (id) => active.add(id),
          show: (id, coverage) => covered.set(id, coverage),
        }),
      paint: (layer) => paints.input(layer),
      patch: (id, strokes) => patches.patch(id, strokes, developed.size),
      // Patches that share a field, such as duplicates, show the one this render synthesizes.
      field: (patch) => {
        const held =
          syntheses.find(({ id }) => id === patch.field) ??
          fields.get(patch.field);
        lacking ||= !held;
        return held;
      },
      keepField: (patch, lattice, texels) =>
        syntheses.push({ id: patch.field, lattice, texels }),
    });
    return { images, syntheses, covered, active, lacking };
  }
  /** Keeps the fields a render synthesized and reads them back for the document. */
  function keepSyntheses(syntheses: readonly Synthesis[], targets: Target[]) {
    for (const [i, { id, lattice }] of syntheses.entries()) {
      fields.keep(id, lattice, targets[i]);
    }
    if (saveField && syntheses.length) {
      capture(syntheses.map(({ id }) => id));
    }
  }
  function render(request: RenderRequest): void {
    const { scene, inputId, rangeSourceId, factor } = request;
    const unchanged =
      last &&
      last.scene === scene &&
      last.inputId === inputId &&
      last.rangeSourceId === rangeSourceId &&
      last.factor === factor;
    // A gesture's proxy shows a patch unfilled; the gesture's end renders it again to synthesize.
    if (unchanged && !(lacked && !request.interactive)) {
      return;
    }
    const developed = raw?.render() ?? source;
    // Every brush and paint layer updates once, bypassed or not, so hidden layers keep their rasters.
    for (const { layer } of walkLayers(scene.layers)) {
      if (layer.kind === "mask" && layer.mask.kind === "brush") {
        brushes.draw(layer.id, layer.mask, developed.size);
      }
      if (layer.kind === "paint") {
        paints.draw(layer.id, layer, developed.size);
      }
    }
    // Remove fields stay while their patches do, shown or not.
    for (const { patch } of removePatches(scene.layers)) {
      fields.retain(patch.field);
    }
    const image =
      factor > 1 ? proxy.render(developed, factor, version) : input(developed);
    let composed = composeAt(request, image, developed);
    synthesized = [];
    // Only a full render synthesizes a field. A proxy that lacks one has it synthesized apart, from
    // the layers below the patch alone, at full size, and composes again with it; a gesture shows the
    // patch unfilled instead.
    if (composed.lacking && factor > 1 && !request.interactive) {
      const { syntheses } = composeAt(
        { ...request, factor: 1 },
        input(developed),
        developed,
      );
      if (syntheses.length) {
        const texels = graph.render(
          syntheses.map(({ texels }) => texels),
          { set: 1, kept: reductions },
        );
        keepSyntheses(syntheses, texels);
        synthesized = graph.inspect().passes;
      }
      composed = composeAt(request, image, developed);
    }
    const { images, syntheses, covered, active } = composed;
    lacked = composed.lacking && factor > 1;
    for (const id of instances) {
      if (!active.has(id)) graph.release(`${id}/`);
    }
    instances = active;
    // The stroke buffer goes last, once the others let go of the strokes they left open.
    masks.sweep();
    patches.sweep();
    paints.sweep();
    strokes.sweep();
    // The inspected input goes first, so the image it reads is let go once the composition is done
    // with it; the two images every render shows, the range source, and coverage follow.
    const inputs = images.input ? [images.input] : [];
    const sources = images.rangeSource ? [images.rangeSource] : [];
    if (factor > 1) {
      reductions = [factor, ...reductions.filter((kept) => kept !== factor)];
      reductions.length = Math.min(reductions.length, keptReductions);
    }
    const targets = graph.render(
      [
        ...inputs,
        images.full,
        images.output,
        ...sources,
        ...covered.values(),
        ...syntheses.map(({ texels }) => texels),
      ],
      { set: factor, kept: reductions },
    );
    proxy.sweep(reductions);
    const [inputTarget] = targets;
    [full, output] = targets.slice(inputs.length);
    const [sourceTarget] = targets.slice(inputs.length + 2);
    const coverages = targets.slice(
      inputs.length + 2 + sources.length,
      inputs.length + 2 + sources.length + covered.size,
    );
    keepSyntheses(syntheses, targets.slice(targets.length - syntheses.length));
    fields.sweep();
    inspected =
      inputId && images.input ? { id: inputId, image: inputTarget } : undefined;
    kept =
      rangeSourceId && images.rangeSource
        ? { id: rangeSourceId, image: sourceTarget }
        : undefined;
    shown = new Map([...covered.keys()].map((id, i) => [id, coverages[i]]));
    rendered = true;
    last = request;
    for (const listener of listeners) {
      listener();
    }
  }
  /** The raster of a paint layer's color or of a brush mask's coverage. */
  function paintRaster(id: string) {
    return paints.get(id) ? paints : brushes;
  }
  /** Remove fields to load before rendering: saved ones the renderer holds nothing for. */
  function staleFields(scene: Scene) {
    const stale = new Map<string, RemoveField>();
    for (const { patch } of removePatches(scene.layers)) {
      const saved = savedField(patch.field);
      if (saved && !fields.get(patch.field)) {
        stale.set(patch.field, saved);
      }
    }
    return [...stale];
  }
  /**
   * Reads the fields a render synthesized back for the document, rendering nothing meanwhile, so no
   * render replaces one first. A field whose readback fails stays held, and `captureFields` reads it
   * again for a snapshot, which reports the failure.
   */
  function capture(ids: readonly string[]) {
    let reading: Promise<unknown> = Promise.resolve();
    for (const id of ids) {
      const read = reading.then(() => fields.read(id));
      capturing.set(id, read);
      reading = read
        .then((field) => field && saveField?.(id, field))
        .catch(() => {})
        .finally(() => {
          if (capturing.get(id) === read) {
            capturing.delete(id);
          }
        });
    }
    const done = reading;
    settling = done;
    void done.then(() => {
      if (settling === done) {
        settling = undefined;
      }
    });
  }
  /** Paintings whose rasters must load settled pixels before they draw. */
  function stalePaint(scene: Scene) {
    const stale = [];
    for (const { layer } of walkLayers(scene.layers)) {
      const painting = paintingOf(layer);
      const rasters = layer.kind === "paint" ? paints : brushes;
      if (painting && rasters.needsBase(layer.id, painting)) {
        stale.push({ id: layer.id, raster: painting.raster, rasters });
      }
    }
    return stale;
  }
  /**
   * Renders the latest request once what it waits for is ready: a settle, a RAW development, whose
   * calibration alone crosses the worker, settled paint, or Remove fields to load.
   */
  async function develop() {
    while (next && !disposed) {
      const request = next;
      next = undefined;
      const { scene } = request;
      if (settling) {
        await settling;
      }
      if (disposed) return;
      if (next) continue;
      const selected = scene.layers[0].whiteBalance ?? resource.raw?.asShot;
      if (raw && selected && !sameBalance(balance, selected)) {
        await raw.prepare(selected);
        if (disposed) {
          return;
        }
        balance = selected;
        version++;
      }
      for (const { id, raster, rasters } of stalePaint(scene)) {
        await rasters.loadBase(id, raster, paintPixels(raster), source.size);
        if (disposed) {
          return;
        }
      }
      for (const [id, field] of staleFields(scene)) {
        await fields.load(id, field);
        if (disposed) {
          return;
        }
      }
      if (!next) {
        render(request);
      }
    }
  }
  /**
   * Renders reduce the source to about `density`, device pixels shown per source pixel, a proxy; one
   * that lacks a Remove patch's field synthesizes it apart in full, outside a gesture. `inputId` keeps
   * a layer's curve input, and `rangeSourceId` a mask's range source, for reading them.
   */
  async function update(
    scene: Scene,
    inputId?: string,
    interactive = false,
    rangeSourceId?: string,
    density = 1,
  ): Promise<void> {
    if (disposed) {
      throw Error("Renderer is closed.");
    }
    const request = {
      scene,
      inputId,
      rangeSourceId,
      density,
      factor: reductionFor(density),
      interactive,
    };
    // Renders wait, in order, for a settle, a RAW development, or settled paint and fields to load.
    if (
      !raw &&
      !pending &&
      !settling &&
      !stalePaint(scene).length &&
      !staleFields(scene).length
    ) {
      render(request);
      return;
    }
    next = request;
    pending ??= develop()
      .catch((error) => {
        if (!next) {
          throw error;
        }
      })
      .finally(() => {
        pending = undefined;
        if (next && !disposed) {
          const { scene, inputId, interactive, rangeSourceId, density } = next;
          return update(scene, inputId, interactive, rangeSourceId, density);
        }
      });
    return pending;
  }

  /** Runs a readback once renders in flight finish, and renders nothing until it does. */
  async function hold<T>(read: () => Promise<T>) {
    while (pending || settling) {
      await (pending ?? settling);
    }
    if (disposed) return undefined;
    const reading = read();
    settling = reading;
    try {
      return await reading;
    } finally {
      settling = undefined;
    }
  }

  function releaseResources() {
    graph.dispose();
    masks.dispose();
    patches.dispose();
    paints.dispose();
    fields.dispose();
    strokes.dispose();
    proxy.dispose();
    raw?.dispose();
    release();
  }

  return {
    fullImage: () => full,
    outputImage: () => output,
    inputImage: (id: string) =>
      inspected?.id === id ? inspected.image : undefined,
    /** The image below a mask, which its ranges read, while an update keeps it. */
    rangeSource: (id: string) => (kept?.id === id ? kept.image : undefined),
    /**
     * A raster by ID and where it sits in the photo: a mask's coverage, for the display overlay and
     * thumbnails, a Healing patch's, or a paint layer's.
     */
    coverage(id: string): { target: Target; origin: Point } | undefined {
      const target = shown.get(id) ?? paints.get(id);
      return target ? { target, origin: [0, 0] } : patches.raster(id);
    },
    inspect: () => ({
      ...graph.inspect(),
      passes: [...synthesized, ...graph.inspect().passes],
      stamped: strokes.stamped(),
      rasters: [
        ...masks.inspect(),
        ...patches.inspect(),
        ...paints.inspect(),
        ...fields.inspect(),
        ...strokes.inspect(),
      ],
    }),
    subscribe(listener: () => void) {
      listeners.add(listener);
      if (rendered) {
        listener();
      }
      return () => {
        listeners.delete(listener);
      };
    },
    update,
    /**
     * Reads a paint layer's raster once renders in flight finish, holding back the next ones, so its
     * strokes, scene, and cached raster settle together before rendering resumes.
     */
    settle(id: string, accept: AcceptPainting) {
      return hold(() =>
        paintRaster(id).settle(id, (painting) =>
          disposed ? undefined : accept(painting),
        ),
      );
    },
    /**
     * The fields among `ids` that the document saved or this renderer holds, once renders and readbacks
     * in flight finish, saving those the document still waits for; rejects when a readback fails. Once
     * the renderer closes, it gives what its readbacks in flight read, and rejects for a field it held
     * that none was reading.
     */
    async captureFields(ids: readonly string[]) {
      const inFlight = ids.flatMap((id) => {
        const read = capturing.get(id);
        return read ? [[id, read] as const] : [];
      });
      const unread = ids.some(
        (id) => !capturing.has(id) && fields.get(id) && !savedField(id),
      );
      const found = await hold(async () => {
        const found = new Map<string, RemoveField>();
        for (const id of ids) {
          const saved = savedField(id);
          if (saved) {
            found.set(id, saved);
            continue;
          }
          const read = await fields.read(id);
          if (read) {
            found.set(id, read);
            saveField?.(id, read);
          }
        }
        return found;
      });
      if (found) {
        return found;
      }
      if (unread) {
        throw Error(
          "The photo closed before its Remove fields were read back.",
        );
      }
      const read = new Map<string, RemoveField>();
      for (const [id, reading] of inFlight) {
        const field = await reading;
        if (field) {
          read.set(id, field);
        }
      }
      return read;
    },
    dispose() {
      if (disposed) {
        return;
      }
      disposed = true;
      listeners.clear();
      // Banded transfers keep their targets until their last GPU operation finishes.
      const finishing = pending ?? settling;
      if (finishing) {
        void finishing.then(releaseResources, releaseResources);
      } else {
        releaseResources();
      }
    },
  };
}