Skip to content

Open source CapCut alternative.

TypeScript
14.9 KB
Raw Blame
import { afterAll, beforeEach, describe, expect, spyOn, test } from "bun:test";
import * as kit from "@donkeycut/effects-kit";
import { applyDetail, applyLutToImageData, buildClipLut } from "@donkeycut/effects-kit";
import { FrameCompositor, type Frame } from "./composite";
import { beginLutDraft, clearClipLuts, clipLutKey, endLutDraft, peekClipLut, requestClipLut } from "./lutBuild";
import { setRasterFactory, type RasterSurface } from "./raster";
import type { VideoClip } from "./types";

// The compositor's color path, drawn onto a pixel canvas that stands in for
// the DOM's: what a clip's pixels are after its source conversion, its grade
// and its detail pass — the same numbers the kit's reference passes produce,
// which is what ties the preview to the export.

/** A canvas of bytes: same-size draws copy pixels source-over, reads and
 * writes go straight to the buffer, and everything else is accepted and
 * ignored. No WebGL, so the GPU passes step aside for the CPU ones. */
class PixelCanvas {
  data: Uint8ClampedArray;
  private w: number;
  private h: number;
  readonly ctx: Record<string, unknown>;
  constructor(w: number, h: number) {
    this.w = Math.max(1, w);
    this.h = Math.max(1, h);
    this.data = new Uint8ClampedArray(this.w * this.h * 4);
    this.ctx = new Proxy(
      {
        canvas: this,
        filter: "none",
        globalAlpha: 1,
        globalCompositeOperation: "source-over",
        fillStyle: "#000000",
        imageSmoothingEnabled: true,
        clearRect: () => this.data.fill(0),
        fillRect: () => {
          const hex = /^#?([0-9a-f]{6})$/i.exec(String(this.ctx.fillStyle))?.[1] ?? "000000";
          const [r, g, b] = [0, 2, 4].map((i) => parseInt(hex.slice(i, i + 2), 16));
          for (let i = 0; i < this.data.length; i += 4) {
            this.data[i] = r;
            this.data[i + 1] = g;
            this.data[i + 2] = b;
            this.data[i + 3] = 255;
          }
        },
        drawImage: (src: PixelCanvas, ...args: number[]) => {
          const [dx, dy, dw, dh] =
            args.length >= 8 ? args.slice(4) : args.length >= 4 ? args : [args[0], args[1], src.w, src.h];
          for (let y = 0; y < this.h; y++) {
            for (let x = 0; x < this.w; x++) {
              const sx = Math.floor(((x - dx) / dw) * src.w);
              const sy = Math.floor(((y - dy) / dh) * src.h);
              if (sx < 0 || sy < 0 || sx >= src.w || sy >= src.h) continue;
              const si = (sy * src.w + sx) * 4;
              const di = (y * this.w + x) * 4;
              const a = src.data[si + 3] / 255;
              for (let c = 0; c < 3; c++) this.data[di + c] = Math.round(src.data[si + c] * a + this.data[di + c] * (1 - a));
              this.data[di + 3] = Math.round(255 * (a + (this.data[di + 3] / 255) * (1 - a)));
            }
          }
        },
        getImageData: (x: number, y: number, w: number, h: number) => ({
          data: this.data.slice(0),
          width: w,
          height: h,
        }),
        putImageData: (img: { data: Uint8ClampedArray }) => this.data.set(img.data),
      } as Record<string, unknown>,
      {
        get: (target, prop) =>
          prop in target ? target[prop as string] : typeof prop === "string" ? () => undefined : undefined,
        set: (target, prop, value) => ((target[prop as string] = value), true),
      }
    );
  }
  get width() {
    return this.w;
  }
  set width(v: number) {
    this.w = Math.max(1, v);
    this.data = new Uint8ClampedArray(this.w * this.h * 4);
  }
  get height() {
    return this.h;
  }
  set height(v: number) {
    this.h = Math.max(1, v);
    this.data = new Uint8ClampedArray(this.w * this.h * 4);
  }
  getContext(kind: string) {
    return kind === "2d" ? this.ctx : null;
  }
}

setRasterFactory({
  createCanvas: (w, h) => new PixelCanvas(w, h) as unknown as RasterSurface,
  decodeImage: async () => null,
  canvasToBlob: async () => new Blob(),
  snapshot: async (canvas) => canvas as unknown as ImageBitmap,
});
afterAll(() => clearClipLuts());

const W = 8;
const H = 8;

/** A picture that walks the cube: red across, green down, blue held. */
function picture(): PixelCanvas {
  const c = new PixelCanvas(W, H);
  for (let y = 0; y < H; y++) {
    for (let x = 0; x < W; x++) {
      const i = (y * W + x) * 4;
      c.data[i] = x * 32;
      c.data[i + 1] = y * 32;
      c.data[i + 2] = 128;
      c.data[i + 3] = 255;
    }
  }
  return c;
}

const frameOf = (c: PixelCanvas): Frame => ({ kind: "ready", image: c as unknown as CanvasImageSource, width: W, height: H });

const clipOf = (over: Partial<VideoClip> = {}): VideoClip => ({
  id: "c1",
  assetId: "a1",
  track: 0,
  start: 0,
  in: 0,
  out: 4,
  muted: false,
  ...over,
});

let canvas: PixelCanvas;
let comp: FrameCompositor;

beforeEach(() => {
  clearClipLuts();
  canvas = new PixelCanvas(W, H);
  comp = new FrameCompositor(canvas as unknown as RasterSurface);
  comp.colorMode = "exact";
});

const drawn = () => Array.from(canvas.data);
const rgb = (px: ArrayLike<number>) => Array.from(px).filter((_, i) => i % 4 !== 3);

describe("the compositor's color path", () => {
  test("a Rec.709 clip with no grade draws its pixels as they are", () => {
    const src = picture();
    comp.drawLayer(frameOf(src), clipOf(), true, 1, 0);
    expect(drawn()).toEqual(Array.from(src.data));
    expect(requestClipLut(comp.recipeFor(clipOf()))).toBeNull();
  });

  test("a log source draws through its conversion LUT, graded or not", () => {
    comp.sourceProvider = () => ({ profile: "apple-log" });
    const src = picture();
    comp.drawLayer(frameOf(src), clipOf(), true, 1, 0);
    const expected = picture().data;
    applyLutToImageData(expected, buildClipLut({ profile: "apple-log", output: "sdr", size: 65 })!);
    expect(rgb(drawn())).toEqual(rgb(expected));
    expect(rgb(drawn())).not.toEqual(rgb(src.data));
  });

  test("a grade bakes into the LUT the export writes, and the pixels match it", () => {
    const grade = { exposure: 10, contrast: 8, saturation: -10 };
    comp.drawLayer(frameOf(picture()), clipOf({ grade }), true, 1, 0);
    const expected = picture().data;
    applyLutToImageData(expected, buildClipLut({ profile: "rec709", grade, output: "sdr", size: 33 })!);
    expect(rgb(drawn())).toEqual(rgb(expected));
  });

  test("sharpen and clarity run after the LUT as the detail pass", () => {
    const grade = { exposure: 10, sharpen: 30, clarity: 20 };
    comp.drawLayer(frameOf(picture()), clipOf({ grade }), true, 1, 0);
    const expected = picture().data;
    applyLutToImageData(expected, buildClipLut({ profile: "rec709", grade: { exposure: 10 }, output: "sdr", size: 33 })!);
    applyDetail(expected, W, H, { sharpen: 30, clarity: 20 });
    expect(rgb(drawn())).toEqual(rgb(expected));
  });

  test("a library LUT that is not on the shelf renders the grade without it, keyed apart", () => {
    const grade = { exposure: 10, lut: { id: "lut:nothere" } };
    comp.drawLayer(frameOf(picture()), clipOf({ grade }), true, 1, 0);
    const expected = picture().data;
    applyLutToImageData(expected, buildClipLut({ profile: "rec709", grade: { exposure: 10 }, output: "sdr", size: 33 })!);
    expect(rgb(drawn())).toEqual(rgb(expected));
    const recipe = comp.recipeFor(clipOf({ grade }));
    expect(clipLutKey(recipe, "lut:nothere", false).endsWith("|lut:nothere:off")).toBe(true);
    expect(clipLutKey(recipe, "lut:nothere", true).endsWith("|lut:nothere:on")).toBe(true);
  });

  test("an HLG composite maps every recipe into HLG, the frame color with it", () => {
    comp.output = "hlg";
    expect(comp.recipeFor(clipOf()).output).toBe("hlg");
    comp.sourceProvider = () => ({ profile: "apple-log" });
    expect(comp.recipeFor(clipOf()).output).toBe("hlg");
    comp.sourceProvider = () => ({ profile: "rec709" });
    // A Rec.709 clip is a conversion now: its pixels land at HLG code values,
    // white at reference white.
    const src = picture();
    comp.drawLayer(frameOf(src), clipOf(), true, 1, 0);
    const expected = picture().data;
    applyLutToImageData(expected, buildClipLut({ profile: "rec709", output: "hlg", size: 33 })!);
    expect(rgb(drawn())).toEqual(rgb(expected));
    expect(rgb(drawn())).not.toEqual(rgb(src.data));
    comp.background = "#ffffff";
    comp.clear();
    expect(Array.from(canvas.data.slice(0, 3))).toEqual([191, 191, 191]);
    comp.output = "sdr";
    expect(comp.recipeFor(clipOf()).output).toBe("sdr");
    comp.clear();
    expect(Array.from(canvas.data.slice(0, 3))).toEqual([255, 255, 255]);
  });

  test("a live drag sizes the cube down and the settle brings the full one back", () => {
    const log = clipOf({ id: "log", assetId: "log" });
    comp.colorMode = "live";
    comp.sourceProvider = (c) => ({ profile: c.assetId === "log" ? "apple-log" : "rec709" });
    expect(comp.recipeFor(clipOf()).size).toBe(33);
    expect(comp.recipeFor(log).size).toBe(65);
    beginLutDraft();
    expect(comp.recipeFor(clipOf()).size).toBe(17);
    expect(comp.recipeFor(log).size).toBe(17);
    endLutDraft();
    expect(comp.recipeFor(clipOf()).size).toBe(33);
  });

  test("a render that writes a picture builds the full cube during a drag", () => {
    // An in-tab export, mask or removal bake running while a slider moves in
    // the editor keeps the full cube: the drag's draft is the preview's alone.
    const log = clipOf({ id: "log", assetId: "log" });
    comp.sourceProvider = (c) => ({ profile: c.assetId === "log" ? "apple-log" : "rec709" });
    beginLutDraft();
    try {
      expect(comp.recipeFor(clipOf()).size).toBe(33);
      expect(comp.recipeFor(log).size).toBe(65);
    } finally {
      endLutDraft();
    }
  });

  test("a frame that changes nothing reuses the clip's recipe and key", () => {
    const log = { profile: "apple-log" as const };
    comp.sourceProvider = () => log;
    const grade = { exposure: 10 };
    const clip = clipOf({ grade });
    const keyOf = spyOn(kit, "recipeKey");
    try {
      const recipe = comp.recipeFor(clip);
      const first = requestClipLut(recipe, { exact: true })!;
      const keyed = keyOf.mock.calls.length;
      expect(keyed).toBeGreaterThan(0);
      for (let i = 0; i < 5; i++) {
        const again = comp.recipeFor(clip);
        expect(again).toBe(recipe);
        expect(requestClipLut(again, { exact: true })).toBe(first);
      }
      expect(keyOf.mock.calls.length).toBe(keyed);
      // A new grade, a new source or a new output is a new recipe and key.
      const regraded = comp.recipeFor(clipOf({ grade: { exposure: 20 } }));
      expect(regraded).not.toBe(recipe);
      expect(clipLutKey(regraded, null, false)).not.toBe(first.key);
      comp.sourceProvider = () => ({ profile: "hlg" });
      const resourced = comp.recipeFor(clip);
      expect(resourced).not.toBe(recipe);
      expect(clipLutKey(resourced, null, false)).not.toBe(first.key);
      comp.sourceProvider = () => log;
      comp.output = "hlg";
      expect(comp.recipeFor(clip)).not.toBe(recipe);
      comp.output = "sdr";
      expect(comp.recipeFor(clip)).toBe(recipe);
    } finally {
      keyOf.mockRestore();
    }
  });

  test("a built LUT is kept and answered from the cache", () => {
    const recipe = comp.recipeFor(clipOf({ grade: { exposure: 10 } }));
    const first = requestClipLut(recipe, { exact: true })!;
    expect(first.lut.size).toBe(33);
    expect(peekClipLut(first.key)?.lut).toBe(first.lut);
    expect(requestClipLut(recipe, { exact: true })!.lut).toBe(first.lut);
    clearClipLuts();
    expect(peekClipLut(first.key)).toBeUndefined();
  });
});

describe("a keyed clip's softening", () => {
  /** Every filter and blend mode the compositor sets while `draw` runs, on
   * the frame and on each scratch it makes. */
  const settingsDuring = (draw: (c: FrameCompositor) => void): string[] => {
    const seen: string[] = [];
    const spy = (c: PixelCanvas): PixelCanvas => {
      const inner = c.ctx;
      (c as unknown as { ctx: unknown }).ctx = new Proxy(inner, {
        set: (target, prop, value) => {
          if (prop === "filter" || prop === "globalCompositeOperation") seen.push(String(value));
          (target as Record<string, unknown>)[prop as string] = value;
          return true;
        },
      });
      return c;
    };
    setRasterFactory({
      createCanvas: (w, h) => spy(new PixelCanvas(w, h)) as unknown as RasterSurface,
      decodeImage: async () => null,
      canvasToBlob: async () => new Blob(),
      snapshot: async (canvas) => canvas as unknown as ImageBitmap,
    });
    try {
      draw(new FrameCompositor(spy(new PixelCanvas(W, H)) as unknown as RasterSurface));
    } finally {
      setRasterFactory({
        createCanvas: (w, h) => new PixelCanvas(w, h) as unknown as RasterSurface,
        decodeImage: async () => null,
        canvasToBlob: async () => new Blob(),
        snapshot: async (canvas) => canvas as unknown as ImageBitmap,
      });
    }
    return seen;
  };
  const key = (t: number, x: number, blur?: number) => ({ t, x, y: 0.5, scale: 1, rotation: 0, opacity: 1, ...(blur !== undefined ? { blur } : {}) });
  const rect = { x: 0.25, y: 0, w: 0.5, h: 1 };

  test("pose keys carrying blur blur the clip as it lands", () => {
    // 540 design px at the 1080 short side is 4 px on this 8 px frame; half
    // way down the ramp it is 2.
    const clip = clipOf({ frame: rect, kf: [key(0, 0.5, 540), key(1, 0.5, 0)] });
    const seen = settingsDuring((c) => c.drawIntoRect(frameOf(picture()), rect, true, 1, 0.5, 1, clip));
    expect(seen).toContain("blur(2.00px)");
  });

  test("motion blur streaks a moving clip and only when it is on", () => {
    const kf = [key(0, 0.2), key(1, 0.8)];
    const still = settingsDuring((c) => c.drawIntoRect(frameOf(picture()), rect, true, 1, 0.5, 1, clipOf({ frame: rect, kf })));
    expect(still).not.toContain("lighter");
    const moving = settingsDuring((c) =>
      c.drawIntoRect(frameOf(picture()), rect, true, 1, 0.5, 1, clipOf({ frame: rect, kf, motionBlur: 1 }))
    );
    expect(moving).toContain("lighter");
  });

  test("a zoomed clip softens from the picture past the frame edge", () => {
    // A full-frame clip zoomed 1.5x hangs past the frame. Its streak and
    // blur have to read that overhang: a frame-sized scratch cut the picture
    // at the edge, and the soft pass pulled transparent black into a rim.
    const full = { x: 0, y: 0, w: 1, h: 1 };
    const zoomed = (t: number, x: number) => ({ t, x, y: 0.5, scale: 1.5, rotation: 0, opacity: 1, blur: 270 });
    const clip = clipOf({ frame: full, kf: [zoomed(0, 0.4), zoomed(1, 0.6)], motionBlur: 1 });
    const frame = new PixelCanvas(W, H);
    const lands: { width: number; dx: number }[] = [];
    const draw = frame.ctx.drawImage as (src: PixelCanvas, ...args: number[]) => void;
    frame.ctx.drawImage = (src: PixelCanvas, ...args: number[]) => {
      lands.push({ width: src.width, dx: args[0] });
      draw(src, ...args);
    };
    new FrameCompositor(frame as unknown as RasterSurface).drawIntoRect(frameOf(picture()), full, true, 1, 0.5, 1, clip);
    const last = lands.at(-1)!;
    expect(last.width).toBeGreaterThan(W);
    expect(last.dx).toBeLessThan(0);
  });
});