Skip to content

Open source CapCut alternative.

TypeScript
9.4 KB
Raw Blame
"use client";

/**
 * A clip's frame as the base rendering sees it: the decoder's code values put
 * through the source conversion alone — Apple Log, HLG or PQ into Rec.709
 * display values — and nothing of the grade. The chat's colour stats,
 * thumbnails, watch frames and freeze frames read this, so a log clip's
 * numbers and pictures describe what the display shows, never the flat log
 * encoding; Auto and a match read the raw frame and render each grade they
 * try through the clip's recipe. The frames are read the way the preview
 * reads them (sourceColor.ts): tags overridden so the decoder draws code
 * values, and the matrix it drew them through undone first.
 */

import {
  applyLutToImageData,
  buildClipLut,
  type ClipColorRecipe,
  type ColorGrade,
  drawnColorKey,
  type DrawnColor,
  type GradeLut,
  type ParsedLut,
  type SourceProfile,
} from "@donkeycut/effects-kit";
import { coveredSrc, frameSink, openMedia, videoTrackOf, type FrameCanvasSink, type FrameSize, type FrameSinkOptions } from "./mediaRead";
import { sampleClipColorRead, sampleClipFrameData } from "./previewCanvas";
import { colorRead, fileAt, previewFile, type ColorRead } from "./sourceColor";
import { useEditor } from "./store";
import { UserError } from "./userError";
import type { InputVideoTrack, WrappedCanvas } from "mediabunny";

export { sourceProfileOf } from "./sourceColor";

/** The source conversion for a readout, sampled once per profile and drawn
 * matrix and kept: a 17³ lattice is plenty for quantiles and means. */
const conversions = new Map<string, GradeLut | null>();

function conversionFor(profile: SourceProfile, drawn: Partial<DrawnColor> | undefined): GradeLut | null {
  const key = `${profile}|${drawnColorKey(drawn)}`;
  let lut = conversions.get(key);
  if (lut === undefined) {
    lut = buildClipLut({ profile, output: "sdr", size: 17, ...drawn });
    conversions.set(key, lut);
  }
  return lut;
}

/** Convert raw code values in place into the base rendering. Identity for a
 * Rec.709 or sRGB source drawn through its own matrix. */
export function toBaseRendering(
  px: Uint8ClampedArray,
  profile: SourceProfile,
  drawn?: Partial<DrawnColor>
): Uint8ClampedArray {
  const lut = conversionFor(profile, drawn);
  if (lut) applyLutToImageData(px, lut);
  return px;
}

/** A grade to read a frame through: the clip's grade and the library LUT it
 * names, loaded. */
export type ReadGrade = { grade: ColorGrade; userLut?: ParsedLut };

/** Thumb-size readouts sample a 33³ lattice: quantiles and means sit within
 * a code of the full-size preview. */
const GRADED_READ_LUT_SIZE = 33;

/** Convert raw code values in place into the picture the grade renders on an
 * SDR project: the clip's whole recipe in one lattice, so a log clip's grade
 * runs in scene light exactly as the preview runs it. */
export function toGradedRendering(
  px: Uint8ClampedArray,
  profile: SourceProfile,
  drawn: Partial<DrawnColor> | undefined,
  read: ReadGrade
): Uint8ClampedArray {
  const lut = buildClipLut({ ...drawn, profile, output: "sdr", size: GRADED_READ_LUT_SIZE, grade: read.grade }, read.userLut);
  if (!lut) return toBaseRendering(px, profile, drawn);
  applyLutToImageData(px, lut);
  return px;
}

/** A frame's code values (RGBA) and the ungraded recipe that draws them on
 * an SDR project: what a match renders each grade it tries through. */
export interface RawFrame {
  px: Uint8ClampedArray;
  recipe: ClipColorRecipe;
}

/** A small readout of a clip's code values as the preview decoded them; null
 * when the clip has no decoded frame ready. */
export function sampleClipRawFrame(clipId: string, w = 96, h = 54): RawFrame | null {
  const px = sampleClipFrameData(clipId, w, h);
  if (!px) return null;
  const s = useEditor.getState();
  const clip = s.clips.find((c) => c.id === clipId);
  const asset = clip ? s.assets.find((a) => a.id === clip.assetId) : undefined;
  // The frame came through the engine's read of the clip; its recipe is the
  // one the compositor draws it with.
  const input = asset ? (sampleClipColorRead(clipId) ?? colorRead(asset, previewFile(asset))).recipe() : undefined;
  return { px, recipe: { ...input, profile: input?.profile ?? "rec709", output: "sdr", size: 0 } };
}

/** How the frames at `url` are to be read: the project asset the URL belongs
 * to, and which of its files the URL is. An unknown URL reads as Rec.709. */
export function colorReadAt(url: string): ColorRead {
  const asset = useEditor.getState().assets.find((a) => a.url === url || a.proxyUrl === url);
  return colorRead(asset ?? {}, asset ? fileAt(asset, url) : "master");
}

function convertInPlace(wrapped: WrappedCanvas, profile: SourceProfile, drawn: DrawnColor): WrappedCanvas {
  const canvas = wrapped.canvas;
  const ctx = canvas.getContext("2d") as CanvasRenderingContext2D | null;
  if (!ctx) return wrapped;
  const image = ctx.getImageData(0, 0, canvas.width, canvas.height);
  toBaseRendering(image.data, profile, drawn);
  ctx.putImageData(image, 0, 0);
  return wrapped;
}

/** A sink whose frames are the base rendering of what `read` reads: the
 * plain sink for a Rec.709 or sRGB source, and for any other the override
 * read followed by the source conversion on each frame. The route's drawn
 * matrix is settled before the first frame, so what comes out never depends
 * on timing. */
export function baseFrameSink(
  track: InputVideoTrack,
  size: FrameSize | undefined,
  read: ColorRead,
  opts?: Omit<FrameSinkOptions, "colorSpace">
): FrameCanvasSink {
  const colorSpace = read.colorSpace;
  const sink = frameSink(track, size, { ...opts, ...(colorSpace ? { colorSpace } : {}) });
  const profile = read.recipe().profile;
  if (!colorSpace && (profile === "rec709" || profile === "srgb")) return sink;
  const ready = read.settled();
  const convert = async (frame: WrappedCanvas | null) => (frame ? convertInPlace(frame, profile, await ready) : frame);
  return {
    getCanvas: async (t) => convert(await sink.getCanvas(t)),
    canvases: async function* (start, end) {
      for await (const frame of sink.canvases(start, end)) yield convertInPlace(frame, profile, await ready);
    },
    canvasesAtTimestamps: async function* (times) {
      for await (const frame of sink.canvasesAtTimestamps(times)) yield await convert(frame);
    },
  };
}

/** One base-rendering frame of the file at `url`. */
export async function baseFrameAt(master: string, time: number, size?: FrameSize): Promise<WrappedCanvas | null> {
  // A stream this page cannot decode reads from its compat proxy, and the
  // proxy's own recipe draws it.
  const url = coveredSrc("video", master);
  const input = openMedia(url);
  try {
    const track = await videoTrackOf(input);
    if (!track) throw new UserError("media_no_readable_video", "This file has no readable video.");
    return await baseFrameSink(track, size, colorReadAt(url)).getCanvas(Math.max(0, time));
  } finally {
    input.dispose();
  }
}

/** The code values of one frame of the file at `url`, read the way the
 * preview reads them, with the recipe that draws them. */
export async function rawFrameAt(master: string, time: number, size: FrameSize): Promise<RawFrame | null> {
  // A stream this page cannot decode reads from its compat proxy, and the
  // proxy's own recipe draws it.
  const url = coveredSrc("video", master);
  const input = openMedia(url);
  try {
    const track = await videoTrackOf(input);
    if (!track) throw new UserError("media_no_readable_video", "This file has no readable video.");
    const read = colorReadAt(url);
    const colorSpace = read.colorSpace;
    const frame = await frameSink(track, size, colorSpace ? { colorSpace } : {}).getCanvas(Math.max(0, time));
    const ctx = frame?.canvas.getContext("2d") as CanvasRenderingContext2D | null | undefined;
    if (!frame || !ctx) return null;
    const drawn = await read.settled();
    return {
      px: ctx.getImageData(0, 0, frame.canvas.width, frame.canvas.height).data,
      recipe: { ...drawn, output: "sdr", size: 0 },
    };
  } finally {
    input.dispose();
  }
}

// The DOMException name WebCodecs gives a decoder that failed on its input.
const DECODER_FAILURE = "EncodingError";

const decoderFailed = (err: unknown) => err instanceof DOMException && err.name === DECODER_FAILURE;

/** Base-rendering frames at ascending `times`, in one decode pass; null for a
 * time the track has no frame for.
 *
 * A machine's hardware decoder can refuse a stream every other decoder plays
 * — Linux VA-API, an Android MediaCodec, a phone out of sessions — and says
 * only "Decoding error.". The pass then goes on in software from the frame it
 * failed on; a software decoder that fails too ends the read. */
export async function* baseFramesAt(master: string, times: number[], size?: FrameSize): AsyncGenerator<WrappedCanvas | null> {
  // A stream this page cannot decode reads from its compat proxy, and the
  // proxy's own recipe draws it.
  const url = coveredSrc("video", master);
  const input = openMedia(url);
  try {
    const track = await videoTrackOf(input);
    if (!track) throw new UserError("media_no_readable_video", "This file has no readable video.");
    let next = 0;
    try {
      for await (const frame of baseFrameSink(track, size, colorReadAt(url)).canvasesAtTimestamps(times)) {
        next++;
        yield frame;
      }
    } catch (err) {
      if (!decoderFailed(err)) throw err;
      yield* baseFrameSink(track, size, colorReadAt(url), { software: true }).canvasesAtTimestamps(times.slice(next));
    }
  } finally {
    input.dispose();
  }
}