Skip to content

AI media harness for planning, generating, and editing locally

TypeScript
16.9 KB
Raw Blame
/// <reference types="@webgpu/types" />
import { backdropKeyUniforms, keyColor, KEY_FEATHER, RGB_DISTANCE_SCALE, usesBackdropKey } from "./chromaKey";
import { BACKDROP_KEY_WGSL, BACKDROP_PLATE_WGSL, BACKDROP_REFINE_WGSL } from "./backdropKeyShader";
import { activeVideoEffects, CREATIVE_LOOKS, type VideoEffects, type LutTable } from "./effectSettings";
import { compileCurves, CURVE_SAMPLES } from "./colorCurves";
import { COLOR_GRADING_WGSL } from "./colorGradingShader";
import type { ClipChromaKey } from "./project";

export type GpuEffects = VideoEffects & { chromaKey?: ClipChromaKey | null };

const VERTEX = /* wgsl */ `
struct VertexOut { @builtin(position) position: vec4f, @location(0) uv: vec2f };
@vertex fn vs(@builtin(vertex_index) index: u32) -> VertexOut {
  let corners = array<vec2f, 3>(vec2f(-1., -1.), vec2f(3., -1.), vec2f(-1., 3.));
  let p = corners[index];
  return VertexOut(vec4f(p, 0., 1.), vec2f(p.x * .5 + .5, .5 - p.y * .5));
}
`;
const PARAMS = /* wgsl */ `
struct Params {
  key: vec4f,
  grade: vec4f,
  detail: vec4f,
  domainMin: vec4f,
  domainMax: vec4f,
  blur: vec4f,
  balance: vec4f,
  tonal: vec4f,
  creative: vec4f,
  vignette: vec4f,
  shadows: vec4f,
  midtones: vec4f,
  highlights: vec4f,
  keyControls: vec4f,
  keyMatte: vec4f,
  keyEdge: vec4f,
};
@group(0) @binding(0) var<uniform> params: Params;
@group(0) @binding(2) var linearSampler: sampler;
`;
const IMPORT = VERTEX + PARAMS + BACKDROP_KEY_WGSL + /* wgsl */ `
@group(0) @binding(1) var picture: texture_external;
@fragment fn fs(@location(0) uv: vec2f) -> @location(0) vec4f {
  let color = textureSampleBaseClampToEdge(picture, linearSampler, uv);
  if (params.detail.w > 0. && params.keyControls.x > 0.) { return backdropKey(color); }
  var alpha = color.a;
  if (params.detail.w > 0.) {
    let difference = distance(color.rgb, params.key.rgb) / ${RGB_DISTANCE_SCALE};
    alpha *= smoothstep(params.key.a, params.key.a + ${KEY_FEATHER}, difference);
  }
  // Premultiply before filtering so removed key colors cannot bleed into edges.
  return vec4f(color.rgb * alpha, alpha);
}
`;
const TEXTURE = "@group(0) @binding(1) var picture: texture_2d<f32>;";
const BLUR = VERTEX + PARAMS + TEXTURE + /* wgsl */ `
@fragment fn fs(@location(0) uv: vec2f) -> @location(0) vec4f {
  let radius = params.blur.x;
  let sigma = max(radius / 3., .333333);
  let step = params.blur.yz / vec2f(textureDimensions(picture));
  var total = vec4f(0.);
  var weights = 0.;
  for (var i = -24; i <= 24; i++) {
    if (abs(f32(i)) > ceil(radius)) { continue; }
    let weight = exp(-f32(i * i) / (2. * sigma * sigma));
    total += textureSampleLevel(picture, linearSampler, uv + f32(i) * step, 0.) * weight;
    weights += weight;
  }
  return total / weights;
}
`;
const FINISH = VERTEX + PARAMS + TEXTURE + COLOR_GRADING_WGSL + /* wgsl */ `
@group(0) @binding(3) var<storage, read> lut: array<f32>;
fn lookup(p: vec3u) -> vec3f {
  let size = u32(params.detail.y);
  let offset = (p.x + size * (p.y + size * p.z)) * 3u;
  return vec3f(lut[offset], lut[offset + 1u], lut[offset + 2u]);
}
fn applyLut(color: vec3f) -> vec3f {
  let size = params.detail.y;
  let position = clamp((color - params.domainMin.xyz) / (params.domainMax.xyz - params.domainMin.xyz), vec3f(0.), vec3f(1.)) * (size - 1.);
  let lo = vec3u(floor(position));
  let hi = min(lo + vec3u(1u), vec3u(u32(size) - 1u));
  let f = fract(position);
  return mix(
    mix(mix(lookup(lo), lookup(vec3u(hi.x, lo.y, lo.z)), f.x),
        mix(lookup(vec3u(lo.x, hi.y, lo.z)), lookup(vec3u(hi.xy, lo.z)), f.x), f.y),
    mix(mix(lookup(vec3u(lo.xy, hi.z)), lookup(vec3u(hi.x, lo.y, hi.z)), f.x),
        mix(lookup(vec3u(lo.x, hi.yz)), lookup(hi), f.x), f.y), f.z);
}
fn straight(color: vec4f) -> vec3f { return color.rgb / max(color.a, .00001); }
@fragment fn fs(@location(0) uv: vec2f) -> @location(0) vec4f {
  let sampled = textureSampleLevel(picture, linearSampler, uv, 0.);
  var rgb = straight(sampled);
  if (params.detail.x > 0.) {
    let pixel = 1. / vec2f(textureDimensions(picture));
    let neighbors = (
      textureSampleLevel(picture, linearSampler, uv + vec2f(pixel.x, 0.), 0.) +
      textureSampleLevel(picture, linearSampler, uv - vec2f(pixel.x, 0.), 0.) +
      textureSampleLevel(picture, linearSampler, uv + vec2f(0., pixel.y), 0.) +
      textureSampleLevel(picture, linearSampler, uv - vec2f(0., pixel.y), 0.)) * .25;
    rgb += (rgb - straight(neighbors)) * params.detail.x;
  }
  rgb *= exp2(params.grade.x);
  rgb *= vec3f(1. + .35 * params.balance.x, 1. - .3 * params.balance.y, 1. - .35 * params.balance.x);
  rgb += vec3f(params.domainMax.w);
  let black = -params.tonal.y * .15;
  let white = 1. - params.tonal.x * .15;
  rgb = (rgb - black) / max(white - black, .001);
  let toneLuma = luma(rgb);
  let tone = clamp(toneLuma, 0., 1.);
  let lift = params.balance.w * .35 * pow(1. - tone, 3.) + params.balance.z * .35 * pow(tone, 3.);
  if (abs(lift) > .000001) {
    if (toneLuma > .00001) { rgb *= max(toneLuma + lift, 0.) / toneLuma; }
    else { rgb += max(lift, 0.); }
  }
  rgb = (rgb - vec3f(.5)) * params.grade.y + vec3f(.5);
  rgb = mix(vec3f(luma(rgb)), rgb, params.grade.z);
  if (params.creative.x > 0. && params.creative.y > 0.) { rgb = mix(rgb, creativeLook(rgb), params.creative.y); }
  rgb = rgb * (1. - .25 * params.creative.z) + .12 * params.creative.z;
  let chroma = clamp(max(rgb.r, max(rgb.g, rgb.b)) - min(rgb.r, min(rgb.g, rgb.b)), 0., 1.);
  rgb = mix(vec3f(luma(rgb)), rgb, 1. + params.tonal.z * (1. - chroma));
  rgb = applyWheels(applyCurves(rgb));
  if (params.detail.y >= 2. && params.detail.z > 0.) { rgb = mix(rgb, applyLut(rgb), params.detail.z); }
  var position = abs((uv - vec2f(.5)) * 1.41421356);
  let aspect = f32(textureDimensions(picture).x) / f32(textureDimensions(picture).y);
  position *= mix(vec2f(1.), vec2f(max(aspect, 1.), max(1. / aspect, 1.)), max(params.vignette.y, 0.));
  let power = 2. + max(-params.vignette.y, 0.) * 6.;
  let distanceFromCenter = pow(pow(position.x, power) + pow(position.y, power), 1. / power);
  let start = params.vignette.x * .6;
  let edge = smoothstep(start, start + max(params.vignette.z * .7, .001), distanceFromCenter);
  if (params.grade.w >= 0.) { rgb *= 1. - params.grade.w * edge; }
  else { rgb += (1. - rgb) * -params.grade.w * edge; }
  return vec4f(clamp(rgb, vec3f(0.), vec3f(1.)) * sampled.a, sampled.a);
}
`;
const BLIT = VERTEX + /* wgsl */ `
@group(0) @binding(0) var picture: texture_2d<f32>;
@group(0) @binding(1) var linearSampler: sampler;
@fragment fn fs(@location(0) uv: vec2f) -> @location(0) vec4f {
  return textureSampleLevel(picture, linearSampler, uv, 0.);
}
`;

function pipeline(device: GPUDevice, code: string, format: GPUTextureFormat): GPURenderPipeline {
  const module = device.createShaderModule({ code });
  return device.createRenderPipeline({
    layout: "auto", vertex: { module, entryPoint: "vs" },
    fragment: { module, entryPoint: "fs", targets: [{ format }] },
    primitive: { topology: "triangle-list" },
  });
}

/** Shared by monitor and export. All pictures stay in GPU textures; output is premultiplied. */
export function createVideoEffectsProcessor(device: GPUDevice) {
  const importPipeline = pipeline(device, IMPORT, "rgba16float");
  const platePipeline = pipeline(device, VERTEX + PARAMS + BACKDROP_PLATE_WGSL, "rgba16float");
  const refinePipeline = pipeline(device, VERTEX + PARAMS + BACKDROP_REFINE_WGSL, "rgba16float");
  const plate = device.createTexture({ size: [1, 1], format: "rgba16float", usage: GPUTextureUsage.RENDER_ATTACHMENT | GPUTextureUsage.TEXTURE_BINDING });
  const blurPipeline = pipeline(device, BLUR, "rgba16float");
  const finishPipeline = pipeline(device, FINISH, "rgba16float");
  const sampler = device.createSampler({ minFilter: "linear", magFilter: "linear" });
  const uniforms = Array.from({ length: 5 }, () => device.createBuffer({ size: 256, usage: GPUBufferUsage.UNIFORM | GPUBufferUsage.COPY_DST }));
  const curveBuffer = device.createBuffer({ size: 9 * CURVE_SAMPLES * 4, usage: GPUBufferUsage.STORAGE | GPUBufferUsage.COPY_DST });
  let curveKey: string | undefined;
  let curveMask = 0;
  const emptyLut = device.createBuffer({ size: 12, usage: GPUBufferUsage.STORAGE });
  const luts: { table: LutTable; buffer: GPUBuffer }[] = [];
  let textures: GPUTexture[] = [];
  let dimensions = "";
  return {
    render(frame: VideoFrame, effects: GpuEffects): GPUTexture {
      effects = { ...effects, ...activeVideoEffects(effects) };
      if (effects.chromaKey?.enabled === false) effects = { ...effects, chromaKey: null };
      const nextCurveKey = JSON.stringify(effects.curves ?? null);
      if (nextCurveKey !== curveKey) {
        const compiled = compileCurves(effects.curves);
        device.queue.writeBuffer(curveBuffer, 0, compiled.values);
        curveMask = compiled.mask;
        curveKey = nextCurveKey;
      }
      const width = frame.displayWidth, height = frame.displayHeight;
      if (dimensions !== `${width}x${height}`) {
        textures.forEach((texture) => texture.destroy());
        textures = Array.from({ length: 3 }, () => device.createTexture({
          size: [width, height], format: "rgba16float", usage: GPUTextureUsage.RENDER_ATTACHMENT | GPUTextureUsage.TEXTURE_BINDING,
        }));
        dimensions = `${width}x${height}`;
      }
      const table = effects.lut?.table;
      let lutBuffer = emptyLut;
      if (table) {
        const cached = luts.find((entry) => entry.table === table);
        if (cached) lutBuffer = cached.buffer;
        else {
          lutBuffer = device.createBuffer({ size: table.values.length * 4, usage: GPUBufferUsage.STORAGE | GPUBufferUsage.COPY_DST });
          device.queue.writeBuffer(lutBuffer, 0, new Float32Array(table.values));
          luts.push({ table, buffer: lutBuffer });
          if (luts.length > 8) luts.shift()!.buffer.destroy();
        }
      }
      const grade = effects.colorCorrection;
      const data = new Float32Array([
        ...(effects.chromaKey ? keyColor(effects.chromaKey.color) : [0, 0, 0]), (effects.chromaKey?.tolerance ?? 0) / 100,
        grade?.exposure ?? 0, 1 + (grade?.contrast ?? 0) / 100, (grade?.saturation ?? 100) / 100, (effects.vignette?.amount ?? 0) / 100,
        ((effects.sharpen?.amount ?? 0) + (effects.creative?.sharpen ?? 0)) / 100, table?.size ?? 0, (effects.lut?.intensity ?? 0) / 100, effects.chromaKey ? 1 : 0,
        // Pack brightness into the unused domainMax vector lane.
        ...(table?.domainMin ?? [0, 0, 0]), 0, ...(table?.domainMax ?? [1, 1, 1]), (grade?.brightness ?? 0) / 100,
        effects.blur?.radius ?? 0, 0, 0, 0,
        (grade?.temperature ?? 0) / 100, (grade?.tint ?? 0) / 100, (grade?.highlights ?? 0) / 100, (grade?.shadows ?? 0) / 100,
        (grade?.whites ?? 0) / 100, (grade?.blacks ?? 0) / 100, (effects.creative?.vibrance ?? 0) / 100, curveMask,
        CREATIVE_LOOKS.indexOf(effects.creative?.look ?? "None"), (effects.creative?.intensity ?? 0) / 100, (effects.creative?.fadedFilm ?? 0) / 100, 0,
        (effects.vignette?.midpoint ?? 50) / 100, (effects.vignette?.roundness ?? 0) / 100, (effects.vignette?.feather ?? 100) / 100, 0,
        ...(["shadows", "midtones", "highlights"] as const).flatMap((name) => {
          const wheel = effects.colorWheels?.[name];
          return [wheel?.x ?? 0, wheel?.y ?? 0, (wheel?.lightness ?? 0) / 100, 0];
        }),
        ...backdropKeyUniforms(effects.chromaKey),
      ]);
      device.queue.writeBuffer(uniforms[0], 0, data);
      const encoder = device.createCommandEncoder();
      const pass = (renderPipeline: GPURenderPipeline, source: GPUExternalTexture | GPUTextureView, target: GPUTexture, uniform: GPUBuffer, lut?: GPUBuffer) => {
        const entries: GPUBindGroupEntry[] = [
          { binding: 0, resource: { buffer: uniform } }, { binding: 1, resource: source }, { binding: 2, resource: sampler },
        ];
        if (lut) entries.push({ binding: 3, resource: { buffer: lut } }, { binding: 4, resource: { buffer: curveBuffer } });
        if (renderPipeline === importPipeline) entries.push({ binding: 3, resource: plate.createView() });
        const group = device.createBindGroup({ layout: renderPipeline.getBindGroupLayout(0), entries });
        const render = encoder.beginRenderPass({ colorAttachments: [{ view: target.createView(), loadOp: "clear", storeOp: "store", clearValue: [0, 0, 0, 0] }] });
        render.setPipeline(renderPipeline); render.setBindGroup(0, group); render.draw(3); render.end();
      };
      const external = device.importExternalTexture({ source: frame });
      const backdrop = usesBackdropKey(effects.chromaKey);
      if (backdrop) pass(platePipeline, external, plate, uniforms[0]);
      pass(importPipeline, external, textures[0], uniforms[0]);
      let source = textures[0];
      let spare = textures[1];
      const key = effects.chromaKey;
      if (backdrop && ((key?.choke ?? 0) !== 0 || key?.despeckle || key?.fillHoles)) {
        const refinement = data.slice();
        refinement[60] = 0;
        device.queue.writeBuffer(uniforms[3], 0, refinement);
        pass(refinePipeline, source.createView(), spare, uniforms[3]);
        [source, spare] = [spare, source];
      }
      if (backdrop && (key?.softness ?? 0) > 0) {
        const refinement = data.slice();
        refinement.fill(0, 61, 64);
        device.queue.writeBuffer(uniforms[4], 0, refinement);
        pass(refinePipeline, source.createView(), spare, uniforms[4]);
        [source, spare] = [spare, source];
      }
      if ((effects.blur?.radius ?? 0) > 0) {
        data[21] = 1;
        device.queue.writeBuffer(uniforms[1], 0, data);
        pass(blurPipeline, source.createView(), spare, uniforms[1]);
        data[21] = 0; data[22] = 1;
        device.queue.writeBuffer(uniforms[2], 0, data);
        pass(blurPipeline, spare.createView(), source, uniforms[2]);
      }
      pass(finishPipeline, source.createView(), textures[2], uniforms[0], lutBuffer);
      device.queue.submit([encoder.finish()]);
      return textures[2];
    },
    dispose() {
      textures.forEach((texture) => texture.destroy());
      uniforms.forEach((buffer) => buffer.destroy());
      luts.forEach(({ buffer }) => buffer.destroy());
      emptyLut.destroy();
      curveBuffer.destroy();
      plate.destroy();
    },
  };
}

// Share a device across monitor layers; each renderer owns its textures.
let previewDevice: Promise<GPUDevice> | undefined;
async function getPreviewDevice(): Promise<GPUDevice> {
  previewDevice ??= (async () => {
    if (!navigator.gpu) throw new Error("Video effects require WebGPU on this computer.");
    const adapter = await navigator.gpu.requestAdapter();
    if (!adapter) throw new Error("No GPU is available for video effects.");
    const device = await adapter.requestDevice();
    void device.lost.then(() => { previewDevice = undefined; });
    return device;
  })().catch((reason) => { previewDevice = undefined; throw reason; });
  return previewDevice;
}

export async function createVideoEffectsPreview(canvas: HTMLCanvasElement) {
  const device = await getPreviewDevice();
  const context = canvas.getContext("webgpu");
  if (!context) throw new Error("Could not create the video effects preview.");
  const format = navigator.gpu.getPreferredCanvasFormat();
  context.configure({ device, format, alphaMode: "premultiplied" });
  const processor = createVideoEffectsProcessor(device);
  const blit = pipeline(device, BLIT, format);
  const sampler = device.createSampler({ minFilter: "linear", magFilter: "linear" });
  let failure: string | null = null;
  const onError = (event: Event) => { failure = (event as GPUUncapturedErrorEvent).error.message; };
  device.addEventListener("uncapturederror", onError);
  void device.lost.then((info) => { failure = info.message || "The GPU stopped responding."; });
  return {
    draw(source: HTMLVideoElement | HTMLImageElement, effects: GpuEffects) {
      if (failure) throw new Error(failure);
      const video = source instanceof HTMLVideoElement;
      const width = video ? source.videoWidth : source.naturalWidth;
      const height = video ? source.videoHeight : source.naturalHeight;
      if (!width || !height || (video ? source.readyState < 2 : !source.complete)) return;
      if (canvas.width !== width) canvas.width = width;
      if (canvas.height !== height) canvas.height = height;
      const frame = new VideoFrame(source, { timestamp: 0 });
      try {
        const result = processor.render(frame, effects);
        const group = device.createBindGroup({ layout: blit.getBindGroupLayout(0), entries: [
          { binding: 0, resource: result.createView() }, { binding: 1, resource: sampler },
        ] });
        const encoder = device.createCommandEncoder();
        const render = encoder.beginRenderPass({ colorAttachments: [{ view: context.getCurrentTexture().createView(), loadOp: "clear", storeOp: "store", clearValue: [0, 0, 0, 0] }] });
        render.setPipeline(blit); render.setBindGroup(0, group); render.draw(3); render.end();
        device.queue.submit([encoder.finish()]);
      } finally { frame.close(); }
    },
    dispose() { device.removeEventListener("uncapturederror", onError); processor.dispose(); context.unconfigure(); },
  };
}