Skip to content

AI media harness for planning, generating, and editing locally

TypeScript
4.8 KB
Raw Blame
import { applyChromaKey, keyColor, KEY_FEATHER, RGB_DISTANCE_SCALE } from "./chromaKey";
import type { ClipChromaKey } from "./project";

type Picture = HTMLVideoElement | HTMLImageElement;
export interface ChromaKeyRenderer {
  draw(source: Picture, key: ClipChromaKey): void;
  dispose(): void;
}

/** Upload media directly as a texture; no pixel readback during GPU playback. */
export function createChromaKeyRenderer(canvas: HTMLCanvasElement): ChromaKeyRenderer {
  const gl = canvas.getContext("webgl", { alpha: true, premultipliedAlpha: false });
  if (!gl) {
    const context = canvas.getContext("2d", { willReadFrequently: true });
    if (!context) throw new Error("This computer could not create a Chroma Key preview.");
    return {
      draw(source, key) {
        const [width, height] = dimensions(source);
        if (!width || !height) return;
        // Bound CPU work to a monitor-sized picture on machines without WebGL.
        const scale = Math.min(1, 1280 / width);
        canvas.width = Math.round(width * scale);
        canvas.height = Math.round(height * scale);
        context.drawImage(source, 0, 0, canvas.width, canvas.height);
        const pixels = context.getImageData(0, 0, canvas.width, canvas.height);
        applyChromaKey(pixels.data, key);
        context.putImageData(pixels, 0, 0);
      },
      dispose() {},
    };
  }
  const shaders: WebGLShader[] = [];
  const program = gl.createProgram();
  const vertices = gl.createBuffer();
  const texture = gl.createTexture();
  const dispose = () => {
    shaders.forEach((shader) => gl.deleteShader(shader));
    gl.deleteProgram(program);
    gl.deleteBuffer(vertices);
    gl.deleteTexture(texture);
  };
  try {
    if (!program || !vertices || !texture) throw new Error("Could not allocate the Chroma Key preview.");
    const compile = (type: number, code: string) => {
      const shader = gl.createShader(type);
      if (!shader) throw new Error("Could not allocate the Chroma Key shader.");
      shaders.push(shader);
      gl.shaderSource(shader, code);
      gl.compileShader(shader);
      if (!gl.getShaderParameter(shader, gl.COMPILE_STATUS)) throw new Error(gl.getShaderInfoLog(shader) ?? "Chroma Key shader failed.");
      gl.attachShader(program, shader);
    };
    compile(gl.VERTEX_SHADER, `
      attribute vec2 position;
      varying vec2 uv;
      void main() { uv = (position + 1.0) * 0.5; gl_Position = vec4(position, 0.0, 1.0); }
    `);
    compile(gl.FRAGMENT_SHADER, `
      precision mediump float;
      varying vec2 uv;
      uniform sampler2D source;
      uniform vec4 key;
      void main() {
        vec4 sampled = texture2D(source, uv);
        float difference = distance(sampled.rgb, key.rgb) / ${RGB_DISTANCE_SCALE};
        float alpha = smoothstep(key.a, key.a + ${KEY_FEATHER}, difference);
        gl_FragColor = vec4(sampled.rgb, sampled.a * alpha);
      }
    `);
    gl.linkProgram(program);
    if (!gl.getProgramParameter(program, gl.LINK_STATUS)) throw new Error(gl.getProgramInfoLog(program) ?? "Chroma Key shader failed to link.");
    gl.useProgram(program);
    gl.bindBuffer(gl.ARRAY_BUFFER, vertices);
    gl.bufferData(gl.ARRAY_BUFFER, new Float32Array([-1, -1, 1, -1, -1, 1, -1, 1, 1, -1, 1, 1]), gl.STATIC_DRAW);
    const position = gl.getAttribLocation(program, "position");
    gl.enableVertexAttribArray(position);
    gl.vertexAttribPointer(position, 2, gl.FLOAT, false, 0, 0);
    gl.bindTexture(gl.TEXTURE_2D, texture);
    gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR);
    gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.LINEAR);
    gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE);
    gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE);
    gl.pixelStorei(gl.UNPACK_FLIP_Y_WEBGL, true);
    gl.uniform1i(gl.getUniformLocation(program, "source"), 0);
    const keyUniform = gl.getUniformLocation(program, "key");
    return {
      draw(source, key) {
        const [width, height] = dimensions(source);
        if (!width || !height) return;
        if (gl.isContextLost()) throw new Error("The GPU stopped responding during Chroma Key preview.");
        if (canvas.width !== width) canvas.width = width;
        if (canvas.height !== height) canvas.height = height;
        gl.viewport(0, 0, width, height);
        gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, source);
        gl.uniform4f(keyUniform, ...keyColor(key.color), key.tolerance / 100);
        gl.drawArrays(gl.TRIANGLES, 0, 6);
      },
      dispose,
    };
  } catch (error) {
    dispose();
    throw error;
  }
}

function dimensions(source: Picture): [number, number] {
  return source instanceof HTMLVideoElement
    ? source.readyState >= 2 ? [source.videoWidth, source.videoHeight] : [0, 0]
    : source.complete ? [source.naturalWidth, source.naturalHeight] : [0, 0];
}