Skip to content

Code-first video & image, rendered natively on WebGPU.

12.7 KB
Raw Blame
/// <reference types="webgpu" />
import type { SignalData } from "@framefields/core";
import type { WebGPUNodeRenderer } from "@framefields/node-sdk";
import {
	createFullscreenPipeline,
	drawChildToTexture,
	drawFullscreenPass,
	drawTextureToTarget,
	FULLSCREEN_QUAD_WGSL,
	fragmentBindGroupLayout,
	perDevice,
	signalTiming,
	texturesLayout,
} from "@framefields/node-sdk/gpu";
import { signalRegistry } from "@framefields/webgpu-renderers";

interface ColorKeyOp {
	op: "ColorKey";
	keyColor?: string;
	similarity?: number;
	smoothness?: number;
	spillSuppression?: number;
	colorSpace?: "YUV" | "RGB";
	spillSuppressionType?: "Desaturate" | "Neutralize" | "None";
	similarityHandleId?: string | null;
	smoothnessHandleId?: string | null;
	spillSuppressionHandleId?: string | null;
	opacity?: number;
	inputs?: Record<
		string,
		{
			connectionValid: boolean;
			outputItem: {
				type: string;
				data: unknown;
			} | null;
		}
	>;
}

type ColorKeySignalData = SignalData & { nodeId?: string };

export const COLORKEY_SHADER = `
struct ColorKeyUniforms {
	similarity           : f32,
	smoothness           : f32,
	spillSuppression     : f32,
	colorSpace           : f32, // 0 for YUV, 1 for RGB
	spillSuppressionType : f32, // 0 for Desaturate, 1 for Neutralize, 2 for None
	keyR                 : f32,
	keyG                 : f32,
	keyB                 : f32,
	hasSimilaritySig     : f32,
	hasSmoothnessSig     : f32,
	hasSpillSig          : f32,
};

@group(0) @binding(0) var<uniform> u : ColorKeyUniforms;
@group(1) @binding(0) var tex        : texture_2d<f32>;
@group(1) @binding(1) var samp       : sampler;

@group(2) @binding(0) var similaritySigTex  : texture_2d<f32>;
@group(2) @binding(1) var smoothnessSigTex  : texture_2d<f32>;
@group(2) @binding(2) var spillSigTex       : texture_2d<f32>;
@group(2) @binding(3) var signalSamp        : sampler;
${FULLSCREEN_QUAD_WGSL}

fn rgb2yuv(rgb: vec3<f32>) -> vec3<f32> {
	let y = 0.299 * rgb.r + 0.587 * rgb.g + 0.114 * rgb.b;
	let u = -0.169 * rgb.r - 0.331 * rgb.g + 0.500 * rgb.b;
	let v = 0.500 * rgb.r - 0.419 * rgb.g - 0.081 * rgb.b;
	return vec3<f32>(y, u, v);
}

fn yuv2rgb(yuv: vec3<f32>) -> vec3<f32> {
	let r = yuv.x + 1.402 * yuv.z;
	let g = yuv.x - 0.344 * yuv.y - 0.714 * yuv.z;
	let b = yuv.x + 1.772 * yuv.y;
	return vec3<f32>(r, g, b);
}

@fragment fn fs(in : VSOut) -> @location(0) vec4<f32> {
	let src_color = textureSampleLevel(tex, samp, in.uv, 0.0);
	if (src_color.a < 1e-5) {
		return src_color;
	}

	// Unpremultiply alpha for keying and grading
	let unpremult_rgb = src_color.rgb / src_color.a;

	// Resolve dynamic signals if connected, otherwise use static uniform values
	var similarity = u.similarity;
	if (u.hasSimilaritySig > 0.5) {
		similarity = textureSampleLevel(similaritySigTex, signalSamp, vec2<f32>(0.5, 0.5), 0.0).r;
	}

	var smoothness = u.smoothness;
	if (u.hasSmoothnessSig > 0.5) {
		smoothness = textureSampleLevel(smoothnessSigTex, signalSamp, vec2<f32>(0.5, 0.5), 0.0).r;
	}

	var spillSuppression = u.spillSuppression;
	if (u.hasSpillSig > 0.5) {
		spillSuppression = textureSampleLevel(spillSigTex, signalSamp, vec2<f32>(0.5, 0.5), 0.0).r;
	}

	let keyColor = vec3<f32>(u.keyR, u.keyG, u.keyB);

	// 1. Calculate similarity distance
	var dist = 0.0;
	if (u.colorSpace < 0.5) {
		// YUV (Chroma Distance)
		let C_yuv = rgb2yuv(unpremult_rgb);
		let K_yuv = rgb2yuv(keyColor);
		dist = distance(C_yuv.yz, K_yuv.yz);
	} else {
		// RGB (Exact Distance)
		dist = distance(unpremult_rgb, keyColor);
	}

	// 2. Calculate transparency mask
	var mask = 0.0;
	if (smoothness < 0.0001) {
		mask = select(0.0, 1.0, dist >= similarity);
	} else {
		mask = smoothstep(similarity, similarity + smoothness, dist);
	}

	// 3. Apply spill suppression
	var final_rgb = unpremult_rgb;
	if (u.spillSuppressionType < 2.5) { // 0 for Desaturate, 1 for Neutralize, 2 for None
		let C_yuv = rgb2yuv(unpremult_rgb);
		let K_yuv = rgb2yuv(keyColor);
		let K_chroma = K_yuv.yz;
		let K_len_sq = dot(K_chroma, K_chroma);

		if (K_len_sq > 1e-6) {
			let C_chroma = C_yuv.yz;
			let proj = dot(C_chroma, K_chroma) / K_len_sq;
			if (proj > 0.0) {
				let C_para = proj * K_chroma;
				var C_chroma_suppressed = C_chroma;

				if (u.spillSuppressionType < 0.5) {
					// Desaturate
					C_chroma_suppressed = C_chroma - spillSuppression * C_para;
				} else {
					// Neutralize (completely shift to neutral gray chroma)
					C_chroma_suppressed = C_chroma - C_para;
				}

				final_rgb = yuv2rgb(vec3<f32>(C_yuv.x, C_chroma_suppressed.x, C_chroma_suppressed.y));
			}
		}
	}

	let final_alpha = src_color.a * mask;
	return vec4<f32>(clamp(final_rgb, vec3<f32>(0.0), vec3<f32>(1.0)) * final_alpha, final_alpha);
}
`;

const colorKeyData = new Float32Array(12);

const getDeviceLayouts = perDevice((device) => ({
	uniformLayout: fragmentBindGroupLayout(device, ["uniform"]),
	singleTextureLayout: fragmentBindGroupLayout(device, ["texture", "sampler"]),
	signalTextureLayout: texturesLayout(device, 3),
	pipelineCache: new Map<string, GPURenderPipeline>(),
}));

function getColorKeyResources(device: GPUDevice, format: GPUTextureFormat) {
	const layouts = getDeviceLayouts(device);

	const cacheKey = `colorkey_${format}`;
	let pipeline = layouts.pipelineCache.get(cacheKey);
	if (!pipeline) {
		const colorKeyModule = device.createShaderModule({
			label: `colorkey_${format}.wgsl`,
			code: COLORKEY_SHADER,
		});

		pipeline = createFullscreenPipeline(device, {
			label: `ColorKeyPipeline_${format}`,
			module: colorKeyModule,
			layouts: [
				layouts.uniformLayout,
				layouts.singleTextureLayout,
				layouts.signalTextureLayout,
			],
			format,
		});
		layouts.pipelineCache.set(cacheKey, pipeline);
	}

	return {
		colorKeyPipeline: pipeline,
		uniformLayout: layouts.uniformLayout,
		singleTextureLayout: layouts.singleTextureLayout,
		signalTextureLayout: layouts.signalTextureLayout,
	};
}

function hexToRgb(hex: string): [number, number, number] {
	let h = hex.replace("#", "");
	if (h.length === 3) {
		h = h
			.split("")
			.map((c) => c + c)
			.join("");
	}
	const r = parseInt(h.substring(0, 2), 16) / 255;
	const g = parseInt(h.substring(2, 4), 16) / 255;
	const b = parseInt(h.substring(4, 6), 16) / 255;
	return [r, g, b];
}

export const ColorKeyWebGPURenderer: WebGPUNodeRenderer = async (args) => {
	const {
		ctx,
		encoder,
		pass,
		targetTexture,
		targetWidth,
		targetHeight,
		props,
	} = args;
	const frame = props.frame ?? 0;
	const fps = props.fps || 30;
	const op = props.virtualMedia?.operation as ColorKeyOp | undefined;
	if (op?.op !== "ColorKey" || !op) return;

	const childMedia = props.virtualMedia.children?.[0];
	if (!childMedia) return;

	// End the provided pass
	pass.end();

	const width = targetWidth;
	const height = targetHeight;
	const nodeId = op.inputs ? Object.keys(op.inputs)[0] : "colorkey_node";

	// Resolve similarity signal
	const similarityInput = op.similarityHandleId
		? op.inputs?.[op.similarityHandleId]
		: null;
	const hasSimilaritySig = !!(
		similarityInput?.connectionValid &&
		(similarityInput.outputItem?.type === "Signal" ||
			similarityInput.outputItem?.type === "Numeric")
	);
	const similaritySd =
		hasSimilaritySig && similarityInput?.outputItem?.data
			? (similarityInput.outputItem.data as ColorKeySignalData)
			: null;

	let similarity = 0.4;
	let hasStaticSimilaritySig = false;
	if (!hasSimilaritySig) {
		if (
			similarityInput?.connectionValid &&
			similarityInput.outputItem?.type === "Number"
		) {
			similarity = Math.max(
				0,
				Math.min(1, Number(similarityInput.outputItem.data ?? 0.4)),
			);
		} else {
			similarity = Math.max(0, Math.min(1, Number(op.similarity ?? 0.4)));
		}
	} else if (similaritySd) {
		similarity = Math.max(0, Math.min(1, Number(similaritySd.offset ?? 0.0)));
		hasStaticSimilaritySig = true;
	}

	// Resolve smoothness signal
	const smoothnessInput = op.smoothnessHandleId
		? op.inputs?.[op.smoothnessHandleId]
		: null;
	const hasSmoothnessSig = !!(
		smoothnessInput?.connectionValid &&
		(smoothnessInput.outputItem?.type === "Signal" ||
			smoothnessInput.outputItem?.type === "Numeric")
	);
	const smoothnessSd =
		hasSmoothnessSig && smoothnessInput?.outputItem?.data
			? (smoothnessInput.outputItem.data as ColorKeySignalData)
			: null;

	let smoothness = 0.1;
	let hasStaticSmoothnessSig = false;
	if (!hasSmoothnessSig) {
		if (
			smoothnessInput?.connectionValid &&
			smoothnessInput.outputItem?.type === "Number"
		) {
			smoothness = Math.max(
				0,
				Math.min(1, Number(smoothnessInput.outputItem.data ?? 0.1)),
			);
		} else {
			smoothness = Math.max(0, Math.min(1, Number(op.smoothness ?? 0.1)));
		}
	} else if (smoothnessSd) {
		smoothness = Math.max(0, Math.min(1, Number(smoothnessSd.offset ?? 0.0)));
		hasStaticSmoothnessSig = true;
	}

	// Resolve spill suppression signal
	const spillInput = op.spillSuppressionHandleId
		? op.inputs?.[op.spillSuppressionHandleId]
		: null;
	const hasSpillSig = !!(
		spillInput?.connectionValid &&
		(spillInput.outputItem?.type === "Signal" ||
			spillInput.outputItem?.type === "Numeric")
	);
	const spillSd =
		hasSpillSig && spillInput?.outputItem?.data
			? (spillInput.outputItem.data as ColorKeySignalData)
			: null;

	let spillSuppression = 0.2;
	let hasStaticSpillSig = false;
	if (!hasSpillSig) {
		if (
			spillInput?.connectionValid &&
			spillInput.outputItem?.type === "Number"
		) {
			spillSuppression = Math.max(
				0,
				Math.min(1, Number(spillInput.outputItem.data ?? 0.2)),
			);
		} else {
			spillSuppression = Math.max(
				0,
				Math.min(1, Number(op.spillSuppression ?? 0.2)),
			);
		}
	} else if (spillSd) {
		spillSuppression = Math.max(0, Math.min(1, Number(spillSd.offset ?? 0.0)));
		hasStaticSpillSig = true;
	}

	const { texture: tmpTex } = await drawChildToTexture(args, childMedia);

	// Fetch static resources and compile pipeline once
	const {
		colorKeyPipeline: pipeline,
		uniformLayout: uLayout,
		singleTextureLayout: tLayout,
		signalTextureLayout: sigLayout,
	} = getColorKeyResources(ctx.device, ctx.renderer.format);

	const sampler = ctx.renderer.samplerCache.getSampler(ctx.device);

	// Fetch dynamic signal texture views on-demand
	const { elapsedSeconds, durationSeconds } = signalTiming(props, frame, fps);

	let similarityView: GPUTextureView;
	if (hasStaticSimilaritySig && similaritySd) {
		similarityView = signalRegistry.getOrCreate2DTextureView(
			ctx.device,
			encoder,
			similaritySd.nodeId ?? `${nodeId}_similarity_sig`,
			elapsedSeconds,
			durationSeconds,
			similaritySd,
			width,
			height,
			props.renderId,
			frame,
			fps,
		);
	} else {
		similarityView = signalRegistry.getDummy1x1TextureView(ctx.device);
	}

	let smoothnessView: GPUTextureView;
	if (hasStaticSmoothnessSig && smoothnessSd) {
		smoothnessView = signalRegistry.getOrCreate2DTextureView(
			ctx.device,
			encoder,
			smoothnessSd.nodeId ?? `${nodeId}_smoothness_sig`,
			elapsedSeconds,
			durationSeconds,
			smoothnessSd,
			width,
			height,
			props.renderId,
			frame,
			fps,
		);
	} else {
		smoothnessView = signalRegistry.getDummy1x1TextureView(ctx.device);
	}

	let spillView: GPUTextureView;
	if (hasStaticSpillSig && spillSd) {
		spillView = signalRegistry.getOrCreate2DTextureView(
			ctx.device,
			encoder,
			spillSd.nodeId ?? `${nodeId}_spill_sig`,
			elapsedSeconds,
			durationSeconds,
			spillSd,
			width,
			height,
			props.renderId,
			frame,
			fps,
		);
	} else {
		spillView = signalRegistry.getDummy1x1TextureView(ctx.device);
	}

	const [keyR, keyG, keyB] = hexToRgb(op.keyColor ?? "#00ff00");

	// Prepare uniform data
	colorKeyData[0] = similarity;
	colorKeyData[1] = smoothness;
	colorKeyData[2] = spillSuppression;
	colorKeyData[3] = op.colorSpace === "RGB" ? 1.0 : 0.0;
	colorKeyData[4] =
		op.spillSuppressionType === "None"
			? 2.0
			: op.spillSuppressionType === "Neutralize"
				? 1.0
				: 0.0;
	colorKeyData[5] = keyR;
	colorKeyData[6] = keyG;
	colorKeyData[7] = keyB;
	colorKeyData[8] = hasStaticSimilaritySig ? 1.0 : 0.0;
	colorKeyData[9] = hasStaticSmoothnessSig ? 1.0 : 0.0;
	colorKeyData[10] = hasStaticSpillSig ? 1.0 : 0.0;
	colorKeyData[11] = 0.0;

	// 1-Pass Filter (ColorKey)
	const outTex = ctx.renderer.getTemporaryTexture(width, height, [
		tmpTex,
		targetTexture,
		...(props.excludeTextures || []),
	]);

	const singleBuffer = ctx.renderer.getTemporaryBuffer(colorKeyData);

	drawFullscreenPass(encoder, outTex.createView(), pipeline, [
		ctx.device.createBindGroup({
			layout: uLayout,
			entries: [{ binding: 0, resource: { buffer: singleBuffer } }],
		}),
		ctx.renderer.bindGroupCache.getBindGroup(
			ctx.device,
			tLayout,
			tmpTex,
			sampler,
		),
		ctx.device.createBindGroup({
			layout: sigLayout,
			entries: [
				{ binding: 0, resource: similarityView },
				{ binding: 1, resource: smoothnessView },
				{ binding: 2, resource: spillView },
				{ binding: 3, resource: sampler },
			],
		}),
	]);

	// Final Pass: Draw result to targetView using standard drawTexture for blending/opacity
	drawTextureToTarget(args, outTex, op.opacity ?? 1);
};