Skip to content

Code-first video & image, rendered natively on WebGPU.

18.1 KB
Raw Blame
/// <reference types="webgpu" />
import type { VirtualMediaData } from "@framefields/core";
import {
	resolveHandleParam,
	type WebGPUNodeRenderer,
} from "@framefields/node-sdk";
import {
	createFullscreenPipeline,
	drawChildToTexture,
	drawFullscreenPass,
	drawTextureToTarget,
	FULLSCREEN_QUAD_WGSL,
	fragmentBindGroupLayout,
	signalTextureView,
	signalTiming,
	texturesLayout,
} from "@framefields/node-sdk/gpu";
import {
	MAX_DECONTAMINATE,
	MAX_FEATHER,
	MAX_RADIUS,
	MAX_SHIFT_EDGE,
	MAX_SMOOTH,
	type MatteChannel,
	MIN_DECONTAMINATE,
	MIN_FEATHER,
	MIN_RADIUS,
	MIN_SHIFT_EDGE,
	MIN_SMOOTH,
	type RefineEdgeOp,
	type RefineEdgeOutputMode,
	type RefineEdgeSignalData,
} from "../shared/index.js";

const CHANNEL_MAP: Record<MatteChannel, number> = {
	Alpha: 0,
	Luminance: 1,
	Red: 2,
	Green: 3,
	Blue: 4,
};

const OUTPUT_MODE_MAP: Record<RefineEdgeOutputMode, number> = {
	Composite: 0,
	MatteOnly: 1,
	DecontaminatedRGB: 2,
};

const WGSL_REFINE_EDGE = `
struct RefineEdgeUniforms {
	dirX                : f32,
	dirY                : f32,
	decontaminateAmount : f32,
	radius              : f32,

	smoothness          : f32,
	feather             : f32,
	shiftEdge           : f32,
	matteChannel        : f32,

	outputMode          : f32,
	hasMatte            : f32,
	hasDecontamSig      : f32,
	hasRadiusSig        : f32,

	hasSmoothSig        : f32,
	hasFeatherSig       : f32,
	hasShiftEdgeSig     : f32,
	isSecondPass        : f32,

	pad0                : f32,
	pad1                : f32,
	pad2                : f32,
	pad3                : f32,
};

@group(0) @binding(0) var<uniform> u          : RefineEdgeUniforms;

@group(1) @binding(0) var texInput            : texture_2d<f32>;
@group(1) @binding(1) var texMatte            : texture_2d<f32>;
@group(1) @binding(2) var origTex             : texture_2d<f32>;
@group(1) @binding(3) var samp                : sampler;

@group(2) @binding(0) var decontamSigTex      : texture_2d<f32>;
@group(2) @binding(1) var radiusSigTex        : texture_2d<f32>;
@group(2) @binding(2) var smoothSigTex        : texture_2d<f32>;
@group(2) @binding(3) var featherSigTex       : texture_2d<f32>;
@group(2) @binding(4) var shiftEdgeSigTex     : texture_2d<f32>;
@group(2) @binding(5) var signalSamp          : sampler;
${FULLSCREEN_QUAD_WGSL}

fn extractChannel(color : vec4<f32>, chCode : f32) -> f32 {
	if (chCode < 0.5) {
		// Alpha
		return color.a;
	} else if (chCode < 1.5) {
		// Luminance
		let luma = dot(color.rgb, vec3<f32>(0.2126, 0.7152, 0.0722));
		if (color.a > 1e-4) {
			return min(1.0, luma / color.a);
		}
		return luma;
	} else if (chCode < 2.5) {
		// Red
		if (color.a > 1e-4) {
			return min(1.0, color.r / color.a);
		}
		return color.r;
	} else if (chCode < 3.5) {
		// Green
		if (color.a > 1e-4) {
			return min(1.0, color.g / color.a);
		}
		return color.g;
	} else {
		// Blue
		if (color.a > 1e-4) {
			return min(1.0, color.b / color.a);
		}
		return color.b;
	}
}

@fragment fn fs(in : VSOut) -> @location(0) vec4<f32> {
	let dimensions = vec2<f32>(textureDimensions(texInput));
	let texelSize = 1.0 / max(dimensions, vec2<f32>(1.0, 1.0));

	var decontam = u.decontaminateAmount;
	if (u.hasDecontamSig > 0.5) {
		decontam = clamp(textureSampleLevel(decontamSigTex, signalSamp, vec2<f32>(0.5, 0.5), 0.0).r, 0.0, 1.0);
	}
	var radius = u.radius;
	if (u.hasRadiusSig > 0.5) {
		radius = clamp(textureSampleLevel(radiusSigTex, signalSamp, vec2<f32>(0.5, 0.5), 0.0).r, 0.5, 50.0);
	}
	var smoothVal = u.smoothness;
	if (u.hasSmoothSig > 0.5) {
		smoothVal = clamp(textureSampleLevel(smoothSigTex, signalSamp, vec2<f32>(0.5, 0.5), 0.0).r, 0.0, 100.0);
	}
	var featherVal = u.feather;
	if (u.hasFeatherSig > 0.5) {
		featherVal = clamp(textureSampleLevel(featherSigTex, signalSamp, vec2<f32>(0.5, 0.5), 0.0).r, 0.0, 50.0);
	}
	var shiftEdgeVal = u.shiftEdge;
	if (u.hasShiftEdgeSig > 0.5) {
		shiftEdgeVal = clamp(textureSampleLevel(shiftEdgeSigTex, signalSamp, vec2<f32>(0.5, 0.5), 0.0).r, -100.0, 100.0);
	}

	let direction = vec2<f32>(u.dirX, u.dirY);

	// Combined sigma for smoothing & feathering
	let smoothSigma = max(0.0, smoothVal * 0.15);
	let featherSigma = max(0.0, featherVal);
	let totalSigma = max(0.1, sqrt(smoothSigma * smoothSigma + featherSigma * featherSigma));

	if (u.isSecondPass <= 0.5) {
		// =============================================================
		// PASS 1: Horizontal Pass (Alpha filter + FG color sampling)
		// =============================================================
		let centerInput = textureSampleLevel(texInput, samp, in.uv, 0.0);
		let centerMatte = textureSampleLevel(texMatte, samp, in.uv, 0.0);

		var centerRawAlpha = centerInput.a;
		if (u.hasMatte > 0.5) {
			centerRawAlpha = extractChannel(centerMatte, u.matteChannel);
		}

		// 1. Horizontal Alpha Blur / Filter
		var alphaAccum = 0.0;
		var alphaWeightSum = 0.0;

		let stepScale = max(1.0, totalSigma / 15.0);
		let effectiveSigma = totalSigma / stepScale;
		let sampleRadius = min(i32(ceil(3.0 * effectiveSigma)), 32);

		for (var i = -sampleRadius; i <= sampleRadius; i++) {
			let offset = vec2<f32>(f32(i) * stepScale) * direction * texelSize;
			let sampleUv = clamp(in.uv + offset, vec2<f32>(0.0), vec2<f32>(1.0));
			let sampleInput = textureSampleLevel(texInput, samp, sampleUv, 0.0);
			var sAlpha = sampleInput.a;
			if (u.hasMatte > 0.5) {
				let sMatte = textureSampleLevel(texMatte, samp, sampleUv, 0.0);
				sAlpha = extractChannel(sMatte, u.matteChannel);
			}

			let dist = f32(i) * stepScale;
			let weight = exp(-0.5 * (dist / totalSigma) * (dist / totalSigma));
			alphaAccum += sAlpha * weight;
			alphaWeightSum += weight;
		}

		let smoothedAlpha = alphaAccum / max(alphaWeightSum, 1e-5);

		// 2. Horizontal Foreground Color Sampling for Defringing
		var fgColorAccum = vec3<f32>(0.0);
		var fgWeightSum = 0.0;

		let rStepScale = max(1.0, radius / 15.0);
		let rSampleCount = min(i32(ceil(radius / rStepScale)), 32);

		for (var i = -rSampleCount; i <= rSampleCount; i++) {
			let offset = vec2<f32>(f32(i) * rStepScale) * direction * texelSize;
			let sampleUv = clamp(in.uv + offset, vec2<f32>(0.0), vec2<f32>(1.0));
			let sampleCol = textureSampleLevel(texInput, samp, sampleUv, 0.0);
			var sampleA = sampleCol.a;
			if (u.hasMatte > 0.5) {
				let sm = textureSampleLevel(texMatte, samp, sampleUv, 0.0);
				sampleA = extractChannel(sm, u.matteChannel);
			}

			let sampleRgb = select(sampleCol.rgb, sampleCol.rgb / max(sampleCol.a, 1e-4), sampleCol.a > 1e-4);
			let spatialDist = f32(i) * rStepScale;
			let spatialWeight = exp(-0.5 * (spatialDist / max(radius, 0.5)) * (spatialDist / max(radius, 0.5)));
			// Weight foreground pixels by alpha confidence
			let fgConfidence = clamp((sampleA - 0.2) / 0.8, 0.0, 1.0);
			let weight = fgConfidence * spatialWeight;

			fgColorAccum += sampleRgb * weight;
			fgWeightSum += weight;
		}

		let centerUnpremult = select(centerInput.rgb, centerInput.rgb / max(centerInput.a, 1e-4), centerInput.a > 1e-4);
		let fgColor = select(centerUnpremult, fgColorAccum / max(fgWeightSum, 1e-5), fgWeightSum > 1e-4);

		return vec4<f32>(fgColor, smoothedAlpha);
	}

	// =============================================================
	// PASS 2: Vertical Pass & Final Defringe/Composite
	// =============================================================
	let pass1Sample = textureSampleLevel(texInput, samp, in.uv, 0.0);
	let origSample = textureSampleLevel(origTex, samp, in.uv, 0.0);

	var rawMatteAlpha = origSample.a;
	if (u.hasMatte > 0.5) {
		let rawMatteSample = textureSampleLevel(texMatte, samp, in.uv, 0.0);
		rawMatteAlpha = extractChannel(rawMatteSample, u.matteChannel);
	}

	// 1. Vertical Alpha Blur / Filter
	var alphaAccum = 0.0;
	var alphaWeightSum = 0.0;

	let stepScale = max(1.0, totalSigma / 15.0);
	let effectiveSigma = totalSigma / stepScale;
	let sampleRadius = min(i32(ceil(3.0 * effectiveSigma)), 32);

	for (var i = -sampleRadius; i <= sampleRadius; i++) {
		let offset = vec2<f32>(f32(i) * stepScale) * direction * texelSize;
		let sampleUv = clamp(in.uv + offset, vec2<f32>(0.0), vec2<f32>(1.0));
		let sPass1 = textureSampleLevel(texInput, samp, sampleUv, 0.0);
		let sAlpha = sPass1.a;

		let dist = f32(i) * stepScale;
		let weight = exp(-0.5 * (dist / totalSigma) * (dist / totalSigma));
		alphaAccum += sAlpha * weight;
		alphaWeightSum += weight;
	}

	var refinedAlpha = alphaAccum / max(alphaWeightSum, 1e-5);

	// 2. Shift Edge (Choke < 0 or Expand > 0)
	if (abs(shiftEdgeVal) > 0.01) {
		let shiftNorm = shiftEdgeVal / 100.0;
		if (shiftNorm < 0.0) {
			// Contract / Choke
			let chokeLimit = abs(shiftNorm);
			refinedAlpha = clamp((refinedAlpha - chokeLimit) / max(1.0 - chokeLimit, 1e-4), 0.0, 1.0);
		} else {
			// Expand / Dilate
			refinedAlpha = clamp(refinedAlpha / max(1.0 - shiftNorm, 1e-4), 0.0, 1.0);
		}
	}

	// 3. Vertical Foreground Color Sampling for Defringing
	var fgColorAccum = vec3<f32>(0.0);
	var fgWeightSum = 0.0;

	let rStepScale = max(1.0, radius / 15.0);
	let rSampleCount = min(i32(ceil(radius / rStepScale)), 32);

	for (var i = -rSampleCount; i <= rSampleCount; i++) {
		let offset = vec2<f32>(f32(i) * rStepScale) * direction * texelSize;
		let sampleUv = clamp(in.uv + offset, vec2<f32>(0.0), vec2<f32>(1.0));
		let sPass1 = textureSampleLevel(texInput, samp, sampleUv, 0.0);
		let sRgb = sPass1.rgb;
		let sA = sPass1.a;

		let spatialDist = f32(i) * rStepScale;
		let spatialWeight = exp(-0.5 * (spatialDist / max(radius, 0.5)) * (spatialDist / max(radius, 0.5)));
		let fgConfidence = clamp((sA - 0.2) / 0.8, 0.0, 1.0);
		let weight = fgConfidence * spatialWeight;

		fgColorAccum += sRgb * weight;
		fgWeightSum += weight;
	}

	let origUnpremult = select(origSample.rgb, origSample.rgb / max(origSample.a, 1e-4), origSample.a > 1e-4);
	let fgColor = select(origUnpremult, fgColorAccum / max(fgWeightSum, 1e-5), fgWeightSum > 1e-4);

	// Edge zone detection: transitions between foreground and background
	let edgeWeight = clamp(4.0 * rawMatteAlpha * (1.0 - rawMatteAlpha), 0.0, 1.0);
	let decontamFactor = clamp(decontam * max(edgeWeight, (1.0 - refinedAlpha) * 0.8), 0.0, 1.0);

	let decontaminatedColor = mix(origUnpremult, fgColor, decontamFactor);

	// Format Output
	let mode = u.outputMode;
	if (mode < 0.5) {
		// Composite: Decontaminated RGB with refined alpha (premultiplied)
		return vec4<f32>(decontaminatedColor * refinedAlpha, refinedAlpha);
	} else if (mode < 1.5) {
		// MatteOnly: Grayscale refined alpha mask
		return vec4<f32>(refinedAlpha, refinedAlpha, refinedAlpha, 1.0);
	} else {
		// DecontaminatedRGB: Clean defringed RGB with full alpha
		return vec4<f32>(decontaminatedColor, 1.0);
	}
}
`;

interface DeviceRefineEdgeResources {
	uniformLayout: GPUBindGroupLayout;
	textureLayout: GPUBindGroupLayout;
	signalTextureLayout: GPUBindGroupLayout;
	pipeline: GPURenderPipeline;
}

const deviceResourceCache = new WeakMap<GPUDevice, DeviceRefineEdgeResources>();

function getRefineEdgeResources(
	device: GPUDevice,
	targetFormat: GPUTextureFormat,
): DeviceRefineEdgeResources {
	const cached = deviceResourceCache.get(device);
	if (cached) return cached;

	const uniformLayout = fragmentBindGroupLayout(device, ["uniform"]);

	const textureLayout = texturesLayout(device, 3);

	const signalTextureLayout = texturesLayout(device, 5);

	const shaderModule = device.createShaderModule({
		code: WGSL_REFINE_EDGE,
	});

	const pipelineLayout = device.createPipelineLayout({
		bindGroupLayouts: [uniformLayout, textureLayout, signalTextureLayout],
	});

	const pipeline = createFullscreenPipeline(device, {
		module: shaderModule,
		layouts: pipelineLayout,
		format: targetFormat,
	});

	const res: DeviceRefineEdgeResources = {
		uniformLayout,
		textureLayout,
		signalTextureLayout,
		pipeline,
	};
	deviceResourceCache.set(device, res);
	return res;
}

export const WebGPURenderer: WebGPUNodeRenderer = async (args) => {
	const {
		ctx,
		encoder,
		pass,
		targetTexture,
		targetWidth,
		targetHeight,
		props,
	} = args;

	const op = props.virtualMedia?.operation as RefineEdgeOp | undefined;
	if (op?.op !== "RefineEdge" || !op) return;

	const childMedia = props.virtualMedia?.children?.[0];
	if (!childMedia) return;

	// End incoming pass
	pass.end();

	const frame = props.frame ?? 0;
	const fps = props.fps || 30;
	const width = targetWidth;
	const height = targetHeight;

	const resolveField = (
		handleId: string | null | undefined,
		fallbackVal: number,
		clampMin = 0,
		clampMax = 1000,
	) =>
		resolveHandleParam<RefineEdgeSignalData>(
			op.inputs,
			handleId,
			fallbackVal,
			clampMin,
			clampMax,
		);

	const decontamRes = resolveField(
		op.decontaminateAmountHandleId,
		Number(op.decontaminateAmount ?? 0.7),
		MIN_DECONTAMINATE,
		MAX_DECONTAMINATE,
	);
	const radiusRes = resolveField(
		op.radiusHandleId,
		Number(op.radius ?? 2.0),
		MIN_RADIUS,
		MAX_RADIUS,
	);
	const smoothRes = resolveField(
		op.smoothHandleId,
		Number(op.smooth ?? 5),
		MIN_SMOOTH,
		MAX_SMOOTH,
	);
	const featherRes = resolveField(
		op.featherHandleId,
		Number(op.feather ?? 0.5),
		MIN_FEATHER,
		MAX_FEATHER,
	);
	const shiftEdgeRes = resolveField(
		op.shiftEdgeHandleId,
		Number(op.shiftEdge ?? 0),
		MIN_SHIFT_EDGE,
		MAX_SHIFT_EDGE,
	);

	// 1. Render primary Input child media
	const { texture: tmpInputTex, view: tmpInputView } = await drawChildToTexture(
		args,
		childMedia,
		{
			props: {
				...props,
				virtualMedia: childMedia,
				renderId: `${props.renderId}-input`,
			},
		},
	);

	// 2. Render optional Matte child media if connected
	let tmpMatteTex: GPUTexture | null = null;
	if (op.matteMedia) {
		({ texture: tmpMatteTex } = await drawChildToTexture(
			args,
			op.matteMedia as VirtualMediaData,
			{
				props: {
					...props,
					virtualMedia: op.matteMedia as VirtualMediaData,
					renderId: `${props.renderId}-matte`,
					excludeTextures: [...(props.excludeTextures || []), tmpInputTex],
				},
				exclude: [tmpInputTex],
			},
		));
	}

	// 3. Compile pipeline resources
	const { pipeline, uniformLayout, textureLayout, signalTextureLayout } =
		getRefineEdgeResources(ctx.device, ctx.renderer.format);
	const sampler = ctx.renderer.samplerCache.getSampler(ctx.device);

	const { elapsedSeconds, durationSeconds } = signalTiming(props, frame, fps);

	const signalContext = {
		args,
		width,
		height,
		frame,
		fps,
		elapsedSeconds,
		durationSeconds,
	};
	const getSignalView = (
		res: {
			val: number;
			hasStaticSig: boolean;
			sd: RefineEdgeSignalData | null;
			handleId?: string | null;
		},
		suffix: string,
	) =>
		signalTextureView(
			signalContext,
			res,
			res.handleId ?? `${props.renderId ?? "refineedge"}_${suffix}`,
		);

	const decontamView = getSignalView(decontamRes, "decontam_sig");
	const radiusView = getSignalView(radiusRes, "radius_sig");
	const smoothView = getSignalView(smoothRes, "smooth_sig");
	const featherView = getSignalView(featherRes, "feather_sig");
	const shiftEdgeView = getSignalView(shiftEdgeRes, "shiftEdge_sig");

	const sigBindGroup = ctx.device.createBindGroup({
		layout: signalTextureLayout,
		entries: [
			{ binding: 0, resource: decontamView },
			{ binding: 1, resource: radiusView },
			{ binding: 2, resource: smoothView },
			{ binding: 3, resource: featherView },
			{ binding: 4, resource: shiftEdgeView },
			{ binding: 5, resource: sampler },
		],
	});

	const matteChannelCode = CHANNEL_MAP[op.matteChannel ?? "Alpha"] ?? 0;
	const outputModeCode = OUTPUT_MODE_MAP[op.outputMode ?? "Composite"] ?? 0;

	const dummyMatteView = tmpMatteTex ? tmpMatteTex.createView() : tmpInputView;

	// Populate separate uniform data buffers for each pass
	const hUniformData = new Float32Array(20);
	const vUniformData = new Float32Array(20);

	const baseUniforms = [
		0.0, // [0] dirX (set per pass)
		0.0, // [1] dirY (set per pass)
		decontamRes.val, // [2] decontaminateAmount
		radiusRes.val, // [3] radius
		smoothRes.val, // [4] smooth
		featherRes.val, // [5] feather
		shiftEdgeRes.val, // [6] shiftEdge
		matteChannelCode, // [7] matteChannel
		outputModeCode, // [8] outputMode
		tmpMatteTex ? 1.0 : 0.0, // [9] hasMatte
		decontamRes.hasStaticSig ? 1.0 : 0.0, // [10] hasDecontamSig
		radiusRes.hasStaticSig ? 1.0 : 0.0, // [11] hasRadiusSig
		smoothRes.hasStaticSig ? 1.0 : 0.0, // [12] hasSmoothSig
		featherRes.hasStaticSig ? 1.0 : 0.0, // [13] hasFeatherSig
		shiftEdgeRes.hasStaticSig ? 1.0 : 0.0, // [14] hasShiftEdgeSig
		0.0, // [15] isSecondPass (set per pass)
		0.0, // [16] pad0
		0.0, // [17] pad1
		0.0, // [18] pad2
		0.0, // [19] pad3
	];

	hUniformData.set(baseUniforms);
	hUniformData[0] = 1.0; // dirX
	hUniformData[1] = 0.0; // dirY
	hUniformData[15] = 0.0; // isSecondPass = false

	vUniformData.set(baseUniforms);
	vUniformData[0] = 0.0; // dirX
	vUniformData[1] = 1.0; // dirY
	vUniformData[15] = 1.0; // isSecondPass = true

	const outTex1 = ctx.renderer.getTemporaryTexture(width, height, [
		tmpInputTex,
		...(tmpMatteTex ? [tmpMatteTex] : []),
		targetTexture,
		...(props.excludeTextures || []),
	]);
	const outTex2 = ctx.renderer.getTemporaryTexture(width, height, [
		tmpInputTex,
		...(tmpMatteTex ? [tmpMatteTex] : []),
		outTex1,
		targetTexture,
		...(props.excludeTextures || []),
	]);

	// Pass 1: Horizontal Pass
	const hBuffer = ctx.renderer.getTemporaryBuffer(hUniformData);

	drawFullscreenPass(encoder, outTex1.createView(), pipeline, [
		ctx.device.createBindGroup({
			layout: uniformLayout,
			entries: [{ binding: 0, resource: { buffer: hBuffer } }],
		}),
		ctx.device.createBindGroup({
			layout: textureLayout,
			entries: [
				{ binding: 0, resource: tmpInputView },
				{ binding: 1, resource: dummyMatteView },
				{ binding: 2, resource: tmpInputView },
				{ binding: 3, resource: sampler },
			],
		}),
		sigBindGroup,
	]);

	// Pass 2: Vertical Pass + Defringe & Composite
	const vBuffer = ctx.renderer.getTemporaryBuffer(vUniformData);

	drawFullscreenPass(encoder, outTex2.createView(), pipeline, [
		ctx.device.createBindGroup({
			layout: uniformLayout,
			entries: [{ binding: 0, resource: { buffer: vBuffer } }],
		}),
		ctx.device.createBindGroup({
			layout: textureLayout,
			entries: [
				{ binding: 0, resource: outTex1.createView() },
				{ binding: 1, resource: dummyMatteView },
				{ binding: 2, resource: tmpInputView },
				{ binding: 3, resource: sampler },
			],
		}),
		sigBindGroup,
	]);

	// 4. Final Pass: draw output into targetView
	const finalOpacity = props.opacity ?? op.opacity ?? 1.0;
	drawTextureToTarget(args, outTex2, finalOpacity);
};