Skip to content

Code-first video & image, rendered natively on WebGPU.

9.4 KB
Raw Blame
/// <reference types="webgpu" />
import type { WebGPUNodeRenderer } from "@framefields/node-sdk";
import {
	createFullscreenPipeline,
	drawChildToTexture,
	drawFullscreenPass,
	drawTextureToTarget,
	FULLSCREEN_QUAD_WGSL,
	fragmentBindGroupLayout,
	perDevice,
	signalTiming,
} from "@framefields/node-sdk/gpu";
import { lutStore, signalRegistry } from "@framefields/webgpu-renderers";

export const LUT_3D_SHADER = `
struct LutUniforms {
	intensity      : f32,
	lutSize        : f32,
	hasSignal      : f32,
	pad            : f32,
};

@group(0) @binding(0) var<uniform> u : LutUniforms;
@group(1) @binding(0) var tex        : texture_2d<f32>;
@group(1) @binding(1) var samp       : sampler;
@group(2) @binding(0) var lutTex     : texture_3d<f32>;
@group(2) @binding(1) var lutSamp    : sampler;
@group(3) @binding(0) var signalTex  : texture_2d<f32>;
@group(3) @binding(1) var signalSamp : sampler;
${FULLSCREEN_QUAD_WGSL}

@fragment fn fs(in : VSOut) -> @location(0) vec4<f32> {
	let inputColor = textureSampleLevel(tex, samp, in.uv, 0.0);
	
	// Coordinate adjustment to sample from texel centers
	let uvw = inputColor.rgb * ((u.lutSize - 1.0) / u.lutSize) + vec3<f32>(0.5 / u.lutSize);
	let mappedColor = textureSampleLevel(lutTex, lutSamp, uvw, 0.0);
	
	var activeIntensity = u.intensity;
	if (u.hasSignal > 0.5) {
		activeIntensity = textureSampleLevel(signalTex, signalSamp, in.uv, 0.0).r;
	}
	
	let finalRgb = inputColor.rgb + activeIntensity  * (mappedColor.rgb - inputColor.rgb);
	return vec4<f32>(finalRgb * inputColor.a, inputColor.a);
}
`;

export const LUT_1D_SHADER = `
struct LutUniforms {
	intensity      : f32,
	lutSize        : f32,
	hasSignal      : f32,
	pad            : f32,
};

@group(0) @binding(0) var<uniform> u : LutUniforms;
@group(1) @binding(0) var tex        : texture_2d<f32>;
@group(1) @binding(1) var samp       : sampler;
@group(2) @binding(0) var lutTex     : texture_1d<f32>;
@group(2) @binding(1) var lutSamp    : sampler;
@group(3) @binding(0) var signalTex  : texture_2d<f32>;
@group(3) @binding(1) var signalSamp : sampler;
${FULLSCREEN_QUAD_WGSL}

@fragment fn fs(in : VSOut) -> @location(0) vec4<f32> {
	let inputColor = textureSampleLevel(tex, samp, in.uv, 0.0);
	
	let rCoord = inputColor.r * ((u.lutSize - 1.0) / u.lutSize) + (0.5 / u.lutSize);
	let gCoord = inputColor.g * ((u.lutSize - 1.0) / u.lutSize) + (0.5 / u.lutSize);
	let bCoord = inputColor.b * ((u.lutSize - 1.0) / u.lutSize) + (0.5 / u.lutSize);
	
	let mappedR = textureSampleLevel(lutTex, lutSamp, rCoord, 0.0).r;
	let mappedG = textureSampleLevel(lutTex, lutSamp, gCoord, 0.0).g;
	let mappedB = textureSampleLevel(lutTex, lutSamp, bCoord, 0.0).b;
	
	var activeIntensity = u.intensity;
	if (u.hasSignal > 0.5) {
		activeIntensity = textureSampleLevel(signalTex, signalSamp, in.uv, 0.0).r;
	}
	
	let finalRgb = inputColor.rgb + activeIntensity  * (vec3<f32>(mappedR, mappedG, mappedB) - inputColor.rgb);
	return vec4<f32>(finalRgb * inputColor.a, inputColor.a);
}
`;

const lutUniformData = new Float32Array(4);

const getDeviceLayouts = perDevice((device) => ({
	lutUniformLayout: fragmentBindGroupLayout(device, ["uniform"]),
	singleTextureLayout: fragmentBindGroupLayout(device, ["texture", "sampler"]),
	lutTextureLayout3D: fragmentBindGroupLayout(device, [
		{ texture: { viewDimension: "3d", sampleType: "float" } },
		"sampler",
	]),
	lutTextureLayout1D: fragmentBindGroupLayout(device, [
		{ texture: { viewDimension: "1d", sampleType: "float" } },
		"sampler",
	]),
	signalTextureLayout: fragmentBindGroupLayout(device, ["texture", "sampler"]),
	pipelineCache: new Map<string, GPURenderPipeline>(),
}));

function getLutResources(
	device: GPUDevice,
	format: GPUTextureFormat,
	type: "1D" | "3D",
) {
	const layouts = getDeviceLayouts(device);
	const cacheKey = `lut_${type}_${format}`;
	let pipeline = layouts.pipelineCache.get(cacheKey);
	if (!pipeline) {
		const shaderCode = type === "3D" ? LUT_3D_SHADER : LUT_1D_SHADER;
		const lutModule = device.createShaderModule({
			label: `lut_${type.toLowerCase()}_${format}.wgsl`,
			code: shaderCode,
		});
		const currentLutLayout =
			type === "3D" ? layouts.lutTextureLayout3D : layouts.lutTextureLayout1D;
		pipeline = createFullscreenPipeline(device, {
			label: `LutPipeline_${type}_${format}`,
			module: lutModule,
			layouts: [
				layouts.lutUniformLayout,
				layouts.singleTextureLayout,
				currentLutLayout,
				layouts.signalTextureLayout,
			],
			format,
		});
		layouts.pipelineCache.set(cacheKey, pipeline);
	}

	return {
		lutPipeline: pipeline,
		lutUniformLayout: layouts.lutUniformLayout,
		singleTextureLayout: layouts.singleTextureLayout,
		lutTextureLayout:
			type === "3D" ? layouts.lutTextureLayout3D : layouts.lutTextureLayout1D,
		signalTextureLayout: layouts.signalTextureLayout,
	};
}

export const LutWebGPURenderer: WebGPUNodeRenderer = async (args) => {
	const {
		ctx,
		encoder,
		pass,
		targetTexture,
		targetWidth,
		targetHeight,
		props,
	} = args;
	const frame = props.frame ?? 0;
	const fps = props.fps || 30;
	const op = props.virtualMedia?.operation as any;
	if (op?.op !== "ApplyLUT" || !op) return;
	const childMedia = props.virtualMedia.children?.[0];
	if (!childMedia) return;

	const lutUrl = op.lutUrl;
	if (!lutUrl) return;

	// End the active pass
	pass.end();

	const width = targetWidth;
	const height = targetHeight;

	// Resolve the intensity signal
	const intensityInput = op.intensityHandleId
		? op.inputs?.[op.intensityHandleId]
		: null;
	const hasSignal = !!(
		intensityInput?.connectionValid &&
		intensityInput.outputItem?.type === "Signal"
	);
	const sd = hasSignal ? intensityInput?.outputItem?.data : null;

	let intensity = 1.0;
	let hasStaticSignal = false;

	if (!hasSignal) {
		if (
			intensityInput?.connectionValid &&
			intensityInput.outputItem?.type === "Number"
		) {
			intensity = Number(intensityInput.outputItem.data ?? 1.0);
		} else {
			intensity = Number(op.intensity ?? 1.0);
		}
	} else if (sd && typeof sd === "object") {
		intensity = Number((sd as any).offset ?? 1.0);
		hasStaticSignal = true;
	}
	intensity = Math.max(0, Math.min(10, intensity));

	// Load the GPU LUT resource without blocking the rendering pipeline.
	let lut = lutStore.get(lutUrl, ctx.device);
	if (!lut) {
		const isHeadless =
			props.isHeadless ??
			(typeof window === "undefined" ||
				(globalThis as any).__IS_HEADLESS_RENDERER__);

		if (isHeadless || props.forceWait) {
			try {
				lut = await lutStore.getOrLoad(lutUrl, ctx.device);
			} catch (err) {
				if (
					!(err instanceof DOMException && err.name === "AbortError") &&
					!(
						err &&
						typeof err === "object" &&
						"name" in err &&
						err.name === "AbortError"
					)
				) {
					console.error("[ApplyLUT] Failed to load LUT synchronously:", err);
				}
			}
		} else {
			// Trigger background load
			lutStore.getOrLoad(lutUrl, ctx.device).catch((err) => {
				if (
					(err instanceof DOMException && err.name === "AbortError") ||
					(err &&
						typeof err === "object" &&
						"name" in err &&
						err.name === "AbortError")
				) {
					return;
				}
				console.error("[ApplyLUT] Failed to load LUT in background:", err);
			});
		}

		// Fallback to any matching cached LUT for the same node
		if (!lut && lutUrl.startsWith("runtime://lut/")) {
			const prefix = lutUrl.substring(0, lutUrl.lastIndexOf("-") + 1);
			if (prefix) {
				lut = lutStore.getAnyMatching(prefix, ctx.device);
			}
		}
	}

	const { texture: tmpTex } = await drawChildToTexture(args, childMedia);

	// If no LUT is ready (and no fallback is available), draw the child frame directly as a passthrough
	if (!lut) {
		drawTextureToTarget(args, tmpTex, op.opacity ?? 1);
		return;
	}

	// Fetch pipeline and layouts
	const {
		lutPipeline: pipeline,
		lutUniformLayout: uLayout,
		singleTextureLayout: tLayout,
		lutTextureLayout: lLayout,
		signalTextureLayout: sigLayout,
	} = getLutResources(ctx.device, ctx.renderer.format, lut.type);

	const sampler = ctx.renderer.samplerCache.getSampler(ctx.device);

	// Setup intermediate output texture
	const outTex = ctx.renderer.getTemporaryTexture(width, height, [
		tmpTex,
		targetTexture,
		...(props.excludeTextures || []),
	]);

	// Fetch dynamic intensity signal texture view
	let signalView: GPUTextureView;
	if (hasStaticSignal) {
		const { elapsedSeconds, durationSeconds } = signalTiming(props, frame, fps);
		signalView = signalRegistry.getOrCreate2DTextureView(
			ctx.device,
			encoder,
			sd.nodeId ?? "lut_intensity_sig",
			elapsedSeconds,
			durationSeconds,
			sd,
			width,
			height,
			props.renderId,
			frame,
			fps,
		);
	} else {
		signalView = signalRegistry.getDummy1x1TextureView(ctx.device);
	}

	// Prepare uniforms
	lutUniformData[0] = intensity;
	lutUniformData[1] = lut.size;
	lutUniformData[2] = hasStaticSignal ? 1 : 0;
	lutUniformData[3] = 0; // padding

	const uniformBuffer = ctx.renderer.getTemporaryBuffer(lutUniformData);

	drawFullscreenPass(encoder, outTex.createView(), pipeline!, [
		ctx.device.createBindGroup({
			layout: uLayout!,
			entries: [{ binding: 0, resource: { buffer: uniformBuffer } }],
		}),
		ctx.renderer.bindGroupCache.getBindGroup(
			ctx.device,
			tLayout!,
			tmpTex,
			sampler,
		),
		ctx.device.createBindGroup({
			layout: lLayout!,
			entries: [
				{ binding: 0, resource: lut.texture.createView() },
				{ binding: 1, resource: sampler },
			],
		}),
		ctx.device.createBindGroup({
			layout: sigLayout!,
			entries: [
				{ binding: 0, resource: signalView },
				{ binding: 1, resource: sampler },
			],
		}),
	]);

	// Blending/opacity draw to the targetView
	drawTextureToTarget(args, outTex, op.opacity ?? 1);
};