Skip to content

Code-first video & image, rendered natively on WebGPU.

13.6 KB
Raw Blame
/// <reference types="webgpu" />
import type { WebGPUNodeRenderer } from "@framefields/node-sdk";
import {
	createFullscreenPipeline,
	drawChildToTexture,
	drawFullscreenPass,
	drawTextureToTarget,
	FULLSCREEN_QUAD_WGSL,
	fragmentBindGroupLayout,
} from "@framefields/node-sdk/gpu";
import { solveMonotonicSpline } from "../shared/spline.js";
import { computeHistogramFromEncoder } from "./histogram-computer.js";

// ─── LUT resolution ───────────────────────────────────────────────────────────
const LUT_SIZE = 1024;
// Use rgba8unorm at 1024 entries — gives 4× finer sampling than 256 while
// avoiding Float16 encoding issues with writeTexture.
const LUT_FORMAT: GPUTextureFormat = "rgba8unorm";
const LUT_BYTES_PER_ROW = LUT_SIZE * 4; // 4 bytes per rgba8unorm texel

// ─── RGB Curves Shader ────────────────────────────────────────────────────────
const CURVES_RGB_SHADER = /* wgsl */ `
${FULLSCREEN_QUAD_WGSL}

@group(0) @binding(0) var t_lut : texture_2d<f32>;
@group(0) @binding(1) var s_lut : sampler;

@group(1) @binding(0) var t_src : texture_2d<f32>;
@group(1) @binding(1) var s_src : sampler;

@fragment fn fs(in: VSOut) -> @location(0) vec4<f32> {
	let src = textureSampleLevel(t_src, s_src, in.uv, 0.0);
	if (src.a < 1e-5) { return src; }

	let rgb = src.rgb / src.a;

	let N = f32(${LUT_SIZE});
	let uv_r = (rgb.r * (N - 1.0) + 0.5) / N;
	let uv_g = (rgb.g * (N - 1.0) + 0.5) / N;
	let uv_b = (rgb.b * (N - 1.0) + 0.5) / N;

	let r_graded = textureSampleLevel(t_lut, s_lut, vec2<f32>(uv_r, 0.5), 0.0).r;
	let g_graded = textureSampleLevel(t_lut, s_lut, vec2<f32>(uv_g, 0.5), 0.0).g;
	let b_graded = textureSampleLevel(t_lut, s_lut, vec2<f32>(uv_b, 0.5), 0.0).b;

	let graded = vec3<f32>(r_graded, g_graded, b_graded);
	return vec4<f32>(graded * src.a, src.a);
}
`;

// ─── HSL Curves Shader ────────────────────────────────────────────────────────
const CURVES_HSL_SHADER = /* wgsl */ `
struct Uniforms {
	mode: u32,
	pad0: u32,
	pad1: u32,
	pad2: u32,
};
@group(0) @binding(0) var<uniform> u : Uniforms;
@group(0) @binding(1) var t_lut      : texture_2d<f32>;
@group(0) @binding(2) var s_lut      : sampler;

@group(1) @binding(0) var t_src : texture_2d<f32>;
@group(1) @binding(1) var s_src : sampler;

${FULLSCREEN_QUAD_WGSL}

fn rgb_to_hsl(rgb: vec3<f32>) -> vec3<f32> {
	let r = rgb.r; let g = rgb.g; let b = rgb.b;
	let cmax = max(r, max(g, b));
	let cmin = min(r, min(g, b));
	let delta = cmax - cmin;

	var h = 0.0;
	var s = 0.0;
	let l = (cmax + cmin) * 0.5;

	if (delta > 1e-5) {
		s = delta / (1.0 - abs(2.0 * l - 1.0));
		if (cmax == r) {
			h = ((g - b) / delta) % 6.0;
		} else if (cmax == g) {
			h = (b - r) / delta + 2.0;
		} else {
			h = (r - g) / delta + 4.0;
		}
		h = h / 6.0;
		if (h < 0.0) { h = h + 1.0; }
	}
	return vec3<f32>(h, s, l);
}

fn hsl_to_rgb(hsl: vec3<f32>) -> vec3<f32> {
	let h = hsl.x; let s = hsl.y; let l = hsl.z;
	let c = (1.0 - abs(2.0 * l - 1.0)) * s;
	let x = c * (1.0 - abs((h * 6.0) % 2.0 - 1.0));
	let m = l - c * 0.5;

	var rgb = vec3<f32>(0.0);
	let hi = u32(h * 6.0) % 6u;
	if (hi == 0u) { rgb = vec3<f32>(c, x, 0.0); }
	else if (hi == 1u) { rgb = vec3<f32>(x, c, 0.0); }
	else if (hi == 2u) { rgb = vec3<f32>(0.0, c, x); }
	else if (hi == 3u) { rgb = vec3<f32>(0.0, x, c); }
	else if (hi == 4u) { rgb = vec3<f32>(x, 0.0, c); }
	else { rgb = vec3<f32>(c, 0.0, x); }
	return rgb + vec3<f32>(m);
}

fn lut_sample(x: f32) -> f32 {
	let N = f32(${LUT_SIZE});
	let uv_x = (x * (N - 1.0) + 0.5) / N;
	return textureSampleLevel(t_lut, s_lut, vec2<f32>(uv_x, 0.5), 0.0).r;
}

@fragment fn fs(in: VSOut) -> @location(0) vec4<f32> {
	let src = textureSampleLevel(t_src, s_src, in.uv, 0.0);
	if (src.a < 1e-5) { return src; }

	let rgb = src.rgb / src.a;
	let hsl = rgb_to_hsl(rgb);

	var out_hsl = hsl;

	if (u.mode == 0u) {
		let adj = lut_sample(hsl.x);
		out_hsl.x = fract(hsl.x + (adj - 0.5) * 2.0);
	} else if (u.mode == 1u) {
		let adj = lut_sample(hsl.x);
		out_hsl.y = clamp(hsl.y * (adj * 2.0), 0.0, 1.0);
	} else if (u.mode == 2u) {
		let adj = lut_sample(hsl.z);
		out_hsl.y = clamp(hsl.y * (adj * 2.0), 0.0, 1.0);
	} else {
		let adj = lut_sample(hsl.y);
		out_hsl.y = clamp(adj, 0.0, 1.0);
	}

	let out_rgb = clamp(hsl_to_rgb(out_hsl), vec3<f32>(0.0), vec3<f32>(1.0));
	return vec4<f32>(out_rgb * src.a, src.a);
}
`;

// ─── Device resource caches ───────────────────────────────────────────────────
interface RgbCurvesResources {
	pipeline: GPURenderPipeline;
	lutBindGroupLayout: GPUBindGroupLayout;
	srcBindGroupLayout: GPUBindGroupLayout;
}

interface HslCurvesResources {
	pipeline: GPURenderPipeline;
	uniformLayout: GPUBindGroupLayout;
	srcBindGroupLayout: GPUBindGroupLayout;
}

const rgbResourceCache = new WeakMap<GPUDevice, RgbCurvesResources>();
const hslResourceCache = new WeakMap<GPUDevice, HslCurvesResources>();

function getRgbResources(
	device: GPUDevice,
	format: GPUTextureFormat,
): RgbCurvesResources {
	let res = rgbResourceCache.get(device);
	if (!res) {
		const lutBGL = fragmentBindGroupLayout(device, ["texture", "sampler"]);
		const srcBGL = fragmentBindGroupLayout(device, ["texture", "sampler"]);
		const module = device.createShaderModule({
			label: "curves-rgb.wgsl",
			code: CURVES_RGB_SHADER,
		});
		const pipeline = createFullscreenPipeline(device, {
			label: "CurvesRgbPipeline",
			module,
			layouts: [lutBGL, srcBGL],
			format,
		});
		res = { pipeline, lutBindGroupLayout: lutBGL, srcBindGroupLayout: srcBGL };
		rgbResourceCache.set(device, res);
	}
	return res;
}

function getHslResources(
	device: GPUDevice,
	format: GPUTextureFormat,
): HslCurvesResources {
	let res = hslResourceCache.get(device);
	if (!res) {
		const uniformLayout = fragmentBindGroupLayout(device, [
			"uniform",
			"texture",
			"sampler",
		]);
		const srcBGL = fragmentBindGroupLayout(device, ["texture", "sampler"]);
		const module = device.createShaderModule({
			label: "curves-hsl.wgsl",
			code: CURVES_HSL_SHADER,
		});
		const pipeline = createFullscreenPipeline(device, {
			label: "CurvesHslPipeline",
			module,
			layouts: [uniformLayout, srcBGL],
			format,
		});
		res = { pipeline, uniformLayout, srcBindGroupLayout: srcBGL };
		hslResourceCache.set(device, res);
	}
	return res;
}

// ─── LUT texture cache ────────────────────────────────────────────────────────
// Content-addressed: instances with equal curves share one LUT, and distinct
// curves never evict each other mid-frame (a node-keyed slot would destroy a
// texture another instance's pending pass still samples). LRU-bounded so
// animated curves don't grow it without limit.
const LUT_CACHE_LIMIT = 32;
const deviceTextureCache = new WeakMap<GPUDevice, Map<string, GPUTexture>>();

function getLut(
	device: GPUDevice,
	configKey: string,
	build: () => GPUTexture,
): GPUTexture {
	let cache = deviceTextureCache.get(device);
	if (!cache) {
		cache = new Map();
		deviceTextureCache.set(device, cache);
	}
	const hit = cache.get(configKey);
	if (hit) {
		cache.delete(configKey);
		cache.set(configKey, hit);
		return hit;
	}
	const texture = build();
	cache.set(configKey, texture);
	if (cache.size > LUT_CACHE_LIMIT) {
		const [oldestKey, oldest] = cache.entries().next().value as [
			string,
			GPUTexture,
		];
		cache.delete(oldestKey);
		oldest.destroy();
	}
	return texture;
}

function buildRgbLut(
	device: GPUDevice,
	op: Record<string, unknown>,
): GPUTexture {
	const masterSpline = solveMonotonicSpline(
		(op.master as { x: number; y: number }[]) || [
			{ x: 0, y: 0 },
			{ x: 1, y: 1 },
		],
	);
	const redSpline = solveMonotonicSpline(
		(op.red as { x: number; y: number }[]) || [
			{ x: 0, y: 0 },
			{ x: 1, y: 1 },
		],
	);
	const greenSpline = solveMonotonicSpline(
		(op.green as { x: number; y: number }[]) || [
			{ x: 0, y: 0 },
			{ x: 1, y: 1 },
		],
	);
	const blueSpline = solveMonotonicSpline(
		(op.blue as { x: number; y: number }[]) || [
			{ x: 0, y: 0 },
			{ x: 1, y: 1 },
		],
	);

	const lutData = new Uint8Array(LUT_SIZE * 4);
	for (let i = 0; i < LUT_SIZE; i++) {
		const u = i / (LUT_SIZE - 1);
		const masterVal = masterSpline(u);
		const r = Math.max(0, Math.min(1, redSpline(masterVal)));
		const g = Math.max(0, Math.min(1, greenSpline(masterVal)));
		const b = Math.max(0, Math.min(1, blueSpline(masterVal)));
		lutData[i * 4 + 0] = Math.round(r * 255);
		lutData[i * 4 + 1] = Math.round(g * 255);
		lutData[i * 4 + 2] = Math.round(b * 255);
		lutData[i * 4 + 3] = 255;
	}

	const texture = device.createTexture({
		size: [LUT_SIZE, 1, 1],
		dimension: "2d",
		format: LUT_FORMAT,
		usage: GPUTextureUsage.TEXTURE_BINDING | GPUTextureUsage.COPY_DST,
	});
	device.queue.writeTexture(
		{ texture },
		lutData,
		{ bytesPerRow: LUT_BYTES_PER_ROW },
		[LUT_SIZE, 1, 1],
	);
	return texture;
}

function buildHslLut(
	device: GPUDevice,
	points: { x: number; y: number }[],
): GPUTexture {
	const spline = solveMonotonicSpline(points);
	const lutData = new Uint8Array(LUT_SIZE * 4);
	for (let i = 0; i < LUT_SIZE; i++) {
		const u = i / (LUT_SIZE - 1);
		const val = Math.round(Math.max(0, Math.min(1, spline(u))) * 255);
		lutData[i * 4 + 0] = val;
		lutData[i * 4 + 1] = val;
		lutData[i * 4 + 2] = val;
		lutData[i * 4 + 3] = 255;
	}
	const texture = device.createTexture({
		size: [LUT_SIZE, 1, 1],
		dimension: "2d",
		format: LUT_FORMAT,
		usage: GPUTextureUsage.TEXTURE_BINDING | GPUTextureUsage.COPY_DST,
	});
	device.queue.writeTexture(
		{ texture },
		lutData,
		{ bytesPerRow: LUT_BYTES_PER_ROW },
		[LUT_SIZE, 1, 1],
	);
	return texture;
}

const HSL_MODE_MAP: Record<string, number> = {
	"hue-vs-hue": 0,
	"hue-vs-sat": 1,
	"lum-vs-sat": 2,
	"sat-vs-sat": 3,
};
const HSL_CURVE_KEY_MAP: Record<string, string> = {
	"hue-vs-hue": "hueVsHue",
	"hue-vs-sat": "hueVsSat",
	"lum-vs-sat": "lumVsSat",
	"sat-vs-sat": "satVsSat",
};

// ─── Main renderer ─────────────────────────────────────────────────────────────
export const CurvesWebGPURenderer: WebGPUNodeRenderer = async (args) => {
	const {
		ctx,
		encoder,
		pass,
		targetTexture,
		targetWidth,
		targetHeight,
		props,
	} = args;

	const op = props.virtualMedia?.operation as Record<string, unknown> & {
		op?: string;
	};
	if (op?.op !== "Curves" || !props.virtualMedia?.children?.[0]) return;

	pass.end();

	const childMedia = props.virtualMedia.children[0];
	const width = targetWidth;
	const height = targetHeight;
	const nodeId = (op.nodeId as string) || "curves_fallback";
	const curveType = (op.curveType as string) || "rgb";

	// ── Capture child input frame ───────────────────────────────────────────
	const { texture: tmpTex } = await drawChildToTexture(args, childMedia);

	// ── LUT / grading pass ──────────────────────────────────────────────────
	const sampler = ctx.renderer.samplerCache.getSampler(ctx.device);
	const outTex = ctx.renderer.getTemporaryTexture(width, height, [
		tmpTex,
		targetTexture,
		...(props.excludeTextures || []),
	]);

	if (curveType === "rgb") {
		const configKey = JSON.stringify({
			curveType,
			master: op.master,
			red: op.red,
			green: op.green,
			blue: op.blue,
		});
		const lut = getLut(ctx.device, configKey, () =>
			buildRgbLut(ctx.device, op),
		);
		const { pipeline, lutBindGroupLayout, srcBindGroupLayout } =
			getRgbResources(ctx.device, ctx.renderer.format);
		drawFullscreenPass(encoder, outTex.createView(), pipeline, [
			ctx.device.createBindGroup({
				layout: lutBindGroupLayout,
				entries: [
					{ binding: 0, resource: lut.createView() },
					{ binding: 1, resource: sampler },
				],
			}),
			ctx.renderer.bindGroupCache.getBindGroup(
				ctx.device,
				srcBindGroupLayout,
				tmpTex,
				sampler,
			),
		]);
	} else {
		const hslCurveKey = HSL_CURVE_KEY_MAP[curveType];
		const hslPoints = (op[hslCurveKey] as { x: number; y: number }[]) || [
			{ x: 0, y: 0.5 },
			{ x: 1, y: 0.5 },
		];
		const configKey = JSON.stringify({ curveType, points: hslPoints });
		const lut = getLut(ctx.device, configKey, () =>
			buildHslLut(ctx.device, hslPoints),
		);
		const uniformData = new Uint32Array([
			HSL_MODE_MAP[curveType] ?? 0,
			0,
			0,
			0,
		]);
		const uniformBuffer = ctx.renderer.getTemporaryBuffer(uniformData);
		const { pipeline, uniformLayout, srcBindGroupLayout } = getHslResources(
			ctx.device,
			ctx.renderer.format,
		);
		drawFullscreenPass(encoder, outTex.createView(), pipeline, [
			ctx.device.createBindGroup({
				layout: uniformLayout,
				entries: [
					{ binding: 0, resource: { buffer: uniformBuffer } },
					{ binding: 1, resource: lut.createView() },
					{ binding: 2, resource: sampler },
				],
			}),
			ctx.renderer.bindGroupCache.getBindGroup(
				ctx.device,
				srcBindGroupLayout,
				tmpTex,
				sampler,
			),
		]);
	}

	// ── Histogram compute — runs in the SAME encoder so outTex data is valid ─
	computeHistogramFromEncoder(ctx.device, encoder, outTex, nodeId);

	// ── Final composite to target ──────────────────────────────────────────
	drawTextureToTarget(args, outTex, props.opacity ?? 1);
};