Skip to content

Code-first video & image, rendered natively on WebGPU.

9.2 KB
Raw Blame
/// <reference types="webgpu" />
import type { SignalData } from "@framefields/core";
import {
	resolveHandleParam,
	type WebGPUNodeRenderer,
} from "@framefields/node-sdk";
import {
	createFullscreenPipeline,
	drawChildToTexture,
	drawFullscreenPass,
	drawTextureToTarget,
	FULLSCREEN_QUAD_WGSL,
	fragmentBindGroupLayout,
	signalTextureView,
	signalTiming,
	texturesLayout,
} from "@framefields/node-sdk/gpu";
import { MAX_RADIUS } from "../shared/config.js";

interface HighPassOp {
	op: "HighPass";
	radius?: number;
	contrastBoost?: number;
	monochrome?: boolean;
	radiusHandleId?: string | null;
	contrastBoostHandleId?: string | null;
	opacity?: number;
	inputs?: Record<
		string,
		{
			connectionValid: boolean;
			outputItem: {
				type: string;
				data: unknown;
			} | null;
		}
	>;
}

type HighPassSignalData = SignalData & { nodeId?: string };

const WGSL_HIGHPASS_UNIFORMS = `
struct HighPassUniforms {
	dirX           : f32,
	dirY           : f32,
	radius         : f32,
	contrastBoost  : f32,

	monochrome     : f32,
	hasRadiusSig   : f32,
	hasContrastSig : f32,
	isSecondPass   : f32,
};

@group(0) @binding(0) var<uniform> u          : HighPassUniforms;
@group(1) @binding(0) var tex                 : texture_2d<f32>;
@group(1) @binding(1) var samp                : sampler;
@group(1) @binding(2) var origTex             : texture_2d<f32>;

@group(2) @binding(0) var radiusSigTex        : texture_2d<f32>;
@group(2) @binding(1) var contrastSigTex      : texture_2d<f32>;
@group(2) @binding(2) var signalSamp          : sampler;
${FULLSCREEN_QUAD_WGSL}

@fragment fn fs(in : VSOut) -> @location(0) vec4<f32> {
	let dimensions = vec2<f32>(textureDimensions(tex));
	var radius = u.radius;
	if (u.hasRadiusSig > 0.5) {
		radius = textureSampleLevel(radiusSigTex, signalSamp, vec2<f32>(0.5, 0.5), 0.0).r;
	}
	var contrastBoost = u.contrastBoost;
	if (u.hasContrastSig > 0.5) {
		contrastBoost = textureSampleLevel(contrastSigTex, signalSamp, vec2<f32>(0.5, 0.5), 0.0).r;
	}

	let origCol = textureSampleLevel(origTex, samp, in.uv, 0.0);
	if (radius <= 0.0) {
		if (u.isSecondPass > 0.5) {
			return vec4<f32>(vec3<f32>(0.5) * origCol.a, origCol.a);
		}
		return origCol;
	}

	let texelSize = 1.0 / dimensions;
	let spatial_sigma = radius;
	let stepScale = max(1.0, spatial_sigma / 20.0);
	let effectiveSigma = spatial_sigma / stepScale;
	let sampleRadius = min(i32(ceil(3.0 * effectiveSigma)), 64);
	let direction = vec2<f32>(u.dirX, u.dirY);

	var blurred = vec4<f32>(0.0);
	var totalWeight = 0.0;

	for (var i = -sampleRadius; i <= sampleRadius; i++) {
		let offset = vec2<f32>(f32(i) * stepScale) * direction * texelSize;
		let x = f32(i) * stepScale;
		let weight = exp(-0.5 * (x / spatial_sigma) * (x / spatial_sigma));
		blurred += textureSampleLevel(tex, samp, in.uv + offset, 0.0) * weight;
		totalWeight += weight;
	}

	let blurResult = blurred / max(totalWeight, 1e-5);

	if (u.isSecondPass <= 0.5) {
		// Pass 1: Intermediate horizontal blur
		return blurResult;
	}

	// Pass 2: Vertical blur complete -> Calculate High Pass difference against original
	if (origCol.a < 1e-5) {
		return vec4<f32>(0.0, 0.0, 0.0, 0.0);
	}

	let origUnpremult = origCol.rgb / origCol.a;
	let blurUnpremult = blurResult.rgb / max(blurResult.a, 1e-5);
	let diff = origUnpremult - blurUnpremult;

	var highPassRgb : vec3<f32>;
	if (u.monochrome > 0.5) {
		let lumaDiff = dot(diff, vec3<f32>(0.2126, 0.7152, 0.0722));
		highPassRgb = vec3<f32>(0.5) + vec3<f32>(lumaDiff * contrastBoost);
	} else {
		highPassRgb = vec3<f32>(0.5) + diff * contrastBoost;
	}

	let clampedRgb = clamp(highPassRgb, vec3<f32>(0.0), vec3<f32>(1.0));
	return vec4<f32>(clampedRgb * origCol.a, origCol.a);
}
`;

interface DeviceHighPassResources {
	highPassUniformLayout: GPUBindGroupLayout;
	singleTextureLayout: GPUBindGroupLayout;
	signalTextureLayout: GPUBindGroupLayout;
	highPassPipeline: GPURenderPipeline;
}

const deviceResourceCache = new WeakMap<GPUDevice, DeviceHighPassResources>();
const highPassUniformData = new Float32Array(8);

function getHighPassResources(
	device: GPUDevice,
	targetFormat: GPUTextureFormat,
): DeviceHighPassResources {
	const cached = deviceResourceCache.get(device);
	if (cached) return cached;

	const highPassUniformLayout = fragmentBindGroupLayout(device, ["uniform"]);

	const singleTextureLayout = fragmentBindGroupLayout(device, [
		"texture",
		"sampler",
		"texture",
	]);

	const signalTextureLayout = texturesLayout(device, 2);

	const shaderModule = device.createShaderModule({
		code: WGSL_HIGHPASS_UNIFORMS,
	});

	const pipelineLayout = device.createPipelineLayout({
		bindGroupLayouts: [
			highPassUniformLayout,
			singleTextureLayout,
			signalTextureLayout,
		],
	});

	const highPassPipeline = createFullscreenPipeline(device, {
		module: shaderModule,
		layouts: pipelineLayout,
		format: targetFormat,
	});

	const res: DeviceHighPassResources = {
		highPassUniformLayout,
		singleTextureLayout,
		signalTextureLayout,
		highPassPipeline,
	};
	deviceResourceCache.set(device, res);
	return res;
}

export const WebGPURenderer: WebGPUNodeRenderer = async (args) => {
	const {
		ctx,
		encoder,
		pass,
		targetTexture,
		targetWidth,
		targetHeight,
		props,
	} = args;

	const op = props.virtualMedia?.operation as HighPassOp | undefined;
	if (op?.op !== "HighPass" || !op) return;

	const childMedia = props.virtualMedia.children?.[0];
	if (!childMedia) return;

	// End the incoming active pass
	pass.end();

	const frame = props.frame ?? 0;
	const fps = props.fps || 30;
	const width = targetWidth;
	const height = targetHeight;
	const nodeId = op.inputs ? Object.keys(op.inputs)[0] : "highpass_node";

	const resolveField = (
		handleId: string | null | undefined,
		fallbackVal: number,
		clampMin = 0,
		clampMax = 1000,
	) =>
		resolveHandleParam<HighPassSignalData>(
			op.inputs,
			handleId,
			fallbackVal,
			clampMin,
			clampMax,
		);

	const radiusRes = resolveField(
		op.radiusHandleId,
		Number(op.radius ?? 3.0),
		0.1,
		MAX_RADIUS,
	);
	const contrastRes = resolveField(
		op.contrastBoostHandleId,
		Number(op.contrastBoost ?? 1.0),
		1.0,
		10.0,
	);

	// 1. Render child media into temporary input texture
	const { texture: tmpTex } = await drawChildToTexture(args, childMedia);

	// 2. Fetch static resources and compile pipeline once
	const {
		highPassPipeline: pipeline,
		highPassUniformLayout: uLayout,
		singleTextureLayout: tLayout,
		signalTextureLayout: sigLayout,
	} = getHighPassResources(ctx.device, ctx.renderer.format);

	const sampler = ctx.renderer.samplerCache.getSampler(ctx.device);

	const { elapsedSeconds, durationSeconds } = signalTiming(props, frame, fps);

	const signalContext = {
		args,
		width,
		height,
		frame,
		fps,
		elapsedSeconds,
		durationSeconds,
	};
	const getSignalView = (
		res: { val: number; hasStaticSig: boolean; sd: HighPassSignalData | null },
		suffix: string,
	) => signalTextureView(signalContext, res, `${nodeId}_${suffix}`);

	const radiusView = getSignalView(radiusRes, "radius_sig");
	const contrastView = getSignalView(contrastRes, "contrast_sig");

	const createHighPassBindGroup1 = (
		inputTex: GPUTexture,
		origTexture: GPUTexture,
	) =>
		ctx.device.createBindGroup({
			layout: tLayout,
			entries: [
				{ binding: 0, resource: inputTex.createView() },
				{ binding: 1, resource: sampler },
				{ binding: 2, resource: origTexture.createView() },
			],
		});

	// Prepare uniform template (32 bytes / 8 floats)
	highPassUniformData[2] = radiusRes.val;
	highPassUniformData[3] = contrastRes.val;
	highPassUniformData[4] = (op.monochrome ?? true) ? 1.0 : 0.0;
	highPassUniformData[5] = radiusRes.hasStaticSig ? 1.0 : 0.0;
	highPassUniformData[6] = contrastRes.hasStaticSig ? 1.0 : 0.0;
	highPassUniformData[7] = 0.0; // isSecondPass

	const outTex1 = ctx.renderer.getTemporaryTexture(width, height, [
		tmpTex,
		targetTexture,
		...(props.excludeTextures || []),
	]);
	const outTex2 = ctx.renderer.getTemporaryTexture(width, height, [
		tmpTex,
		outTex1,
		targetTexture,
		...(props.excludeTextures || []),
	]);

	const sigBindGroup = ctx.device.createBindGroup({
		layout: sigLayout,
		entries: [
			{ binding: 0, resource: radiusView },
			{ binding: 1, resource: contrastView },
			{ binding: 2, resource: sampler },
		],
	});

	// Pass 1: Horizontal Gaussian convolution
	highPassUniformData[0] = 1.0;
	highPassUniformData[1] = 0.0;
	highPassUniformData[7] = 0.0; // isSecondPass = false
	const hBuffer = ctx.renderer.getTemporaryBuffer(highPassUniformData);

	drawFullscreenPass(encoder, outTex1.createView(), pipeline, [
		ctx.device.createBindGroup({
			layout: uLayout,
			entries: [{ binding: 0, resource: { buffer: hBuffer } }],
		}),
		createHighPassBindGroup1(tmpTex, tmpTex),
		sigBindGroup,
	]);

	// Pass 2: Vertical Gaussian convolution + High Pass Subtraction against original tmpTex
	highPassUniformData[0] = 0.0;
	highPassUniformData[1] = 1.0;
	highPassUniformData[7] = 1.0; // isSecondPass = true
	const vBuffer = ctx.renderer.getTemporaryBuffer(highPassUniformData);

	drawFullscreenPass(encoder, outTex2.createView(), pipeline, [
		ctx.device.createBindGroup({
			layout: uLayout,
			entries: [{ binding: 0, resource: { buffer: vBuffer } }],
		}),
		createHighPassBindGroup1(outTex1, tmpTex),
		sigBindGroup,
	]);

	// 3. Final Pass: Draw result back to targetView using standard drawTexture for blending/opacity
	drawTextureToTarget(args, outTex2, op.opacity ?? 1.0);
};