Skip to content

Code-first video & image, rendered natively on WebGPU.

10.7 KB
Raw Blame
/// <reference types="webgpu" />
import type { SignalData } from "@framefields/core";
import {
	unwrapFrameParam,
	type WebGPUNodeRenderer,
} from "@framefields/node-sdk";
import {
	createFullscreenPipeline,
	drawChildToTexture,
	drawFullscreenPass,
	drawTextureToTarget,
	FULLSCREEN_QUAD_WGSL,
	fragmentBindGroupLayout,
	perDevice,
	signalTextureView,
	signalTiming,
	texturesLayout,
} from "@framefields/node-sdk/gpu";

interface VignetteOp {
	op: "Vignette";
	strength?: number;
	radius?: number;
	softness?: number;
	roundness?: number;
	centerX?: number;
	centerY?: number;
	opacity?: number;
	strengthHandleId?: string | null;
	radiusHandleId?: string | null;
	softnessHandleId?: string | null;
	roundnessHandleId?: string | null;
	centerXHandleId?: string | null;
	centerYHandleId?: string | null;
	inputs?: Record<
		string,
		{
			connectionValid: boolean;
			outputItem: {
				type: string;
				data: unknown;
			} | null;
		}
	>;
}

type VignetteSignalData = SignalData & { nodeId?: string };

const VIGNETTE_SHADER = `
struct VignetteUniforms {
    strength       : f32,
    radius         : f32,
    softness       : f32,
    roundness      : f32,

    centerX        : f32,
    centerY        : f32,
    hasStrengthSig : f32,
    hasRadiusSig   : f32,

    hasSoftnessSig : f32,
    hasRoundnessSig: f32,
    hasCenterXSig  : f32,
    hasCenterYSig  : f32,
};

@group(0) @binding(0) var<uniform> u : VignetteUniforms;
@group(1) @binding(0) var tex        : texture_2d<f32>;
@group(1) @binding(1) var samp       : sampler;

@group(2) @binding(0) var strengthSigTex  : texture_2d<f32>;
@group(2) @binding(1) var radiusSigTex    : texture_2d<f32>;
@group(2) @binding(2) var softnessSigTex  : texture_2d<f32>;
@group(2) @binding(3) var roundnessSigTex : texture_2d<f32>;
@group(2) @binding(4) var centerXSigTex   : texture_2d<f32>;
@group(2) @binding(5) var centerYSigTex   : texture_2d<f32>;
@group(2) @binding(6) var signalSamp      : sampler;
${FULLSCREEN_QUAD_WGSL}

@fragment fn fs(in : VSOut) -> @location(0) vec4<f32> {
    let color = textureSampleLevel(tex, samp, in.uv, 0.0);

    if (color.a < 1e-5) {
        return color;
    }

    // Resolve parameter overrides or dynamic signals
    var strength = u.strength;
    if (u.hasStrengthSig > 0.5) {
        strength = textureSampleLevel(strengthSigTex, signalSamp, vec2<f32>(0.5, 0.5), 0.0).r;
    }

    var radius = u.radius;
    if (u.hasRadiusSig > 0.5) {
        radius = textureSampleLevel(radiusSigTex, signalSamp, vec2<f32>(0.5, 0.5), 0.0).r;
    }

    var softness = u.softness;
    if (u.hasSoftnessSig > 0.5) {
        softness = textureSampleLevel(softnessSigTex, signalSamp, vec2<f32>(0.5, 0.5), 0.0).r;
    }

    var roundness = u.roundness;
    if (u.hasRoundnessSig > 0.5) {
        roundness = textureSampleLevel(roundnessSigTex, signalSamp, vec2<f32>(0.5, 0.5), 0.0).r;
    }

    var centerX = u.centerX;
    if (u.hasCenterXSig > 0.5) {
        centerX = textureSampleLevel(centerXSigTex, signalSamp, vec2<f32>(0.5, 0.5), 0.0).r;
    }

    var centerY = u.centerY;
    if (u.hasCenterYSig > 0.5) {
        centerY = textureSampleLevel(centerYSigTex, signalSamp, vec2<f32>(0.5, 0.5), 0.0).r;
    }

    let center = vec2<f32>(centerX, centerY);
    let dimensions = vec2<f32>(textureDimensions(tex));
    let aspect = dimensions.x / dimensions.y;
    let uvOffset = in.uv - center;

    // Calculate aspect ratio corrected coordinate for circle
    let correctedOffset = vec2<f32>(
        uvOffset.x * mix(1.0, aspect, roundness),
        uvOffset.y
    );
    let dist = length(correctedOffset);

    // Calculate vignette edge falloff
    let edge0 = radius;
    let edge1 = max(0.0, radius - softness);
    let vignetteFactor = smoothstep(edge0, edge1, dist);

    // Blend based on strength
    let intensity = strength / 100.0;
    let finalVignette = mix(1.0, vignetteFactor, intensity);

    // Unpremultiply, apply vignette, and repremultiply
    let unpremult_rgb = color.rgb / color.a;
    let final_rgb = unpremult_rgb * finalVignette;

    return vec4<f32>(clamp(final_rgb, vec3<f32>(0.0), vec3<f32>(1.0)) * color.a, color.a);
}
`;

const vignetteUniformData = new Float32Array(12);

const getDeviceLayouts = perDevice((device) => ({
	uniformLayout: fragmentBindGroupLayout(device, ["uniform"]),
	singleTextureLayout: fragmentBindGroupLayout(device, ["texture", "sampler"]),
	signalTextureLayout: texturesLayout(device, 6),
	pipelineCache: new Map<string, GPURenderPipeline>(),
}));

function getVignetteResources(device: GPUDevice, format: GPUTextureFormat) {
	const layouts = getDeviceLayouts(device);

	const cacheKey = `vignette_${format}`;
	let pipeline = layouts.pipelineCache.get(cacheKey);
	if (!pipeline) {
		const vignetteModule = device.createShaderModule({
			label: `vignette_${format}.wgsl`,
			code: VIGNETTE_SHADER,
		});

		pipeline = createFullscreenPipeline(device, {
			label: `VignettePipeline_${format}`,
			module: vignetteModule,
			layouts: [
				layouts.uniformLayout,
				layouts.singleTextureLayout,
				layouts.signalTextureLayout,
			],
			format,
		});
		layouts.pipelineCache.set(cacheKey, pipeline);
	}

	return {
		vignettePipeline: pipeline,
		uniformLayout: layouts.uniformLayout,
		singleTextureLayout: layouts.singleTextureLayout,
		signalTextureLayout: layouts.signalTextureLayout,
	};
}

export const VignetteWebGPURenderer: WebGPUNodeRenderer = async (args) => {
	const {
		ctx,
		encoder,
		pass,
		targetTexture,
		targetWidth,
		targetHeight,
		props,
	} = args;

	const frame = props.frame ?? 0;
	const compositionFrame = props.compositionFrame ?? frame;
	const fps = props.fps || 30;
	const op = props.virtualMedia?.operation as VignetteOp | undefined;
	if (op?.op !== "Vignette" || !op) return;

	const childMedia = props.virtualMedia.children?.[0];
	if (!childMedia) return;

	// End the provided pass
	pass.end();

	const width = targetWidth;
	const height = targetHeight;
	const nodeId = op.inputs ? Object.keys(op.inputs)[0] : "vignette_node";

	// 1. Resolve Bindable Parameters
	const resolveBindable = (
		configKey:
			| "strength"
			| "radius"
			| "softness"
			| "roundness"
			| "centerX"
			| "centerY",
		defaultValue: number,
		minVal: number,
		maxVal: number,
	) => {
		const handleIdKey = `${configKey}HandleId` as keyof VignetteOp;
		const handleId = op[handleIdKey] as string | null | undefined;
		const input = handleId ? op.inputs?.[handleId] : null;

		const hasSignal = !!(
			input?.connectionValid &&
			(input.outputItem?.type === "Signal" ||
				input.outputItem?.type === "Numeric")
		);

		const sd =
			hasSignal && input?.outputItem?.data
				? (input.outputItem.data as VignetteSignalData)
				: null;

		let val = defaultValue;
		let hasStaticSig = false;

		let gpuTextureView: GPUTextureView | undefined;

		if (!hasSignal) {
			if (input?.connectionValid && input.outputItem?.type === "Number") {
				val = Math.max(
					minVal,
					Math.min(maxVal, Number(input.outputItem.data ?? defaultValue)),
				);
			} else {
				const unwrapped = unwrapFrameParam(
					op[configKey],
					compositionFrame,
					fps,
				);
				if (unwrapped.gpuTextureView) {
					gpuTextureView = unwrapped.gpuTextureView;
					hasStaticSig = true;
				}
				const rawVal = unwrapped.value;
				const numVal = Number(rawVal ?? defaultValue);
				val = Math.max(
					minVal,
					Math.min(maxVal, isNaN(numVal) ? defaultValue : numVal),
				);
			}
		} else if (sd) {
			val = Math.max(minVal, Math.min(maxVal, Number(sd.offset ?? 0.0)));
			hasStaticSig = true;
		}

		return { val, hasStaticSig, sd, gpuTextureView };
	};

	const strengthRes = resolveBindable("strength", 50, 0, 100);
	const radiusRes = resolveBindable("radius", 1.0, 0.1, 2.0);
	const softnessRes = resolveBindable("softness", 0.5, 0.0, 1.0);
	const roundnessRes = resolveBindable("roundness", 0.5, 0.0, 1.0);
	const centerXRes = resolveBindable("centerX", 0.5, 0.0, 1.0);
	const centerYRes = resolveBindable("centerY", 0.5, 0.0, 1.0);

	// 2. Draw child media
	const { texture: tmpTex } = await drawChildToTexture(args, childMedia);

	// End any active pass from drawing child
	args.pass.end();

	// 3. Fetch layout and resources
	const {
		vignettePipeline: pipeline,
		uniformLayout: uLayout,
		singleTextureLayout: tLayout,
		signalTextureLayout: sigLayout,
	} = getVignetteResources(ctx.device, ctx.renderer.format);

	const sampler = ctx.renderer.samplerCache.getSampler(ctx.device);

	// Fetch dynamic signal texture views on-demand
	const { elapsedSeconds, durationSeconds } = signalTiming(props, frame, fps);

	const signalContext = {
		args,
		width,
		height,
		frame,
		fps,
		elapsedSeconds,
		durationSeconds,
	};
	const getSignalView = (
		res: {
			val: number;
			hasStaticSig: boolean;
			sd: VignetteSignalData | null;
			gpuTextureView?: GPUTextureView;
		},
		suffix: string,
	) => signalTextureView(signalContext, res, `${nodeId}_${suffix}`);

	const strengthView = getSignalView(strengthRes, "strength_sig");
	const radiusView = getSignalView(radiusRes, "radius_sig");
	const softnessView = getSignalView(softnessRes, "softness_sig");
	const roundnessView = getSignalView(roundnessRes, "roundness_sig");
	const centerXView = getSignalView(centerXRes, "centerX_sig");
	const centerYView = getSignalView(centerYRes, "centerY_sig");

	// 4. Fill uniform array
	vignetteUniformData[0] = strengthRes.val;
	vignetteUniformData[1] = radiusRes.val;
	vignetteUniformData[2] = softnessRes.val;
	vignetteUniformData[3] = roundnessRes.val;
	vignetteUniformData[4] = centerXRes.val;
	vignetteUniformData[5] = centerYRes.val;
	vignetteUniformData[6] = strengthRes.hasStaticSig ? 1.0 : 0.0;
	vignetteUniformData[7] = radiusRes.hasStaticSig ? 1.0 : 0.0;
	vignetteUniformData[8] = softnessRes.hasStaticSig ? 1.0 : 0.0;
	vignetteUniformData[9] = roundnessRes.hasStaticSig ? 1.0 : 0.0;
	vignetteUniformData[10] = centerXRes.hasStaticSig ? 1.0 : 0.0;
	vignetteUniformData[11] = centerYRes.hasStaticSig ? 1.0 : 0.0;

	const uniformBuffer = ctx.renderer.getTemporaryBuffer(vignetteUniformData);

	// 5. Output Texture
	const outTex = ctx.renderer.getTemporaryTexture(width, height, [
		tmpTex,
		targetTexture,
		...(props.excludeTextures || []),
	]);

	// 6. Run vignette pass
	drawFullscreenPass(encoder, outTex.createView(), pipeline, [
		ctx.device.createBindGroup({
			layout: uLayout,
			entries: [{ binding: 0, resource: { buffer: uniformBuffer } }],
		}),
		ctx.renderer.bindGroupCache.getBindGroup(
			ctx.device,
			tLayout,
			tmpTex,
			sampler,
		),
		ctx.device.createBindGroup({
			layout: sigLayout,
			entries: [
				{ binding: 0, resource: strengthView },
				{ binding: 1, resource: radiusView },
				{ binding: 2, resource: softnessView },
				{ binding: 3, resource: roundnessView },
				{ binding: 4, resource: centerXView },
				{ binding: 5, resource: centerYView },
				{ binding: 6, resource: sampler },
			],
		}),
	]);

	// 7. Output result back to targetView
	drawTextureToTarget(args, outTex, op.opacity ?? 1.0);
};