Skip to content

Code-first video & image, rendered natively on WebGPU.

18.6 KB
Raw Blame
/// <reference types="webgpu" />
import type { SignalData, VirtualMediaData } from "@framefields/core";
import {
	resolveSignalField,
	type WebGPUNodeRenderer,
} from "@framefields/node-sdk";
import {
	createFullscreenPipeline,
	drawChildToTexture,
	drawFullscreenPass,
	drawTextureToTarget,
	FULLSCREEN_QUAD_WGSL,
	fragmentBindGroupLayout,
	perDevice,
	texturesLayout,
} from "@framefields/node-sdk/gpu";
import { signalRegistry } from "@framefields/webgpu-renderers";

interface CameraParallax3DOp {
	op: "CameraParallax3D";
	motionPreset?:
		| "Custom"
		| "DollyZoom"
		| "Orbit"
		| "FlyThrough"
		| "HandheldShake"
		| "RackFocus";
	panX?: number;
	panY?: number;
	dollyZ?: number;
	fov?: number;
	parallaxAmount?: number;
	dofAperture?: number;
	focusPlane?: number;
	edgeDilation?: number;
	depthInvert?: boolean;
	opacity?: number;

	panXHandleId?: string | null;
	panYHandleId?: string | null;
	dollyZHandleId?: string | null;
	fovHandleId?: string | null;
	parallaxAmountHandleId?: string | null;
	dofApertureHandleId?: string | null;
	focusPlaneHandleId?: string | null;
	edgeDilationHandleId?: string | null;

	depthMedia?: VirtualMediaData | null;
	inputs?: Record<
		string,
		{
			connectionValid: boolean;
			outputItem: {
				type: string;
				data: unknown;
			} | null;
		}
	>;
}

type CameraParallaxSignalData = SignalData & { nodeId?: string };

const CAMERA_PARALLAX_SHADER = `
struct CameraParallaxUniforms {
    // vec4 0
    motionPreset          : f32, // 0=Custom, 1=DollyZoom, 2=Orbit, 3=FlyThrough, 4=HandheldShake, 5=RackFocus
    panX                  : f32,
    panY                  : f32,
    dollyZ                : f32,

    // vec4 1
    fov                   : f32,
    parallaxAmount        : f32,
    dofAperture           : f32,
    focusPlane            : f32,

    // vec4 2
    edgeDilation          : f32,
    depthInvert           : f32,
    hasDepthMap           : f32,
    timeProgress          : f32, // normalized 0.0 - 1.0 (or continuous)

    // vec4 3
    aspect                : f32,
    _pad1                 : f32,
    _pad2                 : f32,
    _pad3                 : f32,

    // vec4 4: signal flags
    hasSigPanX            : f32,
    hasSigPanY            : f32,
    hasSigDollyZ          : f32,
    hasSigFocusPlane      : f32,

    // vec4 5: signal flags
    hasSigParallax        : f32,
    hasSigAperture        : f32,
    hasSigFOV             : f32,
    hasSigEdgeDilation    : f32,
};

@group(0) @binding(0) var<uniform> u : CameraParallaxUniforms;

@group(1) @binding(0) var srcTex     : texture_2d<f32>;
@group(1) @binding(1) var srcSamp    : sampler;

@group(2) @binding(0) var depthTex   : texture_2d<f32>;
@group(2) @binding(1) var depthSamp  : sampler;

@group(3) @binding(0) var panXSigTex         : texture_2d<f32>;
@group(3) @binding(1) var panYSigTex         : texture_2d<f32>;
@group(3) @binding(2) var dollyZSigTex       : texture_2d<f32>;
@group(3) @binding(3) var focusPlaneSigTex   : texture_2d<f32>;
@group(3) @binding(4) var parallaxSigTex     : texture_2d<f32>;
@group(3) @binding(5) var apertureSigTex     : texture_2d<f32>;
@group(3) @binding(6) var fovSigTex          : texture_2d<f32>;
@group(3) @binding(7) var edgeDilationSigTex : texture_2d<f32>;
@group(3) @binding(8) var sigSamp            : sampler;
${FULLSCREEN_QUAD_WGSL}

fn sampleDepth(uv: vec2<f32>) -> f32 {
    let clampedUV = clamp(uv, vec2<f32>(0.0), vec2<f32>(1.0));
    var raw: f32 = 0.0;
    if (u.hasDepthMap > 0.5) {
        let dSample = textureSampleLevel(depthTex, depthSamp, clampedUV, 0.0);
        raw = dot(dSample.rgb, vec3<f32>(0.299, 0.587, 0.114));
    } else {
        let c = textureSampleLevel(srcTex, srcSamp, clampedUV, 0.0);
        raw = dot(c.rgb, vec3<f32>(0.299, 0.587, 0.114));
    }
    if (u.depthInvert > 0.5) {
        raw = 1.0 - raw;
    }
    return clamp(raw, 0.0, 1.0);
}

fn sampleAlbedo(uv: vec2<f32>) -> vec4<f32> {
    if (uv.x >= 0.0 && uv.x <= 1.0 && uv.y >= 0.0 && uv.y <= 1.0) {
        return textureSampleLevel(srcTex, srcSamp, uv, 0.0);
    }
    // Mirror with subtle fade for border inpainting
    let mirroredUV = abs(fract(uv * 0.5 + 0.5) * 2.0 - 1.0);
    let clampedUV = clamp(mirroredUV, vec2<f32>(0.001), vec2<f32>(0.999));
    return textureSampleLevel(srcTex, srcSamp, clampedUV, 0.0);
}

@fragment fn fs(in: VSOut) -> @location(0) vec4<f32> {
    let dims = vec2<f32>(textureDimensions(srcTex));
    let texelSize = vec2<f32>(1.0 / dims.x, 1.0 / dims.y);
    let aspect = select(u.aspect, dims.x / max(1.0, dims.y), u.aspect <= 0.0);

    // Resolve signals
    var panX = u.panX;
    if (u.hasSigPanX > 0.5) {
        panX = textureSampleLevel(panXSigTex, sigSamp, in.uv, 0.0).r;
    }

    var panY = u.panY;
    if (u.hasSigPanY > 0.5) {
        panY = textureSampleLevel(panYSigTex, sigSamp, in.uv, 0.0).r;
    }

    var dollyZ = u.dollyZ;
    if (u.hasSigDollyZ > 0.5) {
        dollyZ = textureSampleLevel(dollyZSigTex, sigSamp, in.uv, 0.0).r;
    }

    var focusPlane = u.focusPlane;
    if (u.hasSigFocusPlane > 0.5) {
        focusPlane = clamp(textureSampleLevel(focusPlaneSigTex, sigSamp, in.uv, 0.0).r, 0.0, 1.0);
    }

    var parallaxAmount = u.parallaxAmount;
    if (u.hasSigParallax > 0.5) {
        parallaxAmount = max(0.0, textureSampleLevel(parallaxSigTex, sigSamp, in.uv, 0.0).r);
    }

    var dofAperture = u.dofAperture;
    if (u.hasSigAperture > 0.5) {
        dofAperture = clamp(textureSampleLevel(apertureSigTex, sigSamp, in.uv, 0.0).r, 0.0, 1.0);
    }

    var fov = u.fov;
    if (u.hasSigFOV > 0.5) {
        fov = clamp(textureSampleLevel(fovSigTex, sigSamp, in.uv, 0.0).r, 15.0, 120.0);
    }

    var edgeDilation = u.edgeDilation;
    if (u.hasSigEdgeDilation > 0.5) {
        edgeDilation = clamp(textureSampleLevel(edgeDilationSigTex, sigSamp, in.uv, 0.0).r, 0.0, 1.0);
    }

    // Motion Presets Computation
    let t = u.timeProgress;
    let twoPi = 6.2831853;
    let pi = 3.14159265;

    var motionPanX: f32 = 0.0;
    var motionPanY: f32 = 0.0;
    var motionDollyZ: f32 = 0.0;
    var motionFocus: f32 = 0.0;

    let preset = u.motionPreset;

    if (preset > 0.5 && preset < 1.5) {
        // DollyZoom (Vertigo effect)
        let s = sin(t * pi);
        motionDollyZ = (s * 2.0 - 1.0) * 0.45;
        motionPanX = sin(t * twoPi) * 0.06;
        motionPanY = cos(t * twoPi) * 0.03;
    } else if (preset > 1.5 && preset < 2.5) {
        // Orbit
        let angle = t * twoPi;
        motionPanX = cos(angle) * 0.32;
        motionPanY = sin(angle) * 0.18;
        motionDollyZ = sin(angle * 2.0) * 0.12;
    } else if (preset > 2.5 && preset < 3.5) {
        // FlyThrough
        let phase = sin(t * pi);
        motionDollyZ = phase * 0.65;
        motionPanX = sin(t * twoPi) * 0.12;
        motionPanY = cos(t * pi) * 0.08;
        motionFocus = (phase - 0.5) * 0.6;
    } else if (preset > 3.5 && preset < 4.5) {
        // HandheldShake
        motionPanX = (sin(t * twoPi * 1.6) * 0.06 + sin(t * twoPi * 3.7) * 0.03);
        motionPanY = (cos(t * twoPi * 1.3) * 0.05 + sin(t * twoPi * 4.2) * 0.025);
        motionDollyZ = sin(t * twoPi * 0.8) * 0.04;
    } else if (preset > 4.5) {
        // RackFocus
        let focusWave = sin(t * pi);
        motionFocus = (focusWave - 0.5) * 0.8;
        motionPanX = sin(t * twoPi) * 0.04;
        motionPanY = cos(t * twoPi) * 0.02;
        motionDollyZ = sin(t * pi) * 0.06;
    }

    // Cumulative camera parameters
    let effectivePanX = clamp(panX + motionPanX, -1.0, 1.0);
    let effectivePanY = clamp(panY + motionPanY, -1.0, 1.0);
    let effectiveDollyZ = clamp(dollyZ + motionDollyZ, -1.0, 1.0);
    let effectiveFocusPlane = clamp(focusPlane + motionFocus, 0.0, 1.0);

    // FOV & perspective distortion factor
    let fovRad = (fov * pi) / 180.0;
    let fovScale = tan(fovRad * 0.5) / tan(50.0 * pi / 360.0);

    // 3D Camera Raymarch Offset
    let camOffset = vec2<f32>(effectivePanX * aspect, -effectivePanY) * (parallaxAmount * 0.15 * fovScale);
    let zoomFactor = max(0.5, 1.0 + effectiveDollyZ * 0.4);

    let centeredUV = (in.uv - 0.5) / zoomFactor + 0.5;

    // Steep Relief Raymarching (16 linear steps + 4 binary refinement steps)
    let numSteps = 16.0;
    let stepSize = 1.0 / numSteps;
    var currentRayZ: f32 = 1.0;
    var currentUV = centeredUV;
    var lastUV = centeredUV;
    var lastRayZ: f32 = 1.0;
    var lastDepth: f32 = sampleDepth(centeredUV);
    var foundHit = false;

    for (var i = 0; i < 16; i = i + 1) {
        let sampleUV = centeredUV + camOffset * (currentRayZ - effectiveFocusPlane);
        let sceneDepth = sampleDepth(sampleUV);

        if (sceneDepth >= currentRayZ) {
            foundHit = true;
            lastUV = sampleUV;
            break;
        }

        lastRayZ = currentRayZ;
        lastDepth = sceneDepth;
        currentRayZ = currentRayZ - stepSize;
    }

    // Binary refinement search for sub-pixel surface accuracy
    var hitUV = lastUV;
    var hitDepth = lastDepth;
    var minZ = currentRayZ;
    var maxZ = lastRayZ;

    for (var b = 0; b < 4; b = b + 1) {
        let midZ = (minZ + maxZ) * 0.5;
        let testUV = centeredUV + camOffset * (midZ - effectiveFocusPlane);
        let testDepth = sampleDepth(testUV);
        if (testDepth >= midZ) {
            minZ = midZ;
            hitUV = testUV;
            hitDepth = testDepth;
        } else {
            maxZ = midZ;
        }
    }

    // Edge inpainting / dilation clamping
    var finalUV = hitUV;
    if (hitUV.x < 0.0 || hitUV.x > 1.0 || hitUV.y < 0.0 || hitUV.y > 1.0) {
        let clampedBorderUV = clamp(hitUV, vec2<f32>(0.0), vec2<f32>(1.0));
        finalUV = mix(hitUV, clampedBorderUV, edgeDilation);
    }

    // Circle of Confusion (Physical Bokeh Blur)
    let coc = abs(hitDepth - effectiveFocusPlane) * dofAperture * 0.04;

    if (coc < 0.0008) {
        return sampleAlbedo(finalUV);
    }

    // Multi-tap Golden Spiral Bokeh Disk Blur (16 taps)
    var accumColor = vec4<f32>(0.0);
    var totalWeight: f32 = 0.0;
    let goldenAngle = 2.39996323;

    for (var tap = 0; tap < 16; tap = tap + 1) {
        let fTap = f32(tap);
        let r = sqrt((fTap + 0.5) / 16.0);
        let theta = fTap * goldenAngle;
        let offset = vec2<f32>(cos(theta), sin(theta) * aspect) * (r * coc);

        let tapUV = finalUV + offset;
        let tapColor = sampleAlbedo(tapUV);
        let tapDepth = sampleDepth(tapUV);

        // Bokeh depth weighting: prevents foreground blur from bleeding into background
        let depthDelta = tapDepth - hitDepth;
        let bokehWeight = select(1.0, smoothstep(-0.08, 0.04, depthDelta), depthDelta < 0.0) / (1.0 + r * r * 0.5);

        accumColor = accumColor + tapColor * bokehWeight;
        totalWeight = totalWeight + bokehWeight;
    }

    let finalColor = accumColor / max(totalWeight, 0.0001);
    return vec4<f32>(clamp(finalColor.rgb, vec3<f32>(0.0), vec3<f32>(1.0)), finalColor.a);
}
`;

const getDeviceLayouts = perDevice((device) => ({
	uniformLayout: fragmentBindGroupLayout(device, ["uniform"]),
	srcTextureLayout: fragmentBindGroupLayout(device, ["texture", "sampler"]),
	depthTextureLayout: fragmentBindGroupLayout(device, ["texture", "sampler"]),
	signalTextureLayout: texturesLayout(device, 8),
	pipelineCache: new Map<string, GPURenderPipeline>(),
}));

function getCameraParallaxResources(
	device: GPUDevice,
	format: GPUTextureFormat,
) {
	const layouts = getDeviceLayouts(device);
	const cacheKey = `camera_parallax_${format}`;
	let pipeline = layouts.pipelineCache.get(cacheKey);

	if (!pipeline) {
		const module = device.createShaderModule({
			label: `camera_parallax_${format}.wgsl`,
			code: CAMERA_PARALLAX_SHADER,
		});

		pipeline = createFullscreenPipeline(device, {
			label: `CameraParallaxPipeline_${format}`,
			module,
			layouts: [
				layouts.uniformLayout,
				layouts.srcTextureLayout,
				layouts.depthTextureLayout,
				layouts.signalTextureLayout,
			],
			format,
		});

		layouts.pipelineCache.set(cacheKey, pipeline);
	}

	return {
		pipeline,
		uniformLayout: layouts.uniformLayout,
		srcTextureLayout: layouts.srcTextureLayout,
		depthTextureLayout: layouts.depthTextureLayout,
		signalTextureLayout: layouts.signalTextureLayout,
	};
}

export const CameraParallax3DWebGPURenderer: WebGPUNodeRenderer = async (
	args,
) => {
	const {
		ctx,
		encoder,
		pass,
		targetTexture,
		targetWidth,
		targetHeight,
		props,
	} = args;

	const frame = props.frame ?? 0;
	const compositionFrame = props.compositionFrame ?? frame;
	const fps = props.fps || 60;
	const op = props.virtualMedia?.operation as CameraParallax3DOp | undefined;
	if (op?.op !== "CameraParallax3D" || !op) return;

	const sourceMedia = props.virtualMedia.children?.[0];
	if (!sourceMedia) return;

	// End the provided pass
	pass.end();

	const width = targetWidth;
	const height = targetHeight;

	// 1. Draw input source child to temporary texture
	const { texture: srcTex } = await drawChildToTexture(args, sourceMedia, {
		props: {
			...props,
			virtualMedia: sourceMedia,
			containerWidth: width,
			containerHeight: height,
		},
	});

	// End active pass from child drawing
	args.pass.end();

	// 2. Draw depth map if provided
	const depthMedia = op.depthMedia;
	let depthTex: GPUTexture;
	let hasDepthMap = false;

	if (depthMedia) {
		({ texture: depthTex } = await drawChildToTexture(args, depthMedia, {
			props: {
				...props,
				virtualMedia: depthMedia,
				containerWidth: width,
				containerHeight: height,
			},
			exclude: [srcTex],
		}));

		// End active pass from depth drawing
		args.pass.end();
		hasDepthMap = true;
	} else {
		depthTex = srcTex;
		hasDepthMap = false;
	}

	// 3. Fetch static resources and compiled pipeline
	const {
		pipeline,
		uniformLayout: uLayout,
		srcTextureLayout: srcLayout,
		depthTextureLayout: depthLayout,
		signalTextureLayout: sigLayout,
	} = getCameraParallaxResources(ctx.device, ctx.renderer.format);

	const sampler = ctx.renderer.samplerCache.getSampler(ctx.device);

	// 4. Resolve timing and signals
	const elapsedSeconds =
		props.elapsedMs !== undefined
			? props.elapsedMs / 1000
			: compositionFrame / fps;
	const durationSeconds = props.virtualMedia?.metadata?.durationMs
		? props.virtualMedia.metadata.durationMs / 1000
		: props.durationMs !== undefined
			? props.durationMs / 1000
			: 4.0;

	const timeProgress =
		durationSeconds > 0
			? (elapsedSeconds % durationSeconds) / durationSeconds
			: 0.0;

	const panXInfo = resolveSignalField<CameraParallaxSignalData>(
		op,
		"panX",
		0.0,
		-1.0,
		1.0,
	);
	const panYInfo = resolveSignalField<CameraParallaxSignalData>(
		op,
		"panY",
		0.0,
		-1.0,
		1.0,
	);
	const dollyZInfo = resolveSignalField<CameraParallaxSignalData>(
		op,
		"dollyZ",
		0.0,
		-1.0,
		1.0,
	);
	const fovInfo = resolveSignalField<CameraParallaxSignalData>(
		op,
		"fov",
		50.0,
		15.0,
		120.0,
	);
	const parallaxInfo = resolveSignalField<CameraParallaxSignalData>(
		op,
		"parallaxAmount",
		0.6,
		0.0,
		2.0,
	);
	const apertureInfo = resolveSignalField<CameraParallaxSignalData>(
		op,
		"dofAperture",
		0.3,
		0.0,
		1.0,
	);
	const focusPlaneInfo = resolveSignalField<CameraParallaxSignalData>(
		op,
		"focusPlane",
		0.4,
		0.0,
		1.0,
	);
	const edgeDilationInfo = resolveSignalField<CameraParallaxSignalData>(
		op,
		"edgeDilation",
		0.5,
		0.0,
		1.0,
	);

	const getSigTextureView = (
		info: { hasSignal: boolean; sd: CameraParallaxSignalData | null },
		fallbackId: string,
	) => {
		if (info.hasSignal && info.sd) {
			return signalRegistry.getOrCreate2DTextureView(
				ctx.device,
				encoder,
				info.sd.nodeId ?? fallbackId,
				elapsedSeconds,
				durationSeconds,
				info.sd,
				width,
				height,
				props.renderId,
				frame,
				fps,
			);
		}
		return signalRegistry.getDummy1x1TextureView(ctx.device);
	};

	const sigPanXView = getSigTextureView(panXInfo, "cam_panx_sig");
	const sigPanYView = getSigTextureView(panYInfo, "cam_pany_sig");
	const sigDollyZView = getSigTextureView(dollyZInfo, "cam_dollyz_sig");
	const sigFocusPlaneView = getSigTextureView(focusPlaneInfo, "cam_focus_sig");
	const sigParallaxView = getSigTextureView(parallaxInfo, "cam_plax_sig");
	const sigApertureView = getSigTextureView(apertureInfo, "cam_ap_sig");
	const sigFOVView = getSigTextureView(fovInfo, "cam_fov_sig");
	const sigEdgeDilationView = getSigTextureView(
		edgeDilationInfo,
		"cam_edge_sig",
	);

	// 5. Fill Uniforms (24 floats / 6 vec4s)
	const uniformData = new Float32Array(24);

	const presetEnum =
		op.motionPreset === "DollyZoom"
			? 1
			: op.motionPreset === "Orbit"
				? 2
				: op.motionPreset === "FlyThrough"
					? 3
					: op.motionPreset === "HandheldShake"
						? 4
						: op.motionPreset === "RackFocus"
							? 5
							: 0;

	// vec4 0
	uniformData[0] = presetEnum;
	uniformData[1] = panXInfo.value;
	uniformData[2] = panYInfo.value;
	uniformData[3] = dollyZInfo.value;

	// vec4 1
	uniformData[4] = fovInfo.value;
	uniformData[5] = parallaxInfo.value;
	uniformData[6] = apertureInfo.value;
	uniformData[7] = focusPlaneInfo.value;

	// vec4 2
	uniformData[8] = edgeDilationInfo.value;
	uniformData[9] = op.depthInvert ? 1.0 : 0.0;
	uniformData[10] = hasDepthMap ? 1.0 : 0.0;
	uniformData[11] = timeProgress;

	// vec4 3
	uniformData[12] = width / Math.max(1, height);
	uniformData[13] = 0.0;
	uniformData[14] = 0.0;
	uniformData[15] = 0.0;

	// vec4 4: signal flags
	uniformData[16] = panXInfo.hasSignal ? 1.0 : 0.0;
	uniformData[17] = panYInfo.hasSignal ? 1.0 : 0.0;
	uniformData[18] = dollyZInfo.hasSignal ? 1.0 : 0.0;
	uniformData[19] = focusPlaneInfo.hasSignal ? 1.0 : 0.0;

	// vec4 5: signal flags
	uniformData[20] = parallaxInfo.hasSignal ? 1.0 : 0.0;
	uniformData[21] = apertureInfo.hasSignal ? 1.0 : 0.0;
	uniformData[22] = fovInfo.hasSignal ? 1.0 : 0.0;
	uniformData[23] = edgeDilationInfo.hasSignal ? 1.0 : 0.0;

	const uniformBuffer = ctx.renderer.getTemporaryBuffer(uniformData);

	// 6. Render Shader Pass
	const outTex = ctx.renderer.getTemporaryTexture(width, height, [
		srcTex,
		depthTex,
		targetTexture,
		...(props.excludeTextures || []),
	]);

	drawFullscreenPass(encoder, outTex.createView(), pipeline, [
		ctx.device.createBindGroup({
			layout: uLayout,
			entries: [{ binding: 0, resource: { buffer: uniformBuffer } }],
		}),
		ctx.renderer.bindGroupCache.getBindGroup(
			ctx.device,
			srcLayout,
			srcTex,
			sampler,
		),
		ctx.renderer.bindGroupCache.getBindGroup(
			ctx.device,
			depthLayout,
			depthTex,
			sampler,
		),
		ctx.device.createBindGroup({
			layout: sigLayout,
			entries: [
				{ binding: 0, resource: sigPanXView },
				{ binding: 1, resource: sigPanYView },
				{ binding: 2, resource: sigDollyZView },
				{ binding: 3, resource: sigFocusPlaneView },
				{ binding: 4, resource: sigParallaxView },
				{ binding: 5, resource: sigApertureView },
				{ binding: 6, resource: sigFOVView },
				{ binding: 7, resource: sigEdgeDilationView },
				{ binding: 8, resource: sampler },
			],
		}),
	]);

	// 7. Final Pass to targetView
	drawTextureToTarget(args, outTex, op.opacity ?? 1);
};