Skip to content

Code-first video & image, rendered natively on WebGPU.

6.8 KB
Raw Blame
import type { WebGPUNodeRenderer } from "@framefields/node-sdk";
import {
	beginTargetPass,
	drawChildToTexture,
	fragmentBindGroupLayout,
} from "@framefields/node-sdk/gpu";
import { createUniformGrid } from "../shared/utils.js";

const meshWarpWgsl = `
struct MeshUniforms {
	gridSize    : vec2<f32>,
	layerSize   : vec2<f32>,
	matrix_col0 : vec4<f32>,
	matrix_col1 : vec4<f32>,
	points      : array<vec4<f32>, 144>,
};

@group(0) @binding(0) var<uniform> u : MeshUniforms;
@group(1) @binding(0) var tex        : texture_2d<f32>;
@group(1) @binding(1) var samp       : sampler;

struct VSOut {
	@builtin(position) pos : vec4<f32>,
	@location(0) uv        : vec2<f32>,
};

@vertex fn vs(@builtin(vertex_index) vi: u32) -> VSOut {
	let cols = u32(u.gridSize.x);
	let rows = u32(u.gridSize.y);
	
	let cellsCol = cols - 1u;
	let cellsRow = rows - 1u;
	let totalCells = cellsCol * cellsRow;
	
	let cellIndex = vi / 6u;
	let vertexInCell = vi % 6u;
	
	if (cellIndex >= totalCells) {
		return VSOut(vec4<f32>(0.0), vec4<f32>(0.0).xy);
	}
	
	let cellC = cellIndex % cellsCol;
	let cellR = cellIndex / cellsCol;
	
	var colOffset = 0u;
	var rowOffset = 0u;
	
	// Mapping vertex index in cell to quad corner offset
	// Triangle 1: TL (0,0), TR (1,0), BL (0,1)
	// Triangle 2: TR (1,0), BR (1,1), BL (0,1)
	if (vertexInCell == 1u) { // TR
		colOffset = 1u;
		rowOffset = 0u;
	} else if (vertexInCell == 2u) { // BL
		colOffset = 0u;
		rowOffset = 1u;
	} else if (vertexInCell == 3u) { // TR
		colOffset = 1u;
		rowOffset = 0u;
	} else if (vertexInCell == 4u) { // BR
		colOffset = 1u;
		rowOffset = 1u;
	} else if (vertexInCell == 5u) { // BL
		colOffset = 0u;
		rowOffset = 1u;
	}
	// Vertex 0 is TL (colOffset=0, rowOffset=0)
	
	let gridC = cellC + colOffset;
	let gridR = cellR + rowOffset;
	
	let pointIndex = gridR * cols + gridC;
	let warpedPoint = u.points[pointIndex].xy;
	
	let lx = (warpedPoint.x / 100.0) * u.layerSize.x;
	let ly = (warpedPoint.y / 100.0) * u.layerSize.y;
	
	let tx = u.matrix_col0.x * lx + u.matrix_col0.y * ly + u.matrix_col0.z;
	let ty = u.matrix_col1.x * lx + u.matrix_col1.y * ly + u.matrix_col1.z;
	
	let targetW = u.matrix_col0.w;
	let targetH = u.matrix_col1.w;
	
	let x_clip = (tx / targetW) * 2.0 - 1.0;
	let y_clip = 1.0 - (ty / targetH) * 2.0;
	
	let u_coord = f32(gridC) / f32(cellsCol);
	let v_coord = f32(gridR) / f32(cellsRow);
	
	return VSOut(vec4<f32>(x_clip, y_clip, 0.0, 1.0), vec2<f32>(u_coord, v_coord));
}

@fragment fn fs(in : VSOut) -> @location(0) vec4<f32> {
	let color = textureSampleLevel(tex, samp, in.uv, 0.0);
	return color;
}
`;

interface DeviceMeshWarpResources {
	warpPipeline: GPURenderPipeline;
	warpUniformLayout: GPUBindGroupLayout;
	warpTextureLayout: GPUBindGroupLayout;
}

const deviceResourceCache = new WeakMap<GPUDevice, DeviceMeshWarpResources>();

function getMeshWarpResources(device: GPUDevice, format: GPUTextureFormat) {
	let res = deviceResourceCache.get(device);
	if (!res) {
		const warpUniformLayout = device.createBindGroupLayout({
			entries: [
				{
					binding: 0,
					visibility: GPUShaderStage.VERTEX | GPUShaderStage.FRAGMENT,
					buffer: { type: "uniform" },
				},
			],
		});

		const warpTextureLayout = fragmentBindGroupLayout(device, [
			"texture",
			"sampler",
		]);

		const module = device.createShaderModule({
			label: "mesh_warp.wgsl",
			code: meshWarpWgsl,
		});

		const warpPipeline = device.createRenderPipeline({
			label: "MeshWarpPipeline",
			layout: device.createPipelineLayout({
				bindGroupLayouts: [warpUniformLayout, warpTextureLayout],
			}),
			vertex: {
				module,
				entryPoint: "vs",
			},
			fragment: {
				module,
				entryPoint: "fs",
				targets: [
					{
						format,
						blend: {
							color: {
								srcFactor: "src-alpha",
								dstFactor: "one-minus-src-alpha",
								operation: "add",
							},
							alpha: {
								srcFactor: "one",
								dstFactor: "one-minus-src-alpha",
								operation: "add",
							},
						},
					},
				],
			},
			primitive: {
				topology: "triangle-list",
			},
		});

		res = { warpPipeline, warpUniformLayout, warpTextureLayout };
		deviceResourceCache.set(device, res);
	}
	return res;
}

export const MeshWarpWebGPURenderer: WebGPUNodeRenderer = async (args) => {
	const { ctx, props, targetWidth, targetHeight } = args;
	const { virtualMedia } = props;
	const op = virtualMedia?.operation as any;
	if (!op || op.op !== "MeshWarp") return;

	const childMedia = virtualMedia.children?.[0];
	if (!childMedia) return;

	const sourceWidth =
		op.originalWidth ??
		childMedia.metadata?.width ??
		props.containerWidth ??
		targetWidth;
	const sourceHeight =
		op.originalHeight ??
		childMedia.metadata?.height ??
		props.containerHeight ??
		targetHeight;

	// End incoming pass to avoid conflicts
	args.pass.end();

	// 1. Render child into a temporary offscreen texture at its native size
	const { texture: childTex } = await drawChildToTexture(args, childMedia, {
		width: sourceWidth,
		height: sourceHeight,
		props: {
			...props,
			containerWidth: sourceWidth,
			containerHeight: sourceHeight,
		},
	});

	// 2. Setup warp rendering to targetView
	const cols = Number(op.cols) || 3;
	const rows = Number(op.rows) || 3;
	const points = op.points || [];

	const finalPoints =
		points.length === cols * rows ? points : createUniformGrid(cols, rows);

	// Setup uniforms: gridSize (vec2), layerSize (vec2), matrix_col0 (vec4), matrix_col1 (vec4), then 144 points
	const maxPoints = 144;
	const uniformData = new Float32Array(12 + maxPoints * 4);
	uniformData[0] = cols;
	uniformData[1] = rows;
	uniformData[2] = props.containerWidth ?? sourceWidth;
	uniformData[3] = props.containerHeight ?? sourceHeight;

	const matrix = ctx.renderer.getCurrentTransform();
	uniformData[4] = matrix.a;
	uniformData[5] = matrix.c;
	uniformData[6] = matrix.e;
	uniformData[7] = targetWidth;

	uniformData[8] = matrix.b;
	uniformData[9] = matrix.d;
	uniformData[10] = matrix.f;
	uniformData[11] = targetHeight;

	for (let i = 0; i < maxPoints; i++) {
		const pt = finalPoints[i] || { x: 0, y: 0 };
		uniformData[12 + i * 4] = pt.x;
		uniformData[12 + i * 4 + 1] = pt.y;
	}

	const {
		warpPipeline: pipeline,
		warpUniformLayout: uLayout,
		warpTextureLayout: tLayout,
	} = getMeshWarpResources(ctx.device, ctx.renderer.format);

	const sampler = ctx.renderer.samplerCache.getSampler(ctx.device);
	const uniformBuffer = ctx.renderer.getTemporaryBuffer(uniformData);

	const uniformBindGroup = ctx.device.createBindGroup({
		layout: uLayout!,
		entries: [{ binding: 0, resource: { buffer: uniformBuffer } }],
	});

	const textureBindGroup = ctx.renderer.bindGroupCache.getBindGroup(
		ctx.device,
		tLayout!,
		childTex,
		sampler,
	);

	const outPass = beginTargetPass(args);

	outPass.setPipeline(pipeline!);
	outPass.setBindGroup(0, uniformBindGroup);
	outPass.setBindGroup(1, textureBindGroup);

	const cellCount = (cols - 1) * (rows - 1);
	const vertexCount = cellCount * 6;
	outPass.draw(vertexCount);
};