Code-first video & image, rendered natively on WebGPU.
/// <reference types="webgpu" />
import { getActiveMediaMetadata } from "@framefields/core";
import type { WebGPUNodeRenderer } from "@framefields/node-sdk";
import {
beginTargetPass,
createFullscreenPipeline,
drawChildToTexture,
drawFullscreenPass,
FULLSCREEN_QUAD_WGSL,
fragmentBindGroupLayout,
} from "@framefields/node-sdk/gpu";
interface ResizerScalerOp {
op: "ResizerScaler";
aspectRatioPreset?: "9:16" | "16:9" | "1:1" | "4:5" | "21:9" | "custom";
resolutionPreset?: "4k" | "1080p" | "720p" | "480p" | "custom";
targetWidth?: number;
targetHeight?: number;
fitMode?: "cover" | "contain" | "stretch" | "manual";
zoom?: number;
offsetX?: number;
offsetY?: number;
backgroundMode?: "solid" | "blurred" | "gradient" | "transparent";
backgroundColor?: string;
backgroundColor2?: string;
blurRadius?: number;
backgroundBrightness?: number;
anchorX?: "left" | "center" | "right";
anchorY?: "top" | "center" | "bottom";
inputs?: Record<string, unknown>;
}
export const RESIZER_SCALER_SHADER = `
struct ScalerUniforms {
targetSize : vec2<f32>,
srcSize : vec2<f32>,
fitMode : f32, // 0 = Cover, 1 = Contain, 2 = Stretch, 3 = Manual
bgMode : f32, // 0 = Solid, 1 = Blurred, 2 = Gradient, 3 = Transparent
bgColor : vec4<f32>,
bgColor2 : vec4<f32>, // Used for gradient mode
blurRadius : f32,
brightness : f32,
zoom : f32,
offset : vec2<f32>,
anchor : vec2<f32>, // x: 0 (left) - 1 (right), y: 0 (top) - 1 (bottom)
};
@group(0) @binding(0) var<uniform> u : ScalerUniforms;
@group(0) @binding(1) var s : sampler;
@group(0) @binding(2) var t_src : texture_2d<f32>;
${FULLSCREEN_QUAD_WGSL}
@fragment fn fs(in : VSOut) -> @location(0) vec4<f32> {
let uv = in.uv;
// Calculate aspect ratios
let target_ar = u.targetSize.x / u.targetSize.y;
let src_ar = u.srcSize.x / u.srcSize.y;
// Sizing logic
var fg_uv = uv;
var is_outside_fg = false;
if (u.fitMode == 1.0) { // Contain
if (src_ar > target_ar) {
// Letterbox (pads top & bottom)
let scale_y = target_ar / src_ar;
let offset_y = (1.0 - scale_y) * u.anchor.y;
fg_uv.y = (uv.y - offset_y) / scale_y;
if (fg_uv.y < 0.0 || fg_uv.y > 1.0) { is_outside_fg = true; }
} else {
// Pillarbox (pads left & right)
let scale_x = src_ar / target_ar;
let offset_x = (1.0 - scale_x) * u.anchor.x;
fg_uv.x = (uv.x - offset_x) / scale_x;
if (fg_uv.x < 0.0 || fg_uv.x > 1.0) { is_outside_fg = true; }
}
} else if (u.fitMode == 0.0) { // Cover
if (src_ar > target_ar) {
let scale_x = target_ar / src_ar;
let offset_x = (1.0 - scale_x) * u.anchor.x;
fg_uv.x = uv.x * scale_x + offset_x;
} else {
let scale_y = src_ar / target_ar;
let offset_y = (1.0 - scale_y) * u.anchor.y;
fg_uv.y = uv.y * scale_y + offset_y;
}
} else if (u.fitMode == 3.0) { // Manual Scale & Translation
let norm_zoom = u.zoom / 100.0;
fg_uv = (uv - vec2<f32>(0.5)) / norm_zoom + vec2<f32>(0.5) - (u.offset / u.targetSize);
if (fg_uv.x < 0.0 || fg_uv.x > 1.0 || fg_uv.y < 0.0 || fg_uv.y > 1.0) {
is_outside_fg = true;
}
} // (fitMode == 2.0 (Stretch) uses default fg_uv without bounds check)
// Output compositing
if (!is_outside_fg) {
return textureSampleLevel(t_src, s, fg_uv, 0.0);
} else {
// Render background based on mode
if (u.bgMode == 0.0) { // Solid
return u.bgColor;
} else if (u.bgMode == 1.0) { // Blurred Background Copy
// Sample cover-fitted coordinates for background
var bg_uv = uv;
if (src_ar > target_ar) {
let scale_x = target_ar / src_ar;
let offset_x = (1.0 - scale_x) * 0.5; // Always center background
bg_uv.x = uv.x * scale_x + offset_x;
} else {
let scale_y = src_ar / target_ar;
let offset_y = (1.0 - scale_y) * 0.5;
bg_uv.y = uv.y * scale_y + offset_y;
}
// Apply gaussian blur approximation in shader
var color = vec4<f32>(0.0);
let blur_steps = 5.0;
let blur_offset = u.blurRadius / (u.targetSize * blur_steps);
for (var x = -2.0; x <= 2.0; x += 1.0) {
for (var y = -2.0; y <= 2.0; y += 1.0) {
color += textureSampleLevel(t_src, s, bg_uv + vec2<f32>(x, y) * blur_offset, 0.0);
}
}
color = color / 25.0;
// Apply background brightness darken effect
return vec4<f32>(color.rgb * u.brightness, color.a);
} else if (u.bgMode == 2.0) { // Gradient
return mix(u.bgColor, u.bgColor2, uv.y);
} else { // Transparent
return vec4<f32>(0.0, 0.0, 0.0, 0.0);
}
}
}
`;
interface DeviceScalerResources {
scalerUniformLayout: GPUBindGroupLayout;
textureSamplerLayout: GPUBindGroupLayout;
pipelineCache: Map<string, GPURenderPipeline>;
}
const deviceResourceCache = new WeakMap<GPUDevice, DeviceScalerResources>();
function getDeviceLayouts(device: GPUDevice): DeviceScalerResources {
let res = deviceResourceCache.get(device);
if (res) return res;
const scalerUniformLayout = fragmentBindGroupLayout(device, [
"uniform",
"sampler",
"texture",
]);
res = {
scalerUniformLayout,
textureSamplerLayout: scalerUniformLayout, // Shared layout for resources
pipelineCache: new Map(),
};
deviceResourceCache.set(device, res);
return res;
}
function getScalerResources(device: GPUDevice, format: GPUTextureFormat) {
const layouts = getDeviceLayouts(device);
const cacheKey = `scaler_${format}`;
let pipeline = layouts.pipelineCache.get(cacheKey);
if (!pipeline) {
const scalerModule = device.createShaderModule({
label: `scaler_${format}.wgsl`,
code: RESIZER_SCALER_SHADER,
});
pipeline = createFullscreenPipeline(device, {
label: `ResizerScalerPipeline_${format}`,
module: scalerModule,
layouts: [layouts.scalerUniformLayout],
format,
});
layouts.pipelineCache.set(cacheKey, pipeline);
}
return {
scalerPipeline: pipeline,
scalerUniformLayout: layouts.scalerUniformLayout,
};
}
function parseHexToRGBA(hex: string): [number, number, number, number] {
const cleanHex = hex.replace("#", "");
const r = parseInt(cleanHex.substring(0, 2), 16) / 255;
const g = parseInt(cleanHex.substring(2, 4), 16) / 255;
const b = parseInt(cleanHex.substring(4, 6), 16) / 255;
const a =
cleanHex.length === 8 ? parseInt(cleanHex.substring(6, 8), 16) / 255 : 1.0;
return [r, g, b, a];
}
// 24 floats total:
// Float 0,1: targetSize (vec2)
// Float 2,3: srcSize (vec2)
// Float 4: fitMode (f32)
// Float 5: bgMode (f32)
// Float 6,7: padding
// Float 8-11: bgColor (vec4)
// Float 12-15: bgColor2 (vec4)
// Float 16: blurRadius (f32)
// Float 17: brightness (f32)
// Float 18: zoom (f32)
// Float 19: padding
// Float 20,21: offset (vec2)
// Float 22,23: anchor (vec2)
const scalerData = new Float32Array(24);
export const ResizerScalerWebGPURenderer: WebGPUNodeRenderer = async (args) => {
const { ctx, encoder, pass, targetView, targetWidth, targetHeight, props } =
args;
const op = props.virtualMedia?.operation as ResizerScalerOp | undefined;
if (op?.op !== "ResizerScaler" || !op) return;
const childMedia = props.virtualMedia.children?.[0];
if (!childMedia) return;
// End the provided pass
pass.end();
const childMeta = getActiveMediaMetadata(childMedia);
const srcWidth = childMeta?.width || targetWidth || 1920;
const srcHeight = childMeta?.height || targetHeight || 1080;
// Get a temporary texture representing the native child size to draw it into
const { view: tmpView } = await drawChildToTexture(args, childMedia, {
width: srcWidth,
height: srcHeight,
props: {
...props,
virtualMedia: childMedia,
containerWidth: srcWidth,
containerHeight: srcHeight,
},
});
// Get compiled shader pipeline
const { scalerPipeline: pipeline, scalerUniformLayout: uLayout } =
getScalerResources(ctx.device, ctx.renderer.format);
// Pack Uniform data structure precisely
// Float 0,1: targetSize (vec2)
scalerData[0] = targetWidth;
scalerData[1] = targetHeight;
// Float 2,3: srcSize (vec2)
scalerData[2] = srcWidth;
scalerData[3] = srcHeight;
// Float 4: fitMode (0 = cover, 1 = contain, 2 = stretch, 3 = manual)
const fitModeMap = { cover: 0.0, contain: 1.0, stretch: 2.0, manual: 3.0 };
scalerData[4] = fitModeMap[op.fitMode ?? "contain"] ?? 1.0;
// Float 5: bgMode (0 = solid, 1 = blurred, 2 = gradient, 3 = transparent)
const bgModeMap = {
solid: 0.0,
blurred: 1.0,
gradient: 2.0,
transparent: 3.0,
};
scalerData[5] = bgModeMap[op.backgroundMode ?? "solid"] ?? 0.0;
// Float 6,7: padding
scalerData[6] = 0.0;
scalerData[7] = 0.0;
// Float 8-11: bgColor (vec4)
const bgColor = parseHexToRGBA(op.backgroundColor ?? "#000000FF");
scalerData[8] = bgColor[0];
scalerData[9] = bgColor[1];
scalerData[10] = bgColor[2];
scalerData[11] = bgColor[3];
// Float 12-15: bgColor2 (vec4)
const bgColor2 = parseHexToRGBA(op.backgroundColor2 ?? "#000000FF");
scalerData[12] = bgColor2[0];
scalerData[13] = bgColor2[1];
scalerData[14] = bgColor2[2];
scalerData[15] = bgColor2[3];
// Float 16: blurRadius (f32)
scalerData[16] = Number(op.blurRadius ?? 40.0);
// Float 17: brightness (f32)
scalerData[17] = Number(op.backgroundBrightness ?? 0.6);
// Float 18: zoom (f32)
scalerData[18] = Number(op.zoom ?? 100.0);
// Float 19: padding
scalerData[19] = 0.0;
// Float 20,21: offset (vec2)
scalerData[20] = Number(op.offsetX ?? 0.0);
scalerData[21] = Number(op.offsetY ?? 0.0);
// Float 22,23: anchor (vec2)
const anchorXMap = { left: 0.0, center: 0.5, right: 1.0 };
const anchorYMap = { top: 0.0, center: 0.5, bottom: 1.0 };
scalerData[22] = anchorXMap[op.anchorX ?? "center"] ?? 0.5;
scalerData[23] = anchorYMap[op.anchorY ?? "center"] ?? 0.5;
// Allocate a temporary buffer for uniforms
const uBuffer = ctx.renderer.getTemporaryBuffer(scalerData);
const sampler = ctx.renderer.samplerCache.getSampler(ctx.device);
// Render into targetView
drawFullscreenPass(encoder, targetView, pipeline, [
ctx.device.createBindGroup({
layout: uLayout,
entries: [
{ binding: 0, resource: { buffer: uBuffer } },
{ binding: 1, resource: sampler },
{ binding: 2, resource: tmpView },
],
}),
]);
// Create and return load render pass to continue downstream node executions
beginTargetPass(args);
};