Code-first video & image, rendered natively on WebGPU.
refactor(compositor): effects declare their own reach
okanaslbfe5a368e72d
diff --git a/nodes/node-compositor/src/renderers/webgpu-renderer.ts b/nodes/node-compositor/src/renderers/webgpu-renderer.tsindex c5088348..7d69d6a0 100644--- a/nodes/node-compositor/src/renderers/webgpu-renderer.ts+++ b/nodes/node-compositor/src/renderers/webgpu-renderer.ts@@ -16,6 +16,7 @@ import {type VirtualMediaData,} from "@framefields/core";import {+ effectReach,type NodeRenderProps,resolveFrameNumber,solveHomography,@@ -548,8 +549,9 @@ function computeContainerScissor(}/**- * How far a layer's effects draw past its pixels, px, or null when an effect- * is not known to stay local (it then runs over the whole target).+ * How far a layer's effects draw past its pixels, px: the sum of what each+ * effect's renderer declares (`reach`), or null as soon as one doesn't+ * (it then runs over the whole target, as any effect may need to).*/function effectsReach(effects: unknown[]): number | null {let reach = 0;@@ -559,37 +561,13 @@ function effectsReach(effects: unknown[]): number | null {? (e as { toOperation: () => Record<string, unknown> }).toOperation(): e) as Record<string, unknown> | null;- if (op?.op !== "LayerStyle") return null;- reach += layerStyleReach(op);+ const r = op ? effectReach(op) : null;+ if (r === null) return null;+ reach += r;}return reach;}-type StyleEntry = {- enabled?: boolean;- distance?: number;- size?: number;- position?: string;-};--/** A LayerStyle's reach: shadow offset plus blur, glow and outer stroke (the shader caps their kernels at 24 px). */-function layerStyleReach(op: Record<string, unknown>): number {- const kernel = (size?: number) => Math.min(Math.ceil(size ?? 0), 24);- // Each style may be a signal holding its value.- const entry = (v: unknown) =>- (v && typeof v === "object" && "_value" in v- ? (v as { _value: unknown })._value- : v) as StyleEntry | undefined;- const ds = entry(op.dropShadow);- const og = entry(op.outerGlow);- const st = entry(op.stroke);- let r = 0;- if (ds?.enabled) r = Math.max(r, (ds.distance ?? 0) + kernel(ds.size));- if (og?.enabled) r = Math.max(r, kernel(og.size));- if (st?.enabled && st.position !== "inside") r = Math.max(r, kernel(st.size));- return Math.ceil(r) + 2;-}-function expandRect(r: Rect, by: number): Rect {return {x: r.x - by,diff --git a/nodes/node-layer-style/src/renderers/reach.test.ts b/nodes/node-layer-style/src/renderers/reach.test.tsnew file mode 100644index 00000000..3eb95560--- /dev/null+++ b/nodes/node-layer-style/src/renderers/reach.test.ts@@ -0,0 +1,60 @@+import { effectReach, registerWebGPURenderer } from "@framefields/node-sdk";+import { describe, expect, it } from "vitest";+import { layerStyleReach, WebGPURenderer } from "./webgpu-renderer.js";++const signal = <T>(value: T) => ({ _value: value, _subscribers: {} });++describe("LayerStyle reach", () => {+ it("covers the shadow's offset plus its blur, capped like the shader", () => {+ expect(+ layerStyleReach({+ op: "LayerStyle",+ dropShadow: { enabled: true, distance: 10, size: 20 },+ }),+ ).toBe(10 + 20 + 2);+ // The shader's kernels stop at 24 px.+ expect(+ layerStyleReach({+ op: "LayerStyle",+ dropShadow: { enabled: true, distance: 0, size: 80 },+ }),+ ).toBe(24 + 2);+ });++ it("takes the farthest of shadow, glow and an outside stroke; inside strokes don't reach", () => {+ expect(+ layerStyleReach({+ op: "LayerStyle",+ dropShadow: signal({ enabled: true, distance: 2, size: 4 }),+ outerGlow: signal({ enabled: true, size: 16 }),+ stroke: signal({ enabled: true, size: 30, position: "inside" }),+ }),+ ).toBe(16 + 2);+ });++ it("is unknown when a size is driven by a connected signal", () => {+ expect(+ layerStyleReach({+ op: "LayerStyle",+ dropShadow: { enabled: true, distance: 4, size: 4 },+ dropShadowDistanceHandleId: "h1",+ inputs: { h1: { connectionValid: true } },+ }),+ ).toBeNull();+ });+});++describe("effectReach", () => {+ it("asks the registered renderer, and is unknown without one", () => {+ registerWebGPURenderer("LayerStyle", WebGPURenderer);+ expect(+ effectReach({+ op: "LayerStyle",+ dropShadow: { enabled: true, distance: 6, size: 6 },+ }),+ ).toBe(14);+ expect(effectReach({ op: "SomeMirror" })).toBeNull();+ registerWebGPURenderer("NoReach", async () => {});+ expect(effectReach({ op: "NoReach" })).toBeNull();+ });+});diff --git a/nodes/node-layer-style/src/renderers/webgpu-renderer.ts b/nodes/node-layer-style/src/renderers/webgpu-renderer.tsindex 66deaa14..6fc59c0f 100644--- a/nodes/node-layer-style/src/renderers/webgpu-renderer.ts+++ b/nodes/node-layer-style/src/renderers/webgpu-renderer.ts@@ -1,5 +1,5 @@/// <reference types="webgpu" />-import type { WebGPUNodeRenderer } from "@framefields/node-sdk";+import type { EffectReach, WebGPUNodeRenderer } from "@framefields/node-sdk";import {createFullscreenPipeline,drawFullscreenPass,@@ -108,6 +108,9 @@ const STROKE_POS_MAP: Record<string, number> = {inside: 2,};+/** Blur, glow and stroke kernels in the shader stop at this radius, px. */+const MAX_KERNEL = 24;+const WGSL_LAYER_STYLE_SHADER = `struct LayerStyleUniforms {// Drop Shadow (vec4 x 3)@@ -321,7 +324,7 @@ fn compositeLayer(dst: vec4<f32>, src: vec4<f32>, blendMode: u32) -> vec4<f32> {var shadowWeightSum = 0.0;let sigma = max(0.5, size * 0.4);- let sampleRadius = min(i32(ceil(size)), 24);+ let sampleRadius = min(i32(ceil(size)), ${MAX_KERNEL});if (size <= 0.5) {shadowAccum = sampleAlpha(shadowCenterUv);@@ -361,7 +364,7 @@ fn compositeLayer(dst: vec4<f32>, src: vec4<f32>, blendMode: u32) -> vec4<f32> {var glowAccum = 0.0;var glowWeightSum = 0.0;let sigma = max(0.5, size * 0.4);- let sampleRadius = min(i32(ceil(size)), 24);+ let sampleRadius = min(i32(ceil(size)), ${MAX_KERNEL});if (size <= 0.5) {glowAccum = origA;@@ -399,7 +402,7 @@ fn compositeLayer(dst: vec4<f32>, src: vec4<f32>, blendMode: u32) -> vec4<f32> {let blendMode = u32(u.strokeParams2.x);var maxAlpha = 0.0;- let sampleRadius = min(i32(ceil(size)), 24);+ let sampleRadius = min(i32(ceil(size)), ${MAX_KERNEL});let sizeSq = size * size;for (var y = -sampleRadius; y <= sampleRadius; y++) {@@ -557,7 +560,7 @@ fn compositeLayer(dst: vec4<f32>, src: vec4<f32>, blendMode: u32) -> vec4<f32> {let blendMode = u32(u.strokeParams2.x);var minAlpha = 1.0;- let sampleRadius = min(i32(ceil(size)), 24);+ let sampleRadius = min(i32(ceil(size)), ${MAX_KERNEL});let sizeSq = size * size;for (var y = -sampleRadius; y <= sampleRadius; y++) {@@ -592,7 +595,7 @@ fn compositeLayer(dst: vec4<f32>, src: vec4<f32>, blendMode: u32) -> vec4<f32> {var maxAlpha = 0.0;var minAlpha = 1.0;let halfSize = max(0.5, size * 0.5);- let sampleRadius = min(i32(ceil(halfSize)), 24);+ let sampleRadius = min(i32(ceil(halfSize)), ${MAX_KERNEL});let halfSizeSq = halfSize * halfSize;for (var y = -sampleRadius; y <= sampleRadius; y++) {@@ -1096,3 +1099,48 @@ export const WebGPURenderer: WebGPUNodeRenderer = async (args) => {// Blit to targetViewdrawTextureToTarget(args, outTex, op.opacity ?? 1.0);};++/**+ * How far a LayerStyle draws past its layer: the drop shadow's offset plus+ * its blur, the outer glow, an outside or centred stroke. Unknown (null)+ * when one of those sizes is driven by a connected signal. Keep in step with+ * the shader above: anything it draws farther out must count here, or a+ * compositor running the style over the layer's region will clip it.+ */+export const layerStyleReach: EffectReach = (operation) => {+ const op = operation as Record<string, unknown> & {+ inputs?: Record<string, { connectionValid?: boolean } | undefined>;+ };+ const driven = (handle: unknown) =>+ typeof handle === "string" && op.inputs?.[handle]?.connectionValid === true;+ if (+ driven(op.dropShadowDistanceHandleId) ||+ driven(op.dropShadowSizeHandleId) ||+ driven(op.outerGlowSizeHandleId) ||+ driven(op.strokeSizeHandleId)+ )+ return null;+ // A style may be a signal holding its value.+ const style = (v: unknown) =>+ (v && typeof v === "object" && "_value" in v+ ? (v as { _value: unknown })._value+ : v) as+ | {+ enabled?: boolean;+ distance?: number;+ size?: number;+ position?: string;+ }+ | undefined;+ const kernel = (size?: number) => Math.min(Math.ceil(size ?? 0), MAX_KERNEL);+ const ds = style(op.dropShadow);+ const og = style(op.outerGlow);+ const st = style(op.stroke);+ let r = 0;+ if (ds?.enabled) r = Math.max(r, (ds.distance ?? 0) + kernel(ds.size));+ if (og?.enabled) r = Math.max(r, kernel(og.size));+ if (st?.enabled && st.position !== "inside") r = Math.max(r, kernel(st.size));+ return Math.ceil(r) + 2;+};++WebGPURenderer.reach = layerStyleReach;diff --git a/packages/framefields/src/layer-fast-paths.render.test.ts b/packages/framefields/src/layer-fast-paths.render.test.tsindex a9ddcbca..08845ffb 100644--- a/packages/framefields/src/layer-fast-paths.render.test.ts+++ b/packages/framefields/src/layer-fast-paths.render.test.ts@@ -7,7 +7,7 @@import { Canvas, loadImage } from "skia-canvas";import { describe, expect, it } from "vitest";import { LayerStyle } from "./effects/index.js";-import { Composition, Layer } from "./index.js";+import { Composition, Flip, Layer } from "./index.js";async function pixels(png: Buffer) {const img = await loadImage(png);@@ -101,4 +101,30 @@ describe("container fast paths", () => {expect(below[0]).toBeGreaterThan(120);expect(below[1]).toBeLessThan(40);});++ it("an effect that declares no reach still runs over the whole target", async () => {+ // Flip (a mirror) has no `reach`: on a container it mirrors across the+ // whole canvas, so a box on the left lands on the right.+ const box = Layer.box({+ position: "absolute",+ x: 40,+ y: 150,+ width: 100,+ height: 100,+ background: "#ffffff",+ children: [+ Layer.box({+ position: "absolute",+ x: 10,+ y: 10,+ width: 20,+ height: 20,+ background: "#ff0000",+ }),+ ],+ }).withEffect(new Flip({ horizontal: true }));+ const at = await pixels(await scene(box).renderFrame({ frame: 0 }));+ expect(at(90, 200)).toEqual([0, 0, 0]); // where it was laid out+ expect(at(310, 200)).toEqual([255, 255, 255]); // mirrored across the canvas+ });});diff --git a/packages/node-sdk/renderer/index.ts b/packages/node-sdk/renderer/index.tsindex c2ed4abd..13375218 100644--- a/packages/node-sdk/renderer/index.ts+++ b/packages/node-sdk/renderer/index.ts@@ -48,9 +48,23 @@ export interface NodeRenderProps<) => unknown;}+/**+ * How far an effect draws past the pixels it is given, px, for one of its+ * operations; `null` when it isn't local (it reads or writes pixels anywhere,+ * like a mirror or a portal) or can't tell (a size driven by a signal).+ */+export type EffectReach = (operation: Record<string, unknown>) => number | null;++/**+ * A node's GPU renderer. An effect renderer may carry `reach`: a compositor+ * then runs it over the layer's own region plus that margin instead of the+ * whole target. Without it the effect always gets the whole target.+ */export type WebGPUNodeRenderer<TOperation extends MediaOperation = MediaOperation,-> = (args: {+> = WebGPUNodeRenderFn<TOperation> & { reach?: EffectReach };++type WebGPUNodeRenderFn<TOperation extends MediaOperation> = (args: {ctx: RenderContextValue;encoder: GPUCommandEncoder;pass: GPURenderPassEncoder;@@ -336,6 +350,16 @@ export function registerWebGPURenderer(webgpuRegistry.register(op, renderer);}+/**+ * How far an operation's effect draws past its pixels, from its registered+ * renderer; `null` when unknown: no renderer, no `reach`, or no answer.+ */+export function effectReach(operation: Record<string, unknown>): number | null {+ const op = operation.op;+ if (typeof op !== "string") return null;+ return webgpuRegistry.get(op)?.reach?.(operation) ?? null;+}+export type { WebGPUNodeRendererArgs } from "./gpu.js";export interface TextureSourceOp {