Code-first video & image, rendered natively on WebGPU.
fix(text): render rich-text spans on compositor text layers
okanasl53ac1308e8bb
diff --git a/nodes/node-compositor/src/renderers/webgpu-renderer.ts b/nodes/node-compositor/src/renderers/webgpu-renderer.tsindex 11a71316..dbbc3526 100644--- a/nodes/node-compositor/src/renderers/webgpu-renderer.ts+++ b/nodes/node-compositor/src/renderers/webgpu-renderer.ts@@ -9,7 +9,7 @@ import {type LayoutNode,type Rect,} from "@framefields/compositions/program";-import type { MeshAudioDeformConfig } from "@framefields/core";+import type { MeshAudioDeformConfig, TextSpan } from "@framefields/core";import {DEFAULT_DURATION_MS,getActiveMediaMetadata,@@ -39,6 +39,7 @@ import {type DrawQuad3DOpts,type Environment3DOptions,type ExtrudedTextGeometryOptions,+ fitSpansToText,type GPUMeshBuffers,generateExtrudedTextGeometry,getPrimitiveGeometry,@@ -55,12 +56,14 @@ import {poseNodes,type RenderContextValue,Renderer3D,+ resolveTextSpans,ScreenSpaceRelightPipeline,type ShadowCaster3D,type ShapeFillConfig,type ShapeGeometryConfig,SlugFontCache,type SlugGlyphBatch,+ spansText,srtLoader,Vector3Math,} from "@framefields/webgpu-renderers";@@ -3064,7 +3067,29 @@ export const CompositorWebGPURenderer: WebGPUNodeRenderer = async (args) => {lop.dataType === "Caption" ||(content?.operation as any)?.srtText !== undefined;} else if (kind === "text") {- const fullText = lop.text ?? "";+ // Rich-text spans are the drawn text: read their animated fields for+ // this frame and take the plain text from them, so a reveal below+ // slices the same characters the spans hold.+ const rawSpans = (target?.spans ?? lop.spans) as TextSpan[] | undefined;+ const spanFps = fps ?? 24;+ const spans = rawSpans?.length+ ? resolveTextSpans(rawSpans, {+ frame: compositionFrame,+ fps: spanFps,+ time: compositionFrame / spanFps,+ duration: composeDuration / 1000,+ durationMs: composeDuration,+ progress:+ composeDuration > 0+ ? Math.min(+ 1,+ ((compositionFrame / spanFps) * 1000) / composeDuration,+ )+ : 0,+ deltaTime: 1 / spanFps,+ })+ : undefined;+ const fullText: string = spans ? spansText(spans) : (lop.text ?? "");let displayText = fullText;// Kinetic typography animators (Section 5)@@ -3202,6 +3227,10 @@ export const CompositorWebGPURenderer: WebGPUNodeRenderer = async (args) => {operation: {op: "text",text: displayText,+ spans:+ spans && displayText !== fullText+ ? fitSpansToText(spans, displayText)+ : spans,animators: resolvedAnimators,pathOptions: target?.pathOptions ?? lop.pathOptions,typewriter: target?.typewriter ?? lop.typewriter,diff --git a/packages/framefields/src/text-spans-render.test.ts b/packages/framefields/src/text-spans-render.test.tsnew file mode 100644index 00000000..c0fd28b0--- /dev/null+++ b/packages/framefields/src/text-spans-render.test.ts@@ -0,0 +1,108 @@+/**+ * Rich-text spans on a text layer render with their own styles. The node+ * compositor used to build the layer's text op without `spans`, so a span's+ * fill (or size, weight, mark) was dropped and only the plain text drew.+ */+import { Canvas, loadImage } from "skia-canvas";+import { describe, expect, it } from "vitest";+import type { TextSpan } from "./index.js";+import {+ Composition,+ FontManager,+ Layer,+ LayerAnimation,+ rich,+ signal,+} from "./index.js";++const W = 400;+const H = 120;++/** Count of strongly red pixels, and of dark (black-ish) ones. */+async function inkColours(png: Buffer) {+ const img = await loadImage(png);+ const canvas = new Canvas(img.width, img.height);+ const ctx = canvas.getContext("2d");+ ctx.drawImage(img, 0, 0);+ const data = ctx.getImageData(0, 0, img.width, img.height).data;+ let red = 0;+ let dark = 0;+ for (let i = 0; i < data.length; i += 4) {+ const r = data[i] ?? 0;+ const g = data[i + 1] ?? 0;+ const b = data[i + 2] ?? 0;+ if (r > 150 && g < 80 && b < 80) red++;+ if (r < 60 && g < 60 && b < 60) dark++;+ }+ return { red, dark };+}++async function render(+ content: (string | TextSpan)[] | TextSpan[],+ extra: Record<string, unknown> = {},+ frame = 0,+) {+ await FontManager.register("assets/fonts/Inter.ttf");+ const comp = new Composition({+ width: W,+ height: H,+ fps: 30,+ durationFrames: 30,+ backgroundColor: "#ffffff",+ });+ comp.add(+ Layer.text(content, {+ id: "t",+ position: "absolute",+ x: 0,+ y: 0,+ width: W,+ fontFamily: "Inter",+ fontWeight: 700,+ fontSize: 64,+ fill: "#000000",+ ...extra,+ }),+ );+ return inkColours(await comp.renderFrame({ frame }));+}++describe("text layer spans", () => {+ it("draws a coloured span in its own colour", async () => {+ const { red, dark } = await render([+ { text: "HH " },+ { text: "HH", fill: "#dd0000" },+ ]);+ expect(red).toBeGreaterThan(200);+ expect(dark).toBeGreaterThan(200);+ });++ it("draws spans authored with the rich tag", async () => {+ const { red } = await render(rich`HH ${{ text: "HH", fill: "#dd0000" }}`);+ expect(red).toBeGreaterThan(200);+ });++ it("reveals across a span whose text is a signal", async () => {+ // The black head is a signal: the reveal must measure its current text,+ // or the revealed characters slide into the red tail.+ const word = signal("HH");+ const content = [{ text: word }, { text: " HH", fill: "#dd0000" }];+ const extra = { animation: LayerAnimation.typewriter(0, 20) };+ const mid = await render(content, extra, 10);+ expect(mid.dark).toBeGreaterThan(100);+ expect(mid.red).toBe(0);+ expect((await render(content, extra, 25)).red).toBeGreaterThan(200);+ });++ it("keeps spans in step with a text reveal", async () => {+ const content = [{ text: "HH " }, { text: "HH", fill: "#dd0000" }];+ const extra = { animation: LayerAnimation.typewriter(0, 20) };+ // Halfway only the black "HH" is revealed; the red span is not reached.+ const half = await render(content, extra, 10);+ expect(half.dark).toBeGreaterThan(100);+ expect(half.red).toBe(0);+ const done = await render(content, extra, 25);+ expect(done.red).toBeGreaterThan(200);+ expect(done.dark).toBeGreaterThan(200);+ });+});diff --git a/packages/webgpu-renderers/src/nodes/index.ts b/packages/webgpu-renderers/src/nodes/index.tsindex 563cbfb2..7ddbd15b 100644--- a/packages/webgpu-renderers/src/nodes/index.ts+++ b/packages/webgpu-renderers/src/nodes/index.ts@@ -8,5 +8,6 @@ export * from "./media-size.js";export * from "./paragraph.js";export * from "./signal.js";export * from "./svg.js";+export * from "./text-spans.js";export * from "./types.js";export * from "./video.js";diff --git a/packages/webgpu-renderers/src/nodes/paragraph.ts b/packages/webgpu-renderers/src/nodes/paragraph.tsindex 88130522..67664ea1 100644--- a/packages/webgpu-renderers/src/nodes/paragraph.ts+++ b/packages/webgpu-renderers/src/nodes/paragraph.ts@@ -34,6 +34,7 @@ import {sampleAudioSpectrumScale,type TextAnchorConfig,} from "../text-animations/index.js";+import { fitSpansToText, resolveTextSpans } from "./text-spans.js";import type { ParagraphNodeProps, TextAnimator } from "./types.js";function sampleSignalValue(@@ -966,6 +967,11 @@ export function drawParagraphNode();textToRender = twState.currentDisplayString;}+ // Spans carry the text that is drawn: keep them to what the typewriter shows.+ const spans =+ twState && props.spans?.length+ ? fitSpansToText(resolveTextSpans(props.spans), textToRender)+ : props.spans;// Path-following options resolution with animated margin/shift propertiesconst pathOptions = props.pathOptions@@ -998,7 +1004,7 @@ export function drawParagraphNode(letterSpacing,lineHeight,maxWidth,- props.spans,+ spans,);} else {measured = SlugGeometry.measure(@@ -1008,7 +1014,7 @@ export function drawParagraphNode(letterSpacing,lineHeight,undefined,- props.spans,+ spans,);maxWidth = measured.width;}@@ -1036,7 +1042,7 @@ export function drawParagraphNode(animApplyBy,alignWidth,pathOptions,- props.spans,+ spans,);// verticalAlign places the text block inside a box taller than it (CSS// line-box centring). Path text follows its path; captions place themselves.diff --git a/packages/webgpu-renderers/src/nodes/text-spans.test.ts b/packages/webgpu-renderers/src/nodes/text-spans.test.tsnew file mode 100644index 00000000..68c2ad50--- /dev/null+++ b/packages/webgpu-renderers/src/nodes/text-spans.test.ts@@ -0,0 +1,60 @@+import { signal } from "@framefields/core";+import { describe, expect, it } from "vitest";+import { fitSpansToText, resolveTextSpans, spansText } from "./text-spans.js";++const spans = [+ { text: "ab " },+ { text: "cd", fill: "#d00" },+ { text: " ef", fontSize: 10 },+];++describe("resolveTextSpans", () => {+ it("reads signal fields as plain values", () => {+ const out = resolveTextSpans([+ {+ text: signal("hi"),+ fill: signal("#d00"),+ fontSize: signal(12),+ opacity: signal(0.5),+ mark: { background: signal("#ff0"), paddingX: 2 },+ },+ ]);+ expect(out).toEqual([+ {+ text: "hi",+ fill: "#d00",+ fontSize: 12,+ opacity: 0.5,+ mark: { background: "#ff0", paddingX: 2 },+ },+ ]);+ });+});++describe("fitSpansToText", () => {+ it("keeps a revealed prefix in each span's style and drops the rest", () => {+ expect(fitSpansToText(spans, "ab c")).toEqual([+ { text: "ab " },+ { text: "c", fill: "#d00" },+ ]);+ expect(fitSpansToText(spans, "")).toEqual([]);+ });++ it("is the identity on the spans' own text", () => {+ expect(fitSpansToText(spans, spansText(spans))).toEqual(spans);+ });++ it("restyles a same-length rewrite in place", () => {+ expect(fitSpansToText(spans, "xy zw")).toEqual([+ { text: "xy " },+ { text: "zw", fill: "#d00" },+ ]);+ });++ it("gives text past the spans' end the last span's style", () => {+ expect(fitSpansToText(spans, "ab cd efgh").at(-1)).toEqual({+ text: " efgh",+ fontSize: 10,+ });+ });+});diff --git a/packages/webgpu-renderers/src/nodes/text-spans.ts b/packages/webgpu-renderers/src/nodes/text-spans.tsnew file mode 100644index 00000000..6d41d1eb--- /dev/null+++ b/packages/webgpu-renderers/src/nodes/text-spans.ts@@ -0,0 +1,70 @@+import type { FrameContext, TextSpan } from "@framefields/core";++type Gettable<T> = { get: (ctx?: FrameContext) => T };++function isGettable<T>(v: unknown): v is Gettable<T> {+ return (+ typeof v === "object" &&+ v !== null &&+ typeof (v as { get?: unknown }).get === "function"+ );+}++function resolve<T>(v: T | Gettable<T>, ctx?: FrameContext): T {+ return isGettable<T>(v) ? v.get(ctx) : v;+}++/**+ * The spans' animated fields (text, size, fill, opacity, mark background)+ * read for one frame, so the spans can be sliced, compared and cached as+ * plain values.+ */+export function resolveTextSpans(+ spans: readonly TextSpan[],+ ctx?: FrameContext,+): TextSpan[] {+ return spans.map((s) => {+ const out: TextSpan = { ...s, text: String(resolve(s.text, ctx) ?? "") };+ if (s.fontSize !== undefined)+ out.fontSize = Number(resolve(s.fontSize, ctx));+ if (s.fill !== undefined) out.fill = String(resolve(s.fill, ctx));+ if (s.opacity !== undefined) out.opacity = Number(resolve(s.opacity, ctx));+ if (s.mark)+ out.mark = {+ ...s.mark,+ background: String(resolve(s.mark.background, ctx)),+ };+ return out;+ });+}++/** The plain text the spans spell out, in order. */+export function spansText(spans: readonly TextSpan[]): string {+ let text = "";+ for (const s of spans) text += typeof s.text === "string" ? s.text : "";+ return text;+}++/**+ * Re-cut `text` across resolved spans by position: span i keeps the styling+ * of characters [start_i, end_i) of the spans' own text. A revealed prefix+ * (typewriter, word or line reveal) keeps each visible character in its+ * span's style and drops the spans not reached yet; a same-length rewrite+ * (scramble) restyles in place. Text past the spans' end takes the last+ * span's style.+ */+export function fitSpansToText(+ spans: readonly TextSpan[],+ text: string,+): TextSpan[] {+ const out: TextSpan[] = [];+ let offset = 0;+ for (const [i, s] of spans.entries()) {+ if (offset >= text.length) break;+ const len = typeof s.text === "string" ? s.text.length : 0;+ const end = i === spans.length - 1 ? text.length : offset + len;+ if (end > offset) out.push({ ...s, text: text.slice(offset, end) });+ offset = end;+ }+ return out;+}