Skip to content

Code-first video & image, rendered natively on WebGPU.

fix(text): render rich-text spans on compositor text layers

okanasl53ac1308e8bb
diff --git a/nodes/node-compositor/src/renderers/webgpu-renderer.ts b/nodes/node-compositor/src/renderers/webgpu-renderer.ts
index 11a71316..dbbc3526 100644
--- a/nodes/node-compositor/src/renderers/webgpu-renderer.ts
+++ b/nodes/node-compositor/src/renderers/webgpu-renderer.ts
@@ -9,7 +9,7 @@ import {
type LayoutNode,
type Rect,
} from "@framefields/compositions/program";
-import type { MeshAudioDeformConfig } from "@framefields/core";
+import type { MeshAudioDeformConfig, TextSpan } from "@framefields/core";
import {
DEFAULT_DURATION_MS,
getActiveMediaMetadata,
@@ -39,6 +39,7 @@ import {
type DrawQuad3DOpts,
type Environment3DOptions,
type ExtrudedTextGeometryOptions,
+ fitSpansToText,
type GPUMeshBuffers,
generateExtrudedTextGeometry,
getPrimitiveGeometry,
@@ -55,12 +56,14 @@ import {
poseNodes,
type RenderContextValue,
Renderer3D,
+ resolveTextSpans,
ScreenSpaceRelightPipeline,
type ShadowCaster3D,
type ShapeFillConfig,
type ShapeGeometryConfig,
SlugFontCache,
type SlugGlyphBatch,
+ spansText,
srtLoader,
Vector3Math,
} from "@framefields/webgpu-renderers";
@@ -3064,7 +3067,29 @@ export const CompositorWebGPURenderer: WebGPUNodeRenderer = async (args) => {
lop.dataType === "Caption" ||
(content?.operation as any)?.srtText !== undefined;
} else if (kind === "text") {
- const fullText = lop.text ?? "";
+ // Rich-text spans are the drawn text: read their animated fields for
+ // this frame and take the plain text from them, so a reveal below
+ // slices the same characters the spans hold.
+ const rawSpans = (target?.spans ?? lop.spans) as TextSpan[] | undefined;
+ const spanFps = fps ?? 24;
+ const spans = rawSpans?.length
+ ? resolveTextSpans(rawSpans, {
+ frame: compositionFrame,
+ fps: spanFps,
+ time: compositionFrame / spanFps,
+ duration: composeDuration / 1000,
+ durationMs: composeDuration,
+ progress:
+ composeDuration > 0
+ ? Math.min(
+ 1,
+ ((compositionFrame / spanFps) * 1000) / composeDuration,
+ )
+ : 0,
+ deltaTime: 1 / spanFps,
+ })
+ : undefined;
+ const fullText: string = spans ? spansText(spans) : (lop.text ?? "");
let displayText = fullText;
// Kinetic typography animators (Section 5)
@@ -3202,6 +3227,10 @@ export const CompositorWebGPURenderer: WebGPUNodeRenderer = async (args) => {
operation: {
op: "text",
text: displayText,
+ spans:
+ spans && displayText !== fullText
+ ? fitSpansToText(spans, displayText)
+ : spans,
animators: resolvedAnimators,
pathOptions: target?.pathOptions ?? lop.pathOptions,
typewriter: target?.typewriter ?? lop.typewriter,
diff --git a/packages/framefields/src/text-spans-render.test.ts b/packages/framefields/src/text-spans-render.test.ts
new file mode 100644
index 00000000..c0fd28b0
--- /dev/null
+++ b/packages/framefields/src/text-spans-render.test.ts
@@ -0,0 +1,108 @@
+/**
+ * Rich-text spans on a text layer render with their own styles. The node
+ * compositor used to build the layer's text op without `spans`, so a span's
+ * fill (or size, weight, mark) was dropped and only the plain text drew.
+ */
+import { Canvas, loadImage } from "skia-canvas";
+import { describe, expect, it } from "vitest";
+import type { TextSpan } from "./index.js";
+import {
+ Composition,
+ FontManager,
+ Layer,
+ LayerAnimation,
+ rich,
+ signal,
+} from "./index.js";
+
+const W = 400;
+const H = 120;
+
+/** Count of strongly red pixels, and of dark (black-ish) ones. */
+async function inkColours(png: Buffer) {
+ const img = await loadImage(png);
+ const canvas = new Canvas(img.width, img.height);
+ const ctx = canvas.getContext("2d");
+ ctx.drawImage(img, 0, 0);
+ const data = ctx.getImageData(0, 0, img.width, img.height).data;
+ let red = 0;
+ let dark = 0;
+ for (let i = 0; i < data.length; i += 4) {
+ const r = data[i] ?? 0;
+ const g = data[i + 1] ?? 0;
+ const b = data[i + 2] ?? 0;
+ if (r > 150 && g < 80 && b < 80) red++;
+ if (r < 60 && g < 60 && b < 60) dark++;
+ }
+ return { red, dark };
+}
+
+async function render(
+ content: (string | TextSpan)[] | TextSpan[],
+ extra: Record<string, unknown> = {},
+ frame = 0,
+) {
+ await FontManager.register("assets/fonts/Inter.ttf");
+ const comp = new Composition({
+ width: W,
+ height: H,
+ fps: 30,
+ durationFrames: 30,
+ backgroundColor: "#ffffff",
+ });
+ comp.add(
+ Layer.text(content, {
+ id: "t",
+ position: "absolute",
+ x: 0,
+ y: 0,
+ width: W,
+ fontFamily: "Inter",
+ fontWeight: 700,
+ fontSize: 64,
+ fill: "#000000",
+ ...extra,
+ }),
+ );
+ return inkColours(await comp.renderFrame({ frame }));
+}
+
+describe("text layer spans", () => {
+ it("draws a coloured span in its own colour", async () => {
+ const { red, dark } = await render([
+ { text: "HH " },
+ { text: "HH", fill: "#dd0000" },
+ ]);
+ expect(red).toBeGreaterThan(200);
+ expect(dark).toBeGreaterThan(200);
+ });
+
+ it("draws spans authored with the rich tag", async () => {
+ const { red } = await render(rich`HH ${{ text: "HH", fill: "#dd0000" }}`);
+ expect(red).toBeGreaterThan(200);
+ });
+
+ it("reveals across a span whose text is a signal", async () => {
+ // The black head is a signal: the reveal must measure its current text,
+ // or the revealed characters slide into the red tail.
+ const word = signal("HH");
+ const content = [{ text: word }, { text: " HH", fill: "#dd0000" }];
+ const extra = { animation: LayerAnimation.typewriter(0, 20) };
+ const mid = await render(content, extra, 10);
+ expect(mid.dark).toBeGreaterThan(100);
+ expect(mid.red).toBe(0);
+ expect((await render(content, extra, 25)).red).toBeGreaterThan(200);
+ });
+
+ it("keeps spans in step with a text reveal", async () => {
+ const content = [{ text: "HH " }, { text: "HH", fill: "#dd0000" }];
+ const extra = { animation: LayerAnimation.typewriter(0, 20) };
+ // Halfway only the black "HH" is revealed; the red span is not reached.
+ const half = await render(content, extra, 10);
+ expect(half.dark).toBeGreaterThan(100);
+ expect(half.red).toBe(0);
+ const done = await render(content, extra, 25);
+ expect(done.red).toBeGreaterThan(200);
+ expect(done.dark).toBeGreaterThan(200);
+ });
+});
diff --git a/packages/webgpu-renderers/src/nodes/index.ts b/packages/webgpu-renderers/src/nodes/index.ts
index 563cbfb2..7ddbd15b 100644
--- a/packages/webgpu-renderers/src/nodes/index.ts
+++ b/packages/webgpu-renderers/src/nodes/index.ts
@@ -8,5 +8,6 @@ export * from "./media-size.js";
export * from "./paragraph.js";
export * from "./signal.js";
export * from "./svg.js";
+export * from "./text-spans.js";
export * from "./types.js";
export * from "./video.js";
diff --git a/packages/webgpu-renderers/src/nodes/paragraph.ts b/packages/webgpu-renderers/src/nodes/paragraph.ts
index 88130522..67664ea1 100644
--- a/packages/webgpu-renderers/src/nodes/paragraph.ts
+++ b/packages/webgpu-renderers/src/nodes/paragraph.ts
@@ -34,6 +34,7 @@ import {
sampleAudioSpectrumScale,
type TextAnchorConfig,
} from "../text-animations/index.js";
+import { fitSpansToText, resolveTextSpans } from "./text-spans.js";
import type { ParagraphNodeProps, TextAnimator } from "./types.js";
function sampleSignalValue(
@@ -966,6 +967,11 @@ export function drawParagraphNode(
);
textToRender = twState.currentDisplayString;
}
+ // Spans carry the text that is drawn: keep them to what the typewriter shows.
+ const spans =
+ twState && props.spans?.length
+ ? fitSpansToText(resolveTextSpans(props.spans), textToRender)
+ : props.spans;
// Path-following options resolution with animated margin/shift properties
const pathOptions = props.pathOptions
@@ -998,7 +1004,7 @@ export function drawParagraphNode(
letterSpacing,
lineHeight,
maxWidth,
- props.spans,
+ spans,
);
} else {
measured = SlugGeometry.measure(
@@ -1008,7 +1014,7 @@ export function drawParagraphNode(
letterSpacing,
lineHeight,
undefined,
- props.spans,
+ spans,
);
maxWidth = measured.width;
}
@@ -1036,7 +1042,7 @@ export function drawParagraphNode(
animApplyBy,
alignWidth,
pathOptions,
- props.spans,
+ spans,
);
// verticalAlign places the text block inside a box taller than it (CSS
// line-box centring). Path text follows its path; captions place themselves.
diff --git a/packages/webgpu-renderers/src/nodes/text-spans.test.ts b/packages/webgpu-renderers/src/nodes/text-spans.test.ts
new file mode 100644
index 00000000..68c2ad50
--- /dev/null
+++ b/packages/webgpu-renderers/src/nodes/text-spans.test.ts
@@ -0,0 +1,60 @@
+import { signal } from "@framefields/core";
+import { describe, expect, it } from "vitest";
+import { fitSpansToText, resolveTextSpans, spansText } from "./text-spans.js";
+
+const spans = [
+ { text: "ab " },
+ { text: "cd", fill: "#d00" },
+ { text: " ef", fontSize: 10 },
+];
+
+describe("resolveTextSpans", () => {
+ it("reads signal fields as plain values", () => {
+ const out = resolveTextSpans([
+ {
+ text: signal("hi"),
+ fill: signal("#d00"),
+ fontSize: signal(12),
+ opacity: signal(0.5),
+ mark: { background: signal("#ff0"), paddingX: 2 },
+ },
+ ]);
+ expect(out).toEqual([
+ {
+ text: "hi",
+ fill: "#d00",
+ fontSize: 12,
+ opacity: 0.5,
+ mark: { background: "#ff0", paddingX: 2 },
+ },
+ ]);
+ });
+});
+
+describe("fitSpansToText", () => {
+ it("keeps a revealed prefix in each span's style and drops the rest", () => {
+ expect(fitSpansToText(spans, "ab c")).toEqual([
+ { text: "ab " },
+ { text: "c", fill: "#d00" },
+ ]);
+ expect(fitSpansToText(spans, "")).toEqual([]);
+ });
+
+ it("is the identity on the spans' own text", () => {
+ expect(fitSpansToText(spans, spansText(spans))).toEqual(spans);
+ });
+
+ it("restyles a same-length rewrite in place", () => {
+ expect(fitSpansToText(spans, "xy zw")).toEqual([
+ { text: "xy " },
+ { text: "zw", fill: "#d00" },
+ ]);
+ });
+
+ it("gives text past the spans' end the last span's style", () => {
+ expect(fitSpansToText(spans, "ab cd efgh").at(-1)).toEqual({
+ text: " efgh",
+ fontSize: 10,
+ });
+ });
+});
diff --git a/packages/webgpu-renderers/src/nodes/text-spans.ts b/packages/webgpu-renderers/src/nodes/text-spans.ts
new file mode 100644
index 00000000..6d41d1eb
--- /dev/null
+++ b/packages/webgpu-renderers/src/nodes/text-spans.ts
@@ -0,0 +1,70 @@
+import type { FrameContext, TextSpan } from "@framefields/core";
+
+type Gettable<T> = { get: (ctx?: FrameContext) => T };
+
+function isGettable<T>(v: unknown): v is Gettable<T> {
+ return (
+ typeof v === "object" &&
+ v !== null &&
+ typeof (v as { get?: unknown }).get === "function"
+ );
+}
+
+function resolve<T>(v: T | Gettable<T>, ctx?: FrameContext): T {
+ return isGettable<T>(v) ? v.get(ctx) : v;
+}
+
+/**
+ * The spans' animated fields (text, size, fill, opacity, mark background)
+ * read for one frame, so the spans can be sliced, compared and cached as
+ * plain values.
+ */
+export function resolveTextSpans(
+ spans: readonly TextSpan[],
+ ctx?: FrameContext,
+): TextSpan[] {
+ return spans.map((s) => {
+ const out: TextSpan = { ...s, text: String(resolve(s.text, ctx) ?? "") };
+ if (s.fontSize !== undefined)
+ out.fontSize = Number(resolve(s.fontSize, ctx));
+ if (s.fill !== undefined) out.fill = String(resolve(s.fill, ctx));
+ if (s.opacity !== undefined) out.opacity = Number(resolve(s.opacity, ctx));
+ if (s.mark)
+ out.mark = {
+ ...s.mark,
+ background: String(resolve(s.mark.background, ctx)),
+ };
+ return out;
+ });
+}
+
+/** The plain text the spans spell out, in order. */
+export function spansText(spans: readonly TextSpan[]): string {
+ let text = "";
+ for (const s of spans) text += typeof s.text === "string" ? s.text : "";
+ return text;
+}
+
+/**
+ * Re-cut `text` across resolved spans by position: span i keeps the styling
+ * of characters [start_i, end_i) of the spans' own text. A revealed prefix
+ * (typewriter, word or line reveal) keeps each visible character in its
+ * span's style and drops the spans not reached yet; a same-length rewrite
+ * (scramble) restyles in place. Text past the spans' end takes the last
+ * span's style.
+ */
+export function fitSpansToText(
+ spans: readonly TextSpan[],
+ text: string,
+): TextSpan[] {
+ const out: TextSpan[] = [];
+ let offset = 0;
+ for (const [i, s] of spans.entries()) {
+ if (offset >= text.length) break;
+ const len = typeof s.text === "string" ? s.text.length : 0;
+ const end = i === spans.length - 1 ? text.length : offset + len;
+ if (end > offset) out.push({ ...s, text: text.slice(offset, end) });
+ offset = end;
+ }
+ return out;
+}