Code-first video & image, rendered natively on WebGPU.
fix(text): preload and size emoji the way layout draws them
okanasla0f7ac99d6f9
diff --git a/packages/framefields/src/emoji.render.test.ts b/packages/framefields/src/emoji.render.test.tsnew file mode 100644index 00000000..4b000d8f--- /dev/null+++ b/packages/framefields/src/emoji.render.test.ts@@ -0,0 +1,71 @@+/**+ * Emoji in text draw from the registered color emoji font: the renderer+ * preloads each one (text layers reach it as `kind: "text"`, and emoji as+ * whole graphemes such as "❤️"), and draws it on its own advance.+ */+import { fileURLToPath } from "node:url";+import { Canvas, loadImage } from "skia-canvas";+import { beforeAll, describe, expect, it } from "vitest";+import { Composition, FontManager, Layer } from "./index.js";++const asset = (path: string) =>+ fileURLToPath(new URL(`../../../${path}`, import.meta.url));++beforeAll(async () => {+ await FontManager.register("assets/fonts/Inter.ttf");+ await FontManager.registerEmojiFont(+ asset("packages/renderer/src/assets/NotoColorEmoji.ttf"),+ );+});++async function render(text: string) {+ const comp = new Composition({+ width: 300,+ height: 100,+ fps: 30,+ durationFrames: 1,+ backgroundColor: "#000000",+ });+ comp.add(+ Layer.box({+ width: 300,+ height: 100,+ background: "#000000",+ children: [+ Layer.text(text, {+ position: "absolute",+ x: 20,+ y: 20,+ fontFamily: "Inter",+ fontSize: 40,+ fill: "#ffffff",+ }),+ ],+ }),+ );+ const img = await loadImage(await comp.renderFrame({ frame: 0 }));+ const canvas = new Canvas(img.width, img.height);+ const ctx = canvas.getContext("2d");+ ctx.drawImage(img, 0, 0);+ return ctx.getImageData(0, 0, img.width, img.height).data;+}++/** Pixels that are clearly coloured (not black, white or grey). */+function colored(data: Uint8ClampedArray): number {+ let n = 0;+ for (let i = 0; i < data.length; i += 4) {+ const r = data[i] as number;+ const g = data[i + 1] as number;+ const b = data[i + 2] as number;+ if (Math.max(r, g, b) - Math.min(r, g, b) > 60) n++;+ }+ return n;+}++describe("emoji", () => {+ it("draw in color, with variation selectors", async () => {+ expect(colored(await render("ab"))).toBe(0);+ expect(colored(await render("a 🚀"))).toBeGreaterThan(200);+ expect(colored(await render("a ❤️"))).toBeGreaterThan(200);+ });+});diff --git a/packages/renderers/src/preload-fonts.test.ts b/packages/renderers/src/preload-fonts.test.tsindex dec8c0a4..e8acc184 100644--- a/packages/renderers/src/preload-fonts.test.ts+++ b/packages/renderers/src/preload-fonts.test.ts@@ -9,7 +9,13 @@ vi.mock("./utils.js", () => ({ preloadFont: vi.fn(async () => {}) }));const text = (value: string, fontFamily?: string): VirtualMediaData =>({- operation: { op: "text", text: value, fontFamily, fontSize: 32 },+ operation: {+ op: "CompositorLayer",+ kind: "text",+ text: value,+ fontFamily,+ fontSize: 32,+ },}) as unknown as VirtualMediaData;const tree = (...children: VirtualMediaData[]): VirtualMediaData =>@@ -28,12 +34,20 @@ describe("preloadCompositionFonts", () => {.mockResolvedValue({} as never);await preloadCompositionFonts(- tree(text("hi 😀", "Serif"), text("😀 🎉", "Serif"), text("plain")),+ tree(+ text("hi 😀", "Serif"),+ text("😀 🎉", "Serif"),+ text("plain"),+ text("❤️ 👍🏽"),+ ),{} as Device,);expect(slug.mock.calls.map((c) => c[1]).sort()).toEqual(["Inter", "Serif"]);- expect(emoji.mock.calls.map((c) => c[1]).sort()).toEqual(["🎉", "😀"]);+ // Whole graphemes, as layout draws them: with selectors and skin tones.+ expect(emoji.mock.calls.map((c) => c[1]).sort()).toEqual(+ ["🎉", "😀", "❤️", "👍🏽"].sort(),+ );});it("warns once, and does not fail, when emoji have no font", async () => {diff --git a/packages/renderers/src/preload-fonts.ts b/packages/renderers/src/preload-fonts.tsindex bb88aa31..09ee969f 100644--- a/packages/renderers/src/preload-fonts.ts+++ b/packages/renderers/src/preload-fonts.ts@@ -1,10 +1,8 @@import type { VirtualMediaData } from "@framefields/core";-import { SlugFontCache } from "@framefields/webgpu-renderers";+import { emojiGraphemes, SlugFontCache } from "@framefields/webgpu-renderers";import { fontSource, fontUrl } from "./font-source.js";import { preloadFont } from "./utils.js";-const EMOJI_REGEX = /\p{Extended_Pictographic}/gu;-type Device = Parameters<typeof SlugFontCache.preloadSlugFont>[0];let warnedNoEmojiFont = false;@@ -33,9 +31,10 @@ export async function preloadCompositionFonts(typeof op.text === "string";const family = (op.fontFamily as string) || (isText ? "Inter" : "");if (family) families.add(family);- if (op.op === "text" && typeof op.text === "string") {+ // Text layers arrive as `{ op: "CompositorLayer", kind: "text" }`.+ if (isText && typeof op.text === "string") {const fontSize = (op.fontSize as number) ?? 48;- for (const char of op.text.match(EMOJI_REGEX) ?? []) {+ for (const char of emojiGraphemes(op.text)) {emoji.set(`${char}-${fontSize}`, fontSize);}}diff --git a/packages/webgpu-renderers/src/nodes/paragraph.ts b/packages/webgpu-renderers/src/nodes/paragraph.tsindex 4d97181e..a8e8da70 100644--- a/packages/webgpu-renderers/src/nodes/paragraph.ts+++ b/packages/webgpu-renderers/src/nodes/paragraph.ts@@ -1992,7 +1992,10 @@ export function drawParagraphNode(emoji.size,);- const localEmojiSize = emoji.size * 1.5;+ // Layout gives an emoji a 1 em advance. Noto-style bitmaps carry about+ // 13 % padding, so a 1.15 em square puts the ink on that advance+ // instead of over its neighbours.+ const localEmojiSize = emoji.size * 1.15;const offsetAdjustment = (localEmojiSize - emoji.size) / 2;const emojiDstRect = {diff --git a/packages/webgpu-renderers/src/slug/index.ts b/packages/webgpu-renderers/src/slug/index.tsindex d5fb453c..6b6f9fb0 100644--- a/packages/webgpu-renderers/src/slug/index.ts+++ b/packages/webgpu-renderers/src/slug/index.ts@@ -10,4 +10,4 @@ export { SlugGeometry } from "./slug-geometry.js";export type { SlugCodePoint, SlugFont } from "./slug-loader.js";export { SlugLoader } from "./slug-loader.js";export { type SlugGlyphBatch, SlugPipeline } from "./slug-pipeline.js";-export type { TextDirection } from "./text-shaping.js";+export { emojiGraphemes, type TextDirection } from "./text-shaping.js";diff --git a/packages/webgpu-renderers/src/slug/text-shaping.ts b/packages/webgpu-renderers/src/slug/text-shaping.tsindex 257de8a2..84308b9e 100644--- a/packages/webgpu-renderers/src/slug/text-shaping.ts+++ b/packages/webgpu-renderers/src/slug/text-shaping.ts@@ -71,6 +71,17 @@ interface ShapeOptions {const EMOJI_REGEX =/(\p{Extended_Pictographic}|\p{Emoji_Presentation}|[☀-➿])/u;+/**+ * The emoji in `text`, one per grapheme, as layout draws them: "❤️" keeps+ * its variation selector, "👍🏽" its skin tone. Preloading uses these, so its+ * textures are the ones drawing looks up.+ */+export function emojiGraphemes(text: string): string[] {+ return segmentGraphemes(text)+ .map((g) => g.text)+ .filter((g) => EMOJI_REGEX.test(g));+}+/** Characters that make bidi analysis necessary (RTL letters and explicit RTL controls). */const RTL_REGEX = /[-ࣿיִ-﷿ﹰ-\u{10800}-\u{10FFF}\u{1E800}-\u{1EFFF}]/u;