Skip to content

Code-first video & image, rendered natively on WebGPU.

fix(text): preload and size emoji the way layout draws them

okanasla0f7ac99d6f9
diff --git a/packages/framefields/src/emoji.render.test.ts b/packages/framefields/src/emoji.render.test.ts
new file mode 100644
index 00000000..4b000d8f
--- /dev/null
+++ b/packages/framefields/src/emoji.render.test.ts
@@ -0,0 +1,71 @@
+/**
+ * Emoji in text draw from the registered color emoji font: the renderer
+ * preloads each one (text layers reach it as `kind: "text"`, and emoji as
+ * whole graphemes such as "❤️"), and draws it on its own advance.
+ */
+import { fileURLToPath } from "node:url";
+import { Canvas, loadImage } from "skia-canvas";
+import { beforeAll, describe, expect, it } from "vitest";
+import { Composition, FontManager, Layer } from "./index.js";
+
+const asset = (path: string) =>
+ fileURLToPath(new URL(`../../../${path}`, import.meta.url));
+
+beforeAll(async () => {
+ await FontManager.register("assets/fonts/Inter.ttf");
+ await FontManager.registerEmojiFont(
+ asset("packages/renderer/src/assets/NotoColorEmoji.ttf"),
+ );
+});
+
+async function render(text: string) {
+ const comp = new Composition({
+ width: 300,
+ height: 100,
+ fps: 30,
+ durationFrames: 1,
+ backgroundColor: "#000000",
+ });
+ comp.add(
+ Layer.box({
+ width: 300,
+ height: 100,
+ background: "#000000",
+ children: [
+ Layer.text(text, {
+ position: "absolute",
+ x: 20,
+ y: 20,
+ fontFamily: "Inter",
+ fontSize: 40,
+ fill: "#ffffff",
+ }),
+ ],
+ }),
+ );
+ const img = await loadImage(await comp.renderFrame({ frame: 0 }));
+ const canvas = new Canvas(img.width, img.height);
+ const ctx = canvas.getContext("2d");
+ ctx.drawImage(img, 0, 0);
+ return ctx.getImageData(0, 0, img.width, img.height).data;
+}
+
+/** Pixels that are clearly coloured (not black, white or grey). */
+function colored(data: Uint8ClampedArray): number {
+ let n = 0;
+ for (let i = 0; i < data.length; i += 4) {
+ const r = data[i] as number;
+ const g = data[i + 1] as number;
+ const b = data[i + 2] as number;
+ if (Math.max(r, g, b) - Math.min(r, g, b) > 60) n++;
+ }
+ return n;
+}
+
+describe("emoji", () => {
+ it("draw in color, with variation selectors", async () => {
+ expect(colored(await render("ab"))).toBe(0);
+ expect(colored(await render("a 🚀"))).toBeGreaterThan(200);
+ expect(colored(await render("a ❤️"))).toBeGreaterThan(200);
+ });
+});
diff --git a/packages/renderers/src/preload-fonts.test.ts b/packages/renderers/src/preload-fonts.test.ts
index dec8c0a4..e8acc184 100644
--- a/packages/renderers/src/preload-fonts.test.ts
+++ b/packages/renderers/src/preload-fonts.test.ts
@@ -9,7 +9,13 @@ vi.mock("./utils.js", () => ({ preloadFont: vi.fn(async () => {}) }));
const text = (value: string, fontFamily?: string): VirtualMediaData =>
({
- operation: { op: "text", text: value, fontFamily, fontSize: 32 },
+ operation: {
+ op: "CompositorLayer",
+ kind: "text",
+ text: value,
+ fontFamily,
+ fontSize: 32,
+ },
}) as unknown as VirtualMediaData;
const tree = (...children: VirtualMediaData[]): VirtualMediaData =>
@@ -28,12 +34,20 @@ describe("preloadCompositionFonts", () => {
.mockResolvedValue({} as never);
await preloadCompositionFonts(
- tree(text("hi 😀", "Serif"), text("😀 🎉", "Serif"), text("plain")),
+ tree(
+ text("hi 😀", "Serif"),
+ text("😀 🎉", "Serif"),
+ text("plain"),
+ text("❤️ 👍🏽"),
+ ),
{} as Device,
);
expect(slug.mock.calls.map((c) => c[1]).sort()).toEqual(["Inter", "Serif"]);
- expect(emoji.mock.calls.map((c) => c[1]).sort()).toEqual(["🎉", "😀"]);
+ // Whole graphemes, as layout draws them: with selectors and skin tones.
+ expect(emoji.mock.calls.map((c) => c[1]).sort()).toEqual(
+ ["🎉", "😀", "❤️", "👍🏽"].sort(),
+ );
});
it("warns once, and does not fail, when emoji have no font", async () => {
diff --git a/packages/renderers/src/preload-fonts.ts b/packages/renderers/src/preload-fonts.ts
index bb88aa31..09ee969f 100644
--- a/packages/renderers/src/preload-fonts.ts
+++ b/packages/renderers/src/preload-fonts.ts
@@ -1,10 +1,8 @@
import type { VirtualMediaData } from "@framefields/core";
-import { SlugFontCache } from "@framefields/webgpu-renderers";
+import { emojiGraphemes, SlugFontCache } from "@framefields/webgpu-renderers";
import { fontSource, fontUrl } from "./font-source.js";
import { preloadFont } from "./utils.js";
-const EMOJI_REGEX = /\p{Extended_Pictographic}/gu;
-
type Device = Parameters<typeof SlugFontCache.preloadSlugFont>[0];
let warnedNoEmojiFont = false;
@@ -33,9 +31,10 @@ export async function preloadCompositionFonts(
typeof op.text === "string";
const family = (op.fontFamily as string) || (isText ? "Inter" : "");
if (family) families.add(family);
- if (op.op === "text" && typeof op.text === "string") {
+ // Text layers arrive as `{ op: "CompositorLayer", kind: "text" }`.
+ if (isText && typeof op.text === "string") {
const fontSize = (op.fontSize as number) ?? 48;
- for (const char of op.text.match(EMOJI_REGEX) ?? []) {
+ for (const char of emojiGraphemes(op.text)) {
emoji.set(`${char}-${fontSize}`, fontSize);
}
}
diff --git a/packages/webgpu-renderers/src/nodes/paragraph.ts b/packages/webgpu-renderers/src/nodes/paragraph.ts
index 4d97181e..a8e8da70 100644
--- a/packages/webgpu-renderers/src/nodes/paragraph.ts
+++ b/packages/webgpu-renderers/src/nodes/paragraph.ts
@@ -1992,7 +1992,10 @@ export function drawParagraphNode(
emoji.size,
);
- const localEmojiSize = emoji.size * 1.5;
+ // Layout gives an emoji a 1 em advance. Noto-style bitmaps carry about
+ // 13 % padding, so a 1.15 em square puts the ink on that advance
+ // instead of over its neighbours.
+ const localEmojiSize = emoji.size * 1.15;
const offsetAdjustment = (localEmojiSize - emoji.size) / 2;
const emojiDstRect = {
diff --git a/packages/webgpu-renderers/src/slug/index.ts b/packages/webgpu-renderers/src/slug/index.ts
index d5fb453c..6b6f9fb0 100644
--- a/packages/webgpu-renderers/src/slug/index.ts
+++ b/packages/webgpu-renderers/src/slug/index.ts
@@ -10,4 +10,4 @@ export { SlugGeometry } from "./slug-geometry.js";
export type { SlugCodePoint, SlugFont } from "./slug-loader.js";
export { SlugLoader } from "./slug-loader.js";
export { type SlugGlyphBatch, SlugPipeline } from "./slug-pipeline.js";
-export type { TextDirection } from "./text-shaping.js";
+export { emojiGraphemes, type TextDirection } from "./text-shaping.js";
diff --git a/packages/webgpu-renderers/src/slug/text-shaping.ts b/packages/webgpu-renderers/src/slug/text-shaping.ts
index 257de8a2..84308b9e 100644
--- a/packages/webgpu-renderers/src/slug/text-shaping.ts
+++ b/packages/webgpu-renderers/src/slug/text-shaping.ts
@@ -71,6 +71,17 @@ interface ShapeOptions {
const EMOJI_REGEX =
/(\p{Extended_Pictographic}|\p{Emoji_Presentation}|[☀-➿])/u;
+/**
+ * The emoji in `text`, one per grapheme, as layout draws them: "❤️" keeps
+ * its variation selector, "👍🏽" its skin tone. Preloading uses these, so its
+ * textures are the ones drawing looks up.
+ */
+export function emojiGraphemes(text: string): string[] {
+ return segmentGraphemes(text)
+ .map((g) => g.text)
+ .filter((g) => EMOJI_REGEX.test(g));
+}
+
/** Characters that make bidi analysis necessary (RTL letters and explicit RTL controls). */
const RTL_REGEX = /[֐-ࣿיִ-﷿ﹰ-‏‫‮⁧\u{10800}-\u{10FFF}\u{1E800}-\u{1EFFF}]/u;