Skip to content

Open-source AI coworkers that each get a computer of their own: a browser, files and tools, with every action decided before it happens and recorded after. Bring any AG-UI agent.

Cut a roster line between graphemes, not between code points (#455)

kevin5eea77eec562
diff --git a/CHANGELOG.md b/CHANGELOG.md
index daecc5e7..32586d11 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -8,6 +8,16 @@ Newest first. `Unreleased` is what is on `main` and not yet tagged.
## Unreleased
+### A flag or a family emoji in a channel preview is no longer cut in half
+
+The one line a roster draws is cut to a cap, and the cut walked code points -- right for a plain
+emoji, wrong for every emoji built out of more than one. A flag is two regional indicators, a family
+is three people joined by zero-width joiners, a thumbs-up with a skin tone is the thumb plus a
+modifier, and a keycap is a digit plus a variation selector plus an enclosing mark. Landing the cut
+inside any of those left a boxed letter, a dangling joiner or a bare digit in the sidebar, in the
+generated channel title, and in the excerpt the titler is shown. The cut is now taken between
+grapheme clusters, so what a person sees as one character is kept or dropped whole. Plain text is
+cut in exactly the same place as before.
### A scroll with an unusable `deltaY` is refused, rather than scrolling some other distance
`POST /computers/:botId/scroll` and `POST /computers/:botId/human/scroll` accepted any JSON number
diff --git a/server/src/channels/routes.ts b/server/src/channels/routes.ts
index 8be7a28a..a96d8aa0 100644
--- a/server/src/channels/routes.ts
+++ b/server/src/channels/routes.ts
@@ -212,11 +212,11 @@ export type ChannelStore = {
const PRIVATE_AGENT_CHANNEL_DESCRIPTION = "Private agent channel.";
const MAX_CHANNEL_NAME_CODE_POINTS = 120;
-const MAX_ACTIVITY_CODE_POINTS = 200;
+const MAX_ACTIVITY_GRAPHEMES = 200;
/** Reduce a message to the one line a roster draws. See `oneLine` for why it is shared. */
function previewOf(text: string) {
- return oneLine(text, MAX_ACTIVITY_CODE_POINTS);
+ return oneLine(text, MAX_ACTIVITY_GRAPHEMES);
}
function channelName(names: string[]) {
diff --git a/server/src/channels/summary.ts b/server/src/channels/summary.ts
index 9902fbdb..3686ab2b 100644
--- a/server/src/channels/summary.ts
+++ b/server/src/channels/summary.ts
@@ -22,10 +22,10 @@ import { oneLine } from "./text";
export const CHANNEL_SUMMARY_KIND = "channel.summary";
/** A title long enough to truncate says no more than the preview it replaced. */
-const MAX_SUMMARY_CODE_POINTS = 60;
+const MAX_SUMMARY_GRAPHEMES = 60;
/** How much of the opening exchange the model is shown. Enough to see the topic, not the whole run. */
-const MAX_EXCERPT_CODE_POINTS = 600;
+const MAX_EXCERPT_GRAPHEMES = 600;
/** A seam, so a test drives every path with no key and no network. Null means nothing worth writing. */
export type ChannelTitler = (excerpt: string) => Promise<string | null>;
@@ -224,7 +224,7 @@ async function summariseOne(
const answer = await options.title(excerpt);
if (!answer) return "nothing to name it with";
- const title = oneLine(stripWrappingQuotes(answer), MAX_SUMMARY_CODE_POINTS);
+ const title = oneLine(stripWrappingQuotes(answer), MAX_SUMMARY_GRAPHEMES);
if (!title) return "nothing to name it with";
return await options.database.transaction<Attempt>(
@@ -288,7 +288,7 @@ async function openingOf(
return oneLine(
replied ? `Asked: ${asked}\nAnswered: ${replied}` : `Asked: ${asked}`,
- MAX_EXCERPT_CODE_POINTS,
+ MAX_EXCERPT_GRAPHEMES,
);
}
diff --git a/server/src/channels/text.ts b/server/src/channels/text.ts
index e549816f..52d9c947 100644
--- a/server/src/channels/text.ts
+++ b/server/src/channels/text.ts
@@ -1,12 +1,31 @@
/**
- * One line a roster can draw: control characters stripped, whitespace collapsed, cut on code points
- * so an emoji is never split. The caller supplies the cap; a preview and a title want different ones.
+ * The unit a person sees as one character.
+ *
+ * `Array.from` splits on code points, which is right for a plain emoji and wrong for every emoji
+ * built out of more than one. A flag is two regional indicators, a family is three people joined by
+ * zero-width joiners, a thumbs-up with a skin tone is the thumb plus a modifier, and a keycap is a
+ * digit plus a variation selector plus an enclosing mark. Cut between any of those parts and what is
+ * left is not a shorter emoji: it is a boxed letter, a dangling joiner, or a bare digit.
*/
-export function oneLine(text: string, maxCodePoints: number): string {
+const GRAPHEMES = new Intl.Segmenter(undefined, { granularity: "grapheme" });
+
+/**
+ * One line a roster can draw: control characters stripped, whitespace collapsed, cut on grapheme
+ * clusters so an emoji is never split. The caller supplies the cap; a preview and a title want
+ * different ones.
+ */
+export function oneLine(text: string, maxGraphemes: number): string {
// biome-ignore lint/suspicious/noControlCharactersInRegex: stripping them is the point.
const flattened = text.replace(/[\u0000-\u001f\u007f-\u009f]+/g, " ").trim();
const collapsed = flattened.replace(/\s+/g, " ");
- const codePoints = Array.from(collapsed);
- if (codePoints.length <= maxCodePoints) return collapsed;
- return `${codePoints.slice(0, maxCodePoints - 1).join("")}…`;
+ // A string can never hold more graphemes than it holds UTF-16 units, so a line this short is
+ // already under the cap and needs no segmenting. Nearly everything a roster draws is that short,
+ // and segmenting is the expensive part of this function.
+ if (collapsed.length <= maxGraphemes) return collapsed;
+ const graphemes = Array.from(
+ GRAPHEMES.segment(collapsed),
+ (each) => each.segment,
+ );
+ if (graphemes.length <= maxGraphemes) return collapsed;
+ return `${graphemes.slice(0, maxGraphemes - 1).join("")}…`;
}
diff --git a/server/tests/channel-one-line.test.ts b/server/tests/channel-one-line.test.ts
new file mode 100644
index 00000000..d23d1523
--- /dev/null
+++ b/server/tests/channel-one-line.test.ts
@@ -0,0 +1,77 @@
+import { describe, expect, test } from "bun:test";
+import { oneLine } from "../src/channels/text";
+
+const SEGMENTER = new Intl.Segmenter(undefined, { granularity: "grapheme" });
+const graphemesOf = (value: string): string[] =>
+ Array.from(SEGMENTER.segment(value), (each) => each.segment);
+
+/** South Korea: two regional indicators. Cut between them and one letter is left in a box. */
+const FLAG = "\u{1F1F0}\u{1F1F7}";
+/** Man, woman, girl, joined. Cut anywhere and a zero-width joiner dangles off the end. */
+const FAMILY = "\u{1F468}\u{200D}\u{1F469}\u{200D}\u{1F467}";
+/** A thumbs-up plus a skin-tone modifier. Cut and the modifier is dropped. */
+const THUMB = "\u{1F44D}\u{1F3FD}";
+/** Digit, variation selector, enclosing keycap. Cut and it is a bare 1. */
+const KEYCAP = "1\u{FE0F}\u{20E3}";
+
+describe("oneLine", () => {
+ /*
+ * The guard against over-correcting. Every one of these passes on the previous implementation
+ * too: the cut lands in the same place, the ellipsis replaces the same last unit, and control
+ * characters and runs of whitespace are still flattened. Only the definition of "one unit" moved.
+ */
+ test("leaves a short line alone", () => {
+ expect(oneLine("Deploy the staging build", 200)).toBe(
+ "Deploy the staging build",
+ );
+ });
+
+ test("flattens control characters and collapses whitespace", () => {
+ expect(oneLine("one\ttwo three\nfour", 200)).toBe("one two three four");
+ });
+
+ test("cuts plain text to the cap, ellipsis included", () => {
+ expect(oneLine("abcdefghij", 5)).toBe("abcd…");
+ expect(graphemesOf(oneLine("abcdefghij", 5))).toHaveLength(5);
+ });
+
+ test("keeps a single-code-point emoji whole", () => {
+ expect(oneLine(`aaaa\u{1F600}bbbb`, 5)).toBe("aaaa…");
+ });
+
+ /*
+ * The bug. `Array.from` walks code points, and every emoji below is made of more than one, so the
+ * cut landed inside the emoji the docstring promised never to split.
+ */
+ test.each([
+ ["a flag", FLAG],
+ ["a joined family", FAMILY],
+ ["a skin-tone thumb", THUMB],
+ ["a keycap", KEYCAP],
+ ])("never cuts %s in half", (_name, emoji) => {
+ const line = `${emoji} report`;
+ const whole = graphemesOf(line);
+
+ // Every cap from 1 up past the whole line, so no single lucky boundary can carry the test.
+ for (let cap = 1; cap <= whole.length + 2; cap += 1) {
+ const cut = oneLine(line, cap);
+ const kept = cut.endsWith("…") ? cut.slice(0, -1) : cut;
+
+ expect(graphemesOf(cut).length).toBeLessThanOrEqual(cap);
+ // What survives is a whole number of the line's own clusters, taken from the front. Anything
+ // split in half fails here, because its pieces are not clusters of the original.
+ expect(graphemesOf(kept)).toEqual(
+ whole.slice(0, graphemesOf(kept).length),
+ );
+ }
+ });
+
+ test("cuts between emoji rather than inside one", () => {
+ // Three clusters, sixteen UTF-16 units, five code points' worth of joiners and modifiers
+ // between them. A cap of three is the whole line; a cap of two keeps one emoji and the ellipsis.
+ expect(oneLine(`${FLAG}${FAMILY}${THUMB}`, 3)).toBe(
+ `${FLAG}${FAMILY}${THUMB}`,
+ );
+ expect(oneLine(`${FLAG}${FAMILY}${THUMB}`, 2)).toBe(`${FLAG}…`);
+ });
+});