Skip to content

πŸͺ’ Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

fix(shared): truncate() no longer splits surrogate pairs (#16172) (#16234)

greekera10000753d03e9d2d
diff --git a/packages/shared/src/utils/stringChecks.test.ts b/packages/shared/src/utils/stringChecks.test.ts
new file mode 100644
index 0000000000..36dd1b4cf5
--- /dev/null
+++ b/packages/shared/src/utils/stringChecks.test.ts
@@ -0,0 +1,59 @@
+import { describe, expect, it } from "vitest";
+
+import { truncate } from "./stringChecks";
+
+function hasLoneSurrogate(s: string): boolean {
+ for (let i = 0; i < s.length; i++) {
+ const c = s.charCodeAt(i);
+ if (c >= 0xd800 && c <= 0xdbff) {
+ const next = s.charCodeAt(i + 1);
+ if (!(next >= 0xdc00 && next <= 0xdfff)) return true;
+ i++;
+ } else if (c >= 0xdc00 && c <= 0xdfff) {
+ return true;
+ }
+ }
+ return false;
+}
+
+describe("truncate", () => {
+ it("returns the string unchanged when at or under the limit", () => {
+ expect(truncate("hello", 16)).toBe("hello");
+ expect(truncate("exactly-sixteen!", 16)).toBe("exactly-sixteen!");
+ });
+
+ it("appends an ellipsis when over the limit", () => {
+ expect(truncate("this is definitely longer than sixteen", 16)).toBe(
+ "this is definite...",
+ );
+ });
+
+ it("counts code points, not UTF-16 code units, for the limit", () => {
+ expect(truncate("πŸŽ‰".repeat(9), 16)).toBe("πŸŽ‰".repeat(9));
+ });
+
+ it("does not split surrogate pairs when the cut lands mid-pair", () => {
+ const input = "a" + "πŸŽ‰".repeat(20);
+ const out = truncate(input, 16);
+ expect(hasLoneSurrogate(out)).toBe(false);
+ expect(out).toBe("a" + "πŸŽ‰".repeat(15) + "...");
+ });
+
+ it("keeps astral characters intact for realistic model names", () => {
+ const input = "gpt-4o-" + "πŸš€".repeat(30) + "-preview";
+ const out = truncate(input, 30);
+ expect(hasLoneSurrogate(out)).toBe(false);
+ expect(out.endsWith("...")).toBe(true);
+ });
+
+ it("handles mathematical alphanumeric (astral) code points", () => {
+ const input = "πŸ™πŸšπŸ›πŸœπŸπŸžπŸŸπŸ πŸ‘";
+ const out = truncate(input, 5);
+ expect(hasLoneSurrogate(out)).toBe(false);
+ expect(out).toBe("πŸ™πŸšπŸ›πŸœπŸ...");
+ });
+
+ it("leaves BMP (non-astral) strings such as CJK unaffected", () => {
+ expect(truncate("δ½ ε₯½δΈ–η•Œδ½ ε₯½δΈ–η•Œ", 5)).toBe("δ½ ε₯½δΈ–η•Œδ½ ...");
+ });
+});
diff --git a/packages/shared/src/utils/stringChecks.ts b/packages/shared/src/utils/stringChecks.ts
index 5e3a9db448..6bea5411e2 100644
--- a/packages/shared/src/utils/stringChecks.ts
+++ b/packages/shared/src/utils/stringChecks.ts
@@ -43,9 +43,17 @@ export function stringifyValue(value: unknown) {
}
export function truncate(str: string, n = 16) {
- // '...' suffix if the string is longer than n
- if (str.length > n) {
- return str.substring(0, n) + "...";
+ // '...' suffix if the string is longer than n.
+ // Iterate by code point (Array.from) rather than UTF-16 code unit
+ // (String.prototype.substring) so that characters outside the Basic
+ // Multilingual Plane - emoji, CJK extension-B ideographs, mathematical
+ // alphanumerics, etc. - are never split mid-surrogate-pair. Cutting a
+ // surrogate pair in half leaves a lone surrogate that renders as the
+ // Unicode replacement character (U+FFFD). See issue #16172.
+ // Note: `n` counts code points, not UTF-16 code units.
+ const codePoints = Array.from(str);
+ if (codePoints.length > n) {
+ return codePoints.slice(0, n).join("") + "...";
}
return str;
}