πͺ’ Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
fix(shared): truncate() no longer splits surrogate pairs (#16172) (#16234)
greekera10000753d03e9d2d
diff --git a/packages/shared/src/utils/stringChecks.test.ts b/packages/shared/src/utils/stringChecks.test.tsnew file mode 100644index 0000000000..36dd1b4cf5--- /dev/null+++ b/packages/shared/src/utils/stringChecks.test.ts@@ -0,0 +1,59 @@+import { describe, expect, it } from "vitest";++import { truncate } from "./stringChecks";++function hasLoneSurrogate(s: string): boolean {+ for (let i = 0; i < s.length; i++) {+ const c = s.charCodeAt(i);+ if (c >= 0xd800 && c <= 0xdbff) {+ const next = s.charCodeAt(i + 1);+ if (!(next >= 0xdc00 && next <= 0xdfff)) return true;+ i++;+ } else if (c >= 0xdc00 && c <= 0xdfff) {+ return true;+ }+ }+ return false;+}++describe("truncate", () => {+ it("returns the string unchanged when at or under the limit", () => {+ expect(truncate("hello", 16)).toBe("hello");+ expect(truncate("exactly-sixteen!", 16)).toBe("exactly-sixteen!");+ });++ it("appends an ellipsis when over the limit", () => {+ expect(truncate("this is definitely longer than sixteen", 16)).toBe(+ "this is definite...",+ );+ });++ it("counts code points, not UTF-16 code units, for the limit", () => {+ expect(truncate("π".repeat(9), 16)).toBe("π".repeat(9));+ });++ it("does not split surrogate pairs when the cut lands mid-pair", () => {+ const input = "a" + "π".repeat(20);+ const out = truncate(input, 16);+ expect(hasLoneSurrogate(out)).toBe(false);+ expect(out).toBe("a" + "π".repeat(15) + "...");+ });++ it("keeps astral characters intact for realistic model names", () => {+ const input = "gpt-4o-" + "π".repeat(30) + "-preview";+ const out = truncate(input, 30);+ expect(hasLoneSurrogate(out)).toBe(false);+ expect(out.endsWith("...")).toBe(true);+ });++ it("handles mathematical alphanumeric (astral) code points", () => {+ const input = "ππππππππ π‘";+ const out = truncate(input, 5);+ expect(hasLoneSurrogate(out)).toBe(false);+ expect(out).toBe("πππππ...");+ });++ it("leaves BMP (non-astral) strings such as CJK unaffected", () => {+ expect(truncate("δ½ ε₯½δΈηδ½ ε₯½δΈη", 5)).toBe("δ½ ε₯½δΈηδ½ ...");+ });+});diff --git a/packages/shared/src/utils/stringChecks.ts b/packages/shared/src/utils/stringChecks.tsindex 5e3a9db448..6bea5411e2 100644--- a/packages/shared/src/utils/stringChecks.ts+++ b/packages/shared/src/utils/stringChecks.ts@@ -43,9 +43,17 @@ export function stringifyValue(value: unknown) {}export function truncate(str: string, n = 16) {- // '...' suffix if the string is longer than n- if (str.length > n) {- return str.substring(0, n) + "...";+ // '...' suffix if the string is longer than n.+ // Iterate by code point (Array.from) rather than UTF-16 code unit+ // (String.prototype.substring) so that characters outside the Basic+ // Multilingual Plane - emoji, CJK extension-B ideographs, mathematical+ // alphanumerics, etc. - are never split mid-surrogate-pair. Cutting a+ // surrogate pair in half leaves a lone surrogate that renders as the+ // Unicode replacement character (U+FFFD). See issue #16172.+ // Note: `n` counts code points, not UTF-16 code units.+ const codePoints = Array.from(str);+ if (codePoints.length > n) {+ return codePoints.slice(0, n).join("") + "...";}return str;}