Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(branding): send Jev only well-formed Unicode text (#5043)
Abimael Martellff02eeb9937c
diff --git a/apps/api/src/__tests__/lib/branding/jev.test.ts b/apps/api/src/__tests__/lib/branding/jev.test.tsindex c4a5a94ea..5fdcd6168 100644--- a/apps/api/src/__tests__/lib/branding/jev.test.ts+++ b/apps/api/src/__tests__/lib/branding/jev.test.ts@@ -731,6 +731,31 @@ describe("Jev request building", () => {]);});+ it("sends only well-formed Unicode text", () => {+ // "𝐁" (U+1D401) is a surrogate pair. The branding script truncates button+ // text by code unit, which can leave its first half alone at the end.+ const bold = "\u{1D401}";+ const input = baseInput(new CostTracking());+ input.pageTitle = "a".repeat(199) + bold;+ input.buttons![0].text = "Beyond the Storefront " + bold + "\uD835";+ input.buttons![1].text = "x".repeat(39) + bold;++ const request = buildJevRequest(input);+ const page = request.state.page as Record<string, string>;+ const buttons = request.state.buttons as Record<+ string,+ Record<string, string>+ >;++ // clip() drops a pair it would cut in half rather than keep one half.+ expect(page.title).toBe("a".repeat(199));+ expect(buttons.button_0.text).toBe(+ "Beyond the Storefront " + bold + "\uFFFD",+ );+ // JSON.stringify escapes lone surrogates as \udxxx; none may remain.+ expect(JSON.stringify(request.state)).not.toMatch(/\\ud[89a-f]/i);+ });+it("names common colors", () => {expect(describeColor("#E2511A")).toBe("vivid orange");expect(describeColor("#0A2540")).toBe("very dark blue");diff --git a/apps/api/src/lib/branding/jev.ts b/apps/api/src/lib/branding/jev.tsindex e26569150..67e43ac31 100644--- a/apps/api/src/lib/branding/jev.ts+++ b/apps/api/src/lib/branding/jev.ts@@ -179,7 +179,7 @@ function collectColors(input: BrandingLLMInput): ColorCandidate[] {}const buttons = (input.buttons ?? []).slice(0, MAX_BUTTONS);for (const button of buttons) {- const label = button.text?.trim().slice(0, 40) || "unlabeled";+ const label = clip(button.text, 40) || "unlabeled";add(button.background, `background of button "${label}"`);add(button.textColor, `text on button "${label}"`);add(button.borderColor, `border of button "${label}"`);@@ -303,7 +303,7 @@ function sizeLabel(width: number, height: number): string {function fileName(src: string): string {if (src.startsWith("data:")) return "inline SVG";const path = src.split(/[?#]/)[0];- return (path.split("/").pop() || "").slice(0, 80);+ return clip(path.split("/").pop(), 80);}// Cookie-consent controls are page chrome, never the site's call to action.@@ -328,8 +328,32 @@ const buttonStyle = (b: {b.shadow ?? "",].join("/");-const clip = (value: string | undefined, max: number) =>- (value ?? "").replace(/\s+/g, " ").trim().slice(0, max);+const isHighSurrogate = (code: number) => code >= 0xd800 && code <= 0xdbff;++// slice() counts UTF-16 code units, so never end on the first half of a pair.+const clip = (value: string | undefined, max: number) => {+ const text = (value ?? "").replace(/\s+/g, " ").trim();+ return text.slice(+ 0,+ isHighSurrogate(text.charCodeAt(max - 1)) ? max - 1 : max,+ );+};++// TypeSafe rejects text with lone surrogates as invalid Unicode. Page text can+// carry them (the branding script truncates by code unit), so replace them+// with U+FFFD, as String.prototype.toWellFormed does.+const LONE_SURROGATE =+ /[\uD800-\uDBFF](?![\uDC00-\uDFFF])|(?<![\uD800-\uDBFF])[\uDC00-\uDFFF]/g;+const wellFormed = (value: JsonValue): JsonValue =>+ typeof value === "string"+ ? value.replace(LONE_SURROGATE, "\uFFFD")+ : Array.isArray(value)+ ? value.map(wellFormed)+ : value && typeof value === "object"+ ? Object.fromEntries(+ Object.entries(value).map(([k, v]) => [k, wellFormed(v)]),+ )+ : value;type JevRequest = {state: Record<string, JsonValue>;@@ -587,7 +611,7 @@ export function buildJevRequest(input: BrandingLLMInput): JevRequest {};return {- state,+ state: wellFormed(state) as Record<string, JsonValue>,questions,colors,fonts,