Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(branding): keep fallback and duplicate fonts out of Jev's font list (#5045)
Abimael Martell35f64ae2a0e4
diff --git a/apps/api/src/__tests__/lib/branding/jev.test.ts b/apps/api/src/__tests__/lib/branding/jev.test.tsindex 5fdcd6168..356a2dead 100644--- a/apps/api/src/__tests__/lib/branding/jev.test.ts+++ b/apps/api/src/__tests__/lib/branding/jev.test.ts@@ -472,6 +472,87 @@ describe("branding with Jev", () => {]);});+ it("drops fallback fonts and unplaced extras once headings and body are covered", async () => {+ const input = baseInput(new CostTracking());+ input.jsAnalysis.typography = {+ fontFamilies: { primary: "Inter", heading: "Inter" },+ };+ input.jsAnalysis.fonts = [+ { family: "Inter", count: 40 },+ { family: "Segoe UI", count: 20 },+ { family: "Roboto", count: 10 },+ { family: "Canela", count: 5 },+ ];+ respondWith(+ jevResponse({+ font_0_is_brand: { type: "noul", noul: 0.95 },+ font_1_is_brand: { type: "noul", noul: 0.9 },+ font_2_is_brand: { type: "noul", noul: 0.9 },+ font_3_is_brand: { type: "noul", noul: 0.9 },+ font_3_role: {+ type: "choice",+ choice: "unknown",+ probabilities: { unknown: 1 },+ confidence: 0.9,+ },+ }),+ );++ const result = await enhanceBrandingWithLLM(input);++ expect(result.cleanedFonts).toEqual([{ family: "Inter", role: "body" }]);+ });++ it("keeps the typography role of a family merged under its spaced name", async () => {+ const input = baseInput(new CostTracking());+ input.jsAnalysis.typography = {+ fontFamilies: { primary: "Inter", heading: "CormorantGaramond" },+ };+ input.jsAnalysis.fonts = [+ { family: "Inter", count: 40 },+ { family: "CormorantGaramond", count: 10 },+ { family: "Cormorant Garamond", count: 5 },+ { family: "Open-Sans", count: 3 },+ { family: "Open Sans", count: 2 },+ ];++ const request = buildJevRequest(input);++ expect(request.fonts.map(f => [f.family, f.role])).toEqual([+ ["Inter", "body"],+ ["Cormorant Garamond", "heading"],+ ["Open-Sans", undefined],+ ["Open Sans", undefined],+ ]);+ });++ it("keeps a fallback font the page uses for text, or when it is all there is", async () => {+ const input = baseInput(new CostTracking());+ input.jsAnalysis.typography = { fontFamilies: { primary: "Arial" } };+ input.jsAnalysis.fonts = [{ family: "Arial", count: 40 }];+ respondWith(jevResponse({ font_0_is_brand: { type: "noul", noul: 0.9 } }));+ expect((await enhanceBrandingWithLLM(input)).cleanedFonts).toEqual([+ { family: "Arial", role: "body" },+ ]);++ const only = baseInput(new CostTracking());+ only.jsAnalysis.fonts = [{ family: "Helvetica", count: 40 }];+ respondWith(+ jevResponse({+ font_0_is_brand: { type: "noul", noul: 0.9 },+ font_0_role: {+ type: "choice",+ choice: "unknown",+ probabilities: { unknown: 1 },+ confidence: 0.9,+ },+ }),+ );+ expect((await enhanceBrandingWithLLM(only)).cleanedFonts).toEqual([+ { family: "Helvetica", role: "unknown" },+ ]);+ });+it("takes font roles from the page's typography when it has them", async () => {const input = baseInput(new CostTracking());input.jsAnalysis.typography = {@@ -770,5 +851,19 @@ describe("Jev request building", () => {expect(cleanFontFamily("var(--font-sans)")).toBeUndefined();expect(cleanFontFamily("'Söhne'")).toBe("Söhne");expect(cleanFontFamily("ui-sans-serif")).toBeUndefined();+ expect(cleanFontFamily("system-ui, sans-serif")).toBeUndefined();+ expect(cleanFontFamily("Inter, sans-serif")).toBe("Inter");+ expect(cleanFontFamily("Newsreader Variable")).toBe("Newsreader");+ });++ it("merges a family listed with and without spaces", () => {+ const input = baseInput(new CostTracking());+ input.jsAnalysis.fonts = [+ { family: "CormorantGaramond", count: 10 },+ { family: "Cormorant Garamond", count: 5 },+ ];+ expect(buildJevRequest(input).fonts).toEqual([+ { family: "Cormorant Garamond", count: 15, role: undefined },+ ]);});});diff --git a/apps/api/src/lib/branding/jev.ts b/apps/api/src/lib/branding/jev.tsindex 67e43ac31..6d9bc4f87 100644--- a/apps/api/src/lib/branding/jev.ts+++ b/apps/api/src/lib/branding/jev.ts@@ -210,13 +210,48 @@ const GENERIC_FONTS = new Set(["math",]);+// Fonts pages list as fallbacks in their stacks. They are only reported when+// the page's typography uses them, or when nothing else is left.+const FALLBACK_FONTS = new Set([+ "arial",+ "helvetica",+ "helvetica neue",+ "segoe ui",+ "roboto",+ "oxygen",+ "ubuntu",+ "cantarell",+ "noto sans",+ "droid sans",+ "lucida",+ "lucida grande",+ "lucida sans unicode",+ "tahoma",+ "verdana",+ "times",+ "times new roman",+ "georgia",+ "courier",+ "courier new",+ "apple color emoji",+ "segoe ui emoji",+ "segoe ui symbol",+ "noto color emoji",+]);+export function cleanFontFamily(raw: string): string | undefined {- let name = raw.trim().replace(/^["']|["']$/g, "");+ // A whole stack ("system-ui, sans-serif") reports its first family.+ let name = raw+ .split(",")[0]+ .trim()+ .replace(/^["']|["']$/g, "");if (!name || /^var\(/i.test(name)) return undefined;if (/fallback/i.test(name)) return undefined;// next/font obfuscation: "__Roboto_Mono_c8ca7d" → "Roboto Mono"name = name.replace(/^_+/, "").replace(/_[0-9a-f]{6}$/i, "");name = name.replace(/_/g, " ").replace(/\s+/g, " ").trim();+ // Variable-font builds name the same family: "Newsreader Variable".+ name = name.replace(/\s+(variable|vf)$/i, "");if (!name || GENERIC_FONTS.has(name.toLowerCase())) return undefined;if (name === name.toLowerCase()) {name = name.replace(/\b\w/g, ch => ch.toUpperCase());@@ -227,6 +262,9 @@ export function cleanFontFamily(raw: string): string | undefined {type FontRole = BrandingEnhancement["cleanedFonts"][number]["role"];type FontCandidate = { family: string; count: number; role?: FontRole };+/** Matching key for a family: case and spacing don't make a different font. */+const familyKey = (family: string) => family.toLowerCase().replace(/\s/g, "");+/** Role from the page's own typography: the heading and body stacks it measured. */function typographyRole(family: string,@@ -235,9 +273,10 @@ function typographyRole(const t = input.jsAnalysis.typography;const first = (value?: string | string[]) => {const raw = Array.isArray(value) ? value[0] : value;- return raw ? cleanFontFamily(raw)?.toLowerCase() : undefined;+ const cleaned = raw ? cleanFontFamily(raw) : undefined;+ return cleaned ? familyKey(cleaned) : undefined;};- const name = family.toLowerCase();+ const name = familyKey(family);if (/\b(mono|code)\b|consolas|menlo/i.test(family)) return "monospace";const body = [first(t?.fontFamilies?.primary),@@ -264,9 +303,16 @@ function collectFonts(input: BrandingLLMInput): FontCandidate[] {typeof font === "object" && typeof font.count === "number"? font.count: 1;- const existing = byFamily.get(family.toLowerCase());- if (existing) existing.count += count;- else byFamily.set(family.toLowerCase(), { family, count });+ // "CormorantGaramond" and "Cormorant Garamond" are one family; keep the+ // spaced name.+ const key = familyKey(family);+ const existing = byFamily.get(key);+ if (existing) {+ existing.count += count;+ if (!existing.family.includes(" ") && family.includes(" ")) {+ existing.family = family;+ }+ } else byFamily.set(key, { family, count });}return [...byFamily.values()].sort((a, b) => b.count - a.count)@@ -731,17 +777,31 @@ function mapJevAnswers(.map(a => a.confidence);// Fonts: keep the ones Jev calls real typefaces, most used first.- const cleanedFonts = request.fonts+ const realFonts = request.fonts.map((font, i) => {const isBrand = answers[`font_${i}_is_brand`];const role = choiceOf(answers, `font_${i}_role`);return {family: font.family,role: font.role ?? ((role?.choice ?? "unknown") as FontRole),+ // the page's own typography uses it for headings or body+ measured: font.role !== undefined,keep: isBrand?.type === "noul" ? isBrand.noul >= 0.5 : true,};})- .filter(f => f.keep)+ .filter(f => f.keep);+ // Drop fallback faces the page doesn't use for text, and once headings or+ // body are covered, fonts nobody could place.+ const isFallback = (f: (typeof realFonts)[number]) =>+ FALLBACK_FONTS.has(f.family.toLowerCase()) && !f.measured;+ const withoutFallbacks = realFonts.some(f => !isFallback(f))+ ? realFonts.filter(f => !isFallback(f))+ : realFonts;+ const placed = withoutFallbacks.some(+ f => f.role === "heading" || f.role === "body",+ );+ const cleanedFonts = withoutFallbacks+ .filter(f => !placed || f.role !== "unknown").slice(0, 5).map(({ family, role }) => ({ family, role }));