Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(api): make branding script null-safe when document.body is missing (#5443)
Abimael Martellcf938aebf314
diff --git a/apps/api/src/__tests__/lib/branding/branding-script-no-body.test.ts b/apps/api/src/__tests__/lib/branding/branding-script-no-body.test.tsnew file mode 100644index 000000000..145c5119d--- /dev/null+++ b/apps/api/src/__tests__/lib/branding/branding-script-no-body.test.ts@@ -0,0 +1,51 @@+import { describe, it, expect } from "vitest";+import { JSDOM } from "jsdom";++import { getBrandingScript } from "../../../scraper/scrapeURL/engines/fire-engine/brandingScript";++// The branding script can run while a page is still mid-navigation, before+// <body> (or even <html>) has been parsed. It must degrade, not throw.+describe("bundled branding script without a document body", () => {+ const load = (html: string) =>+ new JSDOM(html, { runScripts: "outside-only", url: "https://acme.test/" })+ .window as any;++ it("returns a result when document.body is null", () => {+ const window = load(+ `<html class="dark"><head><title>Acme | Home</title></head>+ <body><h1>Acme</h1><p>Hello</p><button>Go</button></body></html>`,+ );+ window.document.body.remove();+ expect(window.document.body).toBeNull();++ const result = window.eval(getBrandingScript());++ expect(result?.branding?.brandName).toBe("Acme");+ // Scheme indicators on <html> still count when <body> is missing.+ expect(result?.branding?.colorScheme).toBe("dark");+ expect(result?.branding?.errors).toBeUndefined();+ });++ it("falls back to light when no root carries a scheme indicator", () => {+ const window = load(+ `<html><head><title>Acme</title></head><body></body></html>`,+ );+ window.document.body.remove();++ const result = window.eval(getBrandingScript());++ expect(result?.branding?.colorScheme).toBe("light");+ expect(result?.branding?.errors).toBeUndefined();+ });++ it("returns a result when the document has no root element at all", () => {+ const window = load(`<html><head><title>Acme</title></head></html>`);+ window.document.documentElement.remove();+ expect(window.document.documentElement).toBeNull();+ expect(window.document.body).toBeNull();++ expect(() => window.eval(getBrandingScript())).not.toThrow();+ const result = window.eval(getBrandingScript());+ expect(result?.branding?.colorScheme).toBe("light");+ });+});diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/branding-script/brand-utils.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/branding-script/brand-utils.tsindex 0e3e1505f..b595ced5d 100644--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/branding-script/brand-utils.ts+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/branding-script/brand-utils.ts@@ -72,26 +72,29 @@ export const detectFrameworkHints = (): string[] => {};export const detectColorScheme = (): "dark" | "light" => {- const body = document.body;- const html = document.documentElement;+ // Either root may be null when the script runs mid-navigation, before the+ // document (or its <body>) has been parsed. Fall back to whichever exists,+ // then to prefers-color-scheme.+ const html: Element | null = document.documentElement;+ const body: Element | null = document.body;+ const roots = [html, body].filter((el): el is Element => el != null);++ const hasClass = (name: string): boolean =>+ roots.some(el => el.classList.contains(name));+ const hasAttr = (attr: string, value: string): boolean =>+ roots.some(el => el.getAttribute(attr) === value);const hasDarkIndicator =- html.classList.contains("dark") ||- body.classList.contains("dark") ||- html.classList.contains("dark-mode") ||- body.classList.contains("dark-mode") ||- html.getAttribute("data-theme") === "dark" ||- body.getAttribute("data-theme") === "dark" ||- html.getAttribute("data-bs-theme") === "dark";+ hasClass("dark") ||+ hasClass("dark-mode") ||+ hasAttr("data-theme", "dark") ||+ html?.getAttribute("data-bs-theme") === "dark";const hasLightIndicator =- html.classList.contains("light") ||- body.classList.contains("light") ||- html.classList.contains("light-mode") ||- body.classList.contains("light-mode") ||- html.getAttribute("data-theme") === "light" ||- body.getAttribute("data-theme") === "light" ||- html.getAttribute("data-bs-theme") === "light";+ hasClass("light") ||+ hasClass("light-mode") ||+ hasAttr("data-theme", "light") ||+ html?.getAttribute("data-bs-theme") === "light";let prefersDark = false;try {@@ -129,8 +132,8 @@ export const detectColorScheme = (): "dark" | "light" => {return null;};- const bodyBg = getEffectiveBackground(body);- const htmlBg = getEffectiveBackground(html);+ const bodyBg = body ? getEffectiveBackground(body) : null;+ const htmlBg = html ? getEffectiveBackground(html) : null;const effectiveBg = bodyBg || htmlBg;if (effectiveBg) {@@ -287,10 +290,12 @@ export const getBackgroundCandidates = (): BackgroundCandidate[] => {}});- const bodyBg = getComputedStyleCached(document.body).backgroundColor;- const htmlBg = getComputedStyleCached(- document.documentElement,- ).backgroundColor;+ // body/html may be null mid-navigation; a missing root contributes no+ // candidate rather than silently aliasing the other root's background.+ const bodyEl: Element | null = document.body;+ const htmlEl: Element | null = document.documentElement;+ const bodyBg = bodyEl ? getComputedStyleCached(bodyEl).backgroundColor : "";+ const htmlBg = htmlEl ? getComputedStyleCached(htmlEl).backgroundColor : "";if (isValidBackgroundColor(bodyBg)) {const normalized = normalizeColor(bodyBg);@@ -332,7 +337,9 @@ export const getBackgroundCandidates = (): BackgroundCandidate[] => {}try {- const rootStyle = getComputedStyleCached(document.documentElement);+ const rootStyle = getComputedStyleCached(+ document.documentElement as Element,+ );const cssVars = ["--background",diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/branding-script/helpers.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/branding-script/helpers.tsindex f1339f236..cd1c7db71 100644--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/branding-script/helpers.ts+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/branding-script/helpers.ts@@ -80,13 +80,26 @@ function getComputedStyleSafe(el: Element): CSSStyleDeclaration {}),),);- return nativeGetComputedStyle(document.documentElement);+ return getRootStyle();+ }+}++// Computed style of <html>, or an empty style when the document has no root+// yet (script ran before the page was parsed).+function getRootStyle(): CSSStyleDeclaration {+ const root: Element | null = document.documentElement;+ if (!root) return {} as CSSStyleDeclaration;+ try {+ return nativeGetComputedStyle(root);+ } catch (e) {+ recordError("getComputedStyle:root", e);+ return {} as CSSStyleDeclaration;}}export const getComputedStyleCached = (el: Element): CSSStyleDeclaration => {if (!el || typeof el !== "object" || !(el instanceof Element)) {- return getComputedStyleSafe(document.documentElement);+ return getRootStyle();}if (styleCache.has(el)) {return styleCache.get(el)!;@@ -101,21 +114,16 @@ export const toPx = (v: string | null | undefined): number | null => {if (!v || v === "auto") return null;if (v.endsWith("px")) return parseFloat(v);if (v.endsWith("rem"))+ return parseFloat(v) * parseFloat(getRootStyle().fontSize || "16");+ if (v.endsWith("em")) {+ const base: Element | null = document.body ?? document.documentElement;return (parseFloat(v) *parseFloat(- getComputedStyleSafe(document.documentElement).fontSize || "16",- )- );- if (v.endsWith("em"))- return (- parseFloat(v) *- parseFloat(- getComputedStyleSafe(- document.body ?? (document.documentElement as unknown as Element),- ).fontSize || "16",+ (base ? getComputedStyleSafe(base) : getRootStyle()).fontSize || "16",));+ }if (v.endsWith("%")) return null;const num = parseFloat(v);return Number.isFinite(num) ? num : null;