Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(branding): shadow Jev on LLM-answered branding requests (#4968)
Abimael Martell0cd63e62a35d
diff --git a/apps/api/.env.example b/apps/api/.env.exampleindex 7e0b3da9a..767e75161 100644--- a/apps/api/.env.example+++ b/apps/api/.env.example@@ -79,6 +79,8 @@ TYPESAFE_API_KEY=# BRANDING_JEV_ROLLOUT_PERCENT=0# BRANDING_JEV_TIMEOUT_MS=5000# BRANDING_JEV_ESCALATE_BELOW=+# Share of LLM-answered branding requests that also ask Jev in the background to compare answers (response unchanged)+# BRANDING_JEV_SHADOW_PERCENT=0BULL_AUTH_KEY=@# set if you have a llamaparse key you'd like to use to parse pdfsLLAMAPARSE_API_KEY=diff --git a/apps/api/src/__tests__/lib/branding/jev-shadow.test.ts b/apps/api/src/__tests__/lib/branding/jev-shadow.test.tsnew file mode 100644index 000000000..f11bed4a2--- /dev/null+++ b/apps/api/src/__tests__/lib/branding/jev-shadow.test.ts@@ -0,0 +1,346 @@+import { vi, describe, it, expect, beforeEach, afterEach } from "vitest";+import type { Mock } from "vitest";+import { generateObject } from "ai";++const mocks = vi.hoisted(() => ({ systemOne: vi.fn() }));++vi.mock("@typesafe-ai/sdk", async importOriginal => ({+ ...(await importOriginal<typeof import("@typesafe-ai/sdk")>()),+ TypeSafeClient: class {+ systemOne = mocks.systemOne;+ },+}));+vi.mock("ai", async importOriginal => ({+ ...(await importOriginal<typeof import("ai")>()),+ generateObject: vi.fn(),+}));+vi.mock("../../../lib/generic-ai", () => ({+ getModel: vi.fn((name: string) => ({ modelId: name })),+}));++import { config } from "../../../config";+import { compareBrandingAnswers } from "../../../lib/branding/jev-shadow";+import { enhanceBrandingWithLLM } from "../../../lib/branding/llm";+import { BrandingEnhancement } from "../../../lib/branding/schema";+import { BrandingLLMInput } from "../../../lib/branding/types";+import { CostTracking } from "../../../lib/cost-tracking";+import { logger } from "../../../lib/logger";+import * as tracer from "../../../lib/otel-tracer";++const LOGO = {+ src: "https://acme.test/logo.svg",+ alt: "Acme",+ isSvg: true,+ isVisible: true,+ location: "header" as const,+ position: { top: 10, left: 10, width: 120, height: 32 },+ indicators: {+ inHeader: true,+ altMatch: true,+ srcMatch: true,+ classMatch: false,+ hrefMatch: true,+ },+ href: "/",+ source: "header a img",+};++const input = (+ costTracking: CostTracking,+ overrides: Partial<BrandingLLMInput> = {},+): BrandingLLMInput => ({+ jsAnalysis: {+ colorScheme: "light",+ colors: {+ primary: "#6D28D9",+ background: "#FFFFFF",+ textPrimary: "#111111",+ },+ images: { logo: "https://acme.test/old-logo.svg" },+ fonts: [{ family: "Inter", count: 40 }],+ },+ buttons: [+ {+ index: 0,+ text: "Get started",+ html: "",+ classes: "btn",+ background: "#6D28D9",+ textColor: "#FFFFFF",+ },+ {+ index: 1,+ text: "Contact sales",+ html: "",+ classes: "btn-outline",+ background: "transparent",+ textColor: "#111111",+ borderColor: "#111111",+ },+ ],+ logoCandidates: [LOGO],+ brandName: "Acme",+ url: "https://acme.test/",+ teamId: "team-other",+ scrapeId: "scrape-1",+ costTracking,+ logger,+ ...overrides,+});++const LLM_ANSWER: BrandingEnhancement = {+ cleanedFonts: [{ family: "Inter", role: "body" }],+ buttonClassification: {+ primaryButtonIndex: 0,+ primaryButtonReasoning: "violet CTA",+ secondaryButtonIndex: 1,+ secondaryButtonReasoning: "outline",+ confidence: 0.9,+ },+ colorRoles: {+ primaryColor: "#6D28D9",+ accentColor: "#6D28D9",+ backgroundColor: "#FFFFFF",+ textPrimary: "#111111",+ confidence: 0.9,+ },+ logoSelection: {+ selectedLogoIndex: 0,+ selectedLogoReasoning: "header logo",+ confidence: 0.9,+ },+};++// Jev agrees on everything except the logo, which it says is not there.+const jevResponse = () => ({+ model: "jev-1.13.0",+ answers: {+ logo: {+ type: "choice",+ choice: "none",+ probabilities: { logo_0: 0.2, none: 0.8 },+ confidence: 0.8,+ },+ primary_button: {+ type: "choice",+ choice: "button_0",+ probabilities: { button_0: 0.9, button_1: 0.1 },+ confidence: 0.9,+ },+ secondary_button: {+ type: "choice",+ choice: "button_1",+ probabilities: { button_0: 0.1, button_1: 0.9, none: 0 },+ confidence: 0.9,+ },+ primary_color: {+ type: "choice",+ choice: "color_0",+ probabilities: { color_0: 0.9 },+ confidence: 0.9,+ },+ accent_color: {+ type: "choice",+ choice: "color_0",+ probabilities: { color_0: 0.9 },+ confidence: 0.9,+ },+ background_color: {+ type: "choice",+ choice: "color_1",+ probabilities: { color_1: 0.9 },+ confidence: 0.9,+ },+ text_color: {+ type: "choice",+ choice: "color_2",+ probabilities: { color_2: 0.9 },+ confidence: 0.9,+ },+ font_0_is_brand: { type: "noul", noul: 0.9 },+ },+ usage: { input_tokens: 2000, output_tokens: 60 },+});++const saved = {+ key: config.TYPESAFE_API_KEY,+ teamIds: config.BRANDING_JEV_TEAM_IDS,+ shadow: config.BRANDING_JEV_SHADOW_PERCENT,+};++let attributes: Record<string, unknown>[];++beforeEach(() => {+ config.TYPESAFE_API_KEY = "ts-test";+ config.BRANDING_JEV_TEAM_IDS = ["team-jev"];+ config.BRANDING_JEV_SHADOW_PERCENT = 100;+ mocks.systemOne.mockReset().mockResolvedValue(jevResponse());+ (generateObject as Mock).mockReset().mockResolvedValue({+ object: structuredClone(LLM_ANSWER),+ usage: { inputTokens: 6000, outputTokens: 400 },+ });+ attributes = [];+ vi.spyOn(tracer, "setSpanAttributes").mockImplementation((_span, attrs) => {+ attributes.push(attrs as Record<string, unknown>);+ });+});++afterEach(() => {+ config.TYPESAFE_API_KEY = saved.key;+ config.BRANDING_JEV_TEAM_IDS = saved.teamIds;+ config.BRANDING_JEV_SHADOW_PERCENT = saved.shadow;+ vi.restoreAllMocks();+});++const shadowOutcome = () =>+ attributes.find(a => "branding.shadow.outcome" in a);++describe("Jev branding shadow", () => {+ it("asks Jev in the background and still returns the LLM's answer", async () => {+ const costTracking = new CostTracking();+ const withSpan = vi.spyOn(tracer, "withSpan");++ const result = await enhanceBrandingWithLLM(input(costTracking));++ expect(result).toEqual(LLM_ANSWER);+ await vi.waitFor(() => expect(shadowOutcome()).toBeDefined());+ expect(shadowOutcome()).toMatchObject({+ "branding.shadow.outcome": "compared",+ "branding.shadow.logo_agree": false,+ "branding.shadow.button_primary_agree": true,+ "branding.shadow.button_secondary_agree": true,+ "branding.shadow.color_primary_agree": true,+ "branding.shadow.color_background_agree": true,+ "branding.shadow.fonts_overlap": 1,+ });+ expect(+ attributes.find(a => "branding.shadow.llm_model" in a),+ ).toMatchObject({+ "branding.shadow.llm_model": "gpt-4o",+ "branding.shadow.team_id": "team-other",+ });+ // The shadow's cost is not the scrape's.+ expect(costTracking.calls.map(c => c.metadata.method)).toEqual([+ "enhanceBrandingWithLLM",+ ]);+ const jevSpan = withSpan.mock.calls.find(+ c => c[0] === "typesafe.systemone",+ );+ expect(jevSpan?.[2]?.attributes).toMatchObject({+ "branding.jev.shadow": true,+ });+ });++ it("leaves the caller's profile and answer untouched", async () => {+ const request = input(new CostTracking());+ const profileBefore = structuredClone(request.jsAnalysis);++ const result = await enhanceBrandingWithLLM(request);+ const answerBefore = structuredClone(result);+ await vi.waitFor(() => expect(shadowOutcome()).toBeDefined());++ // Merging Jev's "no logo" would delete images.logo on a shared profile.+ expect(request.jsAnalysis).toEqual(profileBefore);+ expect(result).toEqual(answerBefore);+ });++ it("records a failed Jev call without affecting the response", async () => {+ mocks.systemOne.mockReset().mockRejectedValue(new Error("rate limited"));+ const warn = vi.spyOn(logger, "warn");++ const result = await enhanceBrandingWithLLM(input(new CostTracking()));++ expect(result).toEqual(LLM_ANSWER);+ await vi.waitFor(() =>+ expect(shadowOutcome()).toEqual({+ "branding.shadow.outcome": "jev_failed",+ }),+ );+ // Logged by enhanceBrandingWithJev in shadow mode. A rejected call is+ // handled there, so the shadow's own catch ("Jev branding shadow failed")+ // never runs.+ expect(warn).toHaveBeenCalledWith(+ "Jev branding shadow call failed",+ expect.anything(),+ );+ });++ it("skips zero-data-retention requests", async () => {+ await enhanceBrandingWithLLM(+ input(new CostTracking(), { zeroDataRetention: true }),+ );+ await new Promise(r => setTimeout(r, 20));+ expect(mocks.systemOne).not.toHaveBeenCalled();+ });++ it("does nothing at 0%", async () => {+ config.BRANDING_JEV_SHADOW_PERCENT = 0;+ await enhanceBrandingWithLLM(input(new CostTracking()));+ await new Promise(r => setTimeout(r, 20));+ expect(mocks.systemOne).not.toHaveBeenCalled();+ });++ it("does not shadow a request that Jev already answered", async () => {+ await enhanceBrandingWithLLM(+ input(new CostTracking(), { teamId: "team-jev", mode: "fast" }),+ );+ await new Promise(r => setTimeout(r, 20));+ expect(mocks.systemOne).toHaveBeenCalledTimes(1);+ expect(shadowOutcome()).toBeUndefined();+ });+});++describe("comparing branding answers", () => {+ const request = input(new CostTracking());++ it("treats identical answers as agreeing on every field", () => {+ const result = compareBrandingAnswers(request, LLM_ANSWER, LLM_ANSWER);+ expect(result.fields_agreeing).toBe(result.fields_compared);+ expect(result.fields_compared).toBe(7);+ });++ it("counts nearly identical colors as the same and different buttons as not", () => {+ const other: BrandingEnhancement = structuredClone(LLM_ANSWER);+ other.colorRoles.backgroundColor = "#FDFDFD";+ other.buttonClassification.primaryButtonIndex = 1;+ other.buttonClassification.secondaryButtonIndex = 0;++ const result = compareBrandingAnswers(request, LLM_ANSWER, other);++ expect(result.color_background_agree).toBe(true);+ expect(result.button_primary_agree).toBe(false);+ expect(result.button_secondary_agree).toBe(false);+ });++ it("compares fonts after merge, which keeps the page's fonts when an answer has none", () => {+ const noFonts: BrandingEnhancement = {+ ...structuredClone(LLM_ANSWER),+ cleanedFonts: [],+ };++ const result = compareBrandingAnswers(request, noFonts, LLM_ANSWER);++ expect(result.fonts_overlap).toBe(1);+ });++ it("treats buttons that differ only in shape as different", () => {+ const shaped = input(new CostTracking());+ shaped.buttons = [+ { ...shaped.buttons![0], borderRadius: "4px" },+ shaped.buttons![1],+ {+ ...shaped.buttons![0],+ index: 2,+ text: "Start free trial",+ borderRadius: "999px",+ },+ ];+ const pill: BrandingEnhancement = structuredClone(LLM_ANSWER);+ pill.buttonClassification.primaryButtonIndex = 2;++ const result = compareBrandingAnswers(shaped, LLM_ANSWER, pill);++ expect(result.button_primary_agree).toBe(false);+ expect(result.button_secondary_agree).toBe(true);+ });+});diff --git a/apps/api/src/config.ts b/apps/api/src/config.tsindex 4b16b2ddb..eca0463b6 100644--- a/apps/api/src/config.ts+++ b/apps/api/src/config.ts@@ -561,6 +561,10 @@ const configSchema = z.object({BRANDING_JEV_ESCALATE_BELOW: emptyStringAsUndefined(z.coerce.number().min(0).max(1),),+ // Share of LLM-answered branding requests (0-100) that also ask Jev in the+ // background and record how its answer compares (lib/branding/jev-shadow.ts).+ // Customers always get the LLM's answer.+ BRANDING_JEV_SHADOW_PERCENT: z.coerce.number().min(0).max(100).default(0),// AI/MLMODEL_NAME: z.string().optional(),diff --git a/apps/api/src/lib/branding/jev-shadow.ts b/apps/api/src/lib/branding/jev-shadow.tsnew file mode 100644index 000000000..21a57694e--- /dev/null+++ b/apps/api/src/lib/branding/jev-shadow.ts@@ -0,0 +1,182 @@+import { differenceCiede2000, parse } from "culori";++import { config } from "../../config";+import { CostTracking } from "../cost-tracking";+import { setSpanAttributes, withSpan } from "../otel-tracer";+import { enhanceBrandingWithJev } from "./jev";+import { mergeBrandingResults } from "./merge";+import { BrandingEnhancement } from "./schema";+import { BrandingLLMInput } from "./types";+import { BrandingProfile } from "../../types/branding";++// Shadow mode: for a sample of requests the LLM answered, ask Jev the same+// questions in the background and record on a span how the two answers would+// differ in the response. Customers always get the LLM's answer.++// Colors closer than this (CIEDE2000) count as the same; brand-bench's default+// tolerance is 12.+const SAME_COLOR_DELTA_E = 10;+// Backstop for traffic bursts: past this many shadow calls in flight, skip.+const MAX_IN_FLIGHT = 50;++let inFlight = 0;++export function shouldShadowJev(input: BrandingLLMInput): boolean {+ if (!config.TYPESAFE_API_KEY) return false;+ // TypeSafe keeps nothing, but ZDR requests should not do extra work on+ // customer content that is not needed for their response.+ if (input.zeroDataRetention === true) return false;+ const percent = config.BRANDING_JEV_SHADOW_PERCENT;+ if (percent <= 0) return false;+ if (inFlight >= MAX_IN_FLIGHT) return false;+ return Math.random() * 100 < percent;+}++const deltaE = differenceCiede2000();++function sameColor(a?: string, b?: string): boolean {+ if (!a && !b) return true;+ const pa = a ? parse(a) : undefined;+ const pb = b ? parse(b) : undefined;+ if (!pa || !pb) return false;+ return deltaE(pa, pb) <= SAME_COLOR_DELTA_E;+}++const COLOR_ROLES = ["primary", "accent", "background", "textPrimary"] as const;++type ButtonStyle = NonNullable<+ NonNullable<BrandingProfile["components"]>["buttonPrimary"]+>;++// Everything the response carries for a button.+function sameButton(a?: ButtonStyle, b?: ButtonStyle): boolean {+ if (!a && !b) return true;+ if (!a || !b) return false;+ const sameText = (x?: string, y?: string) => (x || "none") === (y || "none");+ return (+ sameColor(a.background, b.background) &&+ sameColor(a.textColor, b.textColor) &&+ sameColor(a.borderColor, b.borderColor) &&+ sameText(a.borderRadius, b.borderRadius) &&+ sameText(a.shadow, b.shadow)+ );+}++// Merge keeps the page's own fonts when an answer cleans none, so compare the+// merged profiles, not the raw answers.+function fontOverlap(a: BrandingProfile, b: BrandingProfile): number {+ const names = (p: BrandingProfile) =>+ new Set((p.fonts ?? []).map(f => f.family.toLowerCase()));+ const x = names(a);+ const y = names(b);+ const union = new Set([...x, ...y]);+ if (union.size === 0) return 1;+ return [...x].filter(f => y.has(f)).length / union.size;+}++/**+ * Agreement between two answers to the same branding questions, measured on+ * what the response would contain after merge: logo, button styles, color roles+ * and fonts.+ */+export function compareBrandingAnswers(+ input: Pick<BrandingLLMInput, "jsAnalysis" | "buttons" | "logoCandidates">,+ llm: BrandingEnhancement,+ jev: BrandingEnhancement,+): Record<string, boolean | number> {+ // merge edits nested objects of the profile it is given; each side gets its own copy.+ const merged = (answer: BrandingEnhancement): BrandingProfile =>+ mergeBrandingResults(+ structuredClone(input.jsAnalysis),+ structuredClone(answer),+ structuredClone(input.buttons ?? []),+ structuredClone(input.logoCandidates),+ );+ const a = merged(llm);+ const b = merged(jev);++ const result: Record<string, boolean | number> = {};+ if ((input.logoCandidates?.length ?? 0) > 0) {+ result.logo_agree = (a.images?.logo ?? null) === (b.images?.logo ?? null);+ }+ result.button_primary_agree = sameButton(+ a.components?.buttonPrimary,+ b.components?.buttonPrimary,+ );+ result.button_secondary_agree = sameButton(+ a.components?.buttonSecondary,+ b.components?.buttonSecondary,+ );+ for (const role of COLOR_ROLES) {+ result[`color_${role}_agree`] = sameColor(+ a.colors?.[role],+ b.colors?.[role],+ );+ }+ result.fonts_overlap = fontOverlap(a, b);++ const flags = Object.values(result).filter(+ (v): v is boolean => typeof v === "boolean",+ );+ result.fields_compared = flags.length;+ result.fields_agreeing = flags.filter(Boolean).length;+ return result;+}++/**+ * Starts the shadow call and returns at once. Inputs are copied before+ * returning: the caller keeps editing the LLM answer and the profile.+ */+export function shadowJev(+ input: BrandingLLMInput,+ llm: BrandingEnhancement,+ llmModel: string,+): Promise<void> {+ const snapshot: BrandingLLMInput = {+ ...input,+ jsAnalysis: structuredClone(input.jsAnalysis),+ buttons: structuredClone(input.buttons),+ logoCandidates: structuredClone(input.logoCandidates),+ // Jev does not read these; don't keep large strings alive.+ screenshot: undefined,+ headerHtmlChunk: undefined,+ // Kept apart from the scrape's own cost record and cost limit.+ costTracking: new CostTracking(),+ };+ const llmAnswer = structuredClone(llm);+ inFlight++;++ return withSpan("branding.jev_shadow", async span => {+ setSpanAttributes(span, {+ feature: "branding",+ "branding.shadow.llm_model": llmModel,+ ...(input.teamId ? { "branding.shadow.team_id": input.teamId } : {}),+ ...(input.scrapeId ? { scrapeId: input.scrapeId } : {}),+ });+ const jev = await enhanceBrandingWithJev(snapshot, { shadow: true });+ if (!jev) {+ setSpanAttributes(span, { "branding.shadow.outcome": "jev_failed" });+ return;+ }+ const comparison = compareBrandingAnswers(+ snapshot,+ llmAnswer,+ jev.enhancement,+ );+ setSpanAttributes(span, {+ "branding.shadow.outcome": "compared",+ ...(jev.logoConfidence !== undefined+ ? { "branding.shadow.jev_logo_confidence": jev.logoConfidence }+ : {}),+ ...Object.fromEntries(+ Object.entries(comparison).map(([k, v]) => [`branding.shadow.${k}`, v]),+ ),+ });+ })+ .catch(error => {+ input.logger.warn("Jev branding shadow failed", { error });+ })+ .finally(() => {+ inFlight--;+ });+}diff --git a/apps/api/src/lib/branding/jev.ts b/apps/api/src/lib/branding/jev.tsindex 2f5a42e33..7e4e7b3fc 100644--- a/apps/api/src/lib/branding/jev.ts+++ b/apps/api/src/lib/branding/jev.ts@@ -766,6 +766,7 @@ function mapJevAnswers(*/export async function enhanceBrandingWithJev(input: BrandingLLMInput,+ options: { shadow?: boolean } = {},): Promise<JevBrandingResult | null> {const typesafe = getTypeSafeClient();if (!typesafe) return null;@@ -802,15 +803,21 @@ export async function enhanceBrandingWithJev(attributes: {feature: "branding","branding.jev.questions": Object.keys(request.questions).length,+ ...(options.shadow ? { "branding.jev.shadow": true } : {}),...(input.scrapeId ? { scrapeId: input.scrapeId } : {}),},},);} catch (error) {- input.logger.warn("Jev branding call failed, falling back to LLM", {- error,- elapsedMs: Date.now() - started,- });+ input.logger.warn(+ options.shadow+ ? "Jev branding shadow call failed"+ : "Jev branding call failed, falling back to LLM",+ {+ error,+ elapsedMs: Date.now() - started,+ },+ );return null;}@@ -829,14 +836,20 @@ export async function enhanceBrandingWithJev(result = mapJevAnswers(request, response);} catch (error) {// A successful call whose answers don't have the expected shape.- input.logger.warn("Jev branding answers unusable, falling back to LLM", {- error,- model: response.model,- });+ input.logger.warn(+ options.shadow+ ? "Jev branding shadow answers unusable"+ : "Jev branding answers unusable, falling back to LLM",+ {+ error,+ model: response.model,+ },+ );return null;}input.logger.info("Jev branding call", {model: response.model,+ shadow: options.shadow === true,elapsedMs: Date.now() - started,inputTokens,questions: Object.keys(request.questions).length,diff --git a/apps/api/src/lib/branding/llm.ts b/apps/api/src/lib/branding/llm.tsindex f96779e39..e3ad72337 100644--- a/apps/api/src/lib/branding/llm.ts+++ b/apps/api/src/lib/branding/llm.ts@@ -5,6 +5,7 @@ import { calculateCost } from "../../scraper/scrapeURL/transformers/llmExtract";import { CostLimitExceededError } from "../cost-tracking";import { BrandingEnhancement, getBrandingEnhancementSchema } from "./schema";import { enhanceBrandingWithJev, isJevBrandingEnabled } from "./jev";+import { shadowJev, shouldShadowJev } from "./jev-shadow";import { buildBrandingPrompt } from "./prompt";import { BrandingLLMInput } from "./types";import { getModel } from "../generic-ai";@@ -107,7 +108,8 @@ export async function enhanceBrandingWithLLM(): Promise<BrandingEnhancement> {const logger = input.logger;- if (isJevBrandingEnabled(input)) {+ const triedJev = isJevBrandingEnabled(input);+ if (triedJev) {const jev = await enhanceBrandingWithJev(input);const escalateBelow = config.BRANDING_JEV_ESCALATE_BELOW;const unsureOfLogo =@@ -284,12 +286,16 @@ export async function enhanceBrandingWithLLM(}// When there are no logo candidates, do not pass logoSelection so downstream treats it as "none"- const resultObject = result.object as BrandingEnhancement;- if (!hasLogoCandidates && resultObject?.logoSelection != null) {- const { logoSelection: _, ...rest } = resultObject;- return rest as BrandingEnhancement;+ let answer = result.object as BrandingEnhancement;+ if (!hasLogoCandidates && answer?.logoSelection != null) {+ const { logoSelection: _, ...rest } = answer;+ answer = rest as BrandingEnhancement;}- return resultObject;+ // Requests that already went to Jev have nothing to compare against.+ if (!triedJev && answer && shouldShadowJev(input)) {+ void shadowJev(input, answer, modelId);+ }+ return answer;} catch (error) {if (error instanceof CostLimitExceededError) {throw error;