Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(branding): answer branding with Jev by default, with one off switch (#5175)
Abimael Martell80fc24271f90
diff --git a/apps/api/.env.example b/apps/api/.env.exampleindex e848b75a9..345b57ade 100644--- a/apps/api/.env.example+++ b/apps/api/.env.example@@ -70,17 +70,11 @@ RATE_LIMIT_TEST_API_KEY_CRAWL=SCRAPING_BEE_API_KEY=# add for LLM dependent features (image alt generation, etc.)OPENAI_API_KEY=-# TypeSafe Jev: the search safe filter, and branding decisions when BRANDING_JEV, BRANDING_JEV_TEAM_IDS or BRANDING_JEV_ROLLOUT_PERCENT selects the team+# TypeSafe Jev: the search safe filter, and branding decisions (with the LLM as fallback)TYPESAFE_API_KEY=-# Branding decisions with Jev: every team, listed teams (comma-separated ids), a stable share of teams (0-100),-# the per-call timeout, and an optional logo-confidence bar (0-1) below which branding reruns on the LLM+# Set BRANDING_JEV=false to send branding to the LLM instead; the per-call Jev timeout# BRANDING_JEV=false-# BRANDING_JEV_TEAM_IDS=-# BRANDING_JEV_ROLLOUT_PERCENT=0# BRANDING_JEV_TIMEOUT_MS=5000-# BRANDING_JEV_ESCALATE_BELOW=-# Share of LLM-answered branding requests that also ask Jev in the background to compare answers (response unchanged)-# BRANDING_JEV_SHADOW_PERCENT=0# optional OpenAI-compatible provider for JSON extraction; routed by team cohort# at ROLLOUT_PERCENT, mirrored at SHADOW_PERCENT, with per-request fallbackEXTRACTION_PROVIDER_BASE_URL=diff --git a/apps/api/src/__tests__/lib/branding/jev-shadow.test.ts b/apps/api/src/__tests__/lib/branding/jev-shadow.test.tsdeleted file mode 100644index 7175d758a..000000000--- a/apps/api/src/__tests__/lib/branding/jev-shadow.test.ts+++ /dev/null@@ -1,366 +0,0 @@-import { vi, describe, it, expect, beforeEach, afterEach } from "vitest";-import type { Mock } from "vitest";-import { generateObject } from "ai";--const mocks = vi.hoisted(() => ({ systemOne: vi.fn() }));--vi.mock("@typesafe-ai/sdk", async importOriginal => ({- ...(await importOriginal<typeof import("@typesafe-ai/sdk")>()),- TypeSafeClient: class {- systemOne = mocks.systemOne;- },-}));-vi.mock("ai", async importOriginal => ({- ...(await importOriginal<typeof import("ai")>()),- generateObject: vi.fn(),-}));-vi.mock("../../../lib/generic-ai", () => ({- getModel: vi.fn((name: string) => ({ modelId: name })),-}));--import { config } from "../../../config";-import { resetJevBreaker } from "../../../lib/branding/jev";-import { compareBrandingAnswers } from "../../../lib/branding/jev-shadow";-import { enhanceBrandingWithLLM } from "../../../lib/branding/llm";-import { BrandingEnhancement } from "../../../lib/branding/schema";-import { BrandingLLMInput } from "../../../lib/branding/types";-import { CostTracking } from "../../../lib/cost-tracking";-import { logger } from "../../../lib/logger";-import * as tracer from "../../../lib/otel-tracer";--const LOGO = {- src: "https://acme.test/logo.svg",- alt: "Acme",- isSvg: true,- isVisible: true,- location: "header" as const,- position: { top: 10, left: 10, width: 120, height: 32 },- indicators: {- inHeader: true,- altMatch: true,- srcMatch: true,- classMatch: false,- hrefMatch: true,- },- href: "/",- source: "header a img",-};--const input = (- costTracking: CostTracking,- overrides: Partial<BrandingLLMInput> = {},-): BrandingLLMInput => ({- jsAnalysis: {- colorScheme: "light",- colors: {- primary: "#6D28D9",- background: "#FFFFFF",- textPrimary: "#111111",- },- images: { logo: "https://acme.test/old-logo.svg" },- fonts: [{ family: "Inter", count: 40 }],- },- buttons: [- {- index: 0,- text: "Get started",- html: "",- classes: "btn",- background: "#6D28D9",- textColor: "#FFFFFF",- },- {- index: 1,- text: "Contact sales",- html: "",- classes: "btn-outline",- background: "transparent",- textColor: "#111111",- borderColor: "#111111",- },- ],- logoCandidates: [LOGO],- brandName: "Acme",- url: "https://acme.test/",- teamId: "team-other",- scrapeId: "scrape-1",- costTracking,- logger,- ...overrides,-});--const LLM_ANSWER: BrandingEnhancement = {- cleanedFonts: [{ family: "Inter", role: "body" }],- buttonClassification: {- primaryButtonIndex: 0,- primaryButtonReasoning: "violet CTA",- secondaryButtonIndex: 1,- secondaryButtonReasoning: "outline",- confidence: 0.9,- },- colorRoles: {- primaryColor: "#6D28D9",- accentColor: "#6D28D9",- backgroundColor: "#FFFFFF",- textPrimary: "#111111",- confidence: 0.9,- },- logoSelection: {- selectedLogoIndex: 0,- selectedLogoReasoning: "header logo",- confidence: 0.9,- },-};--// Jev agrees on everything except the logo, which it says is not there.-const jevResponse = () => ({- model: "jev-1.13.0",- answers: {- logo: {- type: "choice",- choice: "none",- probabilities: { logo_0: 0.2, none: 0.8 },- confidence: 0.8,- },- primary_button: {- type: "choice",- choice: "button_0",- probabilities: { button_0: 0.9, button_1: 0.1 },- confidence: 0.9,- },- secondary_button: {- type: "choice",- choice: "button_1",- probabilities: { button_0: 0.1, button_1: 0.9, none: 0 },- confidence: 0.9,- },- primary_color: {- type: "choice",- choice: "color_0",- probabilities: { color_0: 0.9 },- confidence: 0.9,- },- accent_color: {- type: "choice",- choice: "color_0",- probabilities: { color_0: 0.9 },- confidence: 0.9,- },- background_color: {- type: "choice",- choice: "color_1",- probabilities: { color_1: 0.9 },- confidence: 0.9,- },- text_color: {- type: "choice",- choice: "color_2",- probabilities: { color_2: 0.9 },- confidence: 0.9,- },- font_0_is_brand: { type: "noul", noul: 0.9 },- },- usage: { input_tokens: 2000, output_tokens: 60 },-});--const saved = {- key: config.TYPESAFE_API_KEY,- teamIds: config.BRANDING_JEV_TEAM_IDS,- shadow: config.BRANDING_JEV_SHADOW_PERCENT,-};--let attributes: Record<string, unknown>[];--beforeEach(() => {- resetJevBreaker();- config.TYPESAFE_API_KEY = "ts-test";- config.BRANDING_JEV_TEAM_IDS = ["team-jev"];- config.BRANDING_JEV_SHADOW_PERCENT = 100;- mocks.systemOne.mockReset().mockResolvedValue(jevResponse());- (generateObject as Mock).mockReset().mockResolvedValue({- object: structuredClone(LLM_ANSWER),- usage: { inputTokens: 6000, outputTokens: 400 },- });- attributes = [];- vi.spyOn(tracer, "setSpanAttributes").mockImplementation((_span, attrs) => {- attributes.push(attrs as Record<string, unknown>);- });-});--afterEach(() => {- config.TYPESAFE_API_KEY = saved.key;- config.BRANDING_JEV_TEAM_IDS = saved.teamIds;- config.BRANDING_JEV_SHADOW_PERCENT = saved.shadow;- vi.restoreAllMocks();-});--const shadowOutcome = () =>- attributes.find(a => "branding.shadow.outcome" in a);--describe("Jev branding shadow", () => {- it("asks Jev in the background and still returns the LLM's answer", async () => {- const costTracking = new CostTracking();- const withSpan = vi.spyOn(tracer, "withSpan");-- const result = await enhanceBrandingWithLLM(input(costTracking));-- expect(result).toEqual(LLM_ANSWER);- await vi.waitFor(() => expect(shadowOutcome()).toBeDefined());- expect(shadowOutcome()).toMatchObject({- "branding.shadow.outcome": "compared",- "branding.shadow.logo_agree": false,- "branding.shadow.button_primary_agree": true,- "branding.shadow.button_secondary_agree": true,- "branding.shadow.color_primary_agree": true,- "branding.shadow.color_background_agree": true,- "branding.shadow.fonts_overlap": 1,- });- expect(- attributes.find(a => "branding.shadow.llm_model" in a),- ).toMatchObject({- "branding.shadow.llm_model": "gpt-4o",- "branding.shadow.team_id": "team-other",- });- // The shadow's cost is not the scrape's.- expect(costTracking.calls.map(c => c.metadata.method)).toEqual([- "enhanceBrandingWithLLM",- ]);- const jevSpan = withSpan.mock.calls.find(- c => c[0] === "typesafe.systemone",- );- expect(jevSpan?.[2]?.attributes).toMatchObject({- "branding.jev.shadow": true,- });- });-- it("leaves the caller's profile and answer untouched", async () => {- const request = input(new CostTracking());- const profileBefore = structuredClone(request.jsAnalysis);-- const result = await enhanceBrandingWithLLM(request);- const answerBefore = structuredClone(result);- await vi.waitFor(() => expect(shadowOutcome()).toBeDefined());-- // Merging Jev's "no logo" would delete images.logo on a shared profile.- expect(request.jsAnalysis).toEqual(profileBefore);- expect(result).toEqual(answerBefore);- });-- it("records a failed Jev call without affecting the response", async () => {- mocks.systemOne.mockReset().mockRejectedValue(new Error("rate limited"));- const warn = vi.spyOn(logger, "warn");-- const result = await enhanceBrandingWithLLM(input(new CostTracking()));-- expect(result).toEqual(LLM_ANSWER);- await vi.waitFor(() =>- expect(shadowOutcome()).toEqual({- "branding.shadow.outcome": "jev_failed",- }),- );- // Logged by enhanceBrandingWithJev in shadow mode. A rejected call is- // handled there, so the shadow's own catch ("Jev branding shadow failed")- // never runs.- expect(warn).toHaveBeenCalledWith(- "Jev branding shadow call failed",- expect.anything(),- );- });-- it("pauses while the Jev breaker is open", async () => {- mocks.systemOne- .mockReset()- .mockRejectedValue(new Error("529 high traffic"));- for (let i = 0; i < 5; i++) {- await enhanceBrandingWithLLM(input(new CostTracking()));- await vi.waitFor(() =>- expect(mocks.systemOne).toHaveBeenCalledTimes(i + 1),- );- }- await new Promise(r => setTimeout(r, 20));-- await enhanceBrandingWithLLM(input(new CostTracking()));- await new Promise(r => setTimeout(r, 20));-- expect(mocks.systemOne).toHaveBeenCalledTimes(5);- });-- it("skips zero-data-retention requests", async () => {- await enhanceBrandingWithLLM(- input(new CostTracking(), { zeroDataRetention: true }),- );- await new Promise(r => setTimeout(r, 20));- expect(mocks.systemOne).not.toHaveBeenCalled();- });-- it("does nothing at 0%", async () => {- config.BRANDING_JEV_SHADOW_PERCENT = 0;- await enhanceBrandingWithLLM(input(new CostTracking()));- await new Promise(r => setTimeout(r, 20));- expect(mocks.systemOne).not.toHaveBeenCalled();- });-- it("does not shadow a request that Jev already answered", async () => {- await enhanceBrandingWithLLM(- input(new CostTracking(), { teamId: "team-jev", mode: "fast" }),- );- await new Promise(r => setTimeout(r, 20));- expect(mocks.systemOne).toHaveBeenCalledTimes(1);- expect(shadowOutcome()).toBeUndefined();- });-});--describe("comparing branding answers", () => {- const request = input(new CostTracking());-- it("treats identical answers as agreeing on every field", () => {- const result = compareBrandingAnswers(request, LLM_ANSWER, LLM_ANSWER);- expect(result.fields_agreeing).toBe(result.fields_compared);- expect(result.fields_compared).toBe(7);- });-- it("counts nearly identical colors as the same and different buttons as not", () => {- const other: BrandingEnhancement = structuredClone(LLM_ANSWER);- other.colorRoles.backgroundColor = "#FDFDFD";- other.buttonClassification.primaryButtonIndex = 1;- other.buttonClassification.secondaryButtonIndex = 0;-- const result = compareBrandingAnswers(request, LLM_ANSWER, other);-- expect(result.color_background_agree).toBe(true);- expect(result.button_primary_agree).toBe(false);- expect(result.button_secondary_agree).toBe(false);- });-- it("compares fonts after merge, which keeps the page's fonts when an answer has none", () => {- const noFonts: BrandingEnhancement = {- ...structuredClone(LLM_ANSWER),- cleanedFonts: [],- };-- const result = compareBrandingAnswers(request, noFonts, LLM_ANSWER);-- expect(result.fonts_overlap).toBe(1);- });-- it("treats buttons that differ only in shape as different", () => {- const shaped = input(new CostTracking());- shaped.buttons = [- { ...shaped.buttons![0], borderRadius: "4px" },- shaped.buttons![1],- {- ...shaped.buttons![0],- index: 2,- text: "Start free trial",- borderRadius: "999px",- },- ];- const pill: BrandingEnhancement = structuredClone(LLM_ANSWER);- pill.buttonClassification.primaryButtonIndex = 2;-- const result = compareBrandingAnswers(shaped, LLM_ANSWER, pill);-- expect(result.button_primary_agree).toBe(false);- expect(result.button_secondary_agree).toBe(true);- });-});diff --git a/apps/api/src/__tests__/lib/branding/jev.test.ts b/apps/api/src/__tests__/lib/branding/jev.test.tsindex 56b248c20..fd20b41f7 100644--- a/apps/api/src/__tests__/lib/branding/jev.test.ts+++ b/apps/api/src/__tests__/lib/branding/jev.test.ts@@ -31,7 +31,6 @@ import {brandingRoutedTotal,brandingJevBreakerOpensTotal,brandingJevCallsTotal,- brandingJevRolloutPercent,} from "../../../lib/branding/metrics";import { mergeBrandingResults } from "../../../lib/branding/merge";import { BrandingLLMInput } from "../../../lib/branding/types";@@ -105,7 +104,6 @@ const baseInput = (costTracking: CostTracking): BrandingLLMInput => ({pageTitle: "Acme | Home",url: "https://acme.test/",teamId: "team-jev",- mode: "fast",costTracking,logger,});@@ -214,18 +212,12 @@ const jevResponse = (overrides: Record<string, unknown> = {}) => ({const saved = {key: config.TYPESAFE_API_KEY,global: config.BRANDING_JEV,- teamIds: config.BRANDING_JEV_TEAM_IDS,- rollout: config.BRANDING_JEV_ROLLOUT_PERCENT,- escalate: config.BRANDING_JEV_ESCALATE_BELOW,};beforeEach(() => {resetJevBreaker();config.TYPESAFE_API_KEY = "ts-test";config.BRANDING_JEV = undefined;- config.BRANDING_JEV_TEAM_IDS = ["team-jev"];- config.BRANDING_JEV_ROLLOUT_PERCENT = 0;- config.BRANDING_JEV_ESCALATE_BELOW = undefined;mocks.systemOne.mockReset();(generateObject as Mock).mockReset().mockResolvedValue({object: {@@ -253,9 +245,6 @@ afterEach(() => {vi.useRealTimers();config.TYPESAFE_API_KEY = saved.key;config.BRANDING_JEV = saved.global;- config.BRANDING_JEV_TEAM_IDS = saved.teamIds;- config.BRANDING_JEV_ROLLOUT_PERCENT = saved.rollout;- config.BRANDING_JEV_ESCALATE_BELOW = saved.escalate;});const respondWith = (body: unknown) =>@@ -875,36 +864,18 @@ describe("branding with Jev", () => {]);});- it("escalates to the LLM when Jev is unsure of the logo", async () => {- config.BRANDING_JEV_ESCALATE_BELOW = 0.6;- respondWith(- jevResponse({- logo: {- type: "choice",- choice: "logo_1",- probabilities: { logo_0: 0.45, logo_1: 0.5, none: 0.05 },- confidence: 0.3,- },- }),- );- const costTracking = new CostTracking();-- await enhanceBrandingWithLLM(baseInput(costTracking));-- expect(generateObject).toHaveBeenCalledTimes(1);- expect(costTracking.calls.map(c => c.metadata.method)).toEqual([- "enhanceBrandingWithJev",- "enhanceBrandingWithLLM",- ]);- });-- it("keeps unlisted teams off Jev", async () => {+ it("sends every team to Jev unless it's switched off", async () => {+ respondWith(jevResponse());await enhanceBrandingWithLLM({...baseInput(new CostTracking()),- teamId: "team-unlisted",+ teamId: "team-any",});+ expect(mocks.systemOne).toHaveBeenCalledTimes(1);+ expect(generateObject).not.toHaveBeenCalled();- expect(mocks.systemOne).not.toHaveBeenCalled();+ config.BRANDING_JEV = false;+ await enhanceBrandingWithLLM(baseInput(new CostTracking()));+ expect(mocks.systemOne).toHaveBeenCalledTimes(1);expect(generateObject).toHaveBeenCalledTimes(1);});@@ -942,51 +913,6 @@ describe("branding with Jev", () => {withSpan.mockRestore();});- it("keeps a listed team on the LLM unless the request asks for fast", async () => {- for (const mode of [undefined, "auto", "standard"] as const) {- await enhanceBrandingWithLLM({ ...baseInput(new CostTracking()), mode });- }- expect(mocks.systemOne).not.toHaveBeenCalled();- expect(generateObject).toHaveBeenCalledTimes(3);-- respondWith(jevResponse());- await enhanceBrandingWithLLM({- ...baseInput(new CostTracking()),- mode: "fast",- });- expect(mocks.systemOne).toHaveBeenCalledTimes(1);- });-- it("ignores mode for teams that aren't listed", async () => {- await enhanceBrandingWithLLM({- ...baseInput(new CostTracking()),- teamId: "team-unlisted",- mode: "fast",- });- expect(mocks.systemOne).not.toHaveBeenCalled();-- config.BRANDING_JEV = true;- respondWith(jevResponse());- await enhanceBrandingWithLLM({- ...baseInput(new CostTracking()),- teamId: "team-unlisted",- mode: "standard",- });- expect(mocks.systemOne).toHaveBeenCalledTimes(1);- });-- it("puts a rollout share of teams on Jev without listing them", async () => {- const unlisted = { ...baseInput(new CostTracking()), teamId: "team-a" };-- await enhanceBrandingWithLLM(unlisted);- expect(mocks.systemOne).not.toHaveBeenCalled();-- config.BRANDING_JEV_ROLLOUT_PERCENT = 100;- respondWith(jevResponse());- await enhanceBrandingWithLLM(unlisted);- expect(mocks.systemOne).toHaveBeenCalledTimes(1);- });-it("stays on the LLM when no TypeSafe key is configured", async () => {config.TYPESAFE_API_KEY = undefined;@@ -1155,10 +1081,10 @@ describe("branding metrics", () => {)?.value ?? 0;it("counts which model answered and how each Jev call ended", async () => {- const jevOk = { outcome: "ok", shadow: "false" };- const rateLimited = { outcome: "rate_limited", shadow: "false" };- const timedOut = { outcome: "timeout", shadow: "false" };- const byJev = { model: "jev", reason: "rollout" };+ const jevOk = { outcome: "ok" };+ const rateLimited = { outcome: "rate_limited" };+ const timedOut = { outcome: "timeout" };+ const byJev = { model: "jev", reason: "jev" };const fellBack = { model: "llm", reason: "jev_unavailable" };const before = {jevOk: await value(brandingJevCallsTotal, jevOk),@@ -1191,7 +1117,7 @@ describe("branding metrics", () => {});it("counts the breaker opening and the calls it skips", async () => {- const skipped = { outcome: "breaker_open", shadow: "false" };+ const skipped = { outcome: "breaker_open" };const opensBefore = await value(brandingJevBreakerOpensTotal);const skippedBefore = await value(brandingJevCallsTotal, skipped);mocks.systemOne.mockRejectedValue(new Error("529 high traffic"));@@ -1204,30 +1130,30 @@ describe("branding metrics", () => {expect(await value(brandingJevCallsTotal, skipped)).toBe(skippedBefore + 1);});- it("counts teams outside the rollout as answered by the LLM", async () => {- const outside = { model: "llm", reason: "not_in_rollout" };+ it("counts branding sent to the LLM while Jev is switched off", async () => {+ const outside = { model: "llm", reason: "jev_off" };const before = await value(brandingRoutedTotal, outside);- config.BRANDING_JEV_TEAM_IDS = [];+ config.BRANDING_JEV = false;await enhanceBrandingWithLLM(baseInput(new CostTracking()));expect(await value(brandingRoutedTotal, outside)).toBe(before + 1);expect(mocks.systemOne).not.toHaveBeenCalled();});-});--describe("rollout gauge", () => {- it("reports the share of teams this process sends to Jev", async () => {- const read = async () =>- (await brandingJevRolloutPercent.get()).values[0]?.value;- config.BRANDING_JEV_ROLLOUT_PERCENT = 75;- expect(await read()).toBe(75);+ it("counts branding without a TypeSafe key apart from the switch", async () => {+ const noKey = { model: "llm", reason: "jev_no_key" };+ const off = { model: "llm", reason: "jev_off" };+ const [noKeyBefore, offBefore] = [+ await value(brandingRoutedTotal, noKey),+ await value(brandingRoutedTotal, off),+ ];+ config.TYPESAFE_API_KEY = undefined;- config.BRANDING_JEV = true;- expect(await read()).toBe(100);+ await enhanceBrandingWithLLM(baseInput(new CostTracking()));- config.TYPESAFE_API_KEY = undefined;- expect(await read()).toBe(0);+ expect(await value(brandingRoutedTotal, noKey)).toBe(noKeyBefore + 1);+ expect(await value(brandingRoutedTotal, off)).toBe(offBefore);+ expect(mocks.systemOne).not.toHaveBeenCalled();});});diff --git a/apps/api/src/__tests__/snips/v2/scrape-branding.test.ts b/apps/api/src/__tests__/snips/v2/scrape-branding.test.tsindex 6611568bd..d5c4c5d08 100644--- a/apps/api/src/__tests__/snips/v2/scrape-branding.test.ts+++ b/apps/api/src/__tests__/snips/v2/scrape-branding.test.ts@@ -57,17 +57,13 @@ const isBrandingCall = (call: CostTrackingCall) =>call.metadata?.module === "branding" &&call.metadata?.method === "enhanceBrandingWithLLM";-// Whether the server could route a test team's branding to Jev (all teams, a-// listed team, or a rollout share). The LLM-only assertion below only holds-// when none of these is configured.-const JEV_MAY_APPLY =- !!process.env.TYPESAFE_API_KEY &&- (process.env.BRANDING_JEV === "true" ||- !!process.env.BRANDING_JEV_TEAM_IDS?.trim() ||- Number(process.env.BRANDING_JEV_ROLLOUT_PERCENT || 0) > 0);+// Jev answers branding whenever the server has a TypeSafe key, unless+// BRANDING_JEV=false. The LLM-only assertion below holds only when it doesn't.+const JEV_ON =+ !!process.env.TYPESAFE_API_KEY && process.env.BRANDING_JEV !== "false";describe("Branding cost tracking", () => {- concurrentIf(TEST_PRODUCTION && HAS_JOB_LOG && !JEV_MAY_APPLY)(+ concurrentIf(TEST_PRODUCTION && HAS_JOB_LOG && !JEV_ON)("records the branding LLM call with its model and cost",async () => {const response = await scrape(@@ -111,11 +107,7 @@ describe("Branding cost tracking", () => {);});-// Runs when the API server was started with TYPESAFE_API_KEY and-// BRANDING_JEV=true, which puts every team (including test identities, whose-// ids aren't known ahead of time) on Jev.-const JEV_ON =- !!process.env.TYPESAFE_API_KEY && process.env.BRANDING_JEV === "true";+// Runs when the API server answers branding with Jev (see JEV_ON above).describe("Branding with Jev", () => {concurrentIf(TEST_PRODUCTION && HAS_JOB_LOG && JEV_ON)(diff --git a/apps/api/src/config.ts b/apps/api/src/config.tsindex bf91f975d..23cdf0ba0 100644--- a/apps/api/src/config.ts+++ b/apps/api/src/config.ts@@ -570,24 +570,13 @@ const configSchema = z.object({DISABLE_BLOCKLIST: z.stringbool().optional(),FORCED_ENGINE_DOMAINS: z.string().optional(),DEBUG_BRANDING: z.stringbool().optional(),- // TypeSafe Jev for branding decisions (lib/branding/jev.ts); needs- // TYPESAFE_API_KEY. BRANDING_JEV turns it on for every team,- // BRANDING_JEV_TEAM_IDS (comma-separated) for listed teams, and- // BRANDING_JEV_ROLLOUT_PERCENT for a stable share of teams.+ // TypeSafe Jev answers branding decisions whenever TYPESAFE_API_KEY is set+ // (lib/branding/jev.ts), with the LLM as fallback. BRANDING_JEV=false sends+ // every branding request to the LLM instead.BRANDING_JEV: emptyStringAsUndefined(z.stringbool()),- BRANDING_JEV_TEAM_IDS: delimitedList(",").optional(),- BRANDING_JEV_ROLLOUT_PERCENT: z.coerce.number().min(0).max(100).default(0),BRANDING_JEV_TIMEOUT_MS: emptyStringAsUndefined(z.coerce.number().int().positive(),),- // Rerun branding on the LLM when Jev's logo confidence is below this (0-1).- BRANDING_JEV_ESCALATE_BELOW: emptyStringAsUndefined(- z.coerce.number().min(0).max(1),- ),- // Share of LLM-answered branding requests (0-100) that also ask Jev in the- // background and record how its answer compares (lib/branding/jev-shadow.ts).- // Customers always get the LLM's answer.- BRANDING_JEV_SHADOW_PERCENT: z.coerce.number().min(0).max(100).default(0),// Warn scrapes of search engine results pages (google.*/search, ...) that// /search returns those results directly (lib/serp-url.ts).// SERP_SCRAPE_WARNING turns it on for every team, SERP_SCRAPE_WARNING_TEAM_IDSdiff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.tsindex 8f6403f39..095e2e49c 100644--- a/apps/api/src/controllers/v2/types.ts+++ b/apps/api/src/controllers/v2/types.ts@@ -476,10 +476,8 @@ const queryFormatWithOptions = z.strictObject({type QueryFormatWithOptions = z.output<typeof queryFormatWithOptions>;-// Which engine answers branding decisions: "fast" (Jev), "standard" (the-// LLM) or "auto" (the LLM for now). Internal for now: honored only for teams-// listed in BRANDING_JEV_TEAM_IDS and ignored for everyone else-// (lib/branding/jev.ts).+// Accepted for compatibility and ignored: the same engine answers every+// branding request (lib/branding/jev.ts).const brandingFormatWithOptions = z.strictObject({type: z.literal("branding"),mode: z.enum(["auto", "fast", "standard"]).optional(),diff --git a/apps/api/src/lib/branding/jev-shadow.ts b/apps/api/src/lib/branding/jev-shadow.tsdeleted file mode 100644index 83a9a51ab..000000000--- a/apps/api/src/lib/branding/jev-shadow.ts+++ /dev/null@@ -1,184 +0,0 @@-import { differenceCiede2000, parse } from "culori";--import { config } from "../../config";-import { CostTracking } from "../cost-tracking";-import { setSpanAttributes, withSpan } from "../otel-tracer";-import { enhanceBrandingWithJev, isJevBreakerOpen } from "./jev";-import { mergeBrandingResults } from "./merge";-import { BrandingEnhancement } from "./schema";-import { BrandingLLMInput } from "./types";-import { BrandingProfile } from "../../types/branding";--// Shadow mode: for a sample of requests the LLM answered, ask Jev the same-// questions in the background and record on a span how the two answers would-// differ in the response. Customers always get the LLM's answer.--// Colors closer than this (CIEDE2000) count as the same; brand-bench's default-// tolerance is 12.-const SAME_COLOR_DELTA_E = 10;-// Backstop for traffic bursts: past this many shadow calls in flight, skip.-const MAX_IN_FLIGHT = 50;--let inFlight = 0;--export function shouldShadowJev(input: BrandingLLMInput): boolean {- if (!config.TYPESAFE_API_KEY) return false;- // TypeSafe keeps nothing, but ZDR requests should not do extra work on- // customer content that is not needed for their response.- if (input.zeroDataRetention === true) return false;- const percent = config.BRANDING_JEV_SHADOW_PERCENT;- if (percent <= 0) return false;- if (inFlight >= MAX_IN_FLIGHT) return false;- // No extra load on TypeSafe while branding has stopped calling it.- if (isJevBreakerOpen()) return false;- return Math.random() * 100 < percent;-}--const deltaE = differenceCiede2000();--function sameColor(a?: string, b?: string): boolean {- if (!a && !b) return true;- const pa = a ? parse(a) : undefined;- const pb = b ? parse(b) : undefined;- if (!pa || !pb) return false;- return deltaE(pa, pb) <= SAME_COLOR_DELTA_E;-}--const COLOR_ROLES = ["primary", "accent", "background", "textPrimary"] as const;--type ButtonStyle = NonNullable<- NonNullable<BrandingProfile["components"]>["buttonPrimary"]->;--// Everything the response carries for a button.-function sameButton(a?: ButtonStyle, b?: ButtonStyle): boolean {- if (!a && !b) return true;- if (!a || !b) return false;- const sameText = (x?: string, y?: string) => (x || "none") === (y || "none");- return (- sameColor(a.background, b.background) &&- sameColor(a.textColor, b.textColor) &&- sameColor(a.borderColor, b.borderColor) &&- sameText(a.borderRadius, b.borderRadius) &&- sameText(a.shadow, b.shadow)- );-}--// Merge keeps the page's own fonts when an answer cleans none, so compare the-// merged profiles, not the raw answers.-function fontOverlap(a: BrandingProfile, b: BrandingProfile): number {- const names = (p: BrandingProfile) =>- new Set((p.fonts ?? []).map(f => f.family.toLowerCase()));- const x = names(a);- const y = names(b);- const union = new Set([...x, ...y]);- if (union.size === 0) return 1;- return [...x].filter(f => y.has(f)).length / union.size;-}--/**- * Agreement between two answers to the same branding questions, measured on- * what the response would contain after merge: logo, button styles, color roles- * and fonts.- */-export function compareBrandingAnswers(- input: Pick<BrandingLLMInput, "jsAnalysis" | "buttons" | "logoCandidates">,- llm: BrandingEnhancement,- jev: BrandingEnhancement,-): Record<string, boolean | number> {- // merge edits nested objects of the profile it is given; each side gets its own copy.- const merged = (answer: BrandingEnhancement): BrandingProfile =>- mergeBrandingResults(- structuredClone(input.jsAnalysis),- structuredClone(answer),- structuredClone(input.buttons ?? []),- structuredClone(input.logoCandidates),- );- const a = merged(llm);- const b = merged(jev);-- const result: Record<string, boolean | number> = {};- if ((input.logoCandidates?.length ?? 0) > 0) {- result.logo_agree = (a.images?.logo ?? null) === (b.images?.logo ?? null);- }- result.button_primary_agree = sameButton(- a.components?.buttonPrimary,- b.components?.buttonPrimary,- );- result.button_secondary_agree = sameButton(- a.components?.buttonSecondary,- b.components?.buttonSecondary,- );- for (const role of COLOR_ROLES) {- result[`color_${role}_agree`] = sameColor(- a.colors?.[role],- b.colors?.[role],- );- }- result.fonts_overlap = fontOverlap(a, b);-- const flags = Object.values(result).filter(- (v): v is boolean => typeof v === "boolean",- );- result.fields_compared = flags.length;- result.fields_agreeing = flags.filter(Boolean).length;- return result;-}--/**- * Starts the shadow call and returns at once. Inputs are copied before- * returning: the caller keeps editing the LLM answer and the profile.- */-export function shadowJev(- input: BrandingLLMInput,- llm: BrandingEnhancement,- llmModel: string,-): Promise<void> {- const snapshot: BrandingLLMInput = {- ...input,- jsAnalysis: structuredClone(input.jsAnalysis),- buttons: structuredClone(input.buttons),- logoCandidates: structuredClone(input.logoCandidates),- // Jev does not read these; don't keep large strings alive.- screenshot: undefined,- headerHtmlChunk: undefined,- // Kept apart from the scrape's own cost record and cost limit.- costTracking: new CostTracking(),- };- const llmAnswer = structuredClone(llm);- inFlight++;-- return withSpan("branding.jev_shadow", async span => {- setSpanAttributes(span, {- feature: "branding",- "branding.shadow.llm_model": llmModel,- ...(input.teamId ? { "branding.shadow.team_id": input.teamId } : {}),- ...(input.scrapeId ? { scrapeId: input.scrapeId } : {}),- });- const jev = await enhanceBrandingWithJev(snapshot, { shadow: true });- if (!jev) {- setSpanAttributes(span, { "branding.shadow.outcome": "jev_failed" });- return;- }- const comparison = compareBrandingAnswers(- snapshot,- llmAnswer,- jev.enhancement,- );- setSpanAttributes(span, {- "branding.shadow.outcome": "compared",- ...(jev.logoConfidence !== undefined- ? { "branding.shadow.jev_logo_confidence": jev.logoConfidence }- : {}),- ...Object.fromEntries(- Object.entries(comparison).map(([k, v]) => [`branding.shadow.${k}`, v]),- ),- });- })- .catch(error => {- input.logger.warn("Jev branding shadow failed", { error });- })- .finally(() => {- inFlight--;- });-}diff --git a/apps/api/src/lib/branding/jev.ts b/apps/api/src/lib/branding/jev.tsindex edd45801b..3064ab958 100644--- a/apps/api/src/lib/branding/jev.ts+++ b/apps/api/src/lib/branding/jev.ts@@ -10,7 +10,6 @@ import { formatHex, hsl, parse, wcagContrast } from "culori";import { config } from "../../config";import { setSpanAttributes, SpanKind, withSpan } from "../otel-tracer";-import { sampled } from "../rollout";import { getTypeSafeClient } from "../typesafe";import { APITimeoutError, RateLimitError } from "@typesafe-ai/sdk";import {@@ -59,7 +58,7 @@ const BREAKER_COOLDOWN_MS = 60_000;let consecutiveFailures = 0;let breakerOpenUntil = 0;-export function isJevBreakerOpen(now = Date.now()): boolean {+function isJevBreakerOpen(now = Date.now()): boolean {return now < breakerOpenUntil;}@@ -103,18 +102,9 @@ type JevBrandingResult = {logoConfidence?: number;};-export function isJevBrandingEnabled(input: BrandingLLMInput): boolean {- if (!config.TYPESAFE_API_KEY) return false;- // Listed teams stay on the LLM unless a request asks for mode "fast". Every- // other team ignores `mode` and follows the settings below.- if (input.teamId && config.BRANDING_JEV_TEAM_IDS?.includes(input.teamId)) {- return input.mode === "fast";- }- return (- config.BRANDING_JEV === true ||- (!!input.teamId &&- sampled(`team:${input.teamId}`, config.BRANDING_JEV_ROLLOUT_PERCENT))- );+/** Jev answers branding unless there's no TypeSafe key or BRANDING_JEV is off. */+export function isJevBrandingEnabled(): boolean {+ return !!config.TYPESAFE_API_KEY && config.BRANDING_JEV !== false;}// ---------------------------------------------------------------------------@@ -1022,13 +1012,11 @@ function mapJevAnswers(*/export async function enhanceBrandingWithJev(input: BrandingLLMInput,- options: { shadow?: boolean } = {},): Promise<JevBrandingResult | null> {const typesafe = getTypeSafeClient();if (!typesafe) return null;- const shadow = options.shadow ? "true" : "false";const count = (outcome: JevCallOutcome) =>- brandingJevCallsTotal.inc({ outcome, shadow });+ brandingJevCallsTotal.inc({ outcome });if (isJevBreakerOpen()) {count("breaker_open");return null;@@ -1069,7 +1057,6 @@ export async function enhanceBrandingWithJev(attributes: {feature: "branding","branding.jev.questions": Object.keys(request.questions).length,- ...(options.shadow ? { "branding.jev.shadow": true } : {}),...(input.scrapeId ? { scrapeId: input.scrapeId } : {}),...(input.teamId ? { teamId: input.teamId } : {}),},@@ -1079,10 +1066,7 @@ export async function enhanceBrandingWithJev(// Only TypeSafe's own failures say anything about its health.if (called) {recordJevFailure(input);- brandingJevDurationSeconds.observe(- { shadow },- (Date.now() - started) / 1000,- );+ brandingJevDurationSeconds.observe((Date.now() - started) / 1000);count(error instanceof RateLimitError? "rate_limited"@@ -1091,21 +1075,16 @@ export async function enhanceBrandingWithJev(: "error",);}- input.logger.warn(- options.shadow- ? "Jev branding shadow call failed"- : "Jev branding call failed, falling back to LLM",- {- error,- elapsedMs: Date.now() - started,- },- );+ input.logger.warn("Jev branding call failed, falling back to LLM", {+ error,+ elapsedMs: Date.now() - started,+ });return null;}// A call that started before the breaker opened and succeeds late doesn't// close it: the next failure after the cooldown should reopen it at once.if (!isJevBreakerOpen()) consecutiveFailures = 0;- brandingJevDurationSeconds.observe({ shadow }, (Date.now() - started) / 1000);+ brandingJevDurationSeconds.observe((Date.now() - started) / 1000);const inputTokens = response.usage?.input_tokens ?? 0;const outputTokens = response.usage?.output_tokens ?? 0;@@ -1123,21 +1102,15 @@ export async function enhanceBrandingWithJev(} catch (error) {// A successful call whose answers don't have the expected shape.count("unusable");- input.logger.warn(- options.shadow- ? "Jev branding shadow answers unusable"- : "Jev branding answers unusable, falling back to LLM",- {- error,- model: response.model,- },- );+ input.logger.warn("Jev branding answers unusable, falling back to LLM", {+ error,+ model: response.model,+ });return null;}count("ok");input.logger.info("Jev branding call", {model: response.model,- shadow: options.shadow === true,elapsedMs: Date.now() - started,inputTokens,questions: Object.keys(request.questions).length,diff --git a/apps/api/src/lib/branding/llm.ts b/apps/api/src/lib/branding/llm.tsindex 9cabc4814..0a0fb1378 100644--- a/apps/api/src/lib/branding/llm.ts+++ b/apps/api/src/lib/branding/llm.ts@@ -6,7 +6,6 @@ import { CostLimitExceededError } from "../cost-tracking";import { BrandingEnhancement, getBrandingEnhancementSchema } from "./schema";import { enhanceBrandingWithJev, isJevBrandingEnabled } from "./jev";import { brandingRoutedTotal } from "./metrics";-import { shadowJev, shouldShadowJev } from "./jev-shadow";import { buildBrandingPrompt } from "./prompt";import { BrandingLLMInput } from "./types";import { getModel } from "../generic-ai";@@ -109,30 +108,19 @@ export async function enhanceBrandingWithLLM(): Promise<BrandingEnhancement> {const logger = input.logger;- const triedJev = isJevBrandingEnabled(input);- if (!triedJev) {- brandingRoutedTotal.inc({ model: "llm", reason: "not_in_rollout" });- } else {- const jev = await enhanceBrandingWithJev(input);- const escalateBelow = config.BRANDING_JEV_ESCALATE_BELOW;- const unsureOfLogo =- escalateBelow !== undefined &&- jev?.logoConfidence !== undefined &&- jev.logoConfidence < escalateBelow;- if (jev && !unsureOfLogo) {- brandingRoutedTotal.inc({ model: "jev", reason: "rollout" });- return jev.enhancement;- }+ if (!isJevBrandingEnabled()) {brandingRoutedTotal.inc({model: "llm",- reason: jev ? "logo_escalation" : "jev_unavailable",+ // No key (self-hosted without TypeSafe) is not the switch turned off.+ reason: config.TYPESAFE_API_KEY ? "jev_off" : "jev_no_key",});+ } else {+ const jev = await enhanceBrandingWithJev(input);if (jev) {- logger.info("Jev unsure of the logo, escalating branding to the LLM", {- logoConfidence: jev.logoConfidence,- escalateBelow,- });+ brandingRoutedTotal.inc({ model: "jev", reason: "jev" });+ return jev.enhancement;}+ brandingRoutedTotal.inc({ model: "llm", reason: "jev_unavailable" });}const prompt = buildBrandingPrompt(input);@@ -301,10 +289,6 @@ export async function enhanceBrandingWithLLM(const { logoSelection: _, ...rest } = answer;answer = rest as BrandingEnhancement;}- // Requests that already went to Jev have nothing to compare against.- if (!triedJev && answer && shouldShadowJev(input)) {- void shadowJev(input, answer, modelId);- }return answer;} catch (error) {if (error instanceof CostLimitExceededError) {diff --git a/apps/api/src/lib/branding/metrics.ts b/apps/api/src/lib/branding/metrics.tsindex a9727a556..a1c31a4b1 100644--- a/apps/api/src/lib/branding/metrics.ts+++ b/apps/api/src/lib/branding/metrics.ts@@ -1,18 +1,17 @@-import { Counter, Gauge, Histogram } from "prom-client";-import { config } from "../../config";+import { Counter, Histogram } from "prom-client";// Fixed labels only: never team ids or anything from the page.export const brandingRoutedTotal = new Counter({name: "firecrawl_branding_routed_total",- help: "Each branding request once, by the model it was finally sent to and why: jev (rollout), or llm when the team isn't in the rollout, Jev was unavailable (failed or breaker open), or Jev was unsure of the logo; every Jev attempt is in firecrawl_branding_jev_calls_total",+ help: "Each branding request once, by the model it was finally sent to and why: jev, or llm when BRANDING_JEV=false (jev_off), no TYPESAFE_API_KEY is set (jev_no_key), or Jev was unavailable (failed or breaker open); every Jev attempt is in firecrawl_branding_jev_calls_total",labelNames: ["model", "reason"],});export const brandingJevCallsTotal = new Counter({name: "firecrawl_branding_jev_calls_total",- help: "Jev branding call outcomes; shadow calls are labelled separately",- labelNames: ["outcome", "shadow"],+ help: "Jev branding call outcomes",+ labelNames: ["outcome"],});export const brandingJevBreakerOpensTotal = new Counter({@@ -23,27 +22,10 @@ export const brandingJevBreakerOpensTotal = new Counter({export const brandingJevDurationSeconds = new Histogram({name: "firecrawl_branding_jev_duration_seconds",help: "Jev branding call time, for calls that reached TypeSafe",- labelNames: ["shadow"],// Up to the default timeout with its one retry.buckets: [0.1, 0.15, 0.2, 0.3, 0.5, 0.75, 1, 1.5, 2, 3, 5, 7.5, 10],});-// Each pod reports the share it was started with; pods that disagree are-// running an older config.-export const brandingJevRolloutPercent = new Gauge({- name: "firecrawl_branding_jev_rollout_percent",- help: "Share of teams this process sends to Jev, from its own config: 0 without a TypeSafe key, 100 with BRANDING_JEV on, else BRANDING_JEV_ROLLOUT_PERCENT",- collect() {- this.set(effectiveJevRolloutPercent());- },-});--function effectiveJevRolloutPercent(): number {- if (!config.TYPESAFE_API_KEY) return 0;- if (config.BRANDING_JEV === true) return 100;- return config.BRANDING_JEV_ROLLOUT_PERCENT;-}-export type JevCallOutcome =| "ok"| "timeout"@@ -54,23 +36,21 @@ export type JevCallOutcome =// Export quiet series before their first event, so rates read 0, not missing.for (const [model, reason] of [- ["jev", "rollout"],- ["llm", "not_in_rollout"],+ ["jev", "jev"],+ ["llm", "jev_off"],+ ["llm", "jev_no_key"],["llm", "jev_unavailable"],- ["llm", "logo_escalation"],]) {brandingRoutedTotal.inc({ model, reason }, 0);}-for (const shadow of ["true", "false"]) {- for (const outcome of [- "ok",- "timeout",- "rate_limited",- "error",- "unusable",- "breaker_open",- ] satisfies JevCallOutcome[]) {- brandingJevCallsTotal.inc({ outcome, shadow }, 0);- }+for (const outcome of [+ "ok",+ "timeout",+ "rate_limited",+ "error",+ "unusable",+ "breaker_open",+] satisfies JevCallOutcome[]) {+ brandingJevCallsTotal.inc({ outcome }, 0);}brandingJevBreakerOpensTotal.inc(0);diff --git a/apps/api/src/lib/branding/transformer.ts b/apps/api/src/lib/branding/transformer.tsindex 963dbd69a..09a4b1e82 100644--- a/apps/api/src/lib/branding/transformer.ts+++ b/apps/api/src/lib/branding/transformer.ts@@ -240,7 +240,6 @@ export async function brandingTransformer(scrapeId: meta.id,zeroDataRetention: meta.internalOptions.zeroDataRetention,teamFlags: meta.internalOptions.teamFlags,- mode: hasFormatOfType(meta.options.formats, "branding")?.mode,costTracking: meta.costTracking,logger: meta.logger,});diff --git a/apps/api/src/lib/branding/types.ts b/apps/api/src/lib/branding/types.tsindex d2e00e545..9f79bdb82 100644--- a/apps/api/src/lib/branding/types.ts+++ b/apps/api/src/lib/branding/types.ts@@ -103,8 +103,6 @@ export interface BrandingLLMInput {scrapeId?: string;zeroDataRetention?: boolean;teamFlags?: { debugBranding?: boolean } | null;- /** The branding format's `mode`; see isJevBrandingEnabled. */- mode?: "auto" | "fast" | "standard";costTracking: CostTracking;logger: Logger;}