Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat(branding): answer branding with Jev by default, with one off switch (#5175)

Abimael Martell80fc24271f90
diff --git a/apps/api/.env.example b/apps/api/.env.example
index e848b75a9..345b57ade 100644
--- a/apps/api/.env.example
+++ b/apps/api/.env.example
@@ -70,17 +70,11 @@ RATE_LIMIT_TEST_API_KEY_CRAWL=
SCRAPING_BEE_API_KEY=
# add for LLM dependent features (image alt generation, etc.)
OPENAI_API_KEY=
-# TypeSafe Jev: the search safe filter, and branding decisions when BRANDING_JEV, BRANDING_JEV_TEAM_IDS or BRANDING_JEV_ROLLOUT_PERCENT selects the team
+# TypeSafe Jev: the search safe filter, and branding decisions (with the LLM as fallback)
TYPESAFE_API_KEY=
-# Branding decisions with Jev: every team, listed teams (comma-separated ids), a stable share of teams (0-100),
-# the per-call timeout, and an optional logo-confidence bar (0-1) below which branding reruns on the LLM
+# Set BRANDING_JEV=false to send branding to the LLM instead; the per-call Jev timeout
# BRANDING_JEV=false
-# BRANDING_JEV_TEAM_IDS=
-# BRANDING_JEV_ROLLOUT_PERCENT=0
# BRANDING_JEV_TIMEOUT_MS=5000
-# BRANDING_JEV_ESCALATE_BELOW=
-# Share of LLM-answered branding requests that also ask Jev in the background to compare answers (response unchanged)
-# BRANDING_JEV_SHADOW_PERCENT=0
# optional OpenAI-compatible provider for JSON extraction; routed by team cohort
# at ROLLOUT_PERCENT, mirrored at SHADOW_PERCENT, with per-request fallback
EXTRACTION_PROVIDER_BASE_URL=
diff --git a/apps/api/src/__tests__/lib/branding/jev-shadow.test.ts b/apps/api/src/__tests__/lib/branding/jev-shadow.test.ts
deleted file mode 100644
index 7175d758a..000000000
--- a/apps/api/src/__tests__/lib/branding/jev-shadow.test.ts
+++ /dev/null
@@ -1,366 +0,0 @@
-import { vi, describe, it, expect, beforeEach, afterEach } from "vitest";
-import type { Mock } from "vitest";
-import { generateObject } from "ai";
-
-const mocks = vi.hoisted(() => ({ systemOne: vi.fn() }));
-
-vi.mock("@typesafe-ai/sdk", async importOriginal => ({
- ...(await importOriginal<typeof import("@typesafe-ai/sdk")>()),
- TypeSafeClient: class {
- systemOne = mocks.systemOne;
- },
-}));
-vi.mock("ai", async importOriginal => ({
- ...(await importOriginal<typeof import("ai")>()),
- generateObject: vi.fn(),
-}));
-vi.mock("../../../lib/generic-ai", () => ({
- getModel: vi.fn((name: string) => ({ modelId: name })),
-}));
-
-import { config } from "../../../config";
-import { resetJevBreaker } from "../../../lib/branding/jev";
-import { compareBrandingAnswers } from "../../../lib/branding/jev-shadow";
-import { enhanceBrandingWithLLM } from "../../../lib/branding/llm";
-import { BrandingEnhancement } from "../../../lib/branding/schema";
-import { BrandingLLMInput } from "../../../lib/branding/types";
-import { CostTracking } from "../../../lib/cost-tracking";
-import { logger } from "../../../lib/logger";
-import * as tracer from "../../../lib/otel-tracer";
-
-const LOGO = {
- src: "https://acme.test/logo.svg",
- alt: "Acme",
- isSvg: true,
- isVisible: true,
- location: "header" as const,
- position: { top: 10, left: 10, width: 120, height: 32 },
- indicators: {
- inHeader: true,
- altMatch: true,
- srcMatch: true,
- classMatch: false,
- hrefMatch: true,
- },
- href: "/",
- source: "header a img",
-};
-
-const input = (
- costTracking: CostTracking,
- overrides: Partial<BrandingLLMInput> = {},
-): BrandingLLMInput => ({
- jsAnalysis: {
- colorScheme: "light",
- colors: {
- primary: "#6D28D9",
- background: "#FFFFFF",
- textPrimary: "#111111",
- },
- images: { logo: "https://acme.test/old-logo.svg" },
- fonts: [{ family: "Inter", count: 40 }],
- },
- buttons: [
- {
- index: 0,
- text: "Get started",
- html: "",
- classes: "btn",
- background: "#6D28D9",
- textColor: "#FFFFFF",
- },
- {
- index: 1,
- text: "Contact sales",
- html: "",
- classes: "btn-outline",
- background: "transparent",
- textColor: "#111111",
- borderColor: "#111111",
- },
- ],
- logoCandidates: [LOGO],
- brandName: "Acme",
- url: "https://acme.test/",
- teamId: "team-other",
- scrapeId: "scrape-1",
- costTracking,
- logger,
- ...overrides,
-});
-
-const LLM_ANSWER: BrandingEnhancement = {
- cleanedFonts: [{ family: "Inter", role: "body" }],
- buttonClassification: {
- primaryButtonIndex: 0,
- primaryButtonReasoning: "violet CTA",
- secondaryButtonIndex: 1,
- secondaryButtonReasoning: "outline",
- confidence: 0.9,
- },
- colorRoles: {
- primaryColor: "#6D28D9",
- accentColor: "#6D28D9",
- backgroundColor: "#FFFFFF",
- textPrimary: "#111111",
- confidence: 0.9,
- },
- logoSelection: {
- selectedLogoIndex: 0,
- selectedLogoReasoning: "header logo",
- confidence: 0.9,
- },
-};
-
-// Jev agrees on everything except the logo, which it says is not there.
-const jevResponse = () => ({
- model: "jev-1.13.0",
- answers: {
- logo: {
- type: "choice",
- choice: "none",
- probabilities: { logo_0: 0.2, none: 0.8 },
- confidence: 0.8,
- },
- primary_button: {
- type: "choice",
- choice: "button_0",
- probabilities: { button_0: 0.9, button_1: 0.1 },
- confidence: 0.9,
- },
- secondary_button: {
- type: "choice",
- choice: "button_1",
- probabilities: { button_0: 0.1, button_1: 0.9, none: 0 },
- confidence: 0.9,
- },
- primary_color: {
- type: "choice",
- choice: "color_0",
- probabilities: { color_0: 0.9 },
- confidence: 0.9,
- },
- accent_color: {
- type: "choice",
- choice: "color_0",
- probabilities: { color_0: 0.9 },
- confidence: 0.9,
- },
- background_color: {
- type: "choice",
- choice: "color_1",
- probabilities: { color_1: 0.9 },
- confidence: 0.9,
- },
- text_color: {
- type: "choice",
- choice: "color_2",
- probabilities: { color_2: 0.9 },
- confidence: 0.9,
- },
- font_0_is_brand: { type: "noul", noul: 0.9 },
- },
- usage: { input_tokens: 2000, output_tokens: 60 },
-});
-
-const saved = {
- key: config.TYPESAFE_API_KEY,
- teamIds: config.BRANDING_JEV_TEAM_IDS,
- shadow: config.BRANDING_JEV_SHADOW_PERCENT,
-};
-
-let attributes: Record<string, unknown>[];
-
-beforeEach(() => {
- resetJevBreaker();
- config.TYPESAFE_API_KEY = "ts-test";
- config.BRANDING_JEV_TEAM_IDS = ["team-jev"];
- config.BRANDING_JEV_SHADOW_PERCENT = 100;
- mocks.systemOne.mockReset().mockResolvedValue(jevResponse());
- (generateObject as Mock).mockReset().mockResolvedValue({
- object: structuredClone(LLM_ANSWER),
- usage: { inputTokens: 6000, outputTokens: 400 },
- });
- attributes = [];
- vi.spyOn(tracer, "setSpanAttributes").mockImplementation((_span, attrs) => {
- attributes.push(attrs as Record<string, unknown>);
- });
-});
-
-afterEach(() => {
- config.TYPESAFE_API_KEY = saved.key;
- config.BRANDING_JEV_TEAM_IDS = saved.teamIds;
- config.BRANDING_JEV_SHADOW_PERCENT = saved.shadow;
- vi.restoreAllMocks();
-});
-
-const shadowOutcome = () =>
- attributes.find(a => "branding.shadow.outcome" in a);
-
-describe("Jev branding shadow", () => {
- it("asks Jev in the background and still returns the LLM's answer", async () => {
- const costTracking = new CostTracking();
- const withSpan = vi.spyOn(tracer, "withSpan");
-
- const result = await enhanceBrandingWithLLM(input(costTracking));
-
- expect(result).toEqual(LLM_ANSWER);
- await vi.waitFor(() => expect(shadowOutcome()).toBeDefined());
- expect(shadowOutcome()).toMatchObject({
- "branding.shadow.outcome": "compared",
- "branding.shadow.logo_agree": false,
- "branding.shadow.button_primary_agree": true,
- "branding.shadow.button_secondary_agree": true,
- "branding.shadow.color_primary_agree": true,
- "branding.shadow.color_background_agree": true,
- "branding.shadow.fonts_overlap": 1,
- });
- expect(
- attributes.find(a => "branding.shadow.llm_model" in a),
- ).toMatchObject({
- "branding.shadow.llm_model": "gpt-4o",
- "branding.shadow.team_id": "team-other",
- });
- // The shadow's cost is not the scrape's.
- expect(costTracking.calls.map(c => c.metadata.method)).toEqual([
- "enhanceBrandingWithLLM",
- ]);
- const jevSpan = withSpan.mock.calls.find(
- c => c[0] === "typesafe.systemone",
- );
- expect(jevSpan?.[2]?.attributes).toMatchObject({
- "branding.jev.shadow": true,
- });
- });
-
- it("leaves the caller's profile and answer untouched", async () => {
- const request = input(new CostTracking());
- const profileBefore = structuredClone(request.jsAnalysis);
-
- const result = await enhanceBrandingWithLLM(request);
- const answerBefore = structuredClone(result);
- await vi.waitFor(() => expect(shadowOutcome()).toBeDefined());
-
- // Merging Jev's "no logo" would delete images.logo on a shared profile.
- expect(request.jsAnalysis).toEqual(profileBefore);
- expect(result).toEqual(answerBefore);
- });
-
- it("records a failed Jev call without affecting the response", async () => {
- mocks.systemOne.mockReset().mockRejectedValue(new Error("rate limited"));
- const warn = vi.spyOn(logger, "warn");
-
- const result = await enhanceBrandingWithLLM(input(new CostTracking()));
-
- expect(result).toEqual(LLM_ANSWER);
- await vi.waitFor(() =>
- expect(shadowOutcome()).toEqual({
- "branding.shadow.outcome": "jev_failed",
- }),
- );
- // Logged by enhanceBrandingWithJev in shadow mode. A rejected call is
- // handled there, so the shadow's own catch ("Jev branding shadow failed")
- // never runs.
- expect(warn).toHaveBeenCalledWith(
- "Jev branding shadow call failed",
- expect.anything(),
- );
- });
-
- it("pauses while the Jev breaker is open", async () => {
- mocks.systemOne
- .mockReset()
- .mockRejectedValue(new Error("529 high traffic"));
- for (let i = 0; i < 5; i++) {
- await enhanceBrandingWithLLM(input(new CostTracking()));
- await vi.waitFor(() =>
- expect(mocks.systemOne).toHaveBeenCalledTimes(i + 1),
- );
- }
- await new Promise(r => setTimeout(r, 20));
-
- await enhanceBrandingWithLLM(input(new CostTracking()));
- await new Promise(r => setTimeout(r, 20));
-
- expect(mocks.systemOne).toHaveBeenCalledTimes(5);
- });
-
- it("skips zero-data-retention requests", async () => {
- await enhanceBrandingWithLLM(
- input(new CostTracking(), { zeroDataRetention: true }),
- );
- await new Promise(r => setTimeout(r, 20));
- expect(mocks.systemOne).not.toHaveBeenCalled();
- });
-
- it("does nothing at 0%", async () => {
- config.BRANDING_JEV_SHADOW_PERCENT = 0;
- await enhanceBrandingWithLLM(input(new CostTracking()));
- await new Promise(r => setTimeout(r, 20));
- expect(mocks.systemOne).not.toHaveBeenCalled();
- });
-
- it("does not shadow a request that Jev already answered", async () => {
- await enhanceBrandingWithLLM(
- input(new CostTracking(), { teamId: "team-jev", mode: "fast" }),
- );
- await new Promise(r => setTimeout(r, 20));
- expect(mocks.systemOne).toHaveBeenCalledTimes(1);
- expect(shadowOutcome()).toBeUndefined();
- });
-});
-
-describe("comparing branding answers", () => {
- const request = input(new CostTracking());
-
- it("treats identical answers as agreeing on every field", () => {
- const result = compareBrandingAnswers(request, LLM_ANSWER, LLM_ANSWER);
- expect(result.fields_agreeing).toBe(result.fields_compared);
- expect(result.fields_compared).toBe(7);
- });
-
- it("counts nearly identical colors as the same and different buttons as not", () => {
- const other: BrandingEnhancement = structuredClone(LLM_ANSWER);
- other.colorRoles.backgroundColor = "#FDFDFD";
- other.buttonClassification.primaryButtonIndex = 1;
- other.buttonClassification.secondaryButtonIndex = 0;
-
- const result = compareBrandingAnswers(request, LLM_ANSWER, other);
-
- expect(result.color_background_agree).toBe(true);
- expect(result.button_primary_agree).toBe(false);
- expect(result.button_secondary_agree).toBe(false);
- });
-
- it("compares fonts after merge, which keeps the page's fonts when an answer has none", () => {
- const noFonts: BrandingEnhancement = {
- ...structuredClone(LLM_ANSWER),
- cleanedFonts: [],
- };
-
- const result = compareBrandingAnswers(request, noFonts, LLM_ANSWER);
-
- expect(result.fonts_overlap).toBe(1);
- });
-
- it("treats buttons that differ only in shape as different", () => {
- const shaped = input(new CostTracking());
- shaped.buttons = [
- { ...shaped.buttons![0], borderRadius: "4px" },
- shaped.buttons![1],
- {
- ...shaped.buttons![0],
- index: 2,
- text: "Start free trial",
- borderRadius: "999px",
- },
- ];
- const pill: BrandingEnhancement = structuredClone(LLM_ANSWER);
- pill.buttonClassification.primaryButtonIndex = 2;
-
- const result = compareBrandingAnswers(shaped, LLM_ANSWER, pill);
-
- expect(result.button_primary_agree).toBe(false);
- expect(result.button_secondary_agree).toBe(true);
- });
-});
diff --git a/apps/api/src/__tests__/lib/branding/jev.test.ts b/apps/api/src/__tests__/lib/branding/jev.test.ts
index 56b248c20..fd20b41f7 100644
--- a/apps/api/src/__tests__/lib/branding/jev.test.ts
+++ b/apps/api/src/__tests__/lib/branding/jev.test.ts
@@ -31,7 +31,6 @@ import {
brandingRoutedTotal,
brandingJevBreakerOpensTotal,
brandingJevCallsTotal,
- brandingJevRolloutPercent,
} from "../../../lib/branding/metrics";
import { mergeBrandingResults } from "../../../lib/branding/merge";
import { BrandingLLMInput } from "../../../lib/branding/types";
@@ -105,7 +104,6 @@ const baseInput = (costTracking: CostTracking): BrandingLLMInput => ({
pageTitle: "Acme | Home",
url: "https://acme.test/",
teamId: "team-jev",
- mode: "fast",
costTracking,
logger,
});
@@ -214,18 +212,12 @@ const jevResponse = (overrides: Record<string, unknown> = {}) => ({
const saved = {
key: config.TYPESAFE_API_KEY,
global: config.BRANDING_JEV,
- teamIds: config.BRANDING_JEV_TEAM_IDS,
- rollout: config.BRANDING_JEV_ROLLOUT_PERCENT,
- escalate: config.BRANDING_JEV_ESCALATE_BELOW,
};
beforeEach(() => {
resetJevBreaker();
config.TYPESAFE_API_KEY = "ts-test";
config.BRANDING_JEV = undefined;
- config.BRANDING_JEV_TEAM_IDS = ["team-jev"];
- config.BRANDING_JEV_ROLLOUT_PERCENT = 0;
- config.BRANDING_JEV_ESCALATE_BELOW = undefined;
mocks.systemOne.mockReset();
(generateObject as Mock).mockReset().mockResolvedValue({
object: {
@@ -253,9 +245,6 @@ afterEach(() => {
vi.useRealTimers();
config.TYPESAFE_API_KEY = saved.key;
config.BRANDING_JEV = saved.global;
- config.BRANDING_JEV_TEAM_IDS = saved.teamIds;
- config.BRANDING_JEV_ROLLOUT_PERCENT = saved.rollout;
- config.BRANDING_JEV_ESCALATE_BELOW = saved.escalate;
});
const respondWith = (body: unknown) =>
@@ -875,36 +864,18 @@ describe("branding with Jev", () => {
]);
});
- it("escalates to the LLM when Jev is unsure of the logo", async () => {
- config.BRANDING_JEV_ESCALATE_BELOW = 0.6;
- respondWith(
- jevResponse({
- logo: {
- type: "choice",
- choice: "logo_1",
- probabilities: { logo_0: 0.45, logo_1: 0.5, none: 0.05 },
- confidence: 0.3,
- },
- }),
- );
- const costTracking = new CostTracking();
-
- await enhanceBrandingWithLLM(baseInput(costTracking));
-
- expect(generateObject).toHaveBeenCalledTimes(1);
- expect(costTracking.calls.map(c => c.metadata.method)).toEqual([
- "enhanceBrandingWithJev",
- "enhanceBrandingWithLLM",
- ]);
- });
-
- it("keeps unlisted teams off Jev", async () => {
+ it("sends every team to Jev unless it's switched off", async () => {
+ respondWith(jevResponse());
await enhanceBrandingWithLLM({
...baseInput(new CostTracking()),
- teamId: "team-unlisted",
+ teamId: "team-any",
});
+ expect(mocks.systemOne).toHaveBeenCalledTimes(1);
+ expect(generateObject).not.toHaveBeenCalled();
- expect(mocks.systemOne).not.toHaveBeenCalled();
+ config.BRANDING_JEV = false;
+ await enhanceBrandingWithLLM(baseInput(new CostTracking()));
+ expect(mocks.systemOne).toHaveBeenCalledTimes(1);
expect(generateObject).toHaveBeenCalledTimes(1);
});
@@ -942,51 +913,6 @@ describe("branding with Jev", () => {
withSpan.mockRestore();
});
- it("keeps a listed team on the LLM unless the request asks for fast", async () => {
- for (const mode of [undefined, "auto", "standard"] as const) {
- await enhanceBrandingWithLLM({ ...baseInput(new CostTracking()), mode });
- }
- expect(mocks.systemOne).not.toHaveBeenCalled();
- expect(generateObject).toHaveBeenCalledTimes(3);
-
- respondWith(jevResponse());
- await enhanceBrandingWithLLM({
- ...baseInput(new CostTracking()),
- mode: "fast",
- });
- expect(mocks.systemOne).toHaveBeenCalledTimes(1);
- });
-
- it("ignores mode for teams that aren't listed", async () => {
- await enhanceBrandingWithLLM({
- ...baseInput(new CostTracking()),
- teamId: "team-unlisted",
- mode: "fast",
- });
- expect(mocks.systemOne).not.toHaveBeenCalled();
-
- config.BRANDING_JEV = true;
- respondWith(jevResponse());
- await enhanceBrandingWithLLM({
- ...baseInput(new CostTracking()),
- teamId: "team-unlisted",
- mode: "standard",
- });
- expect(mocks.systemOne).toHaveBeenCalledTimes(1);
- });
-
- it("puts a rollout share of teams on Jev without listing them", async () => {
- const unlisted = { ...baseInput(new CostTracking()), teamId: "team-a" };
-
- await enhanceBrandingWithLLM(unlisted);
- expect(mocks.systemOne).not.toHaveBeenCalled();
-
- config.BRANDING_JEV_ROLLOUT_PERCENT = 100;
- respondWith(jevResponse());
- await enhanceBrandingWithLLM(unlisted);
- expect(mocks.systemOne).toHaveBeenCalledTimes(1);
- });
-
it("stays on the LLM when no TypeSafe key is configured", async () => {
config.TYPESAFE_API_KEY = undefined;
@@ -1155,10 +1081,10 @@ describe("branding metrics", () => {
)?.value ?? 0;
it("counts which model answered and how each Jev call ended", async () => {
- const jevOk = { outcome: "ok", shadow: "false" };
- const rateLimited = { outcome: "rate_limited", shadow: "false" };
- const timedOut = { outcome: "timeout", shadow: "false" };
- const byJev = { model: "jev", reason: "rollout" };
+ const jevOk = { outcome: "ok" };
+ const rateLimited = { outcome: "rate_limited" };
+ const timedOut = { outcome: "timeout" };
+ const byJev = { model: "jev", reason: "jev" };
const fellBack = { model: "llm", reason: "jev_unavailable" };
const before = {
jevOk: await value(brandingJevCallsTotal, jevOk),
@@ -1191,7 +1117,7 @@ describe("branding metrics", () => {
});
it("counts the breaker opening and the calls it skips", async () => {
- const skipped = { outcome: "breaker_open", shadow: "false" };
+ const skipped = { outcome: "breaker_open" };
const opensBefore = await value(brandingJevBreakerOpensTotal);
const skippedBefore = await value(brandingJevCallsTotal, skipped);
mocks.systemOne.mockRejectedValue(new Error("529 high traffic"));
@@ -1204,30 +1130,30 @@ describe("branding metrics", () => {
expect(await value(brandingJevCallsTotal, skipped)).toBe(skippedBefore + 1);
});
- it("counts teams outside the rollout as answered by the LLM", async () => {
- const outside = { model: "llm", reason: "not_in_rollout" };
+ it("counts branding sent to the LLM while Jev is switched off", async () => {
+ const outside = { model: "llm", reason: "jev_off" };
const before = await value(brandingRoutedTotal, outside);
- config.BRANDING_JEV_TEAM_IDS = [];
+ config.BRANDING_JEV = false;
await enhanceBrandingWithLLM(baseInput(new CostTracking()));
expect(await value(brandingRoutedTotal, outside)).toBe(before + 1);
expect(mocks.systemOne).not.toHaveBeenCalled();
});
-});
-
-describe("rollout gauge", () => {
- it("reports the share of teams this process sends to Jev", async () => {
- const read = async () =>
- (await brandingJevRolloutPercent.get()).values[0]?.value;
- config.BRANDING_JEV_ROLLOUT_PERCENT = 75;
- expect(await read()).toBe(75);
+ it("counts branding without a TypeSafe key apart from the switch", async () => {
+ const noKey = { model: "llm", reason: "jev_no_key" };
+ const off = { model: "llm", reason: "jev_off" };
+ const [noKeyBefore, offBefore] = [
+ await value(brandingRoutedTotal, noKey),
+ await value(brandingRoutedTotal, off),
+ ];
+ config.TYPESAFE_API_KEY = undefined;
- config.BRANDING_JEV = true;
- expect(await read()).toBe(100);
+ await enhanceBrandingWithLLM(baseInput(new CostTracking()));
- config.TYPESAFE_API_KEY = undefined;
- expect(await read()).toBe(0);
+ expect(await value(brandingRoutedTotal, noKey)).toBe(noKeyBefore + 1);
+ expect(await value(brandingRoutedTotal, off)).toBe(offBefore);
+ expect(mocks.systemOne).not.toHaveBeenCalled();
});
});
diff --git a/apps/api/src/__tests__/snips/v2/scrape-branding.test.ts b/apps/api/src/__tests__/snips/v2/scrape-branding.test.ts
index 6611568bd..d5c4c5d08 100644
--- a/apps/api/src/__tests__/snips/v2/scrape-branding.test.ts
+++ b/apps/api/src/__tests__/snips/v2/scrape-branding.test.ts
@@ -57,17 +57,13 @@ const isBrandingCall = (call: CostTrackingCall) =>
call.metadata?.module === "branding" &&
call.metadata?.method === "enhanceBrandingWithLLM";
-// Whether the server could route a test team's branding to Jev (all teams, a
-// listed team, or a rollout share). The LLM-only assertion below only holds
-// when none of these is configured.
-const JEV_MAY_APPLY =
- !!process.env.TYPESAFE_API_KEY &&
- (process.env.BRANDING_JEV === "true" ||
- !!process.env.BRANDING_JEV_TEAM_IDS?.trim() ||
- Number(process.env.BRANDING_JEV_ROLLOUT_PERCENT || 0) > 0);
+// Jev answers branding whenever the server has a TypeSafe key, unless
+// BRANDING_JEV=false. The LLM-only assertion below holds only when it doesn't.
+const JEV_ON =
+ !!process.env.TYPESAFE_API_KEY && process.env.BRANDING_JEV !== "false";
describe("Branding cost tracking", () => {
- concurrentIf(TEST_PRODUCTION && HAS_JOB_LOG && !JEV_MAY_APPLY)(
+ concurrentIf(TEST_PRODUCTION && HAS_JOB_LOG && !JEV_ON)(
"records the branding LLM call with its model and cost",
async () => {
const response = await scrape(
@@ -111,11 +107,7 @@ describe("Branding cost tracking", () => {
);
});
-// Runs when the API server was started with TYPESAFE_API_KEY and
-// BRANDING_JEV=true, which puts every team (including test identities, whose
-// ids aren't known ahead of time) on Jev.
-const JEV_ON =
- !!process.env.TYPESAFE_API_KEY && process.env.BRANDING_JEV === "true";
+// Runs when the API server answers branding with Jev (see JEV_ON above).
describe("Branding with Jev", () => {
concurrentIf(TEST_PRODUCTION && HAS_JOB_LOG && JEV_ON)(
diff --git a/apps/api/src/config.ts b/apps/api/src/config.ts
index bf91f975d..23cdf0ba0 100644
--- a/apps/api/src/config.ts
+++ b/apps/api/src/config.ts
@@ -570,24 +570,13 @@ const configSchema = z.object({
DISABLE_BLOCKLIST: z.stringbool().optional(),
FORCED_ENGINE_DOMAINS: z.string().optional(),
DEBUG_BRANDING: z.stringbool().optional(),
- // TypeSafe Jev for branding decisions (lib/branding/jev.ts); needs
- // TYPESAFE_API_KEY. BRANDING_JEV turns it on for every team,
- // BRANDING_JEV_TEAM_IDS (comma-separated) for listed teams, and
- // BRANDING_JEV_ROLLOUT_PERCENT for a stable share of teams.
+ // TypeSafe Jev answers branding decisions whenever TYPESAFE_API_KEY is set
+ // (lib/branding/jev.ts), with the LLM as fallback. BRANDING_JEV=false sends
+ // every branding request to the LLM instead.
BRANDING_JEV: emptyStringAsUndefined(z.stringbool()),
- BRANDING_JEV_TEAM_IDS: delimitedList(",").optional(),
- BRANDING_JEV_ROLLOUT_PERCENT: z.coerce.number().min(0).max(100).default(0),
BRANDING_JEV_TIMEOUT_MS: emptyStringAsUndefined(
z.coerce.number().int().positive(),
),
- // Rerun branding on the LLM when Jev's logo confidence is below this (0-1).
- BRANDING_JEV_ESCALATE_BELOW: emptyStringAsUndefined(
- z.coerce.number().min(0).max(1),
- ),
- // Share of LLM-answered branding requests (0-100) that also ask Jev in the
- // background and record how its answer compares (lib/branding/jev-shadow.ts).
- // Customers always get the LLM's answer.
- BRANDING_JEV_SHADOW_PERCENT: z.coerce.number().min(0).max(100).default(0),
// Warn scrapes of search engine results pages (google.*/search, ...) that
// /search returns those results directly (lib/serp-url.ts).
// SERP_SCRAPE_WARNING turns it on for every team, SERP_SCRAPE_WARNING_TEAM_IDS
diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.ts
index 8f6403f39..095e2e49c 100644
--- a/apps/api/src/controllers/v2/types.ts
+++ b/apps/api/src/controllers/v2/types.ts
@@ -476,10 +476,8 @@ const queryFormatWithOptions = z.strictObject({
type QueryFormatWithOptions = z.output<typeof queryFormatWithOptions>;
-// Which engine answers branding decisions: "fast" (Jev), "standard" (the
-// LLM) or "auto" (the LLM for now). Internal for now: honored only for teams
-// listed in BRANDING_JEV_TEAM_IDS and ignored for everyone else
-// (lib/branding/jev.ts).
+// Accepted for compatibility and ignored: the same engine answers every
+// branding request (lib/branding/jev.ts).
const brandingFormatWithOptions = z.strictObject({
type: z.literal("branding"),
mode: z.enum(["auto", "fast", "standard"]).optional(),
diff --git a/apps/api/src/lib/branding/jev-shadow.ts b/apps/api/src/lib/branding/jev-shadow.ts
deleted file mode 100644
index 83a9a51ab..000000000
--- a/apps/api/src/lib/branding/jev-shadow.ts
+++ /dev/null
@@ -1,184 +0,0 @@
-import { differenceCiede2000, parse } from "culori";
-
-import { config } from "../../config";
-import { CostTracking } from "../cost-tracking";
-import { setSpanAttributes, withSpan } from "../otel-tracer";
-import { enhanceBrandingWithJev, isJevBreakerOpen } from "./jev";
-import { mergeBrandingResults } from "./merge";
-import { BrandingEnhancement } from "./schema";
-import { BrandingLLMInput } from "./types";
-import { BrandingProfile } from "../../types/branding";
-
-// Shadow mode: for a sample of requests the LLM answered, ask Jev the same
-// questions in the background and record on a span how the two answers would
-// differ in the response. Customers always get the LLM's answer.
-
-// Colors closer than this (CIEDE2000) count as the same; brand-bench's default
-// tolerance is 12.
-const SAME_COLOR_DELTA_E = 10;
-// Backstop for traffic bursts: past this many shadow calls in flight, skip.
-const MAX_IN_FLIGHT = 50;
-
-let inFlight = 0;
-
-export function shouldShadowJev(input: BrandingLLMInput): boolean {
- if (!config.TYPESAFE_API_KEY) return false;
- // TypeSafe keeps nothing, but ZDR requests should not do extra work on
- // customer content that is not needed for their response.
- if (input.zeroDataRetention === true) return false;
- const percent = config.BRANDING_JEV_SHADOW_PERCENT;
- if (percent <= 0) return false;
- if (inFlight >= MAX_IN_FLIGHT) return false;
- // No extra load on TypeSafe while branding has stopped calling it.
- if (isJevBreakerOpen()) return false;
- return Math.random() * 100 < percent;
-}
-
-const deltaE = differenceCiede2000();
-
-function sameColor(a?: string, b?: string): boolean {
- if (!a && !b) return true;
- const pa = a ? parse(a) : undefined;
- const pb = b ? parse(b) : undefined;
- if (!pa || !pb) return false;
- return deltaE(pa, pb) <= SAME_COLOR_DELTA_E;
-}
-
-const COLOR_ROLES = ["primary", "accent", "background", "textPrimary"] as const;
-
-type ButtonStyle = NonNullable<
- NonNullable<BrandingProfile["components"]>["buttonPrimary"]
->;
-
-// Everything the response carries for a button.
-function sameButton(a?: ButtonStyle, b?: ButtonStyle): boolean {
- if (!a && !b) return true;
- if (!a || !b) return false;
- const sameText = (x?: string, y?: string) => (x || "none") === (y || "none");
- return (
- sameColor(a.background, b.background) &&
- sameColor(a.textColor, b.textColor) &&
- sameColor(a.borderColor, b.borderColor) &&
- sameText(a.borderRadius, b.borderRadius) &&
- sameText(a.shadow, b.shadow)
- );
-}
-
-// Merge keeps the page's own fonts when an answer cleans none, so compare the
-// merged profiles, not the raw answers.
-function fontOverlap(a: BrandingProfile, b: BrandingProfile): number {
- const names = (p: BrandingProfile) =>
- new Set((p.fonts ?? []).map(f => f.family.toLowerCase()));
- const x = names(a);
- const y = names(b);
- const union = new Set([...x, ...y]);
- if (union.size === 0) return 1;
- return [...x].filter(f => y.has(f)).length / union.size;
-}
-
-/**
- * Agreement between two answers to the same branding questions, measured on
- * what the response would contain after merge: logo, button styles, color roles
- * and fonts.
- */
-export function compareBrandingAnswers(
- input: Pick<BrandingLLMInput, "jsAnalysis" | "buttons" | "logoCandidates">,
- llm: BrandingEnhancement,
- jev: BrandingEnhancement,
-): Record<string, boolean | number> {
- // merge edits nested objects of the profile it is given; each side gets its own copy.
- const merged = (answer: BrandingEnhancement): BrandingProfile =>
- mergeBrandingResults(
- structuredClone(input.jsAnalysis),
- structuredClone(answer),
- structuredClone(input.buttons ?? []),
- structuredClone(input.logoCandidates),
- );
- const a = merged(llm);
- const b = merged(jev);
-
- const result: Record<string, boolean | number> = {};
- if ((input.logoCandidates?.length ?? 0) > 0) {
- result.logo_agree = (a.images?.logo ?? null) === (b.images?.logo ?? null);
- }
- result.button_primary_agree = sameButton(
- a.components?.buttonPrimary,
- b.components?.buttonPrimary,
- );
- result.button_secondary_agree = sameButton(
- a.components?.buttonSecondary,
- b.components?.buttonSecondary,
- );
- for (const role of COLOR_ROLES) {
- result[`color_${role}_agree`] = sameColor(
- a.colors?.[role],
- b.colors?.[role],
- );
- }
- result.fonts_overlap = fontOverlap(a, b);
-
- const flags = Object.values(result).filter(
- (v): v is boolean => typeof v === "boolean",
- );
- result.fields_compared = flags.length;
- result.fields_agreeing = flags.filter(Boolean).length;
- return result;
-}
-
-/**
- * Starts the shadow call and returns at once. Inputs are copied before
- * returning: the caller keeps editing the LLM answer and the profile.
- */
-export function shadowJev(
- input: BrandingLLMInput,
- llm: BrandingEnhancement,
- llmModel: string,
-): Promise<void> {
- const snapshot: BrandingLLMInput = {
- ...input,
- jsAnalysis: structuredClone(input.jsAnalysis),
- buttons: structuredClone(input.buttons),
- logoCandidates: structuredClone(input.logoCandidates),
- // Jev does not read these; don't keep large strings alive.
- screenshot: undefined,
- headerHtmlChunk: undefined,
- // Kept apart from the scrape's own cost record and cost limit.
- costTracking: new CostTracking(),
- };
- const llmAnswer = structuredClone(llm);
- inFlight++;
-
- return withSpan("branding.jev_shadow", async span => {
- setSpanAttributes(span, {
- feature: "branding",
- "branding.shadow.llm_model": llmModel,
- ...(input.teamId ? { "branding.shadow.team_id": input.teamId } : {}),
- ...(input.scrapeId ? { scrapeId: input.scrapeId } : {}),
- });
- const jev = await enhanceBrandingWithJev(snapshot, { shadow: true });
- if (!jev) {
- setSpanAttributes(span, { "branding.shadow.outcome": "jev_failed" });
- return;
- }
- const comparison = compareBrandingAnswers(
- snapshot,
- llmAnswer,
- jev.enhancement,
- );
- setSpanAttributes(span, {
- "branding.shadow.outcome": "compared",
- ...(jev.logoConfidence !== undefined
- ? { "branding.shadow.jev_logo_confidence": jev.logoConfidence }
- : {}),
- ...Object.fromEntries(
- Object.entries(comparison).map(([k, v]) => [`branding.shadow.${k}`, v]),
- ),
- });
- })
- .catch(error => {
- input.logger.warn("Jev branding shadow failed", { error });
- })
- .finally(() => {
- inFlight--;
- });
-}
diff --git a/apps/api/src/lib/branding/jev.ts b/apps/api/src/lib/branding/jev.ts
index edd45801b..3064ab958 100644
--- a/apps/api/src/lib/branding/jev.ts
+++ b/apps/api/src/lib/branding/jev.ts
@@ -10,7 +10,6 @@ import { formatHex, hsl, parse, wcagContrast } from "culori";
import { config } from "../../config";
import { setSpanAttributes, SpanKind, withSpan } from "../otel-tracer";
-import { sampled } from "../rollout";
import { getTypeSafeClient } from "../typesafe";
import { APITimeoutError, RateLimitError } from "@typesafe-ai/sdk";
import {
@@ -59,7 +58,7 @@ const BREAKER_COOLDOWN_MS = 60_000;
let consecutiveFailures = 0;
let breakerOpenUntil = 0;
-export function isJevBreakerOpen(now = Date.now()): boolean {
+function isJevBreakerOpen(now = Date.now()): boolean {
return now < breakerOpenUntil;
}
@@ -103,18 +102,9 @@ type JevBrandingResult = {
logoConfidence?: number;
};
-export function isJevBrandingEnabled(input: BrandingLLMInput): boolean {
- if (!config.TYPESAFE_API_KEY) return false;
- // Listed teams stay on the LLM unless a request asks for mode "fast". Every
- // other team ignores `mode` and follows the settings below.
- if (input.teamId && config.BRANDING_JEV_TEAM_IDS?.includes(input.teamId)) {
- return input.mode === "fast";
- }
- return (
- config.BRANDING_JEV === true ||
- (!!input.teamId &&
- sampled(`team:${input.teamId}`, config.BRANDING_JEV_ROLLOUT_PERCENT))
- );
+/** Jev answers branding unless there's no TypeSafe key or BRANDING_JEV is off. */
+export function isJevBrandingEnabled(): boolean {
+ return !!config.TYPESAFE_API_KEY && config.BRANDING_JEV !== false;
}
// ---------------------------------------------------------------------------
@@ -1022,13 +1012,11 @@ function mapJevAnswers(
*/
export async function enhanceBrandingWithJev(
input: BrandingLLMInput,
- options: { shadow?: boolean } = {},
): Promise<JevBrandingResult | null> {
const typesafe = getTypeSafeClient();
if (!typesafe) return null;
- const shadow = options.shadow ? "true" : "false";
const count = (outcome: JevCallOutcome) =>
- brandingJevCallsTotal.inc({ outcome, shadow });
+ brandingJevCallsTotal.inc({ outcome });
if (isJevBreakerOpen()) {
count("breaker_open");
return null;
@@ -1069,7 +1057,6 @@ export async function enhanceBrandingWithJev(
attributes: {
feature: "branding",
"branding.jev.questions": Object.keys(request.questions).length,
- ...(options.shadow ? { "branding.jev.shadow": true } : {}),
...(input.scrapeId ? { scrapeId: input.scrapeId } : {}),
...(input.teamId ? { teamId: input.teamId } : {}),
},
@@ -1079,10 +1066,7 @@ export async function enhanceBrandingWithJev(
// Only TypeSafe's own failures say anything about its health.
if (called) {
recordJevFailure(input);
- brandingJevDurationSeconds.observe(
- { shadow },
- (Date.now() - started) / 1000,
- );
+ brandingJevDurationSeconds.observe((Date.now() - started) / 1000);
count(
error instanceof RateLimitError
? "rate_limited"
@@ -1091,21 +1075,16 @@ export async function enhanceBrandingWithJev(
: "error",
);
}
- input.logger.warn(
- options.shadow
- ? "Jev branding shadow call failed"
- : "Jev branding call failed, falling back to LLM",
- {
- error,
- elapsedMs: Date.now() - started,
- },
- );
+ input.logger.warn("Jev branding call failed, falling back to LLM", {
+ error,
+ elapsedMs: Date.now() - started,
+ });
return null;
}
// A call that started before the breaker opened and succeeds late doesn't
// close it: the next failure after the cooldown should reopen it at once.
if (!isJevBreakerOpen()) consecutiveFailures = 0;
- brandingJevDurationSeconds.observe({ shadow }, (Date.now() - started) / 1000);
+ brandingJevDurationSeconds.observe((Date.now() - started) / 1000);
const inputTokens = response.usage?.input_tokens ?? 0;
const outputTokens = response.usage?.output_tokens ?? 0;
@@ -1123,21 +1102,15 @@ export async function enhanceBrandingWithJev(
} catch (error) {
// A successful call whose answers don't have the expected shape.
count("unusable");
- input.logger.warn(
- options.shadow
- ? "Jev branding shadow answers unusable"
- : "Jev branding answers unusable, falling back to LLM",
- {
- error,
- model: response.model,
- },
- );
+ input.logger.warn("Jev branding answers unusable, falling back to LLM", {
+ error,
+ model: response.model,
+ });
return null;
}
count("ok");
input.logger.info("Jev branding call", {
model: response.model,
- shadow: options.shadow === true,
elapsedMs: Date.now() - started,
inputTokens,
questions: Object.keys(request.questions).length,
diff --git a/apps/api/src/lib/branding/llm.ts b/apps/api/src/lib/branding/llm.ts
index 9cabc4814..0a0fb1378 100644
--- a/apps/api/src/lib/branding/llm.ts
+++ b/apps/api/src/lib/branding/llm.ts
@@ -6,7 +6,6 @@ import { CostLimitExceededError } from "../cost-tracking";
import { BrandingEnhancement, getBrandingEnhancementSchema } from "./schema";
import { enhanceBrandingWithJev, isJevBrandingEnabled } from "./jev";
import { brandingRoutedTotal } from "./metrics";
-import { shadowJev, shouldShadowJev } from "./jev-shadow";
import { buildBrandingPrompt } from "./prompt";
import { BrandingLLMInput } from "./types";
import { getModel } from "../generic-ai";
@@ -109,30 +108,19 @@ export async function enhanceBrandingWithLLM(
): Promise<BrandingEnhancement> {
const logger = input.logger;
- const triedJev = isJevBrandingEnabled(input);
- if (!triedJev) {
- brandingRoutedTotal.inc({ model: "llm", reason: "not_in_rollout" });
- } else {
- const jev = await enhanceBrandingWithJev(input);
- const escalateBelow = config.BRANDING_JEV_ESCALATE_BELOW;
- const unsureOfLogo =
- escalateBelow !== undefined &&
- jev?.logoConfidence !== undefined &&
- jev.logoConfidence < escalateBelow;
- if (jev && !unsureOfLogo) {
- brandingRoutedTotal.inc({ model: "jev", reason: "rollout" });
- return jev.enhancement;
- }
+ if (!isJevBrandingEnabled()) {
brandingRoutedTotal.inc({
model: "llm",
- reason: jev ? "logo_escalation" : "jev_unavailable",
+ // No key (self-hosted without TypeSafe) is not the switch turned off.
+ reason: config.TYPESAFE_API_KEY ? "jev_off" : "jev_no_key",
});
+ } else {
+ const jev = await enhanceBrandingWithJev(input);
if (jev) {
- logger.info("Jev unsure of the logo, escalating branding to the LLM", {
- logoConfidence: jev.logoConfidence,
- escalateBelow,
- });
+ brandingRoutedTotal.inc({ model: "jev", reason: "jev" });
+ return jev.enhancement;
}
+ brandingRoutedTotal.inc({ model: "llm", reason: "jev_unavailable" });
}
const prompt = buildBrandingPrompt(input);
@@ -301,10 +289,6 @@ export async function enhanceBrandingWithLLM(
const { logoSelection: _, ...rest } = answer;
answer = rest as BrandingEnhancement;
}
- // Requests that already went to Jev have nothing to compare against.
- if (!triedJev && answer && shouldShadowJev(input)) {
- void shadowJev(input, answer, modelId);
- }
return answer;
} catch (error) {
if (error instanceof CostLimitExceededError) {
diff --git a/apps/api/src/lib/branding/metrics.ts b/apps/api/src/lib/branding/metrics.ts
index a9727a556..a1c31a4b1 100644
--- a/apps/api/src/lib/branding/metrics.ts
+++ b/apps/api/src/lib/branding/metrics.ts
@@ -1,18 +1,17 @@
-import { Counter, Gauge, Histogram } from "prom-client";
-import { config } from "../../config";
+import { Counter, Histogram } from "prom-client";
// Fixed labels only: never team ids or anything from the page.
export const brandingRoutedTotal = new Counter({
name: "firecrawl_branding_routed_total",
- help: "Each branding request once, by the model it was finally sent to and why: jev (rollout), or llm when the team isn't in the rollout, Jev was unavailable (failed or breaker open), or Jev was unsure of the logo; every Jev attempt is in firecrawl_branding_jev_calls_total",
+ help: "Each branding request once, by the model it was finally sent to and why: jev, or llm when BRANDING_JEV=false (jev_off), no TYPESAFE_API_KEY is set (jev_no_key), or Jev was unavailable (failed or breaker open); every Jev attempt is in firecrawl_branding_jev_calls_total",
labelNames: ["model", "reason"],
});
export const brandingJevCallsTotal = new Counter({
name: "firecrawl_branding_jev_calls_total",
- help: "Jev branding call outcomes; shadow calls are labelled separately",
- labelNames: ["outcome", "shadow"],
+ help: "Jev branding call outcomes",
+ labelNames: ["outcome"],
});
export const brandingJevBreakerOpensTotal = new Counter({
@@ -23,27 +22,10 @@ export const brandingJevBreakerOpensTotal = new Counter({
export const brandingJevDurationSeconds = new Histogram({
name: "firecrawl_branding_jev_duration_seconds",
help: "Jev branding call time, for calls that reached TypeSafe",
- labelNames: ["shadow"],
// Up to the default timeout with its one retry.
buckets: [0.1, 0.15, 0.2, 0.3, 0.5, 0.75, 1, 1.5, 2, 3, 5, 7.5, 10],
});
-// Each pod reports the share it was started with; pods that disagree are
-// running an older config.
-export const brandingJevRolloutPercent = new Gauge({
- name: "firecrawl_branding_jev_rollout_percent",
- help: "Share of teams this process sends to Jev, from its own config: 0 without a TypeSafe key, 100 with BRANDING_JEV on, else BRANDING_JEV_ROLLOUT_PERCENT",
- collect() {
- this.set(effectiveJevRolloutPercent());
- },
-});
-
-function effectiveJevRolloutPercent(): number {
- if (!config.TYPESAFE_API_KEY) return 0;
- if (config.BRANDING_JEV === true) return 100;
- return config.BRANDING_JEV_ROLLOUT_PERCENT;
-}
-
export type JevCallOutcome =
| "ok"
| "timeout"
@@ -54,23 +36,21 @@ export type JevCallOutcome =
// Export quiet series before their first event, so rates read 0, not missing.
for (const [model, reason] of [
- ["jev", "rollout"],
- ["llm", "not_in_rollout"],
+ ["jev", "jev"],
+ ["llm", "jev_off"],
+ ["llm", "jev_no_key"],
["llm", "jev_unavailable"],
- ["llm", "logo_escalation"],
]) {
brandingRoutedTotal.inc({ model, reason }, 0);
}
-for (const shadow of ["true", "false"]) {
- for (const outcome of [
- "ok",
- "timeout",
- "rate_limited",
- "error",
- "unusable",
- "breaker_open",
- ] satisfies JevCallOutcome[]) {
- brandingJevCallsTotal.inc({ outcome, shadow }, 0);
- }
+for (const outcome of [
+ "ok",
+ "timeout",
+ "rate_limited",
+ "error",
+ "unusable",
+ "breaker_open",
+] satisfies JevCallOutcome[]) {
+ brandingJevCallsTotal.inc({ outcome }, 0);
}
brandingJevBreakerOpensTotal.inc(0);
diff --git a/apps/api/src/lib/branding/transformer.ts b/apps/api/src/lib/branding/transformer.ts
index 963dbd69a..09a4b1e82 100644
--- a/apps/api/src/lib/branding/transformer.ts
+++ b/apps/api/src/lib/branding/transformer.ts
@@ -240,7 +240,6 @@ export async function brandingTransformer(
scrapeId: meta.id,
zeroDataRetention: meta.internalOptions.zeroDataRetention,
teamFlags: meta.internalOptions.teamFlags,
- mode: hasFormatOfType(meta.options.formats, "branding")?.mode,
costTracking: meta.costTracking,
logger: meta.logger,
});
diff --git a/apps/api/src/lib/branding/types.ts b/apps/api/src/lib/branding/types.ts
index d2e00e545..9f79bdb82 100644
--- a/apps/api/src/lib/branding/types.ts
+++ b/apps/api/src/lib/branding/types.ts
@@ -103,8 +103,6 @@ export interface BrandingLLMInput {
scrapeId?: string;
zeroDataRetention?: boolean;
teamFlags?: { debugBranding?: boolean } | null;
- /** The branding format's `mode`; see isJevBrandingEnabled. */
- mode?: "auto" | "fast" | "standard";
costTracking: CostTracking;
logger: Logger;
}