Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat(branding): shadow Jev on LLM-answered branding requests (#4968)

Abimael Martell0cd63e62a35d
diff --git a/apps/api/.env.example b/apps/api/.env.example
index 7e0b3da9a..767e75161 100644
--- a/apps/api/.env.example
+++ b/apps/api/.env.example
@@ -79,6 +79,8 @@ TYPESAFE_API_KEY=
# BRANDING_JEV_ROLLOUT_PERCENT=0
# BRANDING_JEV_TIMEOUT_MS=5000
# BRANDING_JEV_ESCALATE_BELOW=
+# Share of LLM-answered branding requests that also ask Jev in the background to compare answers (response unchanged)
+# BRANDING_JEV_SHADOW_PERCENT=0
BULL_AUTH_KEY=@
# set if you have a llamaparse key you'd like to use to parse pdfs
LLAMAPARSE_API_KEY=
diff --git a/apps/api/src/__tests__/lib/branding/jev-shadow.test.ts b/apps/api/src/__tests__/lib/branding/jev-shadow.test.ts
new file mode 100644
index 000000000..f11bed4a2
--- /dev/null
+++ b/apps/api/src/__tests__/lib/branding/jev-shadow.test.ts
@@ -0,0 +1,346 @@
+import { vi, describe, it, expect, beforeEach, afterEach } from "vitest";
+import type { Mock } from "vitest";
+import { generateObject } from "ai";
+
+const mocks = vi.hoisted(() => ({ systemOne: vi.fn() }));
+
+vi.mock("@typesafe-ai/sdk", async importOriginal => ({
+ ...(await importOriginal<typeof import("@typesafe-ai/sdk")>()),
+ TypeSafeClient: class {
+ systemOne = mocks.systemOne;
+ },
+}));
+vi.mock("ai", async importOriginal => ({
+ ...(await importOriginal<typeof import("ai")>()),
+ generateObject: vi.fn(),
+}));
+vi.mock("../../../lib/generic-ai", () => ({
+ getModel: vi.fn((name: string) => ({ modelId: name })),
+}));
+
+import { config } from "../../../config";
+import { compareBrandingAnswers } from "../../../lib/branding/jev-shadow";
+import { enhanceBrandingWithLLM } from "../../../lib/branding/llm";
+import { BrandingEnhancement } from "../../../lib/branding/schema";
+import { BrandingLLMInput } from "../../../lib/branding/types";
+import { CostTracking } from "../../../lib/cost-tracking";
+import { logger } from "../../../lib/logger";
+import * as tracer from "../../../lib/otel-tracer";
+
+const LOGO = {
+ src: "https://acme.test/logo.svg",
+ alt: "Acme",
+ isSvg: true,
+ isVisible: true,
+ location: "header" as const,
+ position: { top: 10, left: 10, width: 120, height: 32 },
+ indicators: {
+ inHeader: true,
+ altMatch: true,
+ srcMatch: true,
+ classMatch: false,
+ hrefMatch: true,
+ },
+ href: "/",
+ source: "header a img",
+};
+
+const input = (
+ costTracking: CostTracking,
+ overrides: Partial<BrandingLLMInput> = {},
+): BrandingLLMInput => ({
+ jsAnalysis: {
+ colorScheme: "light",
+ colors: {
+ primary: "#6D28D9",
+ background: "#FFFFFF",
+ textPrimary: "#111111",
+ },
+ images: { logo: "https://acme.test/old-logo.svg" },
+ fonts: [{ family: "Inter", count: 40 }],
+ },
+ buttons: [
+ {
+ index: 0,
+ text: "Get started",
+ html: "",
+ classes: "btn",
+ background: "#6D28D9",
+ textColor: "#FFFFFF",
+ },
+ {
+ index: 1,
+ text: "Contact sales",
+ html: "",
+ classes: "btn-outline",
+ background: "transparent",
+ textColor: "#111111",
+ borderColor: "#111111",
+ },
+ ],
+ logoCandidates: [LOGO],
+ brandName: "Acme",
+ url: "https://acme.test/",
+ teamId: "team-other",
+ scrapeId: "scrape-1",
+ costTracking,
+ logger,
+ ...overrides,
+});
+
+const LLM_ANSWER: BrandingEnhancement = {
+ cleanedFonts: [{ family: "Inter", role: "body" }],
+ buttonClassification: {
+ primaryButtonIndex: 0,
+ primaryButtonReasoning: "violet CTA",
+ secondaryButtonIndex: 1,
+ secondaryButtonReasoning: "outline",
+ confidence: 0.9,
+ },
+ colorRoles: {
+ primaryColor: "#6D28D9",
+ accentColor: "#6D28D9",
+ backgroundColor: "#FFFFFF",
+ textPrimary: "#111111",
+ confidence: 0.9,
+ },
+ logoSelection: {
+ selectedLogoIndex: 0,
+ selectedLogoReasoning: "header logo",
+ confidence: 0.9,
+ },
+};
+
+// Jev agrees on everything except the logo, which it says is not there.
+const jevResponse = () => ({
+ model: "jev-1.13.0",
+ answers: {
+ logo: {
+ type: "choice",
+ choice: "none",
+ probabilities: { logo_0: 0.2, none: 0.8 },
+ confidence: 0.8,
+ },
+ primary_button: {
+ type: "choice",
+ choice: "button_0",
+ probabilities: { button_0: 0.9, button_1: 0.1 },
+ confidence: 0.9,
+ },
+ secondary_button: {
+ type: "choice",
+ choice: "button_1",
+ probabilities: { button_0: 0.1, button_1: 0.9, none: 0 },
+ confidence: 0.9,
+ },
+ primary_color: {
+ type: "choice",
+ choice: "color_0",
+ probabilities: { color_0: 0.9 },
+ confidence: 0.9,
+ },
+ accent_color: {
+ type: "choice",
+ choice: "color_0",
+ probabilities: { color_0: 0.9 },
+ confidence: 0.9,
+ },
+ background_color: {
+ type: "choice",
+ choice: "color_1",
+ probabilities: { color_1: 0.9 },
+ confidence: 0.9,
+ },
+ text_color: {
+ type: "choice",
+ choice: "color_2",
+ probabilities: { color_2: 0.9 },
+ confidence: 0.9,
+ },
+ font_0_is_brand: { type: "noul", noul: 0.9 },
+ },
+ usage: { input_tokens: 2000, output_tokens: 60 },
+});
+
+const saved = {
+ key: config.TYPESAFE_API_KEY,
+ teamIds: config.BRANDING_JEV_TEAM_IDS,
+ shadow: config.BRANDING_JEV_SHADOW_PERCENT,
+};
+
+let attributes: Record<string, unknown>[];
+
+beforeEach(() => {
+ config.TYPESAFE_API_KEY = "ts-test";
+ config.BRANDING_JEV_TEAM_IDS = ["team-jev"];
+ config.BRANDING_JEV_SHADOW_PERCENT = 100;
+ mocks.systemOne.mockReset().mockResolvedValue(jevResponse());
+ (generateObject as Mock).mockReset().mockResolvedValue({
+ object: structuredClone(LLM_ANSWER),
+ usage: { inputTokens: 6000, outputTokens: 400 },
+ });
+ attributes = [];
+ vi.spyOn(tracer, "setSpanAttributes").mockImplementation((_span, attrs) => {
+ attributes.push(attrs as Record<string, unknown>);
+ });
+});
+
+afterEach(() => {
+ config.TYPESAFE_API_KEY = saved.key;
+ config.BRANDING_JEV_TEAM_IDS = saved.teamIds;
+ config.BRANDING_JEV_SHADOW_PERCENT = saved.shadow;
+ vi.restoreAllMocks();
+});
+
+const shadowOutcome = () =>
+ attributes.find(a => "branding.shadow.outcome" in a);
+
+describe("Jev branding shadow", () => {
+ it("asks Jev in the background and still returns the LLM's answer", async () => {
+ const costTracking = new CostTracking();
+ const withSpan = vi.spyOn(tracer, "withSpan");
+
+ const result = await enhanceBrandingWithLLM(input(costTracking));
+
+ expect(result).toEqual(LLM_ANSWER);
+ await vi.waitFor(() => expect(shadowOutcome()).toBeDefined());
+ expect(shadowOutcome()).toMatchObject({
+ "branding.shadow.outcome": "compared",
+ "branding.shadow.logo_agree": false,
+ "branding.shadow.button_primary_agree": true,
+ "branding.shadow.button_secondary_agree": true,
+ "branding.shadow.color_primary_agree": true,
+ "branding.shadow.color_background_agree": true,
+ "branding.shadow.fonts_overlap": 1,
+ });
+ expect(
+ attributes.find(a => "branding.shadow.llm_model" in a),
+ ).toMatchObject({
+ "branding.shadow.llm_model": "gpt-4o",
+ "branding.shadow.team_id": "team-other",
+ });
+ // The shadow's cost is not the scrape's.
+ expect(costTracking.calls.map(c => c.metadata.method)).toEqual([
+ "enhanceBrandingWithLLM",
+ ]);
+ const jevSpan = withSpan.mock.calls.find(
+ c => c[0] === "typesafe.systemone",
+ );
+ expect(jevSpan?.[2]?.attributes).toMatchObject({
+ "branding.jev.shadow": true,
+ });
+ });
+
+ it("leaves the caller's profile and answer untouched", async () => {
+ const request = input(new CostTracking());
+ const profileBefore = structuredClone(request.jsAnalysis);
+
+ const result = await enhanceBrandingWithLLM(request);
+ const answerBefore = structuredClone(result);
+ await vi.waitFor(() => expect(shadowOutcome()).toBeDefined());
+
+ // Merging Jev's "no logo" would delete images.logo on a shared profile.
+ expect(request.jsAnalysis).toEqual(profileBefore);
+ expect(result).toEqual(answerBefore);
+ });
+
+ it("records a failed Jev call without affecting the response", async () => {
+ mocks.systemOne.mockReset().mockRejectedValue(new Error("rate limited"));
+ const warn = vi.spyOn(logger, "warn");
+
+ const result = await enhanceBrandingWithLLM(input(new CostTracking()));
+
+ expect(result).toEqual(LLM_ANSWER);
+ await vi.waitFor(() =>
+ expect(shadowOutcome()).toEqual({
+ "branding.shadow.outcome": "jev_failed",
+ }),
+ );
+ // Logged by enhanceBrandingWithJev in shadow mode. A rejected call is
+ // handled there, so the shadow's own catch ("Jev branding shadow failed")
+ // never runs.
+ expect(warn).toHaveBeenCalledWith(
+ "Jev branding shadow call failed",
+ expect.anything(),
+ );
+ });
+
+ it("skips zero-data-retention requests", async () => {
+ await enhanceBrandingWithLLM(
+ input(new CostTracking(), { zeroDataRetention: true }),
+ );
+ await new Promise(r => setTimeout(r, 20));
+ expect(mocks.systemOne).not.toHaveBeenCalled();
+ });
+
+ it("does nothing at 0%", async () => {
+ config.BRANDING_JEV_SHADOW_PERCENT = 0;
+ await enhanceBrandingWithLLM(input(new CostTracking()));
+ await new Promise(r => setTimeout(r, 20));
+ expect(mocks.systemOne).not.toHaveBeenCalled();
+ });
+
+ it("does not shadow a request that Jev already answered", async () => {
+ await enhanceBrandingWithLLM(
+ input(new CostTracking(), { teamId: "team-jev", mode: "fast" }),
+ );
+ await new Promise(r => setTimeout(r, 20));
+ expect(mocks.systemOne).toHaveBeenCalledTimes(1);
+ expect(shadowOutcome()).toBeUndefined();
+ });
+});
+
+describe("comparing branding answers", () => {
+ const request = input(new CostTracking());
+
+ it("treats identical answers as agreeing on every field", () => {
+ const result = compareBrandingAnswers(request, LLM_ANSWER, LLM_ANSWER);
+ expect(result.fields_agreeing).toBe(result.fields_compared);
+ expect(result.fields_compared).toBe(7);
+ });
+
+ it("counts nearly identical colors as the same and different buttons as not", () => {
+ const other: BrandingEnhancement = structuredClone(LLM_ANSWER);
+ other.colorRoles.backgroundColor = "#FDFDFD";
+ other.buttonClassification.primaryButtonIndex = 1;
+ other.buttonClassification.secondaryButtonIndex = 0;
+
+ const result = compareBrandingAnswers(request, LLM_ANSWER, other);
+
+ expect(result.color_background_agree).toBe(true);
+ expect(result.button_primary_agree).toBe(false);
+ expect(result.button_secondary_agree).toBe(false);
+ });
+
+ it("compares fonts after merge, which keeps the page's fonts when an answer has none", () => {
+ const noFonts: BrandingEnhancement = {
+ ...structuredClone(LLM_ANSWER),
+ cleanedFonts: [],
+ };
+
+ const result = compareBrandingAnswers(request, noFonts, LLM_ANSWER);
+
+ expect(result.fonts_overlap).toBe(1);
+ });
+
+ it("treats buttons that differ only in shape as different", () => {
+ const shaped = input(new CostTracking());
+ shaped.buttons = [
+ { ...shaped.buttons![0], borderRadius: "4px" },
+ shaped.buttons![1],
+ {
+ ...shaped.buttons![0],
+ index: 2,
+ text: "Start free trial",
+ borderRadius: "999px",
+ },
+ ];
+ const pill: BrandingEnhancement = structuredClone(LLM_ANSWER);
+ pill.buttonClassification.primaryButtonIndex = 2;
+
+ const result = compareBrandingAnswers(shaped, LLM_ANSWER, pill);
+
+ expect(result.button_primary_agree).toBe(false);
+ expect(result.button_secondary_agree).toBe(true);
+ });
+});
diff --git a/apps/api/src/config.ts b/apps/api/src/config.ts
index 4b16b2ddb..eca0463b6 100644
--- a/apps/api/src/config.ts
+++ b/apps/api/src/config.ts
@@ -561,6 +561,10 @@ const configSchema = z.object({
BRANDING_JEV_ESCALATE_BELOW: emptyStringAsUndefined(
z.coerce.number().min(0).max(1),
),
+ // Share of LLM-answered branding requests (0-100) that also ask Jev in the
+ // background and record how its answer compares (lib/branding/jev-shadow.ts).
+ // Customers always get the LLM's answer.
+ BRANDING_JEV_SHADOW_PERCENT: z.coerce.number().min(0).max(100).default(0),
// AI/ML
MODEL_NAME: z.string().optional(),
diff --git a/apps/api/src/lib/branding/jev-shadow.ts b/apps/api/src/lib/branding/jev-shadow.ts
new file mode 100644
index 000000000..21a57694e
--- /dev/null
+++ b/apps/api/src/lib/branding/jev-shadow.ts
@@ -0,0 +1,182 @@
+import { differenceCiede2000, parse } from "culori";
+
+import { config } from "../../config";
+import { CostTracking } from "../cost-tracking";
+import { setSpanAttributes, withSpan } from "../otel-tracer";
+import { enhanceBrandingWithJev } from "./jev";
+import { mergeBrandingResults } from "./merge";
+import { BrandingEnhancement } from "./schema";
+import { BrandingLLMInput } from "./types";
+import { BrandingProfile } from "../../types/branding";
+
+// Shadow mode: for a sample of requests the LLM answered, ask Jev the same
+// questions in the background and record on a span how the two answers would
+// differ in the response. Customers always get the LLM's answer.
+
+// Colors closer than this (CIEDE2000) count as the same; brand-bench's default
+// tolerance is 12.
+const SAME_COLOR_DELTA_E = 10;
+// Backstop for traffic bursts: past this many shadow calls in flight, skip.
+const MAX_IN_FLIGHT = 50;
+
+let inFlight = 0;
+
+export function shouldShadowJev(input: BrandingLLMInput): boolean {
+ if (!config.TYPESAFE_API_KEY) return false;
+ // TypeSafe keeps nothing, but ZDR requests should not do extra work on
+ // customer content that is not needed for their response.
+ if (input.zeroDataRetention === true) return false;
+ const percent = config.BRANDING_JEV_SHADOW_PERCENT;
+ if (percent <= 0) return false;
+ if (inFlight >= MAX_IN_FLIGHT) return false;
+ return Math.random() * 100 < percent;
+}
+
+const deltaE = differenceCiede2000();
+
+function sameColor(a?: string, b?: string): boolean {
+ if (!a && !b) return true;
+ const pa = a ? parse(a) : undefined;
+ const pb = b ? parse(b) : undefined;
+ if (!pa || !pb) return false;
+ return deltaE(pa, pb) <= SAME_COLOR_DELTA_E;
+}
+
+const COLOR_ROLES = ["primary", "accent", "background", "textPrimary"] as const;
+
+type ButtonStyle = NonNullable<
+ NonNullable<BrandingProfile["components"]>["buttonPrimary"]
+>;
+
+// Everything the response carries for a button.
+function sameButton(a?: ButtonStyle, b?: ButtonStyle): boolean {
+ if (!a && !b) return true;
+ if (!a || !b) return false;
+ const sameText = (x?: string, y?: string) => (x || "none") === (y || "none");
+ return (
+ sameColor(a.background, b.background) &&
+ sameColor(a.textColor, b.textColor) &&
+ sameColor(a.borderColor, b.borderColor) &&
+ sameText(a.borderRadius, b.borderRadius) &&
+ sameText(a.shadow, b.shadow)
+ );
+}
+
+// Merge keeps the page's own fonts when an answer cleans none, so compare the
+// merged profiles, not the raw answers.
+function fontOverlap(a: BrandingProfile, b: BrandingProfile): number {
+ const names = (p: BrandingProfile) =>
+ new Set((p.fonts ?? []).map(f => f.family.toLowerCase()));
+ const x = names(a);
+ const y = names(b);
+ const union = new Set([...x, ...y]);
+ if (union.size === 0) return 1;
+ return [...x].filter(f => y.has(f)).length / union.size;
+}
+
+/**
+ * Agreement between two answers to the same branding questions, measured on
+ * what the response would contain after merge: logo, button styles, color roles
+ * and fonts.
+ */
+export function compareBrandingAnswers(
+ input: Pick<BrandingLLMInput, "jsAnalysis" | "buttons" | "logoCandidates">,
+ llm: BrandingEnhancement,
+ jev: BrandingEnhancement,
+): Record<string, boolean | number> {
+ // merge edits nested objects of the profile it is given; each side gets its own copy.
+ const merged = (answer: BrandingEnhancement): BrandingProfile =>
+ mergeBrandingResults(
+ structuredClone(input.jsAnalysis),
+ structuredClone(answer),
+ structuredClone(input.buttons ?? []),
+ structuredClone(input.logoCandidates),
+ );
+ const a = merged(llm);
+ const b = merged(jev);
+
+ const result: Record<string, boolean | number> = {};
+ if ((input.logoCandidates?.length ?? 0) > 0) {
+ result.logo_agree = (a.images?.logo ?? null) === (b.images?.logo ?? null);
+ }
+ result.button_primary_agree = sameButton(
+ a.components?.buttonPrimary,
+ b.components?.buttonPrimary,
+ );
+ result.button_secondary_agree = sameButton(
+ a.components?.buttonSecondary,
+ b.components?.buttonSecondary,
+ );
+ for (const role of COLOR_ROLES) {
+ result[`color_${role}_agree`] = sameColor(
+ a.colors?.[role],
+ b.colors?.[role],
+ );
+ }
+ result.fonts_overlap = fontOverlap(a, b);
+
+ const flags = Object.values(result).filter(
+ (v): v is boolean => typeof v === "boolean",
+ );
+ result.fields_compared = flags.length;
+ result.fields_agreeing = flags.filter(Boolean).length;
+ return result;
+}
+
+/**
+ * Starts the shadow call and returns at once. Inputs are copied before
+ * returning: the caller keeps editing the LLM answer and the profile.
+ */
+export function shadowJev(
+ input: BrandingLLMInput,
+ llm: BrandingEnhancement,
+ llmModel: string,
+): Promise<void> {
+ const snapshot: BrandingLLMInput = {
+ ...input,
+ jsAnalysis: structuredClone(input.jsAnalysis),
+ buttons: structuredClone(input.buttons),
+ logoCandidates: structuredClone(input.logoCandidates),
+ // Jev does not read these; don't keep large strings alive.
+ screenshot: undefined,
+ headerHtmlChunk: undefined,
+ // Kept apart from the scrape's own cost record and cost limit.
+ costTracking: new CostTracking(),
+ };
+ const llmAnswer = structuredClone(llm);
+ inFlight++;
+
+ return withSpan("branding.jev_shadow", async span => {
+ setSpanAttributes(span, {
+ feature: "branding",
+ "branding.shadow.llm_model": llmModel,
+ ...(input.teamId ? { "branding.shadow.team_id": input.teamId } : {}),
+ ...(input.scrapeId ? { scrapeId: input.scrapeId } : {}),
+ });
+ const jev = await enhanceBrandingWithJev(snapshot, { shadow: true });
+ if (!jev) {
+ setSpanAttributes(span, { "branding.shadow.outcome": "jev_failed" });
+ return;
+ }
+ const comparison = compareBrandingAnswers(
+ snapshot,
+ llmAnswer,
+ jev.enhancement,
+ );
+ setSpanAttributes(span, {
+ "branding.shadow.outcome": "compared",
+ ...(jev.logoConfidence !== undefined
+ ? { "branding.shadow.jev_logo_confidence": jev.logoConfidence }
+ : {}),
+ ...Object.fromEntries(
+ Object.entries(comparison).map(([k, v]) => [`branding.shadow.${k}`, v]),
+ ),
+ });
+ })
+ .catch(error => {
+ input.logger.warn("Jev branding shadow failed", { error });
+ })
+ .finally(() => {
+ inFlight--;
+ });
+}
diff --git a/apps/api/src/lib/branding/jev.ts b/apps/api/src/lib/branding/jev.ts
index 2f5a42e33..7e4e7b3fc 100644
--- a/apps/api/src/lib/branding/jev.ts
+++ b/apps/api/src/lib/branding/jev.ts
@@ -766,6 +766,7 @@ function mapJevAnswers(
*/
export async function enhanceBrandingWithJev(
input: BrandingLLMInput,
+ options: { shadow?: boolean } = {},
): Promise<JevBrandingResult | null> {
const typesafe = getTypeSafeClient();
if (!typesafe) return null;
@@ -802,15 +803,21 @@ export async function enhanceBrandingWithJev(
attributes: {
feature: "branding",
"branding.jev.questions": Object.keys(request.questions).length,
+ ...(options.shadow ? { "branding.jev.shadow": true } : {}),
...(input.scrapeId ? { scrapeId: input.scrapeId } : {}),
},
},
);
} catch (error) {
- input.logger.warn("Jev branding call failed, falling back to LLM", {
- error,
- elapsedMs: Date.now() - started,
- });
+ input.logger.warn(
+ options.shadow
+ ? "Jev branding shadow call failed"
+ : "Jev branding call failed, falling back to LLM",
+ {
+ error,
+ elapsedMs: Date.now() - started,
+ },
+ );
return null;
}
@@ -829,14 +836,20 @@ export async function enhanceBrandingWithJev(
result = mapJevAnswers(request, response);
} catch (error) {
// A successful call whose answers don't have the expected shape.
- input.logger.warn("Jev branding answers unusable, falling back to LLM", {
- error,
- model: response.model,
- });
+ input.logger.warn(
+ options.shadow
+ ? "Jev branding shadow answers unusable"
+ : "Jev branding answers unusable, falling back to LLM",
+ {
+ error,
+ model: response.model,
+ },
+ );
return null;
}
input.logger.info("Jev branding call", {
model: response.model,
+ shadow: options.shadow === true,
elapsedMs: Date.now() - started,
inputTokens,
questions: Object.keys(request.questions).length,
diff --git a/apps/api/src/lib/branding/llm.ts b/apps/api/src/lib/branding/llm.ts
index f96779e39..e3ad72337 100644
--- a/apps/api/src/lib/branding/llm.ts
+++ b/apps/api/src/lib/branding/llm.ts
@@ -5,6 +5,7 @@ import { calculateCost } from "../../scraper/scrapeURL/transformers/llmExtract";
import { CostLimitExceededError } from "../cost-tracking";
import { BrandingEnhancement, getBrandingEnhancementSchema } from "./schema";
import { enhanceBrandingWithJev, isJevBrandingEnabled } from "./jev";
+import { shadowJev, shouldShadowJev } from "./jev-shadow";
import { buildBrandingPrompt } from "./prompt";
import { BrandingLLMInput } from "./types";
import { getModel } from "../generic-ai";
@@ -107,7 +108,8 @@ export async function enhanceBrandingWithLLM(
): Promise<BrandingEnhancement> {
const logger = input.logger;
- if (isJevBrandingEnabled(input)) {
+ const triedJev = isJevBrandingEnabled(input);
+ if (triedJev) {
const jev = await enhanceBrandingWithJev(input);
const escalateBelow = config.BRANDING_JEV_ESCALATE_BELOW;
const unsureOfLogo =
@@ -284,12 +286,16 @@ export async function enhanceBrandingWithLLM(
}
// When there are no logo candidates, do not pass logoSelection so downstream treats it as "none"
- const resultObject = result.object as BrandingEnhancement;
- if (!hasLogoCandidates && resultObject?.logoSelection != null) {
- const { logoSelection: _, ...rest } = resultObject;
- return rest as BrandingEnhancement;
+ let answer = result.object as BrandingEnhancement;
+ if (!hasLogoCandidates && answer?.logoSelection != null) {
+ const { logoSelection: _, ...rest } = answer;
+ answer = rest as BrandingEnhancement;
}
- return resultObject;
+ // Requests that already went to Jev have nothing to compare against.
+ if (!triedJev && answer && shouldShadowJev(input)) {
+ void shadowJev(input, answer, modelId);
+ }
+ return answer;
} catch (error) {
if (error instanceof CostLimitExceededError) {
throw error;