Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(branding): stop calling Jev for a minute after repeated failures (#5032)
Abimael Martelldccff26012ac
diff --git a/apps/api/src/__tests__/lib/branding/jev-shadow.test.ts b/apps/api/src/__tests__/lib/branding/jev-shadow.test.tsindex f11bed4a2..7175d758a 100644--- a/apps/api/src/__tests__/lib/branding/jev-shadow.test.ts+++ b/apps/api/src/__tests__/lib/branding/jev-shadow.test.ts@@ -19,6 +19,7 @@ vi.mock("../../../lib/generic-ai", () => ({}));import { config } from "../../../config";+import { resetJevBreaker } from "../../../lib/branding/jev";import { compareBrandingAnswers } from "../../../lib/branding/jev-shadow";import { enhanceBrandingWithLLM } from "../../../lib/branding/llm";import { BrandingEnhancement } from "../../../lib/branding/schema";@@ -171,6 +172,7 @@ const saved = {let attributes: Record<string, unknown>[];beforeEach(() => {+ resetJevBreaker();config.TYPESAFE_API_KEY = "ts-test";config.BRANDING_JEV_TEAM_IDS = ["team-jev"];config.BRANDING_JEV_SHADOW_PERCENT = 100;@@ -265,6 +267,24 @@ describe("Jev branding shadow", () => {);});+ it("pauses while the Jev breaker is open", async () => {+ mocks.systemOne+ .mockReset()+ .mockRejectedValue(new Error("529 high traffic"));+ for (let i = 0; i < 5; i++) {+ await enhanceBrandingWithLLM(input(new CostTracking()));+ await vi.waitFor(() =>+ expect(mocks.systemOne).toHaveBeenCalledTimes(i + 1),+ );+ }+ await new Promise(r => setTimeout(r, 20));++ await enhanceBrandingWithLLM(input(new CostTracking()));+ await new Promise(r => setTimeout(r, 20));++ expect(mocks.systemOne).toHaveBeenCalledTimes(5);+ });+it("skips zero-data-retention requests", async () => {await enhanceBrandingWithLLM(input(new CostTracking(), { zeroDataRetention: true }),diff --git a/apps/api/src/__tests__/lib/branding/jev.test.ts b/apps/api/src/__tests__/lib/branding/jev.test.tsindex 19d37970b..c4a5a94ea 100644--- a/apps/api/src/__tests__/lib/branding/jev.test.ts+++ b/apps/api/src/__tests__/lib/branding/jev.test.ts@@ -23,6 +23,7 @@ import {buildJevRequest,cleanFontFamily,describeColor,+ resetJevBreaker,} from "../../../lib/branding/jev";import { enhanceBrandingWithLLM } from "../../../lib/branding/llm";import { mergeBrandingResults } from "../../../lib/branding/merge";@@ -212,6 +213,7 @@ const saved = {};beforeEach(() => {+ resetJevBreaker();config.TYPESAFE_API_KEY = "ts-test";config.BRANDING_JEV = undefined;config.BRANDING_JEV_TEAM_IDS = ["team-jev"];@@ -241,6 +243,7 @@ beforeEach(() => {});afterEach(() => {+ vi.useRealTimers();config.TYPESAFE_API_KEY = saved.key;config.BRANDING_JEV = saved.global;config.BRANDING_JEV_TEAM_IDS = saved.teamIds;@@ -377,6 +380,81 @@ describe("branding with Jev", () => {expect(result.buttonClassification.primaryButtonIndex).toBe(-1);});+ it("stops calling Jev for a minute after repeated failures, then tries again", async () => {+ vi.useFakeTimers({ toFake: ["Date"] });+ vi.setSystemTime(Date.parse("2026-10-08T12:00:00Z"));+ mocks.systemOne.mockRejectedValue(new Error("529 high traffic"));++ for (let i = 0; i < 5; i++) {+ await enhanceBrandingWithLLM(baseInput(new CostTracking()));+ }+ expect(mocks.systemOne).toHaveBeenCalledTimes(5);++ // Open: answered by the LLM without waiting on Jev.+ await enhanceBrandingWithLLM(baseInput(new CostTracking()));+ expect(mocks.systemOne).toHaveBeenCalledTimes(5);+ expect(generateObject).toHaveBeenCalledTimes(6);++ // After the cooldown Jev is tried again, and a success keeps it in use.+ vi.setSystemTime(Date.now() + 61_000);+ mocks.systemOne.mockReset().mockResolvedValue(jevResponse());+ await enhanceBrandingWithLLM(baseInput(new CostTracking()));+ await enhanceBrandingWithLLM(baseInput(new CostTracking()));+ expect(mocks.systemOne).toHaveBeenCalledTimes(2);+ expect(generateObject).toHaveBeenCalledTimes(6);+ });++ it("reopens on the first failure after the cooldown", async () => {+ vi.useFakeTimers({ toFake: ["Date"] });+ vi.setSystemTime(Date.parse("2026-10-08T12:00:00Z"));+ mocks.systemOne.mockRejectedValue(new Error("503 no healthy upstream"));+ for (let i = 0; i < 5; i++) {+ await enhanceBrandingWithLLM(baseInput(new CostTracking()));+ }++ vi.setSystemTime(Date.now() + 61_000);+ await enhanceBrandingWithLLM(baseInput(new CostTracking()));+ await enhanceBrandingWithLLM(baseInput(new CostTracking()));++ expect(mocks.systemOne).toHaveBeenCalledTimes(6);+ });++ it("reopens at once after the cooldown even if a call that started earlier succeeded late", async () => {+ vi.useFakeTimers({ toFake: ["Date"] });+ vi.setSystemTime(Date.parse("2026-10-08T12:00:00Z"));+ let finishSlowCall!: (value: unknown) => void;+ mocks.systemOne.mockImplementationOnce(+ () => new Promise(resolve => (finishSlowCall = resolve)),+ );+ const slow = enhanceBrandingWithLLM(baseInput(new CostTracking()));+ mocks.systemOne.mockRejectedValue(new Error("529 high traffic"));+ for (let i = 0; i < 5; i++) {+ await enhanceBrandingWithLLM(baseInput(new CostTracking()));+ }+ finishSlowCall(jevResponse());+ await slow;++ vi.setSystemTime(Date.now() + 61_000);+ await enhanceBrandingWithLLM(baseInput(new CostTracking()));+ await enhanceBrandingWithLLM(baseInput(new CostTracking()));++ // the slow call, five failures, one failure after the cooldown; then open again+ expect(mocks.systemOne).toHaveBeenCalledTimes(7);+ });++ it("does not count unusable answers as failures", async () => {+ mocks.systemOne.mockResolvedValue({+ model: "jev-1.13.0",+ usage: { input_tokens: 10, output_tokens: 0 },+ });++ for (let i = 0; i < 6; i++) {+ await enhanceBrandingWithLLM(baseInput(new CostTracking()));+ }++ expect(mocks.systemOne).toHaveBeenCalledTimes(6);+ });+it("falls back to the LLM when Jev's answers have an unexpected shape", async () => {respondWith({model: "jev-1.13.0",diff --git a/apps/api/src/lib/branding/jev-shadow.ts b/apps/api/src/lib/branding/jev-shadow.tsindex 21a57694e..83a9a51ab 100644--- a/apps/api/src/lib/branding/jev-shadow.ts+++ b/apps/api/src/lib/branding/jev-shadow.ts@@ -3,7 +3,7 @@ import { differenceCiede2000, parse } from "culori";import { config } from "../../config";import { CostTracking } from "../cost-tracking";import { setSpanAttributes, withSpan } from "../otel-tracer";-import { enhanceBrandingWithJev } from "./jev";+import { enhanceBrandingWithJev, isJevBreakerOpen } from "./jev";import { mergeBrandingResults } from "./merge";import { BrandingEnhancement } from "./schema";import { BrandingLLMInput } from "./types";@@ -29,6 +29,8 @@ export function shouldShadowJev(input: BrandingLLMInput): boolean {const percent = config.BRANDING_JEV_SHADOW_PERCENT;if (percent <= 0) return false;if (inFlight >= MAX_IN_FLIGHT) return false;+ // No extra load on TypeSafe while branding has stopped calling it.+ if (isJevBreakerOpen()) return false;return Math.random() * 100 < percent;}diff --git a/apps/api/src/lib/branding/jev.ts b/apps/api/src/lib/branding/jev.tsindex 3dd27af1b..e26569150 100644--- a/apps/api/src/lib/branding/jev.ts+++ b/apps/api/src/lib/branding/jev.ts@@ -25,6 +25,35 @@ const JEV_MODEL = "jev-latest";const JEV_INPUT_USD_PER_MTOK = 0.042;const DEFAULT_TIMEOUT_MS = 5000;+// Circuit breaker: after this many calls in a row fail (timeouts, 5xx), skip+// Jev for the cooldown and answer with the LLM, so an outage doesn't add the+// timeout to every branding request. After the cooldown calls go through+// again; one more failure reopens it.+const BREAKER_FAILURES = 5;+const BREAKER_COOLDOWN_MS = 60_000;+let consecutiveFailures = 0;+let breakerOpenUntil = 0;++export function isJevBreakerOpen(now = Date.now()): boolean {+ return now < breakerOpenUntil;+}++function recordJevFailure(input: BrandingLLMInput): void {+ consecutiveFailures++;+ if (consecutiveFailures >= BREAKER_FAILURES && !isJevBreakerOpen()) {+ breakerOpenUntil = Date.now() + BREAKER_COOLDOWN_MS;+ input.logger.warn("Jev branding calls failing, using the LLM for a while", {+ consecutiveFailures,+ cooldownMs: BREAKER_COOLDOWN_MS,+ });+ }+}++export function resetJevBreaker(): void {+ consecutiveFailures = 0;+ breakerOpenUntil = 0;+}+const MAX_LOGOS = 20;const MAX_BUTTONS = 12;const MAX_COLORS = 24;@@ -781,15 +810,17 @@ export async function enhanceBrandingWithJev(options: { shadow?: boolean } = {},): Promise<JevBrandingResult | null> {const typesafe = getTypeSafeClient();- if (!typesafe) return null;+ if (!typesafe || isJevBreakerOpen()) return null;const started = Date.now();let request: JevRequest;let response: JevResponse;+ let called = false;try {request = buildJevRequest(input);response = await withSpan("typesafe.systemone",async span => {+ called = true;const result = await typesafe.systemOne({model: JEV_MODEL,@@ -823,6 +854,8 @@ export async function enhanceBrandingWithJev(},);} catch (error) {+ // Only TypeSafe's own failures say anything about its health.+ if (called) recordJevFailure(input);input.logger.warn(options.shadow? "Jev branding shadow call failed"@@ -834,6 +867,9 @@ export async function enhanceBrandingWithJev();return null;}+ // A call that started before the breaker opened and succeeds late doesn't+ // close it: the next failure after the cooldown should reopen it at once.+ if (!isJevBreakerOpen()) consecutiveFailures = 0;const inputTokens = response.usage?.input_tokens ?? 0;const outputTokens = response.usage?.output_tokens ?? 0;