Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(branding): ask Jev again without the first screen when TypeSafe refuses the image (#5544)
Abimael Martell49709a12a171
diff --git a/apps/api/src/__tests__/lib/branding/jev.test.ts b/apps/api/src/__tests__/lib/branding/jev.test.tsindex c9674fe49..a4af6b287 100644--- a/apps/api/src/__tests__/lib/branding/jev.test.ts+++ b/apps/api/src/__tests__/lib/branding/jev.test.ts@@ -1,7 +1,11 @@import { vi, describe, it, expect, beforeEach, afterEach } from "vitest";import type { Mock } from "vitest";import { generateObject } from "ai";-import { APITimeoutError, RateLimitError } from "@typesafe-ai/sdk";+import {+ APITimeoutError,+ InternalServerError,+ RateLimitError,+} from "@typesafe-ai/sdk";const mocks = vi.hoisted(() => ({ systemOne: vi.fn() }));@@ -582,6 +586,131 @@ describe("branding with Jev", () => {result.buttonClassification.primaryButtonIndex,).toBeGreaterThanOrEqual(0);});++ const callCount = async (outcome: string) =>+ (await brandingJevCallsTotal.get()).values.find(+ v => v.labels.outcome === outcome,+ )?.value ?? 0;+ const refused = (status: number, message: string) =>+ new InternalServerError(status, {}, new Headers(), message);+ const withScreen = () => {+ const input = baseInput(new CostTracking());+ input.firstScreen = `data:image/png;base64,${PNG}`;+ return input;+ };++ it("asks again from the text alone when TypeSafe refuses the image, instead of paying for the LLM", async () => {+ for (const refusal of [+ refused(503, "503 Image processing is unavailable."),+ refused(+ 529,+ "529 We are currently experiencing high traffic and cannot process your request.",+ ),+ ]) {+ mocks.systemOne.mockReset();+ mocks.systemOne+ .mockRejectedValueOnce(refusal)+ .mockResolvedValueOnce(jevResponse());+ const before = {+ rejected: await callCount("image_rejected"),+ ok: await callCount("ok"),+ };++ const result = await enhanceBrandingWithLLM(withScreen());++ expect(mocks.systemOne).toHaveBeenCalledTimes(2);+ const [first, firstOptions] = mocks.systemOne.mock.calls[0];+ const [second, secondOptions] = mocks.systemOne.mock.calls[1];+ expect(first.assets).toBeDefined();+ expect(firstOptions.headers).toEqual({+ "TypeSafe-Danger-Zone-Images": "true",+ });+ expect(second.assets).toBeUndefined();+ expect(second.state.page.first_screen).toBeUndefined();+ expect(second.questions.primary_button.instructions).not.toContain(+ "<screenshot>",+ );+ expect(secondOptions.headers).toBeUndefined();+ expect(generateObject).not.toHaveBeenCalled();+ expect(+ result.buttonClassification.primaryButtonIndex,+ ).toBeGreaterThanOrEqual(0);+ expect(await callCount("image_rejected")).toBe(before.rejected + 1);+ expect(await callCount("ok")).toBe(before.ok + 1);+ }+ });++ it("does not count a refused image toward the breaker", async () => {+ // Five failures in a row would open it; five refused images must not.+ mocks.systemOne.mockReset();+ for (let i = 0; i < 5; i++) {+ mocks.systemOne+ .mockRejectedValueOnce(+ refused(503, "503 Image processing is unavailable."),+ )+ .mockResolvedValueOnce(jevResponse());+ }+ for (let i = 0; i < 5; i++) await enhanceBrandingWithLLM(withScreen());+ expect(mocks.systemOne).toHaveBeenCalledTimes(10);+ expect(wantsBrandingScreenshot(true, "team-jev", false)).toBe("flag");++ mocks.systemOne.mockResolvedValueOnce(jevResponse());+ await enhanceBrandingWithLLM(withScreen());+ expect(mocks.systemOne).toHaveBeenCalledTimes(11);+ expect(generateObject).not.toHaveBeenCalled();+ });++ it("falls back to the LLM when the text-only retry fails too, and that failure counts toward the breaker", async () => {+ mocks.systemOne.mockReset();+ mocks.systemOne+ .mockRejectedValueOnce(+ refused(503, "503 Image processing is unavailable."),+ )+ .mockRejectedValueOnce(refused(500, "500 Internal server error"));+ const before = {+ rejected: await callCount("image_rejected"),+ error: await callCount("error"),+ };++ await enhanceBrandingWithLLM(withScreen());++ expect(mocks.systemOne).toHaveBeenCalledTimes(2);+ expect(generateObject).toHaveBeenCalledTimes(1);+ expect(await callCount("image_rejected")).toBe(before.rejected + 1);+ expect(await callCount("error")).toBe(before.error + 1);++ // That failure was the first of five: four more plain failures open+ // the breaker, which a refused image alone would not have done.+ mocks.systemOne.mockRejectedValue(+ refused(500, "500 Internal server error"),+ );+ for (let i = 0; i < 4; i++) {+ await enhanceBrandingWithLLM(baseInput(new CostTracking()));+ }+ expect(mocks.systemOne).toHaveBeenCalledTimes(6);+ await enhanceBrandingWithLLM(baseInput(new CostTracking()));+ expect(mocks.systemOne).toHaveBeenCalledTimes(6);+ expect(generateObject).toHaveBeenCalledTimes(6);+ });++ it("still falls back to the LLM on a server error that is not about the image, or a refusal without one", async () => {+ mocks.systemOne.mockReset();+ mocks.systemOne.mockRejectedValueOnce(+ refused(500, "500 Internal server error"),+ );+ await enhanceBrandingWithLLM(withScreen());+ expect(mocks.systemOne).toHaveBeenCalledTimes(1);+ expect(generateObject).toHaveBeenCalledTimes(1);++ // A 529 on a text-only call is TypeSafe asking us to back off: no retry.+ mocks.systemOne.mockReset();+ mocks.systemOne.mockRejectedValueOnce(+ refused(529, "529 We are currently experiencing high traffic"),+ );+ await enhanceBrandingWithLLM(baseInput(new CostTracking()));+ expect(mocks.systemOne).toHaveBeenCalledTimes(1);+ expect(generateObject).toHaveBeenCalledTimes(2);+ });});it("fails fast when TypeSafe asks to back off, and retries a 503 or a timeout once, quickly", async () => {diff --git a/apps/api/src/lib/branding/jev.ts b/apps/api/src/lib/branding/jev.tsindex b3d2367b9..a5af34804 100644--- a/apps/api/src/lib/branding/jev.ts+++ b/apps/api/src/lib/branding/jev.ts@@ -11,7 +11,12 @@ import { formatHex, hsl, parse, wcagContrast } from "culori";import { config } from "../../config";import { setSpanAttributes, SpanKind, withSpan } from "../otel-tracer";import { getTypeSafeClient } from "../typesafe";-import { APITimeoutError, Image, RateLimitError } from "@typesafe-ai/sdk";+import {+ APIError,+ APITimeoutError,+ Image,+ RateLimitError,+} from "@typesafe-ai/sdk";import {brandingJevBreakerOpensTotal,brandingJevCallsTotal,@@ -67,6 +72,20 @@ function isJevBreakerOpen(now = Date.now()): boolean {return now < breakerOpenUntil;}+/**+ * TypeSafe turning the image away: a 529 (it sheds load on the image lane+ * first, text calls keep answering) or a 5xx whose message names the image,+ * such as "503 Image processing is unavailable". A plain 5xx or a timeout+ * is a failure of the call itself and is handled as before.+ */+function isImageRejection(error: unknown): error is APIError {+ return (+ error instanceof APIError &&+ (error.status === 529 ||+ (error.status >= 500 && /\bimage/i.test(error.message)))+ );+}+function recordJevFailure(input: BrandingLLMInput): void {consecutiveFailures++;if (consecutiveFailures >= BREAKER_FAILURES && !isJevBreakerOpen()) {@@ -1231,24 +1250,24 @@ export async function enhanceBrandingWithJev(let request: JevRequest;let response: JevResponse;let called = false;- try {- request = buildJevRequest(input, screenshot);- response = await withSpan(+ let imageRetried = false;+ const callJev = (req: JevRequest) =>+ withSpan("typesafe.systemone",async span => {called = true;const result = await typesafe.systemOne({model: JEV_MODEL,- state: request.state,- questions: request.questions,- ...(request.assets ? { assets: request.assets } : {}),+ state: req.state,+ questions: req.questions,+ ...(req.assets ? { assets: req.assets } : {}),},{timeout: config.BRANDING_JEV_TIMEOUT_MS ?? DEFAULT_TIMEOUT_MS,retry: JEV_RETRY,// Image input is opt-in per request.- ...(request.assets+ ...(req.assets? { headers: { "TypeSafe-Danger-Zone-Images": "true" } }: {}),},@@ -1267,8 +1286,8 @@ export async function enhanceBrandingWithJev(zeroDataRetention: input.zeroDataRetention === true,attributes: {feature: "branding",- "branding.jev.questions": Object.keys(request.questions).length,- "branding.jev.screenshot": !!screenshot,+ "branding.jev.questions": Object.keys(req.questions).length,+ "branding.jev.screenshot": !!req.assets,"branding.jev.screenshot_outcome": firstScreen.outcome,...(firstScreen.bytes !== undefined? { "branding.jev.screenshot_bytes": firstScreen.bytes }@@ -1276,11 +1295,39 @@ export async function enhanceBrandingWithJev(...(firstScreen.outcome !== "none"? { "branding.jev.screenshot_load_ms": firstScreen.loadMs }: {}),+ ...(imageRetried ? { "branding.jev.image_retry": true } : {}),...(input.scrapeId ? { scrapeId: input.scrapeId } : {}),...(input.teamId ? { teamId: input.teamId } : {}),},},);+ try {+ request = buildJevRequest(input, screenshot);+ try {+ response = await callJev(request);+ } catch (error) {+ if (!called || !screenshot || !isImageRejection(error)) throw error;+ error satisfies APIError;+ // TypeSafe's image lane is the part that is overloaded (its text calls+ // keep answering), so ask again from the text alone. Jev without the+ // first screen is still a far better answer per dollar than the LLM+ // fallback, and a refused image says nothing about Jev's health, so+ // it doesn't count toward the breaker.+ count("image_rejected");+ // A warning with a fixed message and the vendor's status, so a log+ // metric can count refusals and page us before the fallback bill does.+ input.logger.warn("TypeSafe refused the first screen", {+ vendor: "typesafe",+ status: error.status,+ error: error.message,+ elapsedMs: Date.now() - started,+ ...screenshotLog,+ });+ imageRetried = true;+ called = false;+ request = buildJevRequest(input, undefined);+ response = await callJev(request);+ }} catch (error) {// Only TypeSafe's own failures say anything about its health.if (called) {@@ -1301,6 +1348,7 @@ export async function enhanceBrandingWithJev(error,elapsedMs: Date.now() - started,...screenshotLog,+ ...(imageRetried ? { imageRetried } : {}),});return null;}@@ -1342,6 +1390,7 @@ export async function enhanceBrandingWithJev(questions: Object.keys(request.questions).length,logoConfidence: result.logoConfidence,...screenshotLog,+ ...(imageRetried ? { imageRetried } : {}),});return result;}diff --git a/apps/api/src/lib/branding/metrics.ts b/apps/api/src/lib/branding/metrics.tsindex 786e87289..341b08afa 100644--- a/apps/api/src/lib/branding/metrics.ts+++ b/apps/api/src/lib/branding/metrics.ts@@ -67,7 +67,9 @@ export type JevCallOutcome =| "rate_limited"| "error"| "unusable"- | "breaker_open";+ | "breaker_open"+ /** TypeSafe refused the first screen (529, or a 5xx about the image); the call was retried without it and its outcome is counted on its own. */+ | "image_rejected";// Export quiet series before their first event, so rates read 0, not missing.for (const [model, reason] of [@@ -85,6 +87,7 @@ for (const outcome of ["error","unusable","breaker_open",+ "image_rejected",] satisfies JevCallOutcome[]) {brandingJevCallsTotal.inc({ outcome }, 0);}