Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(branding): ask Jev again without the first screen when TypeSafe refuses the image (#5544)

Abimael Martell49709a12a171
diff --git a/apps/api/src/__tests__/lib/branding/jev.test.ts b/apps/api/src/__tests__/lib/branding/jev.test.ts
index c9674fe49..a4af6b287 100644
--- a/apps/api/src/__tests__/lib/branding/jev.test.ts
+++ b/apps/api/src/__tests__/lib/branding/jev.test.ts
@@ -1,7 +1,11 @@
import { vi, describe, it, expect, beforeEach, afterEach } from "vitest";
import type { Mock } from "vitest";
import { generateObject } from "ai";
-import { APITimeoutError, RateLimitError } from "@typesafe-ai/sdk";
+import {
+ APITimeoutError,
+ InternalServerError,
+ RateLimitError,
+} from "@typesafe-ai/sdk";
const mocks = vi.hoisted(() => ({ systemOne: vi.fn() }));
@@ -582,6 +586,131 @@ describe("branding with Jev", () => {
result.buttonClassification.primaryButtonIndex,
).toBeGreaterThanOrEqual(0);
});
+
+ const callCount = async (outcome: string) =>
+ (await brandingJevCallsTotal.get()).values.find(
+ v => v.labels.outcome === outcome,
+ )?.value ?? 0;
+ const refused = (status: number, message: string) =>
+ new InternalServerError(status, {}, new Headers(), message);
+ const withScreen = () => {
+ const input = baseInput(new CostTracking());
+ input.firstScreen = `data:image/png;base64,${PNG}`;
+ return input;
+ };
+
+ it("asks again from the text alone when TypeSafe refuses the image, instead of paying for the LLM", async () => {
+ for (const refusal of [
+ refused(503, "503 Image processing is unavailable."),
+ refused(
+ 529,
+ "529 We are currently experiencing high traffic and cannot process your request.",
+ ),
+ ]) {
+ mocks.systemOne.mockReset();
+ mocks.systemOne
+ .mockRejectedValueOnce(refusal)
+ .mockResolvedValueOnce(jevResponse());
+ const before = {
+ rejected: await callCount("image_rejected"),
+ ok: await callCount("ok"),
+ };
+
+ const result = await enhanceBrandingWithLLM(withScreen());
+
+ expect(mocks.systemOne).toHaveBeenCalledTimes(2);
+ const [first, firstOptions] = mocks.systemOne.mock.calls[0];
+ const [second, secondOptions] = mocks.systemOne.mock.calls[1];
+ expect(first.assets).toBeDefined();
+ expect(firstOptions.headers).toEqual({
+ "TypeSafe-Danger-Zone-Images": "true",
+ });
+ expect(second.assets).toBeUndefined();
+ expect(second.state.page.first_screen).toBeUndefined();
+ expect(second.questions.primary_button.instructions).not.toContain(
+ "<screenshot>",
+ );
+ expect(secondOptions.headers).toBeUndefined();
+ expect(generateObject).not.toHaveBeenCalled();
+ expect(
+ result.buttonClassification.primaryButtonIndex,
+ ).toBeGreaterThanOrEqual(0);
+ expect(await callCount("image_rejected")).toBe(before.rejected + 1);
+ expect(await callCount("ok")).toBe(before.ok + 1);
+ }
+ });
+
+ it("does not count a refused image toward the breaker", async () => {
+ // Five failures in a row would open it; five refused images must not.
+ mocks.systemOne.mockReset();
+ for (let i = 0; i < 5; i++) {
+ mocks.systemOne
+ .mockRejectedValueOnce(
+ refused(503, "503 Image processing is unavailable."),
+ )
+ .mockResolvedValueOnce(jevResponse());
+ }
+ for (let i = 0; i < 5; i++) await enhanceBrandingWithLLM(withScreen());
+ expect(mocks.systemOne).toHaveBeenCalledTimes(10);
+ expect(wantsBrandingScreenshot(true, "team-jev", false)).toBe("flag");
+
+ mocks.systemOne.mockResolvedValueOnce(jevResponse());
+ await enhanceBrandingWithLLM(withScreen());
+ expect(mocks.systemOne).toHaveBeenCalledTimes(11);
+ expect(generateObject).not.toHaveBeenCalled();
+ });
+
+ it("falls back to the LLM when the text-only retry fails too, and that failure counts toward the breaker", async () => {
+ mocks.systemOne.mockReset();
+ mocks.systemOne
+ .mockRejectedValueOnce(
+ refused(503, "503 Image processing is unavailable."),
+ )
+ .mockRejectedValueOnce(refused(500, "500 Internal server error"));
+ const before = {
+ rejected: await callCount("image_rejected"),
+ error: await callCount("error"),
+ };
+
+ await enhanceBrandingWithLLM(withScreen());
+
+ expect(mocks.systemOne).toHaveBeenCalledTimes(2);
+ expect(generateObject).toHaveBeenCalledTimes(1);
+ expect(await callCount("image_rejected")).toBe(before.rejected + 1);
+ expect(await callCount("error")).toBe(before.error + 1);
+
+ // That failure was the first of five: four more plain failures open
+ // the breaker, which a refused image alone would not have done.
+ mocks.systemOne.mockRejectedValue(
+ refused(500, "500 Internal server error"),
+ );
+ for (let i = 0; i < 4; i++) {
+ await enhanceBrandingWithLLM(baseInput(new CostTracking()));
+ }
+ expect(mocks.systemOne).toHaveBeenCalledTimes(6);
+ await enhanceBrandingWithLLM(baseInput(new CostTracking()));
+ expect(mocks.systemOne).toHaveBeenCalledTimes(6);
+ expect(generateObject).toHaveBeenCalledTimes(6);
+ });
+
+ it("still falls back to the LLM on a server error that is not about the image, or a refusal without one", async () => {
+ mocks.systemOne.mockReset();
+ mocks.systemOne.mockRejectedValueOnce(
+ refused(500, "500 Internal server error"),
+ );
+ await enhanceBrandingWithLLM(withScreen());
+ expect(mocks.systemOne).toHaveBeenCalledTimes(1);
+ expect(generateObject).toHaveBeenCalledTimes(1);
+
+ // A 529 on a text-only call is TypeSafe asking us to back off: no retry.
+ mocks.systemOne.mockReset();
+ mocks.systemOne.mockRejectedValueOnce(
+ refused(529, "529 We are currently experiencing high traffic"),
+ );
+ await enhanceBrandingWithLLM(baseInput(new CostTracking()));
+ expect(mocks.systemOne).toHaveBeenCalledTimes(1);
+ expect(generateObject).toHaveBeenCalledTimes(2);
+ });
});
it("fails fast when TypeSafe asks to back off, and retries a 503 or a timeout once, quickly", async () => {
diff --git a/apps/api/src/lib/branding/jev.ts b/apps/api/src/lib/branding/jev.ts
index b3d2367b9..a5af34804 100644
--- a/apps/api/src/lib/branding/jev.ts
+++ b/apps/api/src/lib/branding/jev.ts
@@ -11,7 +11,12 @@ import { formatHex, hsl, parse, wcagContrast } from "culori";
import { config } from "../../config";
import { setSpanAttributes, SpanKind, withSpan } from "../otel-tracer";
import { getTypeSafeClient } from "../typesafe";
-import { APITimeoutError, Image, RateLimitError } from "@typesafe-ai/sdk";
+import {
+ APIError,
+ APITimeoutError,
+ Image,
+ RateLimitError,
+} from "@typesafe-ai/sdk";
import {
brandingJevBreakerOpensTotal,
brandingJevCallsTotal,
@@ -67,6 +72,20 @@ function isJevBreakerOpen(now = Date.now()): boolean {
return now < breakerOpenUntil;
}
+/**
+ * TypeSafe turning the image away: a 529 (it sheds load on the image lane
+ * first, text calls keep answering) or a 5xx whose message names the image,
+ * such as "503 Image processing is unavailable". A plain 5xx or a timeout
+ * is a failure of the call itself and is handled as before.
+ */
+function isImageRejection(error: unknown): error is APIError {
+ return (
+ error instanceof APIError &&
+ (error.status === 529 ||
+ (error.status >= 500 && /\bimage/i.test(error.message)))
+ );
+}
+
function recordJevFailure(input: BrandingLLMInput): void {
consecutiveFailures++;
if (consecutiveFailures >= BREAKER_FAILURES && !isJevBreakerOpen()) {
@@ -1231,24 +1250,24 @@ export async function enhanceBrandingWithJev(
let request: JevRequest;
let response: JevResponse;
let called = false;
- try {
- request = buildJevRequest(input, screenshot);
- response = await withSpan(
+ let imageRetried = false;
+ const callJev = (req: JevRequest) =>
+ withSpan(
"typesafe.systemone",
async span => {
called = true;
const result = await typesafe.systemOne(
{
model: JEV_MODEL,
- state: request.state,
- questions: request.questions,
- ...(request.assets ? { assets: request.assets } : {}),
+ state: req.state,
+ questions: req.questions,
+ ...(req.assets ? { assets: req.assets } : {}),
},
{
timeout: config.BRANDING_JEV_TIMEOUT_MS ?? DEFAULT_TIMEOUT_MS,
retry: JEV_RETRY,
// Image input is opt-in per request.
- ...(request.assets
+ ...(req.assets
? { headers: { "TypeSafe-Danger-Zone-Images": "true" } }
: {}),
},
@@ -1267,8 +1286,8 @@ export async function enhanceBrandingWithJev(
zeroDataRetention: input.zeroDataRetention === true,
attributes: {
feature: "branding",
- "branding.jev.questions": Object.keys(request.questions).length,
- "branding.jev.screenshot": !!screenshot,
+ "branding.jev.questions": Object.keys(req.questions).length,
+ "branding.jev.screenshot": !!req.assets,
"branding.jev.screenshot_outcome": firstScreen.outcome,
...(firstScreen.bytes !== undefined
? { "branding.jev.screenshot_bytes": firstScreen.bytes }
@@ -1276,11 +1295,39 @@ export async function enhanceBrandingWithJev(
...(firstScreen.outcome !== "none"
? { "branding.jev.screenshot_load_ms": firstScreen.loadMs }
: {}),
+ ...(imageRetried ? { "branding.jev.image_retry": true } : {}),
...(input.scrapeId ? { scrapeId: input.scrapeId } : {}),
...(input.teamId ? { teamId: input.teamId } : {}),
},
},
);
+ try {
+ request = buildJevRequest(input, screenshot);
+ try {
+ response = await callJev(request);
+ } catch (error) {
+ if (!called || !screenshot || !isImageRejection(error)) throw error;
+ error satisfies APIError;
+ // TypeSafe's image lane is the part that is overloaded (its text calls
+ // keep answering), so ask again from the text alone. Jev without the
+ // first screen is still a far better answer per dollar than the LLM
+ // fallback, and a refused image says nothing about Jev's health, so
+ // it doesn't count toward the breaker.
+ count("image_rejected");
+ // A warning with a fixed message and the vendor's status, so a log
+ // metric can count refusals and page us before the fallback bill does.
+ input.logger.warn("TypeSafe refused the first screen", {
+ vendor: "typesafe",
+ status: error.status,
+ error: error.message,
+ elapsedMs: Date.now() - started,
+ ...screenshotLog,
+ });
+ imageRetried = true;
+ called = false;
+ request = buildJevRequest(input, undefined);
+ response = await callJev(request);
+ }
} catch (error) {
// Only TypeSafe's own failures say anything about its health.
if (called) {
@@ -1301,6 +1348,7 @@ export async function enhanceBrandingWithJev(
error,
elapsedMs: Date.now() - started,
...screenshotLog,
+ ...(imageRetried ? { imageRetried } : {}),
});
return null;
}
@@ -1342,6 +1390,7 @@ export async function enhanceBrandingWithJev(
questions: Object.keys(request.questions).length,
logoConfidence: result.logoConfidence,
...screenshotLog,
+ ...(imageRetried ? { imageRetried } : {}),
});
return result;
}
diff --git a/apps/api/src/lib/branding/metrics.ts b/apps/api/src/lib/branding/metrics.ts
index 786e87289..341b08afa 100644
--- a/apps/api/src/lib/branding/metrics.ts
+++ b/apps/api/src/lib/branding/metrics.ts
@@ -67,7 +67,9 @@ export type JevCallOutcome =
| "rate_limited"
| "error"
| "unusable"
- | "breaker_open";
+ | "breaker_open"
+ /** TypeSafe refused the first screen (529, or a 5xx about the image); the call was retried without it and its outcome is counted on its own. */
+ | "image_rejected";
// Export quiet series before their first event, so rates read 0, not missing.
for (const [model, reason] of [
@@ -85,6 +87,7 @@ for (const outcome of [
"error",
"unusable",
"breaker_open",
+ "image_rejected",
] satisfies JevCallOutcome[]) {
brandingJevCallsTotal.inc({ outcome }, 0);
}