Skip to content

armgit / dub

Public Imported from GitHub
0

The modern link attribution platform. Loved by world-class engineering teams at Twilio, Zillow, Jobber, Vercel, Raycast, and more.

refactor to evaluate

Pedro Ladeira0db1e9502051
diff --git a/apps/web/.env.example b/apps/web/.env.example
index 765dd94c05..c3ac59d0bc 100644
--- a/apps/web/.env.example
+++ b/apps/web/.env.example
@@ -131,7 +131,6 @@ NEXT_PUBLIC_NGROK_URL=
# For AI features
AI_GATEWAY_API_KEY=
ANTHROPIC_API_KEY=
-AI_GATEWAY_API_KEY=
# Axiom – used for logging and monitoring
# Get your Axiom token here: https://axiom.co/docs/rest-api/authentication
diff --git a/apps/web/lib/ai/evaluate-partner-application.ts b/apps/web/lib/ai/evaluate-partner-application.ts
index bd8fcc8fab..fa2bece9d3 100644
--- a/apps/web/lib/ai/evaluate-partner-application.ts
+++ b/apps/web/lib/ai/evaluate-partner-application.ts
@@ -4,14 +4,13 @@ import {
} from "@/lib/partners/format-application-form-data";
import { programLanderSchema } from "@/lib/zod/schemas/program-lander";
import { ProgramApplication } from "@prisma/client";
-import * as z from "zod/v4";
+import { experimental_evaluate as evaluate, type JSONValue } from "ai";
export const JEV_MATCH_THRESHOLD = 0.85;
const MAX_TEXT_LENGTH = 500;
const JEV_EVALUATE_TIMEOUT_MS = 2_000;
const JEV_MODEL_ID = "typesafe-ai/jev";
-const JEV_EVALUATE_URL = "https://ai-gateway.vercel.sh/v4/ai/evaluation-model";
export type EvaluatePartnerApplicationInput = {
program: {
@@ -47,22 +46,6 @@ export type JevEvaluation = {
};
};
-const jevEvaluateResponseSchema = z.object({
- answers: z.record(
- z.string(),
- z.object({
- type: z.literal("boolean"),
- probability: z.number().finite(),
- }),
- ),
- usage: z
- .object({
- inputTokens: z.number().optional(),
- outputTokens: z.number().optional(),
- })
- .optional(),
-});
-
export function isConfidentMatch(probability: number | null | undefined) {
return (
probability != null &&
@@ -213,74 +196,48 @@ async function evaluateBooleanQuestion({
};
tag: string;
}): Promise<JevEvaluation> {
- const apiKey = process.env.AI_GATEWAY_API_KEY;
-
- if (!apiKey) {
+ if (!process.env.AI_GATEWAY_API_KEY) {
return { status: "skipped" };
}
- const abortController = new AbortController();
- const timeout = setTimeout(
- () => abortController.abort(),
- JEV_EVALUATE_TIMEOUT_MS,
- );
-
try {
- const response = await fetch(JEV_EVALUATE_URL, {
- method: "POST",
- headers: {
- Authorization: `Bearer ${apiKey}`,
- "Content-Type": "application/json",
- "ai-gateway-protocol-version": "0.0.1",
- "ai-gateway-auth-method": "api-key",
- "ai-evaluation-model-specification-version": "4",
- "ai-model-id": JEV_MODEL_ID,
- },
- body: JSON.stringify({
- state,
- questions: {
- [questionKey]: {
- type: "boolean",
- instructions,
- criteria,
- },
+ const result = await evaluate({
+ model: JEV_MODEL_ID,
+ state: state as { [key: string]: JSONValue | undefined },
+ questions: {
+ [questionKey]: {
+ type: "boolean",
+ instructions,
+ criteria,
},
- providerOptions: {
- gateway: {
- zeroDataRetention: true,
- disallowPromptTraining: true,
- tags: [tag],
- },
+ },
+ maxRetries: 0,
+ abortSignal: AbortSignal.timeout(JEV_EVALUATE_TIMEOUT_MS),
+ providerOptions: {
+ gateway: {
+ zeroDataRetention: true,
+ disallowPromptTraining: true,
+ tags: [tag],
},
- }),
- signal: abortController.signal,
+ },
});
- if (!response.ok) {
- throw new Error(`Jev evaluate failed with status ${response.status}.`);
- }
-
- const parsed = jevEvaluateResponseSchema.safeParse(await response.json());
- const answer = parsed.success
- ? parsed.data.answers[questionKey]
- : undefined;
+ const answer = result.answers[questionKey];
- if (!parsed.success || !answer) {
+ if (!answer || answer.type !== "boolean") {
throw new Error("Jev evaluate returned an invalid response.");
}
return {
status: isConfidentMatch(answer.probability) ? "matched" : "unmatched",
probability: answer.probability,
- usage: parsed.data.usage,
+ usage: result.usage,
};
} catch (error) {
return {
status: "failed",
error: error instanceof Error ? error.message : String(error),
};
- } finally {
- clearTimeout(timeout);
}
}