The modern link attribution platform. Loved by world-class engineering teams at Twilio, Zillow, Jobber, Vercel, Raycast, and more.
TypeScript clickhousemysqlnextjsopen-sourceplanetscaleprismatailwindcsstinybirdtypescriptupstashvercel
refactor to evaluate
Pedro Ladeira0db1e9502051
diff --git a/apps/web/.env.example b/apps/web/.env.exampleindex 765dd94c05..c3ac59d0bc 100644--- a/apps/web/.env.example+++ b/apps/web/.env.example@@ -131,7 +131,6 @@ NEXT_PUBLIC_NGROK_URL=# For AI featuresAI_GATEWAY_API_KEY=ANTHROPIC_API_KEY=-AI_GATEWAY_API_KEY=# Axiom – used for logging and monitoring# Get your Axiom token here: https://axiom.co/docs/rest-api/authenticationdiff --git a/apps/web/lib/ai/evaluate-partner-application.ts b/apps/web/lib/ai/evaluate-partner-application.tsindex bd8fcc8fab..fa2bece9d3 100644--- a/apps/web/lib/ai/evaluate-partner-application.ts+++ b/apps/web/lib/ai/evaluate-partner-application.ts@@ -4,14 +4,13 @@ import {} from "@/lib/partners/format-application-form-data";import { programLanderSchema } from "@/lib/zod/schemas/program-lander";import { ProgramApplication } from "@prisma/client";-import * as z from "zod/v4";+import { experimental_evaluate as evaluate, type JSONValue } from "ai";export const JEV_MATCH_THRESHOLD = 0.85;const MAX_TEXT_LENGTH = 500;const JEV_EVALUATE_TIMEOUT_MS = 2_000;const JEV_MODEL_ID = "typesafe-ai/jev";-const JEV_EVALUATE_URL = "https://ai-gateway.vercel.sh/v4/ai/evaluation-model";export type EvaluatePartnerApplicationInput = {program: {@@ -47,22 +46,6 @@ export type JevEvaluation = {};};-const jevEvaluateResponseSchema = z.object({- answers: z.record(- z.string(),- z.object({- type: z.literal("boolean"),- probability: z.number().finite(),- }),- ),- usage: z- .object({- inputTokens: z.number().optional(),- outputTokens: z.number().optional(),- })- .optional(),-});-export function isConfidentMatch(probability: number | null | undefined) {return (probability != null &&@@ -213,74 +196,48 @@ async function evaluateBooleanQuestion({};tag: string;}): Promise<JevEvaluation> {- const apiKey = process.env.AI_GATEWAY_API_KEY;-- if (!apiKey) {+ if (!process.env.AI_GATEWAY_API_KEY) {return { status: "skipped" };}- const abortController = new AbortController();- const timeout = setTimeout(- () => abortController.abort(),- JEV_EVALUATE_TIMEOUT_MS,- );-try {- const response = await fetch(JEV_EVALUATE_URL, {- method: "POST",- headers: {- Authorization: `Bearer ${apiKey}`,- "Content-Type": "application/json",- "ai-gateway-protocol-version": "0.0.1",- "ai-gateway-auth-method": "api-key",- "ai-evaluation-model-specification-version": "4",- "ai-model-id": JEV_MODEL_ID,- },- body: JSON.stringify({- state,- questions: {- [questionKey]: {- type: "boolean",- instructions,- criteria,- },+ const result = await evaluate({+ model: JEV_MODEL_ID,+ state: state as { [key: string]: JSONValue | undefined },+ questions: {+ [questionKey]: {+ type: "boolean",+ instructions,+ criteria,},- providerOptions: {- gateway: {- zeroDataRetention: true,- disallowPromptTraining: true,- tags: [tag],- },+ },+ maxRetries: 0,+ abortSignal: AbortSignal.timeout(JEV_EVALUATE_TIMEOUT_MS),+ providerOptions: {+ gateway: {+ zeroDataRetention: true,+ disallowPromptTraining: true,+ tags: [tag],},- }),- signal: abortController.signal,+ },});- if (!response.ok) {- throw new Error(`Jev evaluate failed with status ${response.status}.`);- }-- const parsed = jevEvaluateResponseSchema.safeParse(await response.json());- const answer = parsed.success- ? parsed.data.answers[questionKey]- : undefined;+ const answer = result.answers[questionKey];- if (!parsed.success || !answer) {+ if (!answer || answer.type !== "boolean") {throw new Error("Jev evaluate returned an invalid response.");}return {status: isConfidentMatch(answer.probability) ? "matched" : "unmatched",probability: answer.probability,- usage: parsed.data.usage,+ usage: result.usage,};} catch (error) {return {status: "failed",error: error instanceof Error ? error.message : String(error),};- } finally {- clearTimeout(timeout);}}