Skip to content

A curated collection of practical AI projects implementing OCR systems, RAG, AI agents, and other AI use cases.

fix: tolerate missing Jev confidence from eve evaluate

cyberholicse14ff5693f54
diff --git a/ai_agents/nl_data_analyst_agent/agent/lib/jev.ts b/ai_agents/nl_data_analyst_agent/agent/lib/jev.ts
index 6af78ba..264242a 100644
--- a/ai_agents/nl_data_analyst_agent/agent/lib/jev.ts
+++ b/ai_agents/nl_data_analyst_agent/agent/lib/jev.ts
@@ -53,10 +53,19 @@ export async function reviewSqlRelevance(
},
},
});
- const verdict = result.answers.verdict.choice as "relevant" | "mismatch";
- const confidence = result.answers.verdict.confidence;
+ const verdictAnswer = result.answers.verdict as {
+ choice: string;
+ confidence?: number;
+ probabilities?: Record<string, number>;
+ };
+ const verdict = verdictAnswer.choice as "relevant" | "mismatch";
+ // eve/ai evaluate exposes the choice; confidence may be a scalar, live in the
+ // probability distribution, or be absent. Fall back safely to null.
+ const confidence =
+ verdictAnswer.confidence ?? verdictAnswer.probabilities?.[verdictAnswer.choice] ?? null;
const issueKey = result.answers.issue.choice;
const threshold = Number(process.env.JEV_MIN_CONFIDENCE ?? JEV_MIN_CONFIDENCE);
+ const confident = confidence !== null && confidence >= threshold;
const inconsistent = (verdict === "mismatch") === (issueKey === "none");
const issue = inconsistent
? verdict === "mismatch"
@@ -68,8 +77,8 @@ export async function reviewSqlRelevance(
verdict,
confidence,
issue,
- actionable: verdict === "mismatch" && confidence >= threshold && !inconsistent,
- status: inconsistent ? "inconsistent" : confidence >= threshold ? "ok" : "low_confidence",
+ actionable: verdict === "mismatch" && confident && !inconsistent,
+ status: inconsistent ? "inconsistent" : confident ? "ok" : "low_confidence",
};
} catch (error) {
return {
diff --git a/ai_agents/nl_data_analyst_agent/main.py b/ai_agents/nl_data_analyst_agent/main.py
index 357010b..46a650c 100644
--- a/ai_agents/nl_data_analyst_agent/main.py
+++ b/ai_agents/nl_data_analyst_agent/main.py
@@ -126,7 +126,14 @@ def project_eve_events(
completed_order.append(call_id)
review = output.get("review")
if isinstance(review, dict) and (review.get("actionable") or review.get("status") != "ok"):
- sql_reviews.append(Review(**review))
+ sql_reviews.append(Review(
+ check=review.get("check", "SQL relevance"),
+ verdict=review.get("verdict", "unavailable"),
+ confidence=review.get("confidence"),
+ issue=review.get("issue", ""),
+ actionable=bool(review.get("actionable", False)),
+ status=review.get("status", "ok"),
+ ))
elif kind == "input.requested":
for request in data.get("requests") or []:
action = request.get("action") or {}