🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
fix(evals): base evaluator health on the most recent runs (#19009)
Tobias Wochingerd7e3659aa9d6
diff --git a/packages/shared/src/server/repositories/events.ts b/packages/shared/src/server/repositories/events.tsindex 73b8204f03..6de758985f 100644--- a/packages/shared/src/server/repositories/events.ts+++ b/packages/shared/src/server/repositories/events.ts@@ -3845,7 +3845,13 @@ export const getLatestEvaluatorRunCost = async (return rows[0] ? Number(rows[0].trace_total_cost) : null;};-/** Counts execution traces by outcome over the last seven days, excluding tests. */+const EVALUATOR_HEALTH_RECENT_EXECUTIONS = 50;++/**+ * Counts execution traces by outcome, excluding tests. Only each evaluator's+ * most recent executions from the last seven days count, so an evaluator that+ * recovered from an earlier outage reports its current health.+ */export const getEvaluatorExecutionSummaries = async (projectId: string,evaluatorIds: string[],@@ -3856,7 +3862,9 @@ export const getEvaluatorExecutionSummaries = async (projectId,groupByColumn: "e.trace_id, e.evaluator_id",selectExpression: [+ "e.trace_id as trace_id","e.evaluator_id as evaluator_id",+ "min(e.start_time) as started_at","countIf(e.level = 'ERROR') > 0 as failed",].join(", "),})@@ -3873,9 +3881,19 @@ export const getEvaluatorExecutionSummaries = async (total: string;failed_count: string;}>({- query: `SELECT evaluator_id, count() as total, countIf(failed) as failed_count- FROM (${query}) GROUP BY evaluator_id`,- params,+ // groupArraySorted keeps a bounded top-N per evaluator instead of sorting+ // every execution of the week; trace_id makes ties deterministic.+ query: `SELECT evaluator_id, length(recent) as total,+ arrayCount(x -> x.3, recent) as failed_count+ FROM (+ SELECT evaluator_id,+ groupArraySorted({recentExecutions: UInt32})(+ (-toUnixTimestamp64Micro(started_at), trace_id, failed)+ ) as recent+ FROM (${query})+ GROUP BY evaluator_id+ )`,+ params: { ...params, recentExecutions: EVALUATOR_HEALTH_RECENT_EXECUTIONS },tags: { projectId },preferredClickhouseService: "EventsReadOnly",});diff --git a/web/src/__tests__/server/repositories/event-repository.servertest.ts b/web/src/__tests__/server/repositories/event-repository.servertest.tsindex 55ed23dbe1..7f870316f6 100644--- a/web/src/__tests__/server/repositories/event-repository.servertest.ts+++ b/web/src/__tests__/server/repositories/event-repository.servertest.ts@@ -530,6 +530,51 @@ describe("Clickhouse Events Repository Test", () => {getEvaluatorExecutionSummaries(projectId, [randomUUID()]),).resolves.toEqual([]);});++ it("only counts each evaluator's most recent execution traces so recovered evaluators turn healthy", async () => {+ const evaluatorId = randomUUID();+ const busyEvaluatorId = randomUUID();+ const hourInMicros = 60 * 60 * 1000 * 1000;+ const now = Date.now() * 1000;+ await createEventsCh([+ ...Array.from({ length: 80 }, () =>+ createEvent({+ project_id: projectId,+ evaluator_id: busyEvaluatorId,+ level: "ERROR",+ start_time: now - 30 * 60 * 1000 * 1000,+ }),+ ),+ ...Array.from({ length: 60 }, () =>+ createEvent({+ project_id: projectId,+ evaluator_id: evaluatorId,+ level: "ERROR",+ start_time: now - 48 * hourInMicros,+ }),+ ),+ ...Array.from({ length: 50 }, () =>+ createEvent({+ project_id: projectId,+ evaluator_id: evaluatorId,+ level: "DEFAULT",+ start_time: now - hourInMicros,+ }),+ ),+ ]);++ const summaries = await getEvaluatorExecutionSummaries(projectId, [+ evaluatorId,+ busyEvaluatorId,+ ]);+ expect(summaries).toHaveLength(2);+ expect(summaries).toEqual(+ expect.arrayContaining([+ { evaluatorId, total: 50, failed: 0 },+ { evaluatorId: busyEvaluatorId, total: 50, failed: 50 },+ ]),+ );+ });});maybe("rule execution metrics", () => {diff --git a/web/src/features/evals/v2/components/Evaluators/EvaluatorStatusBadge/EvaluatorStatusBadge.tsx b/web/src/features/evals/v2/components/Evaluators/EvaluatorStatusBadge/EvaluatorStatusBadge.tsxindex bcd81eb34a..d796d56049 100644--- a/web/src/features/evals/v2/components/Evaluators/EvaluatorStatusBadge/EvaluatorStatusBadge.tsx+++ b/web/src/features/evals/v2/components/Evaluators/EvaluatorStatusBadge/EvaluatorStatusBadge.tsx@@ -35,7 +35,7 @@ const colors = {Unknown: "primary",} as const;-/** Displays evaluator health from execution traces over the last seven days. */+/** Displays evaluator health from the most recent execution traces of the last seven days. */export function EvaluatorStatusBadge({ruleCount,summary,@@ -71,9 +71,11 @@ export function EvaluatorStatusBadge({return "No execution traces in the last 7 days.";}if (status === "Healthy") {- return "All runs passed in the last 7 days.";+ return summary?.total === 1+ ? "The only run in the last 7 days passed."+ : `The ${summary?.total} most recent runs in the last 7 days passed.`;}- return `${summary?.failed} of ${summary?.total} execution traces failed in the last 7 days.`;+ return `${summary?.failed} of the ${summary?.total} most recent execution traces in the last 7 days failed.`;}, [status, blockMessage, blockReason, ruleCount, summary]);const badge = (