Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

fix(evals): base evaluator health on the most recent runs (#19009)

Tobias Wochingerd7e3659aa9d6
diff --git a/packages/shared/src/server/repositories/events.ts b/packages/shared/src/server/repositories/events.ts
index 73b8204f03..6de758985f 100644
--- a/packages/shared/src/server/repositories/events.ts
+++ b/packages/shared/src/server/repositories/events.ts
@@ -3845,7 +3845,13 @@ export const getLatestEvaluatorRunCost = async (
return rows[0] ? Number(rows[0].trace_total_cost) : null;
};
-/** Counts execution traces by outcome over the last seven days, excluding tests. */
+const EVALUATOR_HEALTH_RECENT_EXECUTIONS = 50;
+
+/**
+ * Counts execution traces by outcome, excluding tests. Only each evaluator's
+ * most recent executions from the last seven days count, so an evaluator that
+ * recovered from an earlier outage reports its current health.
+ */
export const getEvaluatorExecutionSummaries = async (
projectId: string,
evaluatorIds: string[],
@@ -3856,7 +3862,9 @@ export const getEvaluatorExecutionSummaries = async (
projectId,
groupByColumn: "e.trace_id, e.evaluator_id",
selectExpression: [
+ "e.trace_id as trace_id",
"e.evaluator_id as evaluator_id",
+ "min(e.start_time) as started_at",
"countIf(e.level = 'ERROR') > 0 as failed",
].join(", "),
})
@@ -3873,9 +3881,19 @@ export const getEvaluatorExecutionSummaries = async (
total: string;
failed_count: string;
}>({
- query: `SELECT evaluator_id, count() as total, countIf(failed) as failed_count
- FROM (${query}) GROUP BY evaluator_id`,
- params,
+ // groupArraySorted keeps a bounded top-N per evaluator instead of sorting
+ // every execution of the week; trace_id makes ties deterministic.
+ query: `SELECT evaluator_id, length(recent) as total,
+ arrayCount(x -> x.3, recent) as failed_count
+ FROM (
+ SELECT evaluator_id,
+ groupArraySorted({recentExecutions: UInt32})(
+ (-toUnixTimestamp64Micro(started_at), trace_id, failed)
+ ) as recent
+ FROM (${query})
+ GROUP BY evaluator_id
+ )`,
+ params: { ...params, recentExecutions: EVALUATOR_HEALTH_RECENT_EXECUTIONS },
tags: { projectId },
preferredClickhouseService: "EventsReadOnly",
});
diff --git a/web/src/__tests__/server/repositories/event-repository.servertest.ts b/web/src/__tests__/server/repositories/event-repository.servertest.ts
index 55ed23dbe1..7f870316f6 100644
--- a/web/src/__tests__/server/repositories/event-repository.servertest.ts
+++ b/web/src/__tests__/server/repositories/event-repository.servertest.ts
@@ -530,6 +530,51 @@ describe("Clickhouse Events Repository Test", () => {
getEvaluatorExecutionSummaries(projectId, [randomUUID()]),
).resolves.toEqual([]);
});
+
+ it("only counts each evaluator's most recent execution traces so recovered evaluators turn healthy", async () => {
+ const evaluatorId = randomUUID();
+ const busyEvaluatorId = randomUUID();
+ const hourInMicros = 60 * 60 * 1000 * 1000;
+ const now = Date.now() * 1000;
+ await createEventsCh([
+ ...Array.from({ length: 80 }, () =>
+ createEvent({
+ project_id: projectId,
+ evaluator_id: busyEvaluatorId,
+ level: "ERROR",
+ start_time: now - 30 * 60 * 1000 * 1000,
+ }),
+ ),
+ ...Array.from({ length: 60 }, () =>
+ createEvent({
+ project_id: projectId,
+ evaluator_id: evaluatorId,
+ level: "ERROR",
+ start_time: now - 48 * hourInMicros,
+ }),
+ ),
+ ...Array.from({ length: 50 }, () =>
+ createEvent({
+ project_id: projectId,
+ evaluator_id: evaluatorId,
+ level: "DEFAULT",
+ start_time: now - hourInMicros,
+ }),
+ ),
+ ]);
+
+ const summaries = await getEvaluatorExecutionSummaries(projectId, [
+ evaluatorId,
+ busyEvaluatorId,
+ ]);
+ expect(summaries).toHaveLength(2);
+ expect(summaries).toEqual(
+ expect.arrayContaining([
+ { evaluatorId, total: 50, failed: 0 },
+ { evaluatorId: busyEvaluatorId, total: 50, failed: 50 },
+ ]),
+ );
+ });
});
maybe("rule execution metrics", () => {
diff --git a/web/src/features/evals/v2/components/Evaluators/EvaluatorStatusBadge/EvaluatorStatusBadge.tsx b/web/src/features/evals/v2/components/Evaluators/EvaluatorStatusBadge/EvaluatorStatusBadge.tsx
index bcd81eb34a..d796d56049 100644
--- a/web/src/features/evals/v2/components/Evaluators/EvaluatorStatusBadge/EvaluatorStatusBadge.tsx
+++ b/web/src/features/evals/v2/components/Evaluators/EvaluatorStatusBadge/EvaluatorStatusBadge.tsx
@@ -35,7 +35,7 @@ const colors = {
Unknown: "primary",
} as const;
-/** Displays evaluator health from execution traces over the last seven days. */
+/** Displays evaluator health from the most recent execution traces of the last seven days. */
export function EvaluatorStatusBadge({
ruleCount,
summary,
@@ -71,9 +71,11 @@ export function EvaluatorStatusBadge({
return "No execution traces in the last 7 days.";
}
if (status === "Healthy") {
- return "All runs passed in the last 7 days.";
+ return summary?.total === 1
+ ? "The only run in the last 7 days passed."
+ : `The ${summary?.total} most recent runs in the last 7 days passed.`;
}
- return `${summary?.failed} of ${summary?.total} execution traces failed in the last 7 days.`;
+ return `${summary?.failed} of the ${summary?.total} most recent execution traces in the last 7 days failed.`;
}, [status, blockMessage, blockReason, ruleCount, summary]);
const badge = (