Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

feat(evals): show seven-day execution health in evaluator status (#18113)

Tobias Wochinger9d168c646be6
diff --git a/packages/shared/src/server/repositories/events.ts b/packages/shared/src/server/repositories/events.ts
index 92edbb0fa5..8dc1843911 100644
--- a/packages/shared/src/server/repositories/events.ts
+++ b/packages/shared/src/server/repositories/events.ts
@@ -3692,7 +3692,8 @@ export const getLatestEvaluatorRunCost = async (
return rows[0] ? Number(rows[0].trace_total_cost) : null;
};
-export const getRecentEvaluatorExecutionTraces = async (
+/** Counts execution traces by outcome over the last seven days, excluding tests. */
+export const getEvaluatorExecutionSummaries = async (
projectId: string,
evaluatorIds: string[],
) => {
@@ -3702,81 +3703,34 @@ export const getRecentEvaluatorExecutionTraces = async (
projectId,
groupByColumn: "e.trace_id, e.evaluator_id",
selectExpression: [
- "e.trace_id as id",
"e.evaluator_id as evaluator_id",
- "multiIf(countIf(e.level = 'ERROR') > 0, 'ERROR', countIf(e.level = 'WARNING') > 0, 'WARNING', 'DEFAULT') as level",
- "min(e.start_time) as timestamp",
+ "countIf(e.level = 'ERROR') > 0 as failed",
].join(", "),
})
.whereRaw("e.start_time > now() - INTERVAL 7 DAY")
.whereRaw("e.evaluator_id IN ({evaluatorIds: Array(String)})", {
evaluatorIds,
})
- .havingRaw(`countIf(${evaluatorTestEventCondition}) = 0`)
- .orderBy("ORDER BY timestamp DESC, id DESC")
- .limitByCount(5, "evaluator_id");
+ .havingRaw(`countIf(${evaluatorTestEventCondition}) = 0`);
+ // Aggregate the per-trace outcomes, rather than counting individual observations.
const { query, params } = builder.buildWithParams();
const rows = await queryClickhouse<{
- id: string;
evaluator_id: string;
- level: string;
- timestamp: string;
+ total: string;
+ failed_count: string;
}>({
- query,
+ query: `SELECT evaluator_id, count() as total, countIf(failed) as failed_count
+ FROM (${query}) GROUP BY evaluator_id`,
params,
tags: { projectId },
preferredClickhouseService: "EventsReadOnly",
});
return rows.map((row) => ({
- id: row.id,
evaluatorId: row.evaluator_id,
- level: row.level,
- timestamp: parseClickhouseUTCDateTimeFormat(row.timestamp),
- }));
-};
-
-export const getRecentRuleExecutionTraces = async (
- projectId: string,
- ruleIds: string[],
-) => {
- if (ruleIds.length === 0) return [];
-
- const builder = new EventsAggQueryBuilder({
- projectId,
- groupByColumn: "e.trace_id, e.evaluation_rule_id",
- selectExpression: [
- "e.trace_id as id",
- "e.evaluation_rule_id as evaluation_rule_id",
- "multiIf(countIf(e.level = 'ERROR') > 0, 'ERROR', countIf(e.level = 'WARNING') > 0, 'WARNING', 'DEFAULT') as level",
- "min(e.start_time) as timestamp",
- ].join(", "),
- })
- .whereRaw("e.start_time > now() - INTERVAL 7 DAY")
- .whereRaw("e.evaluation_rule_id IN ({ruleIds: Array(String)})", { ruleIds })
- .orderBy("ORDER BY timestamp DESC, id DESC")
- .limitByCount(5, "evaluation_rule_id");
-
- const { query, params } = builder.buildWithParams();
- const rows = await queryClickhouse<
- {
- id: string;
- level: string;
- timestamp: string;
- } & { evaluation_rule_id: string }
- >({
- query,
- params,
- tags: { projectId },
- preferredClickhouseService: "EventsReadOnly",
- });
-
- return rows.map((row) => ({
- id: row.id,
- ruleId: row.evaluation_rule_id,
- level: row.level,
- timestamp: parseClickhouseUTCDateTimeFormat(row.timestamp),
+ total: Number(row.total),
+ failed: Number(row.failed_count),
}));
};
diff --git a/web/src/__tests__/server/evaluation-rule-v2-service.servertest.ts b/web/src/__tests__/server/evaluation-rule-v2-service.servertest.ts
index 11961f013c..e8fb1802e0 100644
--- a/web/src/__tests__/server/evaluation-rule-v2-service.servertest.ts
+++ b/web/src/__tests__/server/evaluation-rule-v2-service.servertest.ts
@@ -17,13 +17,11 @@ import { RuleService } from "@/src/features/evals/v2/server/rules/ruleService";
import { MAX_ACTIVE_EVALUATION_RULES } from "@/src/features/evals/v2/server/rules/ruleErrors";
const telemetryMocks = vi.hoisted(() => ({
- getRecentRuleExecutionTraces: vi.fn(),
getTotalCostByRule: vi.fn(),
}));
vi.mock("@langfuse/shared/src/server", async (importOriginal) => ({
...(await importOriginal<typeof SharedServer>()),
- getRecentRuleExecutionTraces: telemetryMocks.getRecentRuleExecutionTraces,
getTotalCostByRule: telemetryMocks.getTotalCostByRule,
}));
@@ -43,7 +41,6 @@ beforeAll(async () => {
});
beforeEach(async () => {
- telemetryMocks.getRecentRuleExecutionTraces.mockReset();
telemetryMocks.getTotalCostByRule.mockReset();
defaultEvaluatorId = (await createEvaluator()).id;
});
@@ -378,27 +375,6 @@ describe("RuleService", () => {
});
describe("telemetry", () => {
- it("returns recent event-backed traces grouped by rule", async () => {
- const timestamp = new Date("2026-08-12T12:00:00.000Z");
- telemetryMocks.getRecentRuleExecutionTraces.mockResolvedValue([
- { ruleId: "rule-1", id: "trace-1", level: "DEFAULT", timestamp },
- ]);
-
- await expect(
- createService().listRecent({
- projectId,
- ruleIds: ["rule-1", "rule-2"],
- }),
- ).resolves.toEqual({
- "rule-1": [{ id: "trace-1", level: "DEFAULT", timestamp }],
- "rule-2": [],
- });
- expect(telemetryMocks.getRecentRuleExecutionTraces).toHaveBeenCalledWith(
- projectId,
- ["rule-1", "rule-2"],
- );
- });
-
it("returns event-backed costs by rule", async () => {
telemetryMocks.getTotalCostByRule.mockResolvedValue([
{ ruleId: "rule-1", totalCost: 1.25 },
diff --git a/web/src/__tests__/server/evaluator-v2-service.servertest.ts b/web/src/__tests__/server/evaluator-v2-service.servertest.ts
index 0fa9e8d5b1..ea33d8b56b 100644
--- a/web/src/__tests__/server/evaluator-v2-service.servertest.ts
+++ b/web/src/__tests__/server/evaluator-v2-service.servertest.ts
@@ -36,7 +36,7 @@ import {
const mocks = vi.hoisted(() => ({
generateLangfuseAIText: vi.fn(),
- getRecentEvaluatorExecutionTraces: vi.fn(),
+ getEvaluatorExecutionSummaries: vi.fn(),
getTotalCostByEvaluatorIds: vi.fn(),
invalidateProjectEvalConfigCaches: vi.fn(),
assertEvaluatorConfigurationValid: vi.fn(),
@@ -54,7 +54,7 @@ const mocks = vi.hoisted(() => ({
vi.mock("@langfuse/shared/src/server", async (importOriginal) => ({
...(await importOriginal<typeof SharedServer>()),
generateLangfuseAIText: mocks.generateLangfuseAIText,
- getRecentEvaluatorExecutionTraces: mocks.getRecentEvaluatorExecutionTraces,
+ getEvaluatorExecutionSummaries: mocks.getEvaluatorExecutionSummaries,
getTotalCostByEvaluatorIds: mocks.getTotalCostByEvaluatorIds,
invalidateProjectEvalConfigCaches: mocks.invalidateProjectEvalConfigCaches,
}));
@@ -165,7 +165,7 @@ beforeAll(async () => {
beforeEach(() => {
mocks.generateLangfuseAIText.mockReset();
- mocks.getRecentEvaluatorExecutionTraces.mockReset();
+ mocks.getEvaluatorExecutionSummaries.mockReset();
mocks.getTotalCostByEvaluatorIds.mockReset();
mocks.invalidateProjectEvalConfigCaches.mockReset();
mocks.assertEvaluatorConfigurationValid.mockReset();
@@ -1317,43 +1317,22 @@ describe("EvaluatorService", () => {
);
});
- it("returns recent traces by evaluator id", async () => {
+ it("returns execution summaries by evaluator id, including evaluators without runs", async () => {
const service = createService();
const evaluatorIds = [crypto.randomUUID(), crypto.randomUUID()];
- mocks.getRecentEvaluatorExecutionTraces.mockResolvedValue([
- ...[7, 6, 5, 4, 3].map((day) => ({
- id: `first-${day}`,
- evaluatorId: evaluatorIds[0],
- level: "WARNING",
- timestamp: new Date(`2026-08-0${day}T00:00:00.000Z`),
- })),
- ...[4, 3, 2, 1].map((day) => ({
- id: `second-${day}`,
- evaluatorId: evaluatorIds[1],
- level: "DEFAULT",
- timestamp: new Date(`2026-08-0${day}T00:00:00.000Z`),
- })),
+ mocks.getEvaluatorExecutionSummaries.mockResolvedValue([
+ { evaluatorId: evaluatorIds[0], total: 8, failed: 4 },
]);
-
- const result = await service.listRecent({ projectId, evaluatorIds });
-
- expect(mocks.getRecentEvaluatorExecutionTraces).toHaveBeenCalledWith(
+ await expect(
+ service.getExecutionSummaries({ projectId, evaluatorIds }),
+ ).resolves.toEqual({
+ [evaluatorIds[0]]: { total: 8, failed: 4 },
+ [evaluatorIds[1]]: { total: 0, failed: 0 },
+ });
+ expect(mocks.getEvaluatorExecutionSummaries).toHaveBeenCalledWith(
projectId,
evaluatorIds,
);
- expect(result[evaluatorIds[0]]?.map(({ id }) => id)).toEqual([
- "first-7",
- "first-6",
- "first-5",
- "first-4",
- "first-3",
- ]);
- expect(result[evaluatorIds[1]]?.map(({ id }) => id)).toEqual([
- "second-4",
- "second-3",
- "second-2",
- "second-1",
- ]);
});
it("returns total costs by evaluator id", async () => {
diff --git a/web/src/__tests__/server/repositories/event-repository.servertest.ts b/web/src/__tests__/server/repositories/event-repository.servertest.ts
index b415d2e92c..3b30208785 100644
--- a/web/src/__tests__/server/repositories/event-repository.servertest.ts
+++ b/web/src/__tests__/server/repositories/event-repository.servertest.ts
@@ -18,8 +18,7 @@ import {
getEventsExactFilterOptionsForColumns,
getEventsFilterOptionValuesPage,
getLatestEvaluatorRunCost,
- getRecentEvaluatorExecutionTraces,
- getRecentRuleExecutionTraces,
+ getEvaluatorExecutionSummaries,
getTotalCostByEvaluatorIds,
getTotalCostByRule,
createScoresCh,
@@ -424,7 +423,7 @@ describe("Clickhouse Events Repository Test", () => {
).resolves.toBeNull();
});
- it("returns recent evaluator traces without test runs", async () => {
+ it("counts evaluator execution health without tagged or legacy test runs", async () => {
const traceId = randomUUID();
const testTraceId = randomUUID();
const failedTestSpanId = randomUUID();
@@ -468,14 +467,63 @@ describe("Clickhouse Events Repository Test", () => {
]);
await expect(
- getRecentEvaluatorExecutionTraces(projectId, [evaluatorId]),
- ).resolves.toEqual([
- expect.objectContaining({
- id: traceId,
- evaluatorId,
+ getEvaluatorExecutionSummaries(projectId, [evaluatorId]),
+ ).resolves.toEqual([{ evaluatorId, total: 1, failed: 1 }]);
+ });
+ });
+
+ maybe("evaluator execution health", () => {
+ it("counts all seven-day execution traces once and excludes tests, old traces, and other projects", async () => {
+ const evaluatorId = randomUUID();
+ const otherEvaluatorId = randomUUID();
+ const traceIds = Array.from({ length: 8 }, () => randomUUID());
+ await createEventsCh([
+ ...traceIds.flatMap((traceId, index) => [
+ createEvent({
+ project_id: projectId,
+ trace_id: traceId,
+ evaluator_id: evaluatorId,
+ level: "DEFAULT",
+ }),
+ createEvent({
+ project_id: projectId,
+ trace_id: traceId,
+ evaluator_id: evaluatorId,
+ level: index < 4 ? "ERROR" : "DEFAULT",
+ }),
+ ]),
+ createEvent({
+ project_id: projectId,
+ evaluator_id: evaluatorId,
+ level: "ERROR",
+ start_time: (Date.now() - 8 * 24 * 60 * 60 * 1000) * 1000,
+ }),
+ createEvent({
+ project_id: randomUUID(),
+ evaluator_id: evaluatorId,
+ level: "ERROR",
+ }),
+ createEvent({
+ project_id: projectId,
+ evaluator_id: evaluatorId,
+ trace_name: "Test evaluator: Example",
+ level: "ERROR",
+ }),
+ createEvent({
+ project_id: projectId,
+ evaluator_id: otherEvaluatorId,
level: "ERROR",
}),
]);
+ await expect(
+ getEvaluatorExecutionSummaries(projectId, [evaluatorId]),
+ ).resolves.toEqual([{ evaluatorId, total: 8, failed: 4 }]);
+ await expect(
+ getEvaluatorExecutionSummaries(projectId, []),
+ ).resolves.toEqual([]);
+ await expect(
+ getEvaluatorExecutionSummaries(projectId, [randomUUID()]),
+ ).resolves.toEqual([]);
});
});
@@ -518,45 +566,6 @@ describe("Clickhouse Events Repository Test", () => {
]),
);
});
-
- it("returns the last five traces, falling back to job_configuration_id", async () => {
- const ruleId = randomUUID();
- const legacyRuleId = randomUUID();
- const traceIds = Array.from({ length: 6 }, () => randomUUID());
- const legacyTraceId = randomUUID();
- const now = Date.now() * 1000;
-
- await createEventsCh([
- ...traceIds.map((traceId, index) =>
- createEvent({
- project_id: projectId,
- trace_id: traceId,
- start_time: now - index * 1_000_000,
- metadata_names: ["evaluation_rule_id", "job_configuration_id"],
- metadata_values: [ruleId, randomUUID()],
- }),
- ),
- createEvent({
- project_id: projectId,
- trace_id: legacyTraceId,
- metadata_names: ["job_configuration_id"],
- metadata_values: [legacyRuleId],
- }),
- ]);
-
- const traces = await getRecentRuleExecutionTraces(projectId, [
- ruleId,
- legacyRuleId,
- ]);
-
- expect(traces.filter((trace) => trace.ruleId === ruleId)).toHaveLength(5);
- expect(traces.map(({ id }) => id)).not.toContain(traceIds[5]);
- expect(traces).toEqual(
- expect.arrayContaining([
- expect.objectContaining({ id: legacyTraceId, ruleId: legacyRuleId }),
- ]),
- );
- });
});
maybe("getObservationsWithModelDataFromEventsTable", () => {
diff --git a/web/src/components/design-system/Tooltip/Tooltip.stories.tsx b/web/src/components/design-system/Tooltip/Tooltip.stories.tsx
index 6a25299c5a..2c879c1055 100644
--- a/web/src/components/design-system/Tooltip/Tooltip.stories.tsx
+++ b/web/src/components/design-system/Tooltip/Tooltip.stories.tsx
@@ -53,6 +53,14 @@ export const TestHoverAndFocus = meta.story({
);
await userEvent.unhover(trigger);
+ const viewport = canvasElement.ownerDocument.documentElement;
+ await userEvent.pointer({
+ target: canvasElement.ownerDocument.body,
+ coords: {
+ clientX: viewport.clientWidth - 1,
+ clientY: viewport.clientHeight - 1,
+ },
+ });
await waitFor(() =>
expect(body.queryByRole("tooltip")).not.toBeInTheDocument(),
);
diff --git a/web/src/features/evals/v2/components/Evaluators/EvaluatorStatusBadge/EvaluatorStatusBadge.stories.tsx b/web/src/features/evals/v2/components/Evaluators/EvaluatorStatusBadge/EvaluatorStatusBadge.stories.tsx
index df46428f59..88b8a80b4a 100644
--- a/web/src/features/evals/v2/components/Evaluators/EvaluatorStatusBadge/EvaluatorStatusBadge.stories.tsx
+++ b/web/src/features/evals/v2/components/Evaluators/EvaluatorStatusBadge/EvaluatorStatusBadge.stories.tsx
@@ -1,39 +1,39 @@
import preview from "../../../../../../../.storybook/preview";
import { EvaluatorStatusBadge } from "./EvaluatorStatusBadge";
-const meta = preview.meta({ component: EvaluatorStatusBadge });
-
-export const Active = meta.story({
- args: { ruleCount: 2, active: true },
+const meta = preview.meta({
+ component: EvaluatorStatusBadge,
+ args: {
+ ruleCount: 2,
+ summary: { total: 10, failed: 0 },
+ executionsHref: "/project/demo/traces?dateRange=7d",
+ },
});
+export const Healthy = meta.story({});
export const Inactive = meta.story({
- args: { ruleCount: 1, active: false },
+ args: { summary: { total: 0, failed: 0 } },
});
-
-/** No reason stored: the badge stays a plain badge with no hover target. */
-export const Blocked = meta.story({
- args: { ruleCount: 1, active: true, blocked: true },
+export const NoRule = meta.story({ args: { ruleCount: 0 } });
+export const Degraded = meta.story({
+ args: { summary: { total: 10, failed: 4 } },
});
-
-/** Hovering explains why the evaluator was paused and how to fix it. */
+export const Failing = meta.story({
+ args: { summary: { total: 10, failed: 5 } },
+});
+export const Unavailable = meta.story({ args: { summary: undefined } });
+export const WithoutExecutionAccess = meta.story({
+ args: { summary: undefined, executionsHref: null },
+});
+export const Blocked = meta.story({ args: { blocked: true } });
export const BlockedWithReason = meta.story({
args: {
- ruleCount: 1,
- active: true,
blocked: true,
blockReason: "LLM_CONNECTION_MISSING",
blockMessage:
"Evaluator paused: the LLM connection it used was deleted. Recreate the connection or point the evaluator at another one, then reactivate it.",
},
});
-
-/** Rows blocked before messages were persisted fall back to the reason copy. */
export const BlockedReasonOnly = meta.story({
- args: {
- ruleCount: 1,
- active: true,
- blocked: true,
- blockReason: "LLM_CONNECTION_AUTH_INVALID",
- },
+ args: { blocked: true, blockReason: "LLM_CONNECTION_AUTH_INVALID" },
});
diff --git a/web/src/features/evals/v2/components/Evaluators/EvaluatorStatusBadge/EvaluatorStatusBadge.tsx b/web/src/features/evals/v2/components/Evaluators/EvaluatorStatusBadge/EvaluatorStatusBadge.tsx
index 502013b72e..7492de8779 100644
--- a/web/src/features/evals/v2/components/Evaluators/EvaluatorStatusBadge/EvaluatorStatusBadge.tsx
+++ b/web/src/features/evals/v2/components/Evaluators/EvaluatorStatusBadge/EvaluatorStatusBadge.tsx
@@ -1,68 +1,113 @@
-/* eslint-disable no-nested-ternary */
-import { Circle } from "lucide-react";
-
+import Link from "next/link";
import {
getEvaluatorBlockMetadata,
type EvaluatorBlockReason,
} from "@langfuse/shared";
-
-import { Badge } from "@/src/components/ui/badge";
+import { Badge } from "@/src/components/design-system/Badge/Badge";
import {
- Tooltip,
- TooltipContent,
- TooltipTrigger,
-} from "@/src/components/ui/tooltip";
-import { cn } from "@/src/utils/tailwind";
+ HoverCard,
+ HoverCardContent,
+ HoverCardTrigger,
+} from "@/src/components/ui/hover-card";
-/** Displays whether an evaluator is active, inactive, or blocked. */
+type ExecutionSummary = { total: number; failed: number };
+
+function getStatus({
+ ruleCount,
+ summary,
+ blocked,
+}: {
+ ruleCount: number;
+ summary: ExecutionSummary | undefined;
+ blocked: boolean;
+}) {
+ if (blocked) return "Blocked";
+ if (ruleCount === 0) return "Inactive";
+ if (!summary) return "Unknown";
+ if (summary.total === 0) return "Inactive";
+ if (summary.failed === 0) return "Healthy";
+ return summary.failed / summary.total >= 0.5 ? "Failing" : "Degraded";
+}
+
+const colors = {
+ Blocked: "red",
+ Inactive: "primary",
+ Healthy: "green",
+ Degraded: "yellow",
+ Failing: "red",
+ Unknown: "primary",
+} as const;
+
+/** Displays evaluator health from execution traces over the last seven days. */
export function EvaluatorStatusBadge({
ruleCount,
- active,
+ summary,
blocked = false,
blockReason = null,
blockMessage = null,
+ executionsHref,
}: {
ruleCount: number;
- active: boolean;
+ summary: ExecutionSummary | undefined;
blocked?: boolean;
blockReason?: EvaluatorBlockReason | null;
blockMessage?: string | null;
+ executionsHref: string | null;
}) {
+ const status = getStatus({ ruleCount, summary, blocked });
+ let explanation: string;
+ if (status === "Blocked") {
+ explanation =
+ blockMessage ??
+ (blockReason
+ ? getEvaluatorBlockMetadata(blockReason).message
+ : "This evaluator is blocked.");
+ } else if (ruleCount === 0) {
+ explanation = "No rule is attached to this evaluator.";
+ } else if (status === "Unknown") {
+ explanation = "Execution status is unavailable.";
+ } else if (status === "Inactive") {
+ explanation = "No execution traces in the last 7 days.";
+ } else if (status === "Healthy") {
+ explanation = "All runs passed in the last 7 days.";
+ } else {
+ explanation = `${summary?.failed} of ${summary?.total} execution traces failed in the last 7 days.`;
+ }
+
const badge = (
- <Badge
- variant={blocked ? "warning" : active ? "default" : "secondary"}
- className={cn(
- "gap-1.5 whitespace-nowrap",
- active &&
- !blocked &&
- "bg-light-green text-dark-green hover:bg-light-green",
- )}
- >
- <Circle className="h-2 w-2 fill-current" />
- {blocked ? "Blocked" : active ? "Active" : "Inactive"} · {ruleCount}
- </Badge>
+ <Badge text={status === "Unknown" ? "—" : status} color={colors[status]} />
);
- // Prefer the message stored when the evaluator was paused — that is what the
- // project was notified with — and fall back to the reason's copy for rows
- // blocked before messages were persisted.
- const explanation = blocked
- ? (blockMessage ??
- (blockReason ? getEvaluatorBlockMetadata(blockReason).message : null))
- : null;
-
- if (!explanation) {
- return badge;
- }
-
return (
- <Tooltip>
- <TooltipTrigger asChild>
- <span className="inline-flex" tabIndex={0}>
- {badge}
- </span>
- </TooltipTrigger>
- <TooltipContent className="max-w-xs">{explanation}</TooltipContent>
- </Tooltip>
+ <HoverCard openDelay={200}>
+ <HoverCardTrigger asChild>
+ {executionsHref ? (
+ <Link
+ href={executionsHref}
+ onClick={(event) => event.stopPropagation()}
+ aria-label={`View executions: ${status}`}
+ className="focus-visible:ring-ring inline-flex rounded-sm focus-visible:ring-2 focus-visible:outline-none"
+ >
+ {badge}
+ </Link>
+ ) : (
+ <span className="inline-flex" tabIndex={0}>
+ {badge}
+ </span>
+ )}
+ </HoverCardTrigger>
+ <HoverCardContent align="start" className="w-80 text-sm">
+ <p>{explanation}</p>
+ {executionsHref && (
+ <Link
+ href={executionsHref}
+ onClick={(event) => event.stopPropagation()}
+ className="text-primary mt-2 inline-block underline"
+ >
+ View executions
+ </Link>
+ )}
+ </HoverCardContent>
+ </HoverCard>
);
}
diff --git a/web/src/features/evals/v2/components/Rules/EvaluatorExecutionHistory/EvaluatorExecutionHistory.stories.tsx b/web/src/features/evals/v2/components/Rules/EvaluatorExecutionHistory/EvaluatorExecutionHistory.stories.tsx
deleted file mode 100644
index fb4f25ab98..0000000000
--- a/web/src/features/evals/v2/components/Rules/EvaluatorExecutionHistory/EvaluatorExecutionHistory.stories.tsx
+++ /dev/null
@@ -1,30 +0,0 @@
-import preview from "../../../../../../../.storybook/preview";
-import { EvaluatorExecutionHistory } from "./EvaluatorExecutionHistory";
-
-const meta = preview.meta({ component: EvaluatorExecutionHistory });
-
-export const Runs = meta.story({
- args: {
- traces: [
- {
- id: "trace-1",
- level: "DEFAULT",
- timestamp: new Date("2026-07-01T10:00:00Z"),
- },
- {
- id: "trace-2",
- level: "WARNING",
- timestamp: new Date("2026-07-01T11:00:00Z"),
- },
- {
- id: "trace-3",
- level: "ERROR",
- timestamp: new Date("2026-07-01T12:00:00Z"),
- },
- ],
- },
-});
-
-export const NoRuns = meta.story({
- args: { traces: [] },
-});
diff --git a/web/src/features/evals/v2/components/Rules/EvaluatorExecutionHistory/EvaluatorExecutionHistory.tsx b/web/src/features/evals/v2/components/Rules/EvaluatorExecutionHistory/EvaluatorExecutionHistory.tsx
deleted file mode 100644
index 7d263006c8..0000000000
--- a/web/src/features/evals/v2/components/Rules/EvaluatorExecutionHistory/EvaluatorExecutionHistory.tsx
+++ /dev/null
@@ -1,49 +0,0 @@
-/* eslint-disable no-nested-ternary */
-import { Tooltip } from "@/src/components/design-system/Tooltip/Tooltip";
-import { cn } from "@/src/utils/tailwind";
-
-export function EvaluatorExecutionHistory({
- traces,
-}: {
- traces: Array<{ id: string; level: string; timestamp: Date }>;
-}) {
- if (traces.length === 0) {
- return (
- <span className="text-muted-foreground text-xs">No recent runs</span>
- );
- }
-
- return (
- <div
- className="flex items-center gap-1"
- role="img"
- aria-label={traces
- .map(
- (trace) =>
- `${trace.level.toLowerCase()} at ${trace.timestamp.toLocaleString()}`,
- )
- .join(", ")}
- >
- {[...traces].reverse().map((trace) => (
- <Tooltip
- key={trace.id}
- label={`${trace.level.toLowerCase()} at ${trace.timestamp.toLocaleString()}`}
- >
- {({ getTriggerProps }) => (
- <span
- {...getTriggerProps()}
- className={cn(
- "block h-4 w-1.5 rounded-full",
- trace.level === "ERROR"
- ? "bg-destructive"
- : trace.level === "WARNING"
- ? "bg-dark-yellow"
- : "bg-dark-green",
- )}
- />
- )}
- </Tooltip>
- ))}
- </div>
- );
-}
diff --git a/web/src/features/evals/v2/components/Rules/RulesTable/RulesTable.tsx b/web/src/features/evals/v2/components/Rules/RulesTable/RulesTable.tsx
index 84bad3339a..f258f3d3a4 100644
--- a/web/src/features/evals/v2/components/Rules/RulesTable/RulesTable.tsx
+++ b/web/src/features/evals/v2/components/Rules/RulesTable/RulesTable.tsx
@@ -34,9 +34,7 @@ import {
useColumnOrder,
useColumnVisibility,
} from "@/src/features/column-visibility";
-import { EvaluatorExecutionHistory } from "@/src/features/evals/v2/components/Rules/EvaluatorExecutionHistory/EvaluatorExecutionHistory";
import type { RuleTableRow } from "@/src/features/evals/v2/types/rules";
-import { Skeleton } from "@/src/components/ui/skeleton";
import {
Tooltip,
TooltipContent,
@@ -218,10 +216,6 @@ export function RulesTable({
{ projectId, ruleIds },
{ enabled: ruleIds.length > 0, meta: { silentHttpCodes: [503] } },
);
- const recentExecutions = api.evalsV2.rules.recentExecutions.useQuery(
- { projectId, ruleIds },
- { enabled: ruleIds.length > 0, meta: { silentHttpCodes: [503] } },
- );
const deleteMany = api.evalsV2.rules.deleteMany.useMutation({
onError: trpcErrorToast,
onSuccess: async (result) => {
@@ -315,33 +309,6 @@ export function RulesTable({
return value;
},
}),
- {
- accessorKey: "executionTraces",
- id: "executionTraces",
- header: "Last 5 runs",
- size: 140,
- enableHiding: true,
- cell: ({ row }) => {
- if (recentExecutions.isPending) {
- return <Skeleton className="h-4 w-16" />;
- }
- return (
- <button
- type="button"
- className="focus-visible:ring-ring rounded-sm focus-visible:ring-2 focus-visible:outline-none"
- aria-label={`View runs for ${row.original.name}`}
- onClick={(event) => {
- event.stopPropagation();
- router.push(ruleExecutionsUrl(projectId, row.original.id));
- }}
- >
- <EvaluatorExecutionHistory
- traces={recentExecutions.data?.[row.original.id] ?? []}
- />
- </button>
- );
- },
- },
{
accessorKey: "assignments",
id: "assignments",
@@ -486,8 +453,6 @@ export function RulesTable({
costs.data,
costs.isPending,
projectId,
- recentExecutions.data,
- recentExecutions.isPending,
router,
capture,
selectActionColumn,
diff --git a/web/src/features/evals/v2/fns/evaluators/evaluatorScoresUrl.clienttest.ts b/web/src/features/evals/v2/fns/evaluators/evaluatorScoresUrl.clienttest.ts
index 751abf0ea3..058a89b8ad 100644
--- a/web/src/features/evals/v2/fns/evaluators/evaluatorScoresUrl.clienttest.ts
+++ b/web/src/features/evals/v2/fns/evaluators/evaluatorScoresUrl.clienttest.ts
@@ -5,6 +5,10 @@ import {
LangfuseInternalTraceEnvironment,
} from "@langfuse/shared";
import { describe, expect, it } from "vitest";
+import {
+ rangeFromString,
+ TABLE_AGGREGATION_OPTIONS,
+} from "@/src/utils/date-range-utils";
import {
evaluatorExecutionsUrl,
evaluatorScoresUrl,
@@ -75,13 +79,18 @@ describe("evaluatorScoresUrl", () => {
const getFilters = (evaluatorType: EvalTemplateType) => {
const url = new URL(
- evaluatorExecutionsUrl("project/id", "Quality", evaluatorType),
+ evaluatorExecutionsUrl("project/id", "evaluator-id", evaluatorType),
"https://langfuse.local",
);
return {
pathname: url.pathname,
filters: decodeFiltersGeneric(url.searchParams.get("filter") ?? ""),
+ range: rangeFromString(
+ url.searchParams.get("dateRange") ?? "",
+ TABLE_AGGREGATION_OPTIONS,
+ "last1Day",
+ ),
};
};
@@ -97,12 +106,13 @@ describe("evaluatorExecutionsUrl", () => {
(type, environment) => {
expect(getFilters(type)).toEqual({
pathname: "/project/project%2Fid/traces",
+ range: { range: "last7Days" },
filters: [
{
- column: "traceName",
+ column: "evaluatorId",
type: "stringOptions",
operator: "any of",
- value: ["Execute evaluator: Quality"],
+ value: ["evaluator-id"],
},
{
column: "environment",
diff --git a/web/src/features/evals/v2/fns/evaluators/evaluatorScoresUrl.ts b/web/src/features/evals/v2/fns/evaluators/evaluatorScoresUrl.ts
index 38c3a83613..1250362852 100644
--- a/web/src/features/evals/v2/fns/evaluators/evaluatorScoresUrl.ts
+++ b/web/src/features/evals/v2/fns/evaluators/evaluatorScoresUrl.ts
@@ -45,7 +45,7 @@ export function evaluatorScoresUrl(
export function evaluatorExecutionsUrl(
projectId: string,
- evaluatorName: string,
+ evaluatorId: string,
evaluatorType: EvalTemplateType,
) {
const environment =
@@ -54,10 +54,10 @@ export function evaluatorExecutionsUrl(
: LangfuseInternalTraceEnvironment.LLMJudge;
const filter: FilterState = [
{
- column: "traceName",
+ column: "evaluatorId",
type: "stringOptions",
operator: "any of",
- value: [`Execute evaluator: ${evaluatorName}`],
+ value: [evaluatorId],
},
{
column: "environment",
@@ -72,5 +72,5 @@ export function evaluatorExecutionsUrl(
value: true,
},
];
- return `/project/${encodeURIComponent(projectId)}/traces?filter=${encodeURIComponent(encodeFiltersGeneric(filter))}`;
+ return `/project/${encodeURIComponent(projectId)}/traces?dateRange=7d&filter=${encodeURIComponent(encodeFiltersGeneric(filter))}`;
}
diff --git a/web/src/features/evals/v2/pages/EvaluatorsPage.tsx b/web/src/features/evals/v2/pages/EvaluatorsPage.tsx
index 3f9effe33f..e9aaf41ca4 100644
--- a/web/src/features/evals/v2/pages/EvaluatorsPage.tsx
+++ b/web/src/features/evals/v2/pages/EvaluatorsPage.tsx
@@ -31,7 +31,6 @@ import { EvaluatorBulkDeleteDialog } from "../components/Evaluators/EvaluatorBul
import { EvaluatorGalleryDialog } from "../components/EvaluatorGalleryDialog/EvaluatorGalleryDialog";
import { EvaluatorStatusBadge } from "../components/Evaluators/EvaluatorStatusBadge/EvaluatorStatusBadge";
import { EvaluatorTypeBadge } from "../components/Evaluators/EvaluatorTypeBadge/EvaluatorTypeBadge";
-import { EvaluatorExecutionHistory } from "@/src/features/evals/v2/components/Rules/EvaluatorExecutionHistory/EvaluatorExecutionHistory";
import { OverviewSelectionBar } from "../components/OverviewSelectionBar/OverviewSelectionBar";
import { usePostHogClientCapture } from "@/src/features/posthog-analytics";
import {
@@ -322,7 +321,7 @@ export default function EvaluatorsPage() {
meta: { silentHttpCodes: [503] },
},
);
- const recentExecutions = api.evalsV2.recentExecutions.useQuery(
+ const executionSummaries = api.evalsV2.executionSummaries.useQuery(
{ projectId, evaluatorIds },
{
enabled: hasExecutionReadAccess && evaluatorIds.length > 0,
@@ -384,50 +383,32 @@ export default function EvaluatorsPage() {
header: "Status",
size: 130,
enableHiding: true,
- cell: ({ row }) => (
- <EvaluatorStatusBadge
- ruleCount={row.original._count.assignments}
- active={row.original.hasActiveRules}
- blocked={Boolean(row.original.blockedAt)}
- blockReason={row.original.blockReason}
- blockMessage={row.original.blockMessage}
- />
- ),
- },
- {
- accessorKey: "executionTraces",
- id: "executionTraces",
- header: "Last 5 runs",
- size: 130,
- enableHiding: true,
cell: ({ row }) => {
- if (recentExecutions.isPending && hasExecutionReadAccess) {
+ if (
+ executionSummaries.isPending &&
+ hasExecutionReadAccess &&
+ !row.original.blockedAt &&
+ row.original._count.assignments > 0
+ ) {
return <Skeleton className="h-4 w-16" />;
}
- const history = (
- <EvaluatorExecutionHistory
- traces={recentExecutions.data?.[row.original.id] ?? []}
- />
- );
- return hasExecutionReadAccess ? (
- <button
- type="button"
- className="focus-visible:ring-ring rounded-sm focus-visible:ring-2 focus-visible:outline-none"
- aria-label={`View executions for ${row.original.name}`}
- onClick={() =>
- router.push(
- evaluatorExecutionsUrl(
- projectId,
- row.original.name,
- row.original.type,
- ),
- )
+ return (
+ <EvaluatorStatusBadge
+ ruleCount={row.original._count.assignments}
+ summary={executionSummaries.data?.[row.original.id]}
+ blocked={Boolean(row.original.blockedAt)}
+ blockReason={row.original.blockReason}
+ blockMessage={row.original.blockMessage}
+ executionsHref={
+ hasExecutionReadAccess
+ ? evaluatorExecutionsUrl(
+ projectId,
+ row.original.id,
+ row.original.type,
+ )
+ : null
}
- >
- {history}
- </button>
- ) : (
- history
+ />
);
},
},
@@ -526,7 +507,7 @@ export default function EvaluatorsPage() {
router.push(
evaluatorExecutionsUrl(
projectId,
- row.original.name,
+ row.original.id,
row.original.type,
),
)
@@ -555,8 +536,8 @@ export default function EvaluatorsPage() {
costs.isPending,
hasExecutionReadAccess,
projectId,
- recentExecutions.data,
- recentExecutions.isPending,
+ executionSummaries.data,
+ executionSummaries.isPending,
router,
selectActionColumn,
],
diff --git a/web/src/features/evals/v2/server/evaluators/evaluatorRouter.ts b/web/src/features/evals/v2/server/evaluators/evaluatorRouter.ts
index 9c5e014eea..8b4fef9934 100644
--- a/web/src/features/evals/v2/server/evaluators/evaluatorRouter.ts
+++ b/web/src/features/evals/v2/server/evaluators/evaluatorRouter.ts
@@ -145,7 +145,7 @@ export const evaluatorRouter = createTRPCRouter({
});
}),
- recentExecutions: protectedProjectProcedure
+ executionSummaries: protectedProjectProcedure
.input(EvaluatorIdsSchema)
.query(({ input, ctx }) => {
throwIfNoProjectAccess({
@@ -153,7 +153,7 @@ export const evaluatorRouter = createTRPCRouter({
projectId: ctx.session.projectId,
scope: "evalJobExecution:read",
});
- return serviceForContext(ctx).listRecent({
+ return serviceForContext(ctx).getExecutionSummaries({
...input,
projectId: ctx.session.projectId,
});
diff --git a/web/src/features/evals/v2/server/evaluators/evaluatorService.ts b/web/src/features/evals/v2/server/evaluators/evaluatorService.ts
index 69c71cf413..cee41c01ef 100644
--- a/web/src/features/evals/v2/server/evaluators/evaluatorService.ts
+++ b/web/src/features/evals/v2/server/evaluators/evaluatorService.ts
@@ -20,7 +20,7 @@ import {
ChatMessageType,
generateLangfuseAIText,
getClientInitiatedNonStreamingLlmTimeoutMs,
- getRecentEvaluatorExecutionTraces,
+ getEvaluatorExecutionSummaries,
getTotalCostByEvaluatorIds,
invalidateProjectEvalConfigCaches,
logger,
@@ -129,12 +129,6 @@ function prepareEvaluatorDefinitionForPersistence(
};
}
-type EvaluatorExecutionTrace = {
- id: string;
- level: string;
- timestamp: Date;
-};
-
export type EvaluatorAuditEvent = {
action: "create" | "update" | "delete";
projectId: string;
@@ -260,25 +254,21 @@ export class EvaluatorService {
};
}
- async listRecent(params: { projectId: string; evaluatorIds: string[] }) {
- const result = Object.fromEntries(
- params.evaluatorIds.map((evaluatorId) => [evaluatorId, []]),
- ) as Record<string, EvaluatorExecutionTrace[]>;
- if (params.evaluatorIds.length === 0) return result;
-
- const traces = await getRecentEvaluatorExecutionTraces(
+ async getExecutionSummaries(params: {
+ projectId: string;
+ evaluatorIds: string[];
+ }) {
+ const result: Record<string, { total: number; failed: number }> =
+ Object.fromEntries(
+ params.evaluatorIds.map((id) => [id, { total: 0, failed: 0 }]),
+ );
+ const summaries = await getEvaluatorExecutionSummaries(
params.projectId,
params.evaluatorIds,
);
-
- for (const trace of traces) {
- result[trace.evaluatorId]?.push({
- id: trace.id,
- level: trace.level,
- timestamp: trace.timestamp,
- });
+ for (const { evaluatorId, total, failed } of summaries) {
+ result[evaluatorId] = { total, failed };
}
-
return result;
}
diff --git a/web/src/features/evals/v2/server/rules/ruleRouter.ts b/web/src/features/evals/v2/server/rules/ruleRouter.ts
index e3e8f8d5c9..1091fbc659 100644
--- a/web/src/features/evals/v2/server/rules/ruleRouter.ts
+++ b/web/src/features/evals/v2/server/rules/ruleRouter.ts
@@ -80,20 +80,6 @@ export const ruleRouter = createTRPCRouter({
return serviceForContext(ctx).get(ctx.session.projectId, input.ruleId);
}),
- recentExecutions: protectedProjectProcedure
- .input(RuleIdsSchema)
- .query(({ input, ctx }) => {
- throwIfNoProjectAccess({
- session: ctx.session,
- projectId: ctx.session.projectId,
- scope: "evalJobExecution:read",
- });
- return serviceForContext(ctx).listRecent({
- ...input,
- projectId: ctx.session.projectId,
- });
- }),
-
costByRuleIds: protectedProjectProcedure
.input(RuleIdsSchema)
.query(({ input, ctx }) => {
diff --git a/web/src/features/evals/v2/server/rules/ruleService.ts b/web/src/features/evals/v2/server/rules/ruleService.ts
index b39801f3f7..f65f3aa2e5 100644
--- a/web/src/features/evals/v2/server/rules/ruleService.ts
+++ b/web/src/features/evals/v2/server/rules/ruleService.ts
@@ -22,7 +22,6 @@ import {
getClientInitiatedNonStreamingLlmTimeoutMs,
invalidateProjectEvalConfigCaches,
logger,
- getRecentRuleExecutionTraces,
getTotalCostByRule,
} from "@langfuse/shared/src/server";
import { resolveLangfuseAiFeatureAvailability } from "@/src/features/ai-features/server";
@@ -104,23 +103,6 @@ export class RuleService {
return toRuleResponse(rule);
}
- async listRecent(params: { projectId: string; ruleIds: string[] }) {
- const result = Object.fromEntries(
- params.ruleIds.map((ruleId) => [ruleId, []]),
- ) as Record<string, Array<{ id: string; level: string; timestamp: Date }>>;
- if (params.ruleIds.length === 0) return result;
-
- const traces = await getRecentRuleExecutionTraces(
- params.projectId,
- params.ruleIds,
- );
-
- for (const { ruleId, ...trace } of traces) {
- result[ruleId]?.push(trace);
- }
- return result;
- }
-
async getTotalCosts(params: { projectId: string; ruleIds: string[] }) {
const costs = await getTotalCostByRule(params.projectId, params.ruleIds);
return Object.fromEntries(