🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
feat(evals): show seven-day execution health in evaluator status (#18113)
Tobias Wochinger9d168c646be6
diff --git a/packages/shared/src/server/repositories/events.ts b/packages/shared/src/server/repositories/events.tsindex 92edbb0fa5..8dc1843911 100644--- a/packages/shared/src/server/repositories/events.ts+++ b/packages/shared/src/server/repositories/events.ts@@ -3692,7 +3692,8 @@ export const getLatestEvaluatorRunCost = async (return rows[0] ? Number(rows[0].trace_total_cost) : null;};-export const getRecentEvaluatorExecutionTraces = async (+/** Counts execution traces by outcome over the last seven days, excluding tests. */+export const getEvaluatorExecutionSummaries = async (projectId: string,evaluatorIds: string[],) => {@@ -3702,81 +3703,34 @@ export const getRecentEvaluatorExecutionTraces = async (projectId,groupByColumn: "e.trace_id, e.evaluator_id",selectExpression: [- "e.trace_id as id","e.evaluator_id as evaluator_id",- "multiIf(countIf(e.level = 'ERROR') > 0, 'ERROR', countIf(e.level = 'WARNING') > 0, 'WARNING', 'DEFAULT') as level",- "min(e.start_time) as timestamp",+ "countIf(e.level = 'ERROR') > 0 as failed",].join(", "),}).whereRaw("e.start_time > now() - INTERVAL 7 DAY").whereRaw("e.evaluator_id IN ({evaluatorIds: Array(String)})", {evaluatorIds,})- .havingRaw(`countIf(${evaluatorTestEventCondition}) = 0`)- .orderBy("ORDER BY timestamp DESC, id DESC")- .limitByCount(5, "evaluator_id");+ .havingRaw(`countIf(${evaluatorTestEventCondition}) = 0`);+ // Aggregate the per-trace outcomes, rather than counting individual observations.const { query, params } = builder.buildWithParams();const rows = await queryClickhouse<{- id: string;evaluator_id: string;- level: string;- timestamp: string;+ total: string;+ failed_count: string;}>({- query,+ query: `SELECT evaluator_id, count() as total, countIf(failed) as failed_count+ FROM (${query}) GROUP BY evaluator_id`,params,tags: { projectId },preferredClickhouseService: "EventsReadOnly",});return rows.map((row) => ({- id: row.id,evaluatorId: row.evaluator_id,- level: row.level,- timestamp: parseClickhouseUTCDateTimeFormat(row.timestamp),- }));-};--export const getRecentRuleExecutionTraces = async (- projectId: string,- ruleIds: string[],-) => {- if (ruleIds.length === 0) return [];-- const builder = new EventsAggQueryBuilder({- projectId,- groupByColumn: "e.trace_id, e.evaluation_rule_id",- selectExpression: [- "e.trace_id as id",- "e.evaluation_rule_id as evaluation_rule_id",- "multiIf(countIf(e.level = 'ERROR') > 0, 'ERROR', countIf(e.level = 'WARNING') > 0, 'WARNING', 'DEFAULT') as level",- "min(e.start_time) as timestamp",- ].join(", "),- })- .whereRaw("e.start_time > now() - INTERVAL 7 DAY")- .whereRaw("e.evaluation_rule_id IN ({ruleIds: Array(String)})", { ruleIds })- .orderBy("ORDER BY timestamp DESC, id DESC")- .limitByCount(5, "evaluation_rule_id");-- const { query, params } = builder.buildWithParams();- const rows = await queryClickhouse<- {- id: string;- level: string;- timestamp: string;- } & { evaluation_rule_id: string }- >({- query,- params,- tags: { projectId },- preferredClickhouseService: "EventsReadOnly",- });-- return rows.map((row) => ({- id: row.id,- ruleId: row.evaluation_rule_id,- level: row.level,- timestamp: parseClickhouseUTCDateTimeFormat(row.timestamp),+ total: Number(row.total),+ failed: Number(row.failed_count),}));};diff --git a/web/src/__tests__/server/evaluation-rule-v2-service.servertest.ts b/web/src/__tests__/server/evaluation-rule-v2-service.servertest.tsindex 11961f013c..e8fb1802e0 100644--- a/web/src/__tests__/server/evaluation-rule-v2-service.servertest.ts+++ b/web/src/__tests__/server/evaluation-rule-v2-service.servertest.ts@@ -17,13 +17,11 @@ import { RuleService } from "@/src/features/evals/v2/server/rules/ruleService";import { MAX_ACTIVE_EVALUATION_RULES } from "@/src/features/evals/v2/server/rules/ruleErrors";const telemetryMocks = vi.hoisted(() => ({- getRecentRuleExecutionTraces: vi.fn(),getTotalCostByRule: vi.fn(),}));vi.mock("@langfuse/shared/src/server", async (importOriginal) => ({...(await importOriginal<typeof SharedServer>()),- getRecentRuleExecutionTraces: telemetryMocks.getRecentRuleExecutionTraces,getTotalCostByRule: telemetryMocks.getTotalCostByRule,}));@@ -43,7 +41,6 @@ beforeAll(async () => {});beforeEach(async () => {- telemetryMocks.getRecentRuleExecutionTraces.mockReset();telemetryMocks.getTotalCostByRule.mockReset();defaultEvaluatorId = (await createEvaluator()).id;});@@ -378,27 +375,6 @@ describe("RuleService", () => {});describe("telemetry", () => {- it("returns recent event-backed traces grouped by rule", async () => {- const timestamp = new Date("2026-08-12T12:00:00.000Z");- telemetryMocks.getRecentRuleExecutionTraces.mockResolvedValue([- { ruleId: "rule-1", id: "trace-1", level: "DEFAULT", timestamp },- ]);-- await expect(- createService().listRecent({- projectId,- ruleIds: ["rule-1", "rule-2"],- }),- ).resolves.toEqual({- "rule-1": [{ id: "trace-1", level: "DEFAULT", timestamp }],- "rule-2": [],- });- expect(telemetryMocks.getRecentRuleExecutionTraces).toHaveBeenCalledWith(- projectId,- ["rule-1", "rule-2"],- );- });-it("returns event-backed costs by rule", async () => {telemetryMocks.getTotalCostByRule.mockResolvedValue([{ ruleId: "rule-1", totalCost: 1.25 },diff --git a/web/src/__tests__/server/evaluator-v2-service.servertest.ts b/web/src/__tests__/server/evaluator-v2-service.servertest.tsindex 0fa9e8d5b1..ea33d8b56b 100644--- a/web/src/__tests__/server/evaluator-v2-service.servertest.ts+++ b/web/src/__tests__/server/evaluator-v2-service.servertest.ts@@ -36,7 +36,7 @@ import {const mocks = vi.hoisted(() => ({generateLangfuseAIText: vi.fn(),- getRecentEvaluatorExecutionTraces: vi.fn(),+ getEvaluatorExecutionSummaries: vi.fn(),getTotalCostByEvaluatorIds: vi.fn(),invalidateProjectEvalConfigCaches: vi.fn(),assertEvaluatorConfigurationValid: vi.fn(),@@ -54,7 +54,7 @@ const mocks = vi.hoisted(() => ({vi.mock("@langfuse/shared/src/server", async (importOriginal) => ({...(await importOriginal<typeof SharedServer>()),generateLangfuseAIText: mocks.generateLangfuseAIText,- getRecentEvaluatorExecutionTraces: mocks.getRecentEvaluatorExecutionTraces,+ getEvaluatorExecutionSummaries: mocks.getEvaluatorExecutionSummaries,getTotalCostByEvaluatorIds: mocks.getTotalCostByEvaluatorIds,invalidateProjectEvalConfigCaches: mocks.invalidateProjectEvalConfigCaches,}));@@ -165,7 +165,7 @@ beforeAll(async () => {beforeEach(() => {mocks.generateLangfuseAIText.mockReset();- mocks.getRecentEvaluatorExecutionTraces.mockReset();+ mocks.getEvaluatorExecutionSummaries.mockReset();mocks.getTotalCostByEvaluatorIds.mockReset();mocks.invalidateProjectEvalConfigCaches.mockReset();mocks.assertEvaluatorConfigurationValid.mockReset();@@ -1317,43 +1317,22 @@ describe("EvaluatorService", () => {);});- it("returns recent traces by evaluator id", async () => {+ it("returns execution summaries by evaluator id, including evaluators without runs", async () => {const service = createService();const evaluatorIds = [crypto.randomUUID(), crypto.randomUUID()];- mocks.getRecentEvaluatorExecutionTraces.mockResolvedValue([- ...[7, 6, 5, 4, 3].map((day) => ({- id: `first-${day}`,- evaluatorId: evaluatorIds[0],- level: "WARNING",- timestamp: new Date(`2026-08-0${day}T00:00:00.000Z`),- })),- ...[4, 3, 2, 1].map((day) => ({- id: `second-${day}`,- evaluatorId: evaluatorIds[1],- level: "DEFAULT",- timestamp: new Date(`2026-08-0${day}T00:00:00.000Z`),- })),+ mocks.getEvaluatorExecutionSummaries.mockResolvedValue([+ { evaluatorId: evaluatorIds[0], total: 8, failed: 4 },]);-- const result = await service.listRecent({ projectId, evaluatorIds });-- expect(mocks.getRecentEvaluatorExecutionTraces).toHaveBeenCalledWith(+ await expect(+ service.getExecutionSummaries({ projectId, evaluatorIds }),+ ).resolves.toEqual({+ [evaluatorIds[0]]: { total: 8, failed: 4 },+ [evaluatorIds[1]]: { total: 0, failed: 0 },+ });+ expect(mocks.getEvaluatorExecutionSummaries).toHaveBeenCalledWith(projectId,evaluatorIds,);- expect(result[evaluatorIds[0]]?.map(({ id }) => id)).toEqual([- "first-7",- "first-6",- "first-5",- "first-4",- "first-3",- ]);- expect(result[evaluatorIds[1]]?.map(({ id }) => id)).toEqual([- "second-4",- "second-3",- "second-2",- "second-1",- ]);});it("returns total costs by evaluator id", async () => {diff --git a/web/src/__tests__/server/repositories/event-repository.servertest.ts b/web/src/__tests__/server/repositories/event-repository.servertest.tsindex b415d2e92c..3b30208785 100644--- a/web/src/__tests__/server/repositories/event-repository.servertest.ts+++ b/web/src/__tests__/server/repositories/event-repository.servertest.ts@@ -18,8 +18,7 @@ import {getEventsExactFilterOptionsForColumns,getEventsFilterOptionValuesPage,getLatestEvaluatorRunCost,- getRecentEvaluatorExecutionTraces,- getRecentRuleExecutionTraces,+ getEvaluatorExecutionSummaries,getTotalCostByEvaluatorIds,getTotalCostByRule,createScoresCh,@@ -424,7 +423,7 @@ describe("Clickhouse Events Repository Test", () => {).resolves.toBeNull();});- it("returns recent evaluator traces without test runs", async () => {+ it("counts evaluator execution health without tagged or legacy test runs", async () => {const traceId = randomUUID();const testTraceId = randomUUID();const failedTestSpanId = randomUUID();@@ -468,14 +467,63 @@ describe("Clickhouse Events Repository Test", () => {]);await expect(- getRecentEvaluatorExecutionTraces(projectId, [evaluatorId]),- ).resolves.toEqual([- expect.objectContaining({- id: traceId,- evaluatorId,+ getEvaluatorExecutionSummaries(projectId, [evaluatorId]),+ ).resolves.toEqual([{ evaluatorId, total: 1, failed: 1 }]);+ });+ });++ maybe("evaluator execution health", () => {+ it("counts all seven-day execution traces once and excludes tests, old traces, and other projects", async () => {+ const evaluatorId = randomUUID();+ const otherEvaluatorId = randomUUID();+ const traceIds = Array.from({ length: 8 }, () => randomUUID());+ await createEventsCh([+ ...traceIds.flatMap((traceId, index) => [+ createEvent({+ project_id: projectId,+ trace_id: traceId,+ evaluator_id: evaluatorId,+ level: "DEFAULT",+ }),+ createEvent({+ project_id: projectId,+ trace_id: traceId,+ evaluator_id: evaluatorId,+ level: index < 4 ? "ERROR" : "DEFAULT",+ }),+ ]),+ createEvent({+ project_id: projectId,+ evaluator_id: evaluatorId,+ level: "ERROR",+ start_time: (Date.now() - 8 * 24 * 60 * 60 * 1000) * 1000,+ }),+ createEvent({+ project_id: randomUUID(),+ evaluator_id: evaluatorId,+ level: "ERROR",+ }),+ createEvent({+ project_id: projectId,+ evaluator_id: evaluatorId,+ trace_name: "Test evaluator: Example",+ level: "ERROR",+ }),+ createEvent({+ project_id: projectId,+ evaluator_id: otherEvaluatorId,level: "ERROR",}),]);+ await expect(+ getEvaluatorExecutionSummaries(projectId, [evaluatorId]),+ ).resolves.toEqual([{ evaluatorId, total: 8, failed: 4 }]);+ await expect(+ getEvaluatorExecutionSummaries(projectId, []),+ ).resolves.toEqual([]);+ await expect(+ getEvaluatorExecutionSummaries(projectId, [randomUUID()]),+ ).resolves.toEqual([]);});});@@ -518,45 +566,6 @@ describe("Clickhouse Events Repository Test", () => {]),);});-- it("returns the last five traces, falling back to job_configuration_id", async () => {- const ruleId = randomUUID();- const legacyRuleId = randomUUID();- const traceIds = Array.from({ length: 6 }, () => randomUUID());- const legacyTraceId = randomUUID();- const now = Date.now() * 1000;-- await createEventsCh([- ...traceIds.map((traceId, index) =>- createEvent({- project_id: projectId,- trace_id: traceId,- start_time: now - index * 1_000_000,- metadata_names: ["evaluation_rule_id", "job_configuration_id"],- metadata_values: [ruleId, randomUUID()],- }),- ),- createEvent({- project_id: projectId,- trace_id: legacyTraceId,- metadata_names: ["job_configuration_id"],- metadata_values: [legacyRuleId],- }),- ]);-- const traces = await getRecentRuleExecutionTraces(projectId, [- ruleId,- legacyRuleId,- ]);-- expect(traces.filter((trace) => trace.ruleId === ruleId)).toHaveLength(5);- expect(traces.map(({ id }) => id)).not.toContain(traceIds[5]);- expect(traces).toEqual(- expect.arrayContaining([- expect.objectContaining({ id: legacyTraceId, ruleId: legacyRuleId }),- ]),- );- });});maybe("getObservationsWithModelDataFromEventsTable", () => {diff --git a/web/src/components/design-system/Tooltip/Tooltip.stories.tsx b/web/src/components/design-system/Tooltip/Tooltip.stories.tsxindex 6a25299c5a..2c879c1055 100644--- a/web/src/components/design-system/Tooltip/Tooltip.stories.tsx+++ b/web/src/components/design-system/Tooltip/Tooltip.stories.tsx@@ -53,6 +53,14 @@ export const TestHoverAndFocus = meta.story({);await userEvent.unhover(trigger);+ const viewport = canvasElement.ownerDocument.documentElement;+ await userEvent.pointer({+ target: canvasElement.ownerDocument.body,+ coords: {+ clientX: viewport.clientWidth - 1,+ clientY: viewport.clientHeight - 1,+ },+ });await waitFor(() =>expect(body.queryByRole("tooltip")).not.toBeInTheDocument(),);diff --git a/web/src/features/evals/v2/components/Evaluators/EvaluatorStatusBadge/EvaluatorStatusBadge.stories.tsx b/web/src/features/evals/v2/components/Evaluators/EvaluatorStatusBadge/EvaluatorStatusBadge.stories.tsxindex df46428f59..88b8a80b4a 100644--- a/web/src/features/evals/v2/components/Evaluators/EvaluatorStatusBadge/EvaluatorStatusBadge.stories.tsx+++ b/web/src/features/evals/v2/components/Evaluators/EvaluatorStatusBadge/EvaluatorStatusBadge.stories.tsx@@ -1,39 +1,39 @@import preview from "../../../../../../../.storybook/preview";import { EvaluatorStatusBadge } from "./EvaluatorStatusBadge";-const meta = preview.meta({ component: EvaluatorStatusBadge });--export const Active = meta.story({- args: { ruleCount: 2, active: true },+const meta = preview.meta({+ component: EvaluatorStatusBadge,+ args: {+ ruleCount: 2,+ summary: { total: 10, failed: 0 },+ executionsHref: "/project/demo/traces?dateRange=7d",+ },});+export const Healthy = meta.story({});export const Inactive = meta.story({- args: { ruleCount: 1, active: false },+ args: { summary: { total: 0, failed: 0 } },});--/** No reason stored: the badge stays a plain badge with no hover target. */-export const Blocked = meta.story({- args: { ruleCount: 1, active: true, blocked: true },+export const NoRule = meta.story({ args: { ruleCount: 0 } });+export const Degraded = meta.story({+ args: { summary: { total: 10, failed: 4 } },});--/** Hovering explains why the evaluator was paused and how to fix it. */+export const Failing = meta.story({+ args: { summary: { total: 10, failed: 5 } },+});+export const Unavailable = meta.story({ args: { summary: undefined } });+export const WithoutExecutionAccess = meta.story({+ args: { summary: undefined, executionsHref: null },+});+export const Blocked = meta.story({ args: { blocked: true } });export const BlockedWithReason = meta.story({args: {- ruleCount: 1,- active: true,blocked: true,blockReason: "LLM_CONNECTION_MISSING",blockMessage:"Evaluator paused: the LLM connection it used was deleted. Recreate the connection or point the evaluator at another one, then reactivate it.",},});--/** Rows blocked before messages were persisted fall back to the reason copy. */export const BlockedReasonOnly = meta.story({- args: {- ruleCount: 1,- active: true,- blocked: true,- blockReason: "LLM_CONNECTION_AUTH_INVALID",- },+ args: { blocked: true, blockReason: "LLM_CONNECTION_AUTH_INVALID" },});diff --git a/web/src/features/evals/v2/components/Evaluators/EvaluatorStatusBadge/EvaluatorStatusBadge.tsx b/web/src/features/evals/v2/components/Evaluators/EvaluatorStatusBadge/EvaluatorStatusBadge.tsxindex 502013b72e..7492de8779 100644--- a/web/src/features/evals/v2/components/Evaluators/EvaluatorStatusBadge/EvaluatorStatusBadge.tsx+++ b/web/src/features/evals/v2/components/Evaluators/EvaluatorStatusBadge/EvaluatorStatusBadge.tsx@@ -1,68 +1,113 @@-/* eslint-disable no-nested-ternary */-import { Circle } from "lucide-react";-+import Link from "next/link";import {getEvaluatorBlockMetadata,type EvaluatorBlockReason,} from "@langfuse/shared";--import { Badge } from "@/src/components/ui/badge";+import { Badge } from "@/src/components/design-system/Badge/Badge";import {- Tooltip,- TooltipContent,- TooltipTrigger,-} from "@/src/components/ui/tooltip";-import { cn } from "@/src/utils/tailwind";+ HoverCard,+ HoverCardContent,+ HoverCardTrigger,+} from "@/src/components/ui/hover-card";-/** Displays whether an evaluator is active, inactive, or blocked. */+type ExecutionSummary = { total: number; failed: number };++function getStatus({+ ruleCount,+ summary,+ blocked,+}: {+ ruleCount: number;+ summary: ExecutionSummary | undefined;+ blocked: boolean;+}) {+ if (blocked) return "Blocked";+ if (ruleCount === 0) return "Inactive";+ if (!summary) return "Unknown";+ if (summary.total === 0) return "Inactive";+ if (summary.failed === 0) return "Healthy";+ return summary.failed / summary.total >= 0.5 ? "Failing" : "Degraded";+}++const colors = {+ Blocked: "red",+ Inactive: "primary",+ Healthy: "green",+ Degraded: "yellow",+ Failing: "red",+ Unknown: "primary",+} as const;++/** Displays evaluator health from execution traces over the last seven days. */export function EvaluatorStatusBadge({ruleCount,- active,+ summary,blocked = false,blockReason = null,blockMessage = null,+ executionsHref,}: {ruleCount: number;- active: boolean;+ summary: ExecutionSummary | undefined;blocked?: boolean;blockReason?: EvaluatorBlockReason | null;blockMessage?: string | null;+ executionsHref: string | null;}) {+ const status = getStatus({ ruleCount, summary, blocked });+ let explanation: string;+ if (status === "Blocked") {+ explanation =+ blockMessage ??+ (blockReason+ ? getEvaluatorBlockMetadata(blockReason).message+ : "This evaluator is blocked.");+ } else if (ruleCount === 0) {+ explanation = "No rule is attached to this evaluator.";+ } else if (status === "Unknown") {+ explanation = "Execution status is unavailable.";+ } else if (status === "Inactive") {+ explanation = "No execution traces in the last 7 days.";+ } else if (status === "Healthy") {+ explanation = "All runs passed in the last 7 days.";+ } else {+ explanation = `${summary?.failed} of ${summary?.total} execution traces failed in the last 7 days.`;+ }+const badge = (- <Badge- variant={blocked ? "warning" : active ? "default" : "secondary"}- className={cn(- "gap-1.5 whitespace-nowrap",- active &&- !blocked &&- "bg-light-green text-dark-green hover:bg-light-green",- )}- >- <Circle className="h-2 w-2 fill-current" />- {blocked ? "Blocked" : active ? "Active" : "Inactive"} · {ruleCount}- </Badge>+ <Badge text={status === "Unknown" ? "—" : status} color={colors[status]} />);- // Prefer the message stored when the evaluator was paused — that is what the- // project was notified with — and fall back to the reason's copy for rows- // blocked before messages were persisted.- const explanation = blocked- ? (blockMessage ??- (blockReason ? getEvaluatorBlockMetadata(blockReason).message : null))- : null;-- if (!explanation) {- return badge;- }-return (- <Tooltip>- <TooltipTrigger asChild>- <span className="inline-flex" tabIndex={0}>- {badge}- </span>- </TooltipTrigger>- <TooltipContent className="max-w-xs">{explanation}</TooltipContent>- </Tooltip>+ <HoverCard openDelay={200}>+ <HoverCardTrigger asChild>+ {executionsHref ? (+ <Link+ href={executionsHref}+ onClick={(event) => event.stopPropagation()}+ aria-label={`View executions: ${status}`}+ className="focus-visible:ring-ring inline-flex rounded-sm focus-visible:ring-2 focus-visible:outline-none"+ >+ {badge}+ </Link>+ ) : (+ <span className="inline-flex" tabIndex={0}>+ {badge}+ </span>+ )}+ </HoverCardTrigger>+ <HoverCardContent align="start" className="w-80 text-sm">+ <p>{explanation}</p>+ {executionsHref && (+ <Link+ href={executionsHref}+ onClick={(event) => event.stopPropagation()}+ className="text-primary mt-2 inline-block underline"+ >+ View executions+ </Link>+ )}+ </HoverCardContent>+ </HoverCard>);}diff --git a/web/src/features/evals/v2/components/Rules/EvaluatorExecutionHistory/EvaluatorExecutionHistory.stories.tsx b/web/src/features/evals/v2/components/Rules/EvaluatorExecutionHistory/EvaluatorExecutionHistory.stories.tsxdeleted file mode 100644index fb4f25ab98..0000000000--- a/web/src/features/evals/v2/components/Rules/EvaluatorExecutionHistory/EvaluatorExecutionHistory.stories.tsx+++ /dev/null@@ -1,30 +0,0 @@-import preview from "../../../../../../../.storybook/preview";-import { EvaluatorExecutionHistory } from "./EvaluatorExecutionHistory";--const meta = preview.meta({ component: EvaluatorExecutionHistory });--export const Runs = meta.story({- args: {- traces: [- {- id: "trace-1",- level: "DEFAULT",- timestamp: new Date("2026-07-01T10:00:00Z"),- },- {- id: "trace-2",- level: "WARNING",- timestamp: new Date("2026-07-01T11:00:00Z"),- },- {- id: "trace-3",- level: "ERROR",- timestamp: new Date("2026-07-01T12:00:00Z"),- },- ],- },-});--export const NoRuns = meta.story({- args: { traces: [] },-});diff --git a/web/src/features/evals/v2/components/Rules/EvaluatorExecutionHistory/EvaluatorExecutionHistory.tsx b/web/src/features/evals/v2/components/Rules/EvaluatorExecutionHistory/EvaluatorExecutionHistory.tsxdeleted file mode 100644index 7d263006c8..0000000000--- a/web/src/features/evals/v2/components/Rules/EvaluatorExecutionHistory/EvaluatorExecutionHistory.tsx+++ /dev/null@@ -1,49 +0,0 @@-/* eslint-disable no-nested-ternary */-import { Tooltip } from "@/src/components/design-system/Tooltip/Tooltip";-import { cn } from "@/src/utils/tailwind";--export function EvaluatorExecutionHistory({- traces,-}: {- traces: Array<{ id: string; level: string; timestamp: Date }>;-}) {- if (traces.length === 0) {- return (- <span className="text-muted-foreground text-xs">No recent runs</span>- );- }-- return (- <div- className="flex items-center gap-1"- role="img"- aria-label={traces- .map(- (trace) =>- `${trace.level.toLowerCase()} at ${trace.timestamp.toLocaleString()}`,- )- .join(", ")}- >- {[...traces].reverse().map((trace) => (- <Tooltip- key={trace.id}- label={`${trace.level.toLowerCase()} at ${trace.timestamp.toLocaleString()}`}- >- {({ getTriggerProps }) => (- <span- {...getTriggerProps()}- className={cn(- "block h-4 w-1.5 rounded-full",- trace.level === "ERROR"- ? "bg-destructive"- : trace.level === "WARNING"- ? "bg-dark-yellow"- : "bg-dark-green",- )}- />- )}- </Tooltip>- ))}- </div>- );-}diff --git a/web/src/features/evals/v2/components/Rules/RulesTable/RulesTable.tsx b/web/src/features/evals/v2/components/Rules/RulesTable/RulesTable.tsxindex 84bad3339a..f258f3d3a4 100644--- a/web/src/features/evals/v2/components/Rules/RulesTable/RulesTable.tsx+++ b/web/src/features/evals/v2/components/Rules/RulesTable/RulesTable.tsx@@ -34,9 +34,7 @@ import {useColumnOrder,useColumnVisibility,} from "@/src/features/column-visibility";-import { EvaluatorExecutionHistory } from "@/src/features/evals/v2/components/Rules/EvaluatorExecutionHistory/EvaluatorExecutionHistory";import type { RuleTableRow } from "@/src/features/evals/v2/types/rules";-import { Skeleton } from "@/src/components/ui/skeleton";import {Tooltip,TooltipContent,@@ -218,10 +216,6 @@ export function RulesTable({{ projectId, ruleIds },{ enabled: ruleIds.length > 0, meta: { silentHttpCodes: [503] } },);- const recentExecutions = api.evalsV2.rules.recentExecutions.useQuery(- { projectId, ruleIds },- { enabled: ruleIds.length > 0, meta: { silentHttpCodes: [503] } },- );const deleteMany = api.evalsV2.rules.deleteMany.useMutation({onError: trpcErrorToast,onSuccess: async (result) => {@@ -315,33 +309,6 @@ export function RulesTable({return value;},}),- {- accessorKey: "executionTraces",- id: "executionTraces",- header: "Last 5 runs",- size: 140,- enableHiding: true,- cell: ({ row }) => {- if (recentExecutions.isPending) {- return <Skeleton className="h-4 w-16" />;- }- return (- <button- type="button"- className="focus-visible:ring-ring rounded-sm focus-visible:ring-2 focus-visible:outline-none"- aria-label={`View runs for ${row.original.name}`}- onClick={(event) => {- event.stopPropagation();- router.push(ruleExecutionsUrl(projectId, row.original.id));- }}- >- <EvaluatorExecutionHistory- traces={recentExecutions.data?.[row.original.id] ?? []}- />- </button>- );- },- },{accessorKey: "assignments",id: "assignments",@@ -486,8 +453,6 @@ export function RulesTable({costs.data,costs.isPending,projectId,- recentExecutions.data,- recentExecutions.isPending,router,capture,selectActionColumn,diff --git a/web/src/features/evals/v2/fns/evaluators/evaluatorScoresUrl.clienttest.ts b/web/src/features/evals/v2/fns/evaluators/evaluatorScoresUrl.clienttest.tsindex 751abf0ea3..058a89b8ad 100644--- a/web/src/features/evals/v2/fns/evaluators/evaluatorScoresUrl.clienttest.ts+++ b/web/src/features/evals/v2/fns/evaluators/evaluatorScoresUrl.clienttest.ts@@ -5,6 +5,10 @@ import {LangfuseInternalTraceEnvironment,} from "@langfuse/shared";import { describe, expect, it } from "vitest";+import {+ rangeFromString,+ TABLE_AGGREGATION_OPTIONS,+} from "@/src/utils/date-range-utils";import {evaluatorExecutionsUrl,evaluatorScoresUrl,@@ -75,13 +79,18 @@ describe("evaluatorScoresUrl", () => {const getFilters = (evaluatorType: EvalTemplateType) => {const url = new URL(- evaluatorExecutionsUrl("project/id", "Quality", evaluatorType),+ evaluatorExecutionsUrl("project/id", "evaluator-id", evaluatorType),"https://langfuse.local",);return {pathname: url.pathname,filters: decodeFiltersGeneric(url.searchParams.get("filter") ?? ""),+ range: rangeFromString(+ url.searchParams.get("dateRange") ?? "",+ TABLE_AGGREGATION_OPTIONS,+ "last1Day",+ ),};};@@ -97,12 +106,13 @@ describe("evaluatorExecutionsUrl", () => {(type, environment) => {expect(getFilters(type)).toEqual({pathname: "/project/project%2Fid/traces",+ range: { range: "last7Days" },filters: [{- column: "traceName",+ column: "evaluatorId",type: "stringOptions",operator: "any of",- value: ["Execute evaluator: Quality"],+ value: ["evaluator-id"],},{column: "environment",diff --git a/web/src/features/evals/v2/fns/evaluators/evaluatorScoresUrl.ts b/web/src/features/evals/v2/fns/evaluators/evaluatorScoresUrl.tsindex 38c3a83613..1250362852 100644--- a/web/src/features/evals/v2/fns/evaluators/evaluatorScoresUrl.ts+++ b/web/src/features/evals/v2/fns/evaluators/evaluatorScoresUrl.ts@@ -45,7 +45,7 @@ export function evaluatorScoresUrl(export function evaluatorExecutionsUrl(projectId: string,- evaluatorName: string,+ evaluatorId: string,evaluatorType: EvalTemplateType,) {const environment =@@ -54,10 +54,10 @@ export function evaluatorExecutionsUrl(: LangfuseInternalTraceEnvironment.LLMJudge;const filter: FilterState = [{- column: "traceName",+ column: "evaluatorId",type: "stringOptions",operator: "any of",- value: [`Execute evaluator: ${evaluatorName}`],+ value: [evaluatorId],},{column: "environment",@@ -72,5 +72,5 @@ export function evaluatorExecutionsUrl(value: true,},];- return `/project/${encodeURIComponent(projectId)}/traces?filter=${encodeURIComponent(encodeFiltersGeneric(filter))}`;+ return `/project/${encodeURIComponent(projectId)}/traces?dateRange=7d&filter=${encodeURIComponent(encodeFiltersGeneric(filter))}`;}diff --git a/web/src/features/evals/v2/pages/EvaluatorsPage.tsx b/web/src/features/evals/v2/pages/EvaluatorsPage.tsxindex 3f9effe33f..e9aaf41ca4 100644--- a/web/src/features/evals/v2/pages/EvaluatorsPage.tsx+++ b/web/src/features/evals/v2/pages/EvaluatorsPage.tsx@@ -31,7 +31,6 @@ import { EvaluatorBulkDeleteDialog } from "../components/Evaluators/EvaluatorBulimport { EvaluatorGalleryDialog } from "../components/EvaluatorGalleryDialog/EvaluatorGalleryDialog";import { EvaluatorStatusBadge } from "../components/Evaluators/EvaluatorStatusBadge/EvaluatorStatusBadge";import { EvaluatorTypeBadge } from "../components/Evaluators/EvaluatorTypeBadge/EvaluatorTypeBadge";-import { EvaluatorExecutionHistory } from "@/src/features/evals/v2/components/Rules/EvaluatorExecutionHistory/EvaluatorExecutionHistory";import { OverviewSelectionBar } from "../components/OverviewSelectionBar/OverviewSelectionBar";import { usePostHogClientCapture } from "@/src/features/posthog-analytics";import {@@ -322,7 +321,7 @@ export default function EvaluatorsPage() {meta: { silentHttpCodes: [503] },},);- const recentExecutions = api.evalsV2.recentExecutions.useQuery(+ const executionSummaries = api.evalsV2.executionSummaries.useQuery({ projectId, evaluatorIds },{enabled: hasExecutionReadAccess && evaluatorIds.length > 0,@@ -384,50 +383,32 @@ export default function EvaluatorsPage() {header: "Status",size: 130,enableHiding: true,- cell: ({ row }) => (- <EvaluatorStatusBadge- ruleCount={row.original._count.assignments}- active={row.original.hasActiveRules}- blocked={Boolean(row.original.blockedAt)}- blockReason={row.original.blockReason}- blockMessage={row.original.blockMessage}- />- ),- },- {- accessorKey: "executionTraces",- id: "executionTraces",- header: "Last 5 runs",- size: 130,- enableHiding: true,cell: ({ row }) => {- if (recentExecutions.isPending && hasExecutionReadAccess) {+ if (+ executionSummaries.isPending &&+ hasExecutionReadAccess &&+ !row.original.blockedAt &&+ row.original._count.assignments > 0+ ) {return <Skeleton className="h-4 w-16" />;}- const history = (- <EvaluatorExecutionHistory- traces={recentExecutions.data?.[row.original.id] ?? []}- />- );- return hasExecutionReadAccess ? (- <button- type="button"- className="focus-visible:ring-ring rounded-sm focus-visible:ring-2 focus-visible:outline-none"- aria-label={`View executions for ${row.original.name}`}- onClick={() =>- router.push(- evaluatorExecutionsUrl(- projectId,- row.original.name,- row.original.type,- ),- )+ return (+ <EvaluatorStatusBadge+ ruleCount={row.original._count.assignments}+ summary={executionSummaries.data?.[row.original.id]}+ blocked={Boolean(row.original.blockedAt)}+ blockReason={row.original.blockReason}+ blockMessage={row.original.blockMessage}+ executionsHref={+ hasExecutionReadAccess+ ? evaluatorExecutionsUrl(+ projectId,+ row.original.id,+ row.original.type,+ )+ : null}- >- {history}- </button>- ) : (- history+ />);},},@@ -526,7 +507,7 @@ export default function EvaluatorsPage() {router.push(evaluatorExecutionsUrl(projectId,- row.original.name,+ row.original.id,row.original.type,),)@@ -555,8 +536,8 @@ export default function EvaluatorsPage() {costs.isPending,hasExecutionReadAccess,projectId,- recentExecutions.data,- recentExecutions.isPending,+ executionSummaries.data,+ executionSummaries.isPending,router,selectActionColumn,],diff --git a/web/src/features/evals/v2/server/evaluators/evaluatorRouter.ts b/web/src/features/evals/v2/server/evaluators/evaluatorRouter.tsindex 9c5e014eea..8b4fef9934 100644--- a/web/src/features/evals/v2/server/evaluators/evaluatorRouter.ts+++ b/web/src/features/evals/v2/server/evaluators/evaluatorRouter.ts@@ -145,7 +145,7 @@ export const evaluatorRouter = createTRPCRouter({});}),- recentExecutions: protectedProjectProcedure+ executionSummaries: protectedProjectProcedure.input(EvaluatorIdsSchema).query(({ input, ctx }) => {throwIfNoProjectAccess({@@ -153,7 +153,7 @@ export const evaluatorRouter = createTRPCRouter({projectId: ctx.session.projectId,scope: "evalJobExecution:read",});- return serviceForContext(ctx).listRecent({+ return serviceForContext(ctx).getExecutionSummaries({...input,projectId: ctx.session.projectId,});diff --git a/web/src/features/evals/v2/server/evaluators/evaluatorService.ts b/web/src/features/evals/v2/server/evaluators/evaluatorService.tsindex 69c71cf413..cee41c01ef 100644--- a/web/src/features/evals/v2/server/evaluators/evaluatorService.ts+++ b/web/src/features/evals/v2/server/evaluators/evaluatorService.ts@@ -20,7 +20,7 @@ import {ChatMessageType,generateLangfuseAIText,getClientInitiatedNonStreamingLlmTimeoutMs,- getRecentEvaluatorExecutionTraces,+ getEvaluatorExecutionSummaries,getTotalCostByEvaluatorIds,invalidateProjectEvalConfigCaches,logger,@@ -129,12 +129,6 @@ function prepareEvaluatorDefinitionForPersistence(};}-type EvaluatorExecutionTrace = {- id: string;- level: string;- timestamp: Date;-};-export type EvaluatorAuditEvent = {action: "create" | "update" | "delete";projectId: string;@@ -260,25 +254,21 @@ export class EvaluatorService {};}- async listRecent(params: { projectId: string; evaluatorIds: string[] }) {- const result = Object.fromEntries(- params.evaluatorIds.map((evaluatorId) => [evaluatorId, []]),- ) as Record<string, EvaluatorExecutionTrace[]>;- if (params.evaluatorIds.length === 0) return result;-- const traces = await getRecentEvaluatorExecutionTraces(+ async getExecutionSummaries(params: {+ projectId: string;+ evaluatorIds: string[];+ }) {+ const result: Record<string, { total: number; failed: number }> =+ Object.fromEntries(+ params.evaluatorIds.map((id) => [id, { total: 0, failed: 0 }]),+ );+ const summaries = await getEvaluatorExecutionSummaries(params.projectId,params.evaluatorIds,);-- for (const trace of traces) {- result[trace.evaluatorId]?.push({- id: trace.id,- level: trace.level,- timestamp: trace.timestamp,- });+ for (const { evaluatorId, total, failed } of summaries) {+ result[evaluatorId] = { total, failed };}-return result;}diff --git a/web/src/features/evals/v2/server/rules/ruleRouter.ts b/web/src/features/evals/v2/server/rules/ruleRouter.tsindex e3e8f8d5c9..1091fbc659 100644--- a/web/src/features/evals/v2/server/rules/ruleRouter.ts+++ b/web/src/features/evals/v2/server/rules/ruleRouter.ts@@ -80,20 +80,6 @@ export const ruleRouter = createTRPCRouter({return serviceForContext(ctx).get(ctx.session.projectId, input.ruleId);}),- recentExecutions: protectedProjectProcedure- .input(RuleIdsSchema)- .query(({ input, ctx }) => {- throwIfNoProjectAccess({- session: ctx.session,- projectId: ctx.session.projectId,- scope: "evalJobExecution:read",- });- return serviceForContext(ctx).listRecent({- ...input,- projectId: ctx.session.projectId,- });- }),-costByRuleIds: protectedProjectProcedure.input(RuleIdsSchema).query(({ input, ctx }) => {diff --git a/web/src/features/evals/v2/server/rules/ruleService.ts b/web/src/features/evals/v2/server/rules/ruleService.tsindex b39801f3f7..f65f3aa2e5 100644--- a/web/src/features/evals/v2/server/rules/ruleService.ts+++ b/web/src/features/evals/v2/server/rules/ruleService.ts@@ -22,7 +22,6 @@ import {getClientInitiatedNonStreamingLlmTimeoutMs,invalidateProjectEvalConfigCaches,logger,- getRecentRuleExecutionTraces,getTotalCostByRule,} from "@langfuse/shared/src/server";import { resolveLangfuseAiFeatureAvailability } from "@/src/features/ai-features/server";@@ -104,23 +103,6 @@ export class RuleService {return toRuleResponse(rule);}- async listRecent(params: { projectId: string; ruleIds: string[] }) {- const result = Object.fromEntries(- params.ruleIds.map((ruleId) => [ruleId, []]),- ) as Record<string, Array<{ id: string; level: string; timestamp: Date }>>;- if (params.ruleIds.length === 0) return result;-- const traces = await getRecentRuleExecutionTraces(- params.projectId,- params.ruleIds,- );-- for (const { ruleId, ...trace } of traces) {- result[ruleId]?.push(trace);- }- return result;- }-async getTotalCosts(params: { projectId: string; ruleIds: string[] }) {const costs = await getTotalCostByRule(params.projectId, params.ruleIds);return Object.fromEntries(