🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
feat(traces): link evaluator execution traces (#16839)
Tobias Wochinger02c77e6586e5
diff --git a/web/src/__tests__/server/evaluator-v2-service.servertest.ts b/web/src/__tests__/server/evaluator-v2-service.servertest.tsindex 5d6e84b21c..3f14fd47db 100644--- a/web/src/__tests__/server/evaluator-v2-service.servertest.ts+++ b/web/src/__tests__/server/evaluator-v2-service.servertest.ts@@ -817,6 +817,7 @@ describe("EvaluatorService", () => {expect.objectContaining({orgId,evaluatorId: newEvaluatorId,+ includeEvaluatorLink: false,...observation,}),);@@ -824,6 +825,7 @@ describe("EvaluatorService", () => {expect.objectContaining({orgId,evaluatorId: evaluator.id,+ includeEvaluatorLink: true,...observation,}),);diff --git a/web/src/features/evals/v2/server/evaluators/evaluatorService.ts b/web/src/features/evals/v2/server/evaluators/evaluatorService.tsindex c6087723b1..4d2eb7b16f 100644--- a/web/src/features/evals/v2/server/evaluators/evaluatorService.ts+++ b/web/src/features/evals/v2/server/evaluators/evaluatorService.ts@@ -551,7 +551,10 @@ export class EvaluatorService {if (!evaluator && !isPregeneratedEvaluatorId(params.evaluatorId)) {throw new LangfuseNotFoundError("Evaluator not found");}- return executeEvaluatorTest(params);+ return executeEvaluatorTest({+ ...params,+ includeEvaluatorLink: Boolean(evaluator),+ });}async suggestName(params: SuggestEvaluatorTextParams) {diff --git a/web/src/features/evals/v2/server/evaluators/testEvaluator.ts b/web/src/features/evals/v2/server/evaluators/testEvaluator.tsindex 9b177800cc..760d3be631 100644--- a/web/src/features/evals/v2/server/evaluators/testEvaluator.ts+++ b/web/src/features/evals/v2/server/evaluators/testEvaluator.ts@@ -25,6 +25,7 @@ export async function testEvaluator(params: {orgId: string;projectId: string;evaluatorId: string;+ includeEvaluatorLink?: boolean;definition: EvaluatorDefinition;observationId: string;traceId: string;@@ -49,7 +50,8 @@ export async function testEvaluator(params: {});const metadata = buildEvalExecutionMetadata({type: "TEST",- evaluatorId: params.evaluatorId,+ evaluatorId:+ params.includeEvaluatorLink === false ? null : params.evaluatorId,targetTraceId: params.traceId,targetObservationId: params.observationId,});diff --git a/web/src/features/traces/components/ObservationDetailView/ConnectedObservationDetailView.tsx b/web/src/features/traces/components/ObservationDetailView/ConnectedObservationDetailView.tsxindex cbd21b688b..088c2cbaed 100644--- a/web/src/features/traces/components/ObservationDetailView/ConnectedObservationDetailView.tsx+++ b/web/src/features/traces/components/ObservationDetailView/ConnectedObservationDetailView.tsx@@ -54,7 +54,7 @@ import { useCommentedPaths } from "@/src/features/comments/hooks/useCommentedPatimport { api } from "@/src/utils/api";// Extracted components-import { ObservationDetailViewHeader } from "./components/ObservationDetailViewHeader";+import { ObservationDetailViewHeader } from "@/src/features/traces/components/ObservationDetailView/components/ObservationDetailViewHeader/ObservationDetailViewHeader";import { TraceLogView } from "../TraceLogView/TraceLogView";import { useV4Beta } from "@/src/features/events/hooks/useV4Beta";import { TRACE_VIEW_CONFIG } from "@/src/features/traces/constants/traceViewConfig";diff --git a/web/src/features/traces/components/ObservationDetailView/components/ObservationDetailViewHeader.tsx b/web/src/features/traces/components/ObservationDetailView/components/ObservationDetailViewHeader/ObservationDetailViewHeader.tsxsimilarity index 95%rename from web/src/features/traces/components/ObservationDetailView/components/ObservationDetailViewHeader.tsxrename to web/src/features/traces/components/ObservationDetailView/components/ObservationDetailViewHeader/ObservationDetailViewHeader.tsxindex c8fae57000..0f6ae8e45e 100644--- a/web/src/features/traces/components/ObservationDetailView/components/ObservationDetailViewHeader.tsx+++ b/web/src/features/traces/components/ObservationDetailView/components/ObservationDetailViewHeader/ObservationDetailViewHeader.tsx@@ -14,6 +14,7 @@ import {type ObservationType,AnnotationQueueObjectType,isGenerationLike,+ LangfuseInternalTraceEnvironment,type ScoreDomain,} from "@langfuse/shared";import { type SelectionData } from "@/src/features/comments/contexts/InlineCommentSelectionContext";@@ -34,12 +35,19 @@ import {EnvironmentBadge,ReleaseBadge,VersionBadge,-} from "../../ObservationMetadataBadgesSimple/ObservationMetadataBadgesSimple";+} from "@/src/features/traces/components/ObservationMetadataBadgesSimple/ObservationMetadataBadgesSimple";import { ObservationLevelBadge } from "@/src/features/traces/components/ObservationLevelBadge";-import { SessionBadge, UserIdBadge } from "../../TraceMetadataBadges";-import { CostBadge, UsageBadge } from "../../ObservationMetadataBadgesTooltip";-import { ModelBadge } from "./ModelBadge";-import { ModelParametersBadges } from "./ModelParametersBadges";+import {+ SessionBadge,+ UserIdBadge,+} from "@/src/features/traces/components/TraceMetadataBadges";+import { EvaluatorBadge } from "@/src/features/traces/components/ObservationDetailView/components/ObservationDetailViewHeader/components/EvaluatorBadge/EvaluatorBadge";+import {+ CostBadge,+ UsageBadge,+} from "@/src/features/traces/components/ObservationMetadataBadgesTooltip";+import { ModelBadge } from "@/src/features/traces/components/ObservationDetailView/components/ModelBadge";+import { ModelParametersBadges } from "@/src/features/traces/components/ObservationDetailView/components/ModelParametersBadges";import {type WithStringifiedMetadata,type MetadataDomainClient,@@ -79,6 +87,8 @@ import { DualAnnotationContent } from "@/src/features/scores/components/DualAnnoimport { CollapsibleBadgeRow } from "@/src/features/traces/components/CollapsibleBadgeRow";import { useIsMobile } from "@/src/hooks/use-mobile";import { cn } from "@/src/utils/tailwind";+import { resolveEvaluatorIdMetadata } from "@/src/features/traces/fns/resolveEvaluatorIdMetadata";+import { api } from "@/src/utils/api";import { buildLocalIsoDatePresentation } from "@/src/utils/dates";export interface ObservationDetailViewHeaderProps {@@ -155,6 +165,22 @@ export const ObservationDetailViewHeader = memo(const totalUsage = observation.totalUsage;const inputUsage = observation.inputUsage;const outputUsage = observation.outputUsage;+ const evaluatorId = resolveEvaluatorIdMetadata(+ observationWithIO?.metadata ?? observation.metadata,+ );+ const isEvaluatorExecution =+ observation.environment === LangfuseInternalTraceEnvironment.LLMJudge ||+ observation.environment === LangfuseInternalTraceEnvironment.CodeEval;+ const evaluator = api.evalsV2.get.useQuery(+ { projectId, evaluatorId: evaluatorId ?? "" },+ {+ enabled: Boolean(+ evaluatorId &&+ isEvaluatorExecution &&+ !evaluatorId.startsWith("managed:"),+ ),+ },+ );const preparedDate = buildLocalIsoDatePresentation({date: observation.startTime,@@ -680,6 +706,12 @@ export const ObservationDetailViewHeader = memo(userId={observation.userId ?? null}projectId={projectId}/>+ <EvaluatorBadge+ evaluatorId={evaluatorId}+ evaluatorName={evaluator.data?.name}+ environment={observation.environment}+ projectId={projectId}+ /><EnvironmentBadge environment={observation.environment} /><ReleaseBadge release={observation.release} /><CostBadgediff --git a/web/src/features/traces/components/ObservationDetailView/components/ObservationDetailViewHeader/components/EvaluatorBadge/EvaluatorBadge.stories.tsx b/web/src/features/traces/components/ObservationDetailView/components/ObservationDetailViewHeader/components/EvaluatorBadge/EvaluatorBadge.stories.tsxnew file mode 100644index 0000000000..9462ea4ce7--- /dev/null+++ b/web/src/features/traces/components/ObservationDetailView/components/ObservationDetailViewHeader/components/EvaluatorBadge/EvaluatorBadge.stories.tsx@@ -0,0 +1,20 @@+import preview from "../../../../../../../../../.storybook/preview";+import { EvaluatorBadge } from "./EvaluatorBadge";++const meta = preview.meta({+ component: EvaluatorBadge,+ args: {+ evaluatorId: "evaluator-id",+ evaluatorName: "Quality check",+ environment: "langfuse-llm-as-a-judge",+ projectId: "project-id",+ },+});++export const Named = meta.story({});++export const WithoutName = meta.story({+ args: {+ evaluatorName: null,+ },+});diff --git a/web/src/features/traces/components/ObservationDetailView/components/ObservationDetailViewHeader/components/EvaluatorBadge/EvaluatorBadge.tsx b/web/src/features/traces/components/ObservationDetailView/components/ObservationDetailViewHeader/components/EvaluatorBadge/EvaluatorBadge.tsxnew file mode 100644index 0000000000..2fb5187a94--- /dev/null+++ b/web/src/features/traces/components/ObservationDetailView/components/ObservationDetailViewHeader/components/EvaluatorBadge/EvaluatorBadge.tsx@@ -0,0 +1,37 @@+import { LangfuseInternalTraceEnvironment } from "@langfuse/shared";+import { ExternalLinkIcon } from "lucide-react";+import Link from "next/link";++import { Badge } from "@/src/components/design-system/Badge/Badge";++export function EvaluatorBadge({+ evaluatorId,+ evaluatorName,+ environment,+ projectId,+}: {+ evaluatorId: string | null;+ evaluatorName?: string | null;+ environment: string;+ projectId: string;+}) {+ const isEvaluatorExecution =+ environment === LangfuseInternalTraceEnvironment.LLMJudge ||+ environment === LangfuseInternalTraceEnvironment.CodeEval;+ const isManagedTemplate = evaluatorId?.startsWith("managed:") ?? false;+ if (!evaluatorId || !isEvaluatorExecution || isManagedTemplate) return null;++ return (+ <Link+ href={`/project/${projectId}/evals/v2/${encodeURIComponent(evaluatorId)}`}+ target="_blank"+ rel="noopener noreferrer"+ className="ph-no-capture inline-flex"+ >+ <Badge+ text={evaluatorName ? `Evaluator: ${evaluatorName}` : "Evaluator"}+ trailingIcon={ExternalLinkIcon}+ />+ </Link>+ );+}diff --git a/web/src/features/traces/components/TraceMetadataBadges.clienttest.tsx b/web/src/features/traces/components/TraceMetadataBadges.clienttest.tsxindex 6695940569..bbb5af452b 100644--- a/web/src/features/traces/components/TraceMetadataBadges.clienttest.tsx+++ b/web/src/features/traces/components/TraceMetadataBadges.clienttest.tsx@@ -29,7 +29,6 @@ describe("TraceMetadataBadges session replay privacy", () => {expect(screen.getByText("Target Trace: target-trace").closest("a"),).toHaveClass("ph-no-capture");-expect(screen.getByText("Session: customer-session").parentElement,).toHaveClass("bg-primary");diff --git a/web/src/features/traces/fns/resolveEvaluatorIdMetadata.clienttest.ts b/web/src/features/traces/fns/resolveEvaluatorIdMetadata.clienttest.tsnew file mode 100644index 0000000000..ecc4e1de99--- /dev/null+++ b/web/src/features/traces/fns/resolveEvaluatorIdMetadata.clienttest.ts@@ -0,0 +1,13 @@+import { describe, expect, it } from "vitest";++import { resolveEvaluatorIdMetadata } from "./resolveEvaluatorIdMetadata";++describe("resolveEvaluatorIdMetadata", () => {+ it("resolves evaluator ids from serialized observation metadata", () => {+ expect(+ resolveEvaluatorIdMetadata(+ JSON.stringify({ evaluator_id: "evaluator-id" }),+ ),+ ).toBe("evaluator-id");+ });+});diff --git a/web/src/features/traces/fns/resolveEvaluatorIdMetadata.ts b/web/src/features/traces/fns/resolveEvaluatorIdMetadata.tsnew file mode 100644index 0000000000..6bb52a7988--- /dev/null+++ b/web/src/features/traces/fns/resolveEvaluatorIdMetadata.ts@@ -0,0 +1,19 @@+export function resolveEvaluatorIdMetadata(metadata: unknown): string | null {+ let parsedMetadata = metadata;++ if (typeof parsedMetadata === "string") {+ try {+ parsedMetadata = JSON.parse(parsedMetadata);+ } catch {+ return null;+ }+ }++ if (typeof parsedMetadata !== "object" || parsedMetadata === null)+ return null;++ const evaluatorId = (parsedMetadata as Record<string, unknown>).evaluator_id;+ return typeof evaluatorId === "string" && evaluatorId.length > 0+ ? evaluatorId+ : null;+}