Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

feat(traces): link evaluator execution traces (#16839)

Tobias Wochinger02c77e6586e5
diff --git a/web/src/__tests__/server/evaluator-v2-service.servertest.ts b/web/src/__tests__/server/evaluator-v2-service.servertest.ts
index 5d6e84b21c..3f14fd47db 100644
--- a/web/src/__tests__/server/evaluator-v2-service.servertest.ts
+++ b/web/src/__tests__/server/evaluator-v2-service.servertest.ts
@@ -817,6 +817,7 @@ describe("EvaluatorService", () => {
expect.objectContaining({
orgId,
evaluatorId: newEvaluatorId,
+ includeEvaluatorLink: false,
...observation,
}),
);
@@ -824,6 +825,7 @@ describe("EvaluatorService", () => {
expect.objectContaining({
orgId,
evaluatorId: evaluator.id,
+ includeEvaluatorLink: true,
...observation,
}),
);
diff --git a/web/src/features/evals/v2/server/evaluators/evaluatorService.ts b/web/src/features/evals/v2/server/evaluators/evaluatorService.ts
index c6087723b1..4d2eb7b16f 100644
--- a/web/src/features/evals/v2/server/evaluators/evaluatorService.ts
+++ b/web/src/features/evals/v2/server/evaluators/evaluatorService.ts
@@ -551,7 +551,10 @@ export class EvaluatorService {
if (!evaluator && !isPregeneratedEvaluatorId(params.evaluatorId)) {
throw new LangfuseNotFoundError("Evaluator not found");
}
- return executeEvaluatorTest(params);
+ return executeEvaluatorTest({
+ ...params,
+ includeEvaluatorLink: Boolean(evaluator),
+ });
}
async suggestName(params: SuggestEvaluatorTextParams) {
diff --git a/web/src/features/evals/v2/server/evaluators/testEvaluator.ts b/web/src/features/evals/v2/server/evaluators/testEvaluator.ts
index 9b177800cc..760d3be631 100644
--- a/web/src/features/evals/v2/server/evaluators/testEvaluator.ts
+++ b/web/src/features/evals/v2/server/evaluators/testEvaluator.ts
@@ -25,6 +25,7 @@ export async function testEvaluator(params: {
orgId: string;
projectId: string;
evaluatorId: string;
+ includeEvaluatorLink?: boolean;
definition: EvaluatorDefinition;
observationId: string;
traceId: string;
@@ -49,7 +50,8 @@ export async function testEvaluator(params: {
});
const metadata = buildEvalExecutionMetadata({
type: "TEST",
- evaluatorId: params.evaluatorId,
+ evaluatorId:
+ params.includeEvaluatorLink === false ? null : params.evaluatorId,
targetTraceId: params.traceId,
targetObservationId: params.observationId,
});
diff --git a/web/src/features/traces/components/ObservationDetailView/ConnectedObservationDetailView.tsx b/web/src/features/traces/components/ObservationDetailView/ConnectedObservationDetailView.tsx
index cbd21b688b..088c2cbaed 100644
--- a/web/src/features/traces/components/ObservationDetailView/ConnectedObservationDetailView.tsx
+++ b/web/src/features/traces/components/ObservationDetailView/ConnectedObservationDetailView.tsx
@@ -54,7 +54,7 @@ import { useCommentedPaths } from "@/src/features/comments/hooks/useCommentedPat
import { api } from "@/src/utils/api";
// Extracted components
-import { ObservationDetailViewHeader } from "./components/ObservationDetailViewHeader";
+import { ObservationDetailViewHeader } from "@/src/features/traces/components/ObservationDetailView/components/ObservationDetailViewHeader/ObservationDetailViewHeader";
import { TraceLogView } from "../TraceLogView/TraceLogView";
import { useV4Beta } from "@/src/features/events/hooks/useV4Beta";
import { TRACE_VIEW_CONFIG } from "@/src/features/traces/constants/traceViewConfig";
diff --git a/web/src/features/traces/components/ObservationDetailView/components/ObservationDetailViewHeader.tsx b/web/src/features/traces/components/ObservationDetailView/components/ObservationDetailViewHeader/ObservationDetailViewHeader.tsx
similarity index 95%
rename from web/src/features/traces/components/ObservationDetailView/components/ObservationDetailViewHeader.tsx
rename to web/src/features/traces/components/ObservationDetailView/components/ObservationDetailViewHeader/ObservationDetailViewHeader.tsx
index c8fae57000..0f6ae8e45e 100644
--- a/web/src/features/traces/components/ObservationDetailView/components/ObservationDetailViewHeader.tsx
+++ b/web/src/features/traces/components/ObservationDetailView/components/ObservationDetailViewHeader/ObservationDetailViewHeader.tsx
@@ -14,6 +14,7 @@ import {
type ObservationType,
AnnotationQueueObjectType,
isGenerationLike,
+ LangfuseInternalTraceEnvironment,
type ScoreDomain,
} from "@langfuse/shared";
import { type SelectionData } from "@/src/features/comments/contexts/InlineCommentSelectionContext";
@@ -34,12 +35,19 @@ import {
EnvironmentBadge,
ReleaseBadge,
VersionBadge,
-} from "../../ObservationMetadataBadgesSimple/ObservationMetadataBadgesSimple";
+} from "@/src/features/traces/components/ObservationMetadataBadgesSimple/ObservationMetadataBadgesSimple";
import { ObservationLevelBadge } from "@/src/features/traces/components/ObservationLevelBadge";
-import { SessionBadge, UserIdBadge } from "../../TraceMetadataBadges";
-import { CostBadge, UsageBadge } from "../../ObservationMetadataBadgesTooltip";
-import { ModelBadge } from "./ModelBadge";
-import { ModelParametersBadges } from "./ModelParametersBadges";
+import {
+ SessionBadge,
+ UserIdBadge,
+} from "@/src/features/traces/components/TraceMetadataBadges";
+import { EvaluatorBadge } from "@/src/features/traces/components/ObservationDetailView/components/ObservationDetailViewHeader/components/EvaluatorBadge/EvaluatorBadge";
+import {
+ CostBadge,
+ UsageBadge,
+} from "@/src/features/traces/components/ObservationMetadataBadgesTooltip";
+import { ModelBadge } from "@/src/features/traces/components/ObservationDetailView/components/ModelBadge";
+import { ModelParametersBadges } from "@/src/features/traces/components/ObservationDetailView/components/ModelParametersBadges";
import {
type WithStringifiedMetadata,
type MetadataDomainClient,
@@ -79,6 +87,8 @@ import { DualAnnotationContent } from "@/src/features/scores/components/DualAnno
import { CollapsibleBadgeRow } from "@/src/features/traces/components/CollapsibleBadgeRow";
import { useIsMobile } from "@/src/hooks/use-mobile";
import { cn } from "@/src/utils/tailwind";
+import { resolveEvaluatorIdMetadata } from "@/src/features/traces/fns/resolveEvaluatorIdMetadata";
+import { api } from "@/src/utils/api";
import { buildLocalIsoDatePresentation } from "@/src/utils/dates";
export interface ObservationDetailViewHeaderProps {
@@ -155,6 +165,22 @@ export const ObservationDetailViewHeader = memo(
const totalUsage = observation.totalUsage;
const inputUsage = observation.inputUsage;
const outputUsage = observation.outputUsage;
+ const evaluatorId = resolveEvaluatorIdMetadata(
+ observationWithIO?.metadata ?? observation.metadata,
+ );
+ const isEvaluatorExecution =
+ observation.environment === LangfuseInternalTraceEnvironment.LLMJudge ||
+ observation.environment === LangfuseInternalTraceEnvironment.CodeEval;
+ const evaluator = api.evalsV2.get.useQuery(
+ { projectId, evaluatorId: evaluatorId ?? "" },
+ {
+ enabled: Boolean(
+ evaluatorId &&
+ isEvaluatorExecution &&
+ !evaluatorId.startsWith("managed:"),
+ ),
+ },
+ );
const preparedDate = buildLocalIsoDatePresentation({
date: observation.startTime,
@@ -680,6 +706,12 @@ export const ObservationDetailViewHeader = memo(
userId={observation.userId ?? null}
projectId={projectId}
/>
+ <EvaluatorBadge
+ evaluatorId={evaluatorId}
+ evaluatorName={evaluator.data?.name}
+ environment={observation.environment}
+ projectId={projectId}
+ />
<EnvironmentBadge environment={observation.environment} />
<ReleaseBadge release={observation.release} />
<CostBadge
diff --git a/web/src/features/traces/components/ObservationDetailView/components/ObservationDetailViewHeader/components/EvaluatorBadge/EvaluatorBadge.stories.tsx b/web/src/features/traces/components/ObservationDetailView/components/ObservationDetailViewHeader/components/EvaluatorBadge/EvaluatorBadge.stories.tsx
new file mode 100644
index 0000000000..9462ea4ce7
--- /dev/null
+++ b/web/src/features/traces/components/ObservationDetailView/components/ObservationDetailViewHeader/components/EvaluatorBadge/EvaluatorBadge.stories.tsx
@@ -0,0 +1,20 @@
+import preview from "../../../../../../../../../.storybook/preview";
+import { EvaluatorBadge } from "./EvaluatorBadge";
+
+const meta = preview.meta({
+ component: EvaluatorBadge,
+ args: {
+ evaluatorId: "evaluator-id",
+ evaluatorName: "Quality check",
+ environment: "langfuse-llm-as-a-judge",
+ projectId: "project-id",
+ },
+});
+
+export const Named = meta.story({});
+
+export const WithoutName = meta.story({
+ args: {
+ evaluatorName: null,
+ },
+});
diff --git a/web/src/features/traces/components/ObservationDetailView/components/ObservationDetailViewHeader/components/EvaluatorBadge/EvaluatorBadge.tsx b/web/src/features/traces/components/ObservationDetailView/components/ObservationDetailViewHeader/components/EvaluatorBadge/EvaluatorBadge.tsx
new file mode 100644
index 0000000000..2fb5187a94
--- /dev/null
+++ b/web/src/features/traces/components/ObservationDetailView/components/ObservationDetailViewHeader/components/EvaluatorBadge/EvaluatorBadge.tsx
@@ -0,0 +1,37 @@
+import { LangfuseInternalTraceEnvironment } from "@langfuse/shared";
+import { ExternalLinkIcon } from "lucide-react";
+import Link from "next/link";
+
+import { Badge } from "@/src/components/design-system/Badge/Badge";
+
+export function EvaluatorBadge({
+ evaluatorId,
+ evaluatorName,
+ environment,
+ projectId,
+}: {
+ evaluatorId: string | null;
+ evaluatorName?: string | null;
+ environment: string;
+ projectId: string;
+}) {
+ const isEvaluatorExecution =
+ environment === LangfuseInternalTraceEnvironment.LLMJudge ||
+ environment === LangfuseInternalTraceEnvironment.CodeEval;
+ const isManagedTemplate = evaluatorId?.startsWith("managed:") ?? false;
+ if (!evaluatorId || !isEvaluatorExecution || isManagedTemplate) return null;
+
+ return (
+ <Link
+ href={`/project/${projectId}/evals/v2/${encodeURIComponent(evaluatorId)}`}
+ target="_blank"
+ rel="noopener noreferrer"
+ className="ph-no-capture inline-flex"
+ >
+ <Badge
+ text={evaluatorName ? `Evaluator: ${evaluatorName}` : "Evaluator"}
+ trailingIcon={ExternalLinkIcon}
+ />
+ </Link>
+ );
+}
diff --git a/web/src/features/traces/components/TraceMetadataBadges.clienttest.tsx b/web/src/features/traces/components/TraceMetadataBadges.clienttest.tsx
index 6695940569..bbb5af452b 100644
--- a/web/src/features/traces/components/TraceMetadataBadges.clienttest.tsx
+++ b/web/src/features/traces/components/TraceMetadataBadges.clienttest.tsx
@@ -29,7 +29,6 @@ describe("TraceMetadataBadges session replay privacy", () => {
expect(
screen.getByText("Target Trace: target-trace").closest("a"),
).toHaveClass("ph-no-capture");
-
expect(
screen.getByText("Session: customer-session").parentElement,
).toHaveClass("bg-primary");
diff --git a/web/src/features/traces/fns/resolveEvaluatorIdMetadata.clienttest.ts b/web/src/features/traces/fns/resolveEvaluatorIdMetadata.clienttest.ts
new file mode 100644
index 0000000000..ecc4e1de99
--- /dev/null
+++ b/web/src/features/traces/fns/resolveEvaluatorIdMetadata.clienttest.ts
@@ -0,0 +1,13 @@
+import { describe, expect, it } from "vitest";
+
+import { resolveEvaluatorIdMetadata } from "./resolveEvaluatorIdMetadata";
+
+describe("resolveEvaluatorIdMetadata", () => {
+ it("resolves evaluator ids from serialized observation metadata", () => {
+ expect(
+ resolveEvaluatorIdMetadata(
+ JSON.stringify({ evaluator_id: "evaluator-id" }),
+ ),
+ ).toBe("evaluator-id");
+ });
+});
diff --git a/web/src/features/traces/fns/resolveEvaluatorIdMetadata.ts b/web/src/features/traces/fns/resolveEvaluatorIdMetadata.ts
new file mode 100644
index 0000000000..6bb52a7988
--- /dev/null
+++ b/web/src/features/traces/fns/resolveEvaluatorIdMetadata.ts
@@ -0,0 +1,19 @@
+export function resolveEvaluatorIdMetadata(metadata: unknown): string | null {
+ let parsedMetadata = metadata;
+
+ if (typeof parsedMetadata === "string") {
+ try {
+ parsedMetadata = JSON.parse(parsedMetadata);
+ } catch {
+ return null;
+ }
+ }
+
+ if (typeof parsedMetadata !== "object" || parsedMetadata === null)
+ return null;
+
+ const evaluatorId = (parsedMetadata as Record<string, unknown>).evaluator_id;
+ return typeof evaluatorId === "string" && evaluatorId.length > 0
+ ? evaluatorId
+ : null;
+}