🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
import { Alert } from "@/src/components/design-system/Alert/Alert";
import { AlertTriangle } from "@langfuse/icons";
import { type EvalCapabilities } from "@/src/features/evals/hooks/useEvalCapabilities";
import {
isTraceTarget,
isEventTarget,
isExperimentTarget,
isDatasetTarget,
} from "@/src/features/evals/utils/typeHelpers";
interface EvalVersionCalloutProps {
content: CalloutContent;
}
interface CalloutContent {
title: string;
description: React.ReactNode;
}
export const getEvalVersionCalloutContent = (
targetObject: string,
evalCapabilities: EvalCapabilities,
) => {
// For event/observation target
if (isEventTarget(targetObject)) {
if (evalCapabilities.isNewCompatible) {
return null;
}
return {
title: "Please verify your SDK version",
description: (
<>
This evaluator targets observations, which require JS SDK v4+ or
Python SDK v3+. You can still configure this evaluator now—it will
start running once you upgrade.{" "}
<a
href="https://langfuse.com/docs/observability/sdk/upgrade-path"
target="_blank"
rel="noopener noreferrer"
className="text-dark-blue font-bold hover:opacity-80"
>
Learn more
</a>
.
</>
),
};
}
// For experiment target (Experiment Runner SDK)
if (isExperimentTarget(targetObject)) {
if (!evalCapabilities.isNewCompatible) {
return {
title: "Please verify you are using the Experiment Runner SDK",
description: (
<>
The Experiment Runner SDK requires JS SDK v4.4+ or Python SDK v3.9+.
You can still configure this evaluator now—it will start running
once you upgrade.{" "}
<a
href="https://langfuse.com/docs/evaluation/experiments/experiments-via-sdk#experiment-runner-sdk"
target="_blank"
rel="noopener noreferrer"
className="text-dark-blue font-bold hover:opacity-80"
>
Learn more about the Experiment Runner SDK.
</a>
.
</>
),
};
}
return null;
}
// For dataset target (legacy dataset run methods)
if (isDatasetTarget(targetObject)) {
// Forced-v3 projects keep trace evaluators as their intended experience —
// no upgrade nag.
if (evalCapabilities.forceV3Experience) {
return null;
}
return {
title: "Legacy low-level SDK methods",
description: (
<>
This evaluator targets traces from legacy low-level SDK methods for
dataset runs that manually linked dataset items to traces. Consider
upgrading to the Experiment Runner SDK for improved performance and
features.{" "}
<a
href="https://langfuse.com/docs/evaluation/experiments/experiments-via-sdk#experiment-runner-sdk"
target="_blank"
rel="noopener noreferrer"
className="text-dark-blue font-bold hover:opacity-80"
>
Learn more
</a>
.
</>
),
};
}
// For trace target
if (isTraceTarget(targetObject)) {
// Forced-v3 projects keep trace evaluators as their intended experience —
// no upgrade nag.
if (evalCapabilities.forceV3Experience) {
return null;
}
return {
title: "Consider upgrading to observation evaluators",
description: (
<>
Observation evaluators provide more granular control and an easier
workflow. We strongly recommend upgrading to observation evaluators.{" "}
<a
href="https://langfuse.com/faq/all/llm-as-a-judge-migration"
target="_blank"
rel="noopener noreferrer"
className="text-dark-blue font-bold hover:opacity-80"
>
Learn more
</a>
.
</>
),
};
}
return null;
};
export function EvalVersionCallout({ content }: EvalVersionCalloutProps) {
return (
<div className="mt-2 w-full max-w-4xl">
<Alert variant="warning" icon={AlertTriangle}>
<Alert.Description>
<div className="flex flex-col gap-2">
<div className="flex flex-col gap-1">
<span className="text-foreground font-bold">{content.title}</span>
<span className="text-foreground text-sm">
{content.description}
</span>
</div>
</div>
</Alert.Description>
</Alert>
</div>
);
}