🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
import {
EvalTargetObject,
EvalTemplateType,
InvalidRequestError,
LangfuseNotFoundError,
validateEvaluatorFiltersForTarget,
type FilterState,
} from "@langfuse/shared";
import { prisma } from "@langfuse/shared/src/db";
import {
getLatestEvaluatorRunCost,
getObservationsCountFromEventsTable,
getObservationsWithModelDataFromEventsTable,
logger,
} from "@langfuse/shared/src/server";
import { findEvaluatorsByIds } from "./evaluatorRepository";
import { toEvaluatorDefinition } from "./evaluatorService";
import { testEvaluator } from "./testEvaluator";
const SEVEN_DAYS_MS = 7 * 24 * 60 * 60 * 1_000;
const TEST_COST_RETRY_DELAYS_MS = [
0, 250, 500, 1_000, 2_000, 4_000, 8_000,
] as const;
export async function getActivationCostEstimates(params: {
orgId: string;
projectId: string;
evaluatorIds: string[];
filter: FilterState;
sampling: number;
knownTestRunCostUsd?: number;
missingCostMode?: "probe" | "wait" | "skip";
shouldReadFromObservationsTable: boolean;
timeRange?: {
from: Date;
to: Date;
};
}) {
const validation = validateEvaluatorFiltersForTarget({
targetObject: EvalTargetObject.EVENT,
filter: params.filter,
});
if (!validation.isValid) {
throw new InvalidRequestError(
validation.issues[0]?.message ?? "Evaluation rule filters are invalid.",
);
}
const evaluators = await findEvaluatorsByIds({
prisma,
projectId: params.projectId,
evaluatorIds: params.evaluatorIds,
});
if (evaluators.length !== new Set(params.evaluatorIds).size) {
throw new LangfuseNotFoundError("Evaluator not found");
}
const llmEvaluators = evaluators.filter(
(evaluator) => evaluator.type === EvalTemplateType.LLM_AS_JUDGE,
);
const since = params.timeRange?.from ?? new Date(Date.now() - SEVEN_DAYS_MS);
const filter: FilterState = [
...validation.validatedFilters,
{
column: "startTime",
type: "datetime",
operator: ">=",
value: since,
},
...(params.timeRange
? [
{
column: "startTime" as const,
type: "datetime" as const,
operator: "<=" as const,
value: params.timeRange.to,
},
]
: []),
];
const [matchingObservations, recentRunCosts] = await Promise.all([
getObservationsCountFromEventsTable({
projectId: params.projectId,
filter,
// Required by the shared query shape; the count aggregate ignores paging.
limit: 1,
offset: 0,
}),
Promise.all(
llmEvaluators.map(async (evaluator) =>
getLatestEvaluatorRunCost(params.projectId, evaluator.id),
),
),
]);
const costsByEvaluatorId = new Map<string, number | null>(
evaluators.map((evaluator) => [
evaluator.id,
evaluator.type === EvalTemplateType.LLM_AS_JUDGE ? null : 0,
]),
);
llmEvaluators.forEach((evaluator, index) =>
costsByEvaluatorId.set(evaluator.id, recentRunCosts[index] ?? null),
);
const knownCostEvaluatorId = params.evaluatorIds[0];
if (
knownCostEvaluatorId &&
costsByEvaluatorId.get(knownCostEvaluatorId) === null &&
params.knownTestRunCostUsd !== undefined
) {
costsByEvaluatorId.set(knownCostEvaluatorId, params.knownTestRunCostUsd);
}
const evaluatorsWithoutCost = llmEvaluators.filter(
(evaluator) => costsByEvaluatorId.get(evaluator.id) === null,
);
if (
params.missingCostMode === "wait" &&
matchingObservations > 0 &&
evaluatorsWithoutCost.length > 0
) {
const availableCosts = await Promise.all(
evaluatorsWithoutCost.map(async (evaluator) => ({
evaluatorId: evaluator.id,
cost: await waitForEvaluatorRunCost(params.projectId, evaluator.id),
})),
);
availableCosts.forEach(({ evaluatorId, cost }) =>
costsByEvaluatorId.set(evaluatorId, cost),
);
} else if (
params.missingCostMode !== "skip" &&
matchingObservations > 0 &&
evaluatorsWithoutCost.length > 0
) {
const sample = (
await getObservationsWithModelDataFromEventsTable({
projectId: params.projectId,
filter,
orderBy: { column: "startTime", order: "DESC" },
limit: 1,
offset: 0,
selectIOAndMetadata: false,
})
)[0];
if (sample?.traceId) {
const generatedCosts = await Promise.all(
evaluatorsWithoutCost.map(async (evaluator) => ({
evaluatorId: evaluator.id,
cost: await runTestAndWaitForCost({ params, evaluator, sample }),
})),
);
generatedCosts.forEach(({ evaluatorId, cost }) =>
costsByEvaluatorId.set(evaluatorId, cost),
);
}
}
return params.evaluatorIds.map((evaluatorId) => {
const testRunCostUsd = costsByEvaluatorId.get(evaluatorId) ?? null;
return {
evaluatorId,
matchingObservations,
sampling: params.sampling,
testRunCostUsd,
estimatedCostUsd:
testRunCostUsd === null
? null
: matchingObservations * params.sampling * testRunCostUsd,
};
});
}
async function runTestAndWaitForCost({
params,
evaluator,
sample,
}: {
params: Parameters<typeof getActivationCostEstimates>[0];
evaluator: Awaited<ReturnType<typeof findEvaluatorsByIds>>[number];
sample: Awaited<
ReturnType<typeof getObservationsWithModelDataFromEventsTable>
>[number];
}) {
const latestVersion = evaluator.versions[0];
if (!sample.traceId || !latestVersion) return null;
try {
const result = await testEvaluator({
orgId: params.orgId,
projectId: params.projectId,
evaluatorId: evaluator.id,
definition: toEvaluatorDefinition(evaluator.type, latestVersion),
observationId: sample.id,
traceId: sample.traceId,
startTime: sample.startTime,
shouldReadFromObservationsTable: params.shouldReadFromObservationsTable,
});
if (!("executionTraceId" in result)) return null;
if (
"estimatedCostUsd" in result &&
typeof result.estimatedCostUsd === "number"
) {
return result.estimatedCostUsd;
}
return waitForEvaluatorRunCost(params.projectId, evaluator.id);
} catch (error) {
logger.warn("Automatic evaluator test for cost estimation failed", {
projectId: params.projectId,
evaluatorId: evaluator.id,
error,
});
}
return null;
}
async function waitForEvaluatorRunCost(projectId: string, evaluatorId: string) {
for (const delayMs of TEST_COST_RETRY_DELAYS_MS) {
if (delayMs > 0) {
await new Promise((resolve) => setTimeout(resolve, delayMs));
}
const cost = await getLatestEvaluatorRunCost(projectId, evaluatorId);
if (cost !== null) return cost;
}
return null;
}