🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
/* eslint-disable no-nested-ternary */
import { useMemo } from "react";
import {
type EvaluatorBlockReason,
type FilterState,
singleFilterList,
type OrderByState,
} from "@langfuse/shared";
import { api } from "@/src/utils/api";
import {
getLazyEvaluatorDisplayStatus,
useLazyEvaluatorExecutionCountsByIds,
} from "@/src/features/evals/hooks/useLazyEvaluatorExecutionCounts";
import { generateJobExecutionCounts } from "@/src/features/evals/utils/job-execution-utils";
import { requiresLegacyMigrationAction } from "@/src/features/evals/utils/typeHelpers";
import { RAGAS_TEMPLATE_PREFIX } from "@/src/features/evals/types";
export type EvaluatorDataRow = {
id: string;
status: string;
createdAt: Date;
updatedAt: Date;
maintainer: string;
rawStatus: string;
template?: {
id: string;
name: string;
version: number;
};
blockMessage?: string | null;
blockReason?: EvaluatorBlockReason | null;
scoreName: string;
target: string;
filter: FilterState;
result: {
level: string;
count: number;
symbol: string;
}[];
isCostLoading: boolean;
isResultLoading: boolean;
logs?: string;
actions?: string;
totalCost?: number | null;
isLegacy?: boolean;
};
type UseEvaluatorTableDataParams = {
projectId: string;
page: number;
limit: number;
filter: FilterState;
orderBy: OrderByState;
searchQuery: string | null;
};
export const useEvaluatorTableData = ({
projectId,
page,
limit,
filter,
orderBy,
searchQuery,
}: UseEvaluatorTableDataParams) => {
const evaluators = api.evals.allConfigs.useQuery({
page,
limit,
projectId,
filter,
orderBy,
searchQuery,
});
const evaluatorIds =
evaluators.data?.configs.map((config) => config.id) ?? [];
const costs = api.evals.costByEvaluatorIds.useQuery(
{
projectId,
evaluatorIds,
},
{
enabled: evaluators.isSuccess && evaluatorIds.length > 0,
meta: {
silentHttpCodes: [503],
},
},
);
const lazyExecutionCounts = useLazyEvaluatorExecutionCountsByIds({
projectId,
evaluatorIds,
enabled: evaluators.isSuccess,
});
const rows = useMemo(
() =>
(evaluators.data?.configs ?? []).map((jobConfig) => {
const executionCounts =
lazyExecutionCounts.jobExecutionCountsByEvaluatorId[jobConfig.id];
const costData = costs.data?.[jobConfig.id];
const status =
getLazyEvaluatorDisplayStatus({
evaluator: jobConfig,
jobExecutionCounts: executionCounts,
}) ?? jobConfig.displayStatus;
return {
id: jobConfig.id,
status,
rawStatus: jobConfig.status,
createdAt: jobConfig.createdAt,
updatedAt: jobConfig.updatedAt,
template: jobConfig.evalTemplate
? {
id: jobConfig.evalTemplate.id,
name: jobConfig.evalTemplate.name,
version: jobConfig.evalTemplate.version,
}
: undefined,
blockMessage: jobConfig.blockMessage,
blockReason: jobConfig.blockReason,
scoreName: jobConfig.scoreName,
target: jobConfig.targetObject,
filter: singleFilterList.parse(jobConfig.filter),
result: generateJobExecutionCounts(executionCounts),
isCostLoading: !costs.data,
isResultLoading:
lazyExecutionCounts.isLoading &&
!lazyExecutionCounts.jobExecutionCountsByEvaluatorId[jobConfig.id],
maintainer: jobConfig.evalTemplate
? jobConfig.evalTemplate.projectId
? "User maintained"
: jobConfig.evalTemplate.name.startsWith(RAGAS_TEMPLATE_PREFIX)
? "Langfuse and Ragas maintained"
: "Langfuse maintained"
: "Not available",
totalCost: costData,
isLegacy: requiresLegacyMigrationAction({
targetObject: jobConfig.targetObject,
status: jobConfig.status,
timeScope: jobConfig.timeScope,
}),
} satisfies EvaluatorDataRow;
}),
[
costs.data,
evaluators.data?.configs,
lazyExecutionCounts.isLoading,
lazyExecutionCounts.jobExecutionCountsByEvaluatorId,
],
);
return {
evaluators,
rows,
totalCount: evaluators.data?.totalCount ?? null,
// isLegacy already requires ACTIVE status and a NEW time scope
hasLegacyEvals: rows.some((row) => Boolean(row.isLegacy)),
};
};