🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
import { api, type RouterOutputs } from "@/src/utils/api";
import { useMemo } from "react";
import { isEvalRuleExecutable, singleFilterList } from "@langfuse/shared";
import { getEvalTemplateFamilyKey } from "@/src/features/evals/utils/eval-template-family";
/**
* One entry per evaluator family that already has a job config for this
* dataset. A non-executable config (deactivated or blocked) renders as
* "paused"; an active config wins if a family has several configs.
*/
export const getExistingEvaluators = (
jobConfigs: RouterOutputs["evals"]["jobConfigsByTarget"] | undefined,
datasetId: string,
) => {
const result: Record<
string,
{
evalTemplateId: string;
targetObject: string;
templateName: string;
isActive: boolean;
}
> = {};
for (const jobConfig of jobConfigs ?? []) {
const parsedFilter = singleFilterList.safeParse(jobConfig.filter);
if (!parsedFilter.success) continue;
const matchesDataset =
parsedFilter.data.length === 0 ||
parsedFilter.data.some(
({ type, value }) =>
type === "stringOptions" && value.includes(datasetId),
);
if (!matchesDataset || !jobConfig.evalTemplate) continue;
const familyKey = getEvalTemplateFamilyKey(jobConfig.evalTemplate);
const isActive = isEvalRuleExecutable({
status: jobConfig.status,
blockedAt: jobConfig.blockedAt,
});
if (result[familyKey]?.isActive && !isActive) continue;
result[familyKey] = {
evalTemplateId: jobConfig.evalTemplate.id,
targetObject: jobConfig.targetObject,
templateName: jobConfig.evalTemplate.name,
isActive,
};
}
return result;
};
/**
* Owns the existing-evaluator state for the template selector: fetches the
* dataset's job configs, derives per-family selection state, and toggles
* existing configs. Selecting a family without a config is delegated to
* `onSelectEvaluator` (opens the create form).
*/
export function useExperimentEvaluatorSelection({
projectId,
datasetId,
onSelectEvaluator,
}: {
projectId: string;
datasetId: string;
onSelectEvaluator?: (templateId: string) => void;
}) {
const utils = api.useUtils();
const jobConfigs = api.evals.jobConfigsByTarget.useQuery(
{ projectId, targetObject: ["dataset", "experiment"] },
{ enabled: !!datasetId },
);
const existingEvaluators = useMemo(
() => getExistingEvaluators(jobConfigs.data, datasetId),
[jobConfigs.data, datasetId],
);
const updateStatus =
api.evals.updateAllDatasetEvalJobStatusByTemplateId.useMutation({
// Refreshes every consumer of the query, incl. the parent forms.
onSuccess: () => utils.evals.jobConfigsByTarget.invalidate(),
});
const isTemplateActive = (familyKey: string) =>
Boolean(existingEvaluators[familyKey]?.isActive);
const isTemplateInactive = (familyKey: string) => {
const evaluator = existingEvaluators[familyKey];
return Boolean(evaluator && !evaluator.isActive);
};
const handleRowClick = (templateId: string, familyKey: string) => {
const existingEvaluator = existingEvaluators[familyKey];
if (!existingEvaluator) {
onSelectEvaluator?.(templateId);
return;
}
updateStatus.mutate({
projectId,
evalTemplateId: existingEvaluator.evalTemplateId,
datasetId,
newStatus: existingEvaluator.isActive ? "INACTIVE" : "ACTIVE",
});
};
return {
// State
existingEvaluators,
// Action
handleRowClick,
// Status checks
isTemplateActive,
isTemplateInactive,
// Loading
isLoading: updateStatus.isPending,
};
}