🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
import { useState, useEffect } from "react";
import { api } from "@/src/utils/api";
import { useIsCodeEvalEnabled } from "@/src/features/evals/hooks/useIsCodeEvalEnabled";
import {
isCodeEvalTemplate,
shouldShowEvalTemplate,
} from "@/src/features/evals/utils/code-eval-template-utils";
export function useTemplatesValidation({
projectId,
selectedTemplateIds = [],
}: {
projectId: string;
selectedTemplateIds?: string[];
}) {
const [isSelectionValid, setIsSelectionValid] = useState(true);
const codeEvalCapabilities = useIsCodeEvalEnabled();
const { enabled: isCodeEvalEnabled, supportedSourceCodeLanguages } =
codeEvalCapabilities;
// Fetch default model
const { data: defaultModel, isLoading: isLoadingDefaultModel } =
api.defaultLlmModel.fetchDefaultModel.useQuery(
{ projectId },
{ enabled: !!projectId },
);
// Fetch all templates
const { data: templatesData } = api.evals.latestTemplates.useQuery(
{ projectId },
{ enabled: !!projectId },
);
useEffect(() => {
if (isLoadingDefaultModel) return;
// Find selected templates
const selectedTemplates = (templatesData?.templates || []).filter(
(template) => selectedTemplateIds.includes(template.id),
);
if (
selectedTemplates.some(
(template) =>
!shouldShowEvalTemplate(template, {
enabled: isCodeEvalEnabled,
supportedSourceCodeLanguages,
}),
)
) {
setIsSelectionValid(false);
return;
}
// If there's no default model, check if any of the selected templates requires one
if (!defaultModel) {
// Check if any LLM-as-a-judge template requires a default model (has no provider/model specified)
const requiresDefaultModel = selectedTemplates.some(
(template) =>
!isCodeEvalTemplate(template) &&
(!template.provider || !template.model),
);
setIsSelectionValid(!requiresDefaultModel);
} else {
// If there is a default model, selection is valid
setIsSelectionValid(true);
}
}, [
defaultModel,
isLoadingDefaultModel,
isCodeEvalEnabled,
supportedSourceCodeLanguages,
selectedTemplateIds,
templatesData?.templates,
]);
/**
* Check if a specific template is valid (has a default model if needed)
*/
const isTemplateValid = (templateId: string): boolean => {
if (!templatesData?.templates) return true;
// Find the template
const template = templatesData.templates.find((t) => t.id === templateId);
if (!template) return true;
if (isCodeEvalTemplate(template)) {
return shouldShowEvalTemplate(template, codeEvalCapabilities);
}
// If we have a default model, all LLM-as-a-judge templates are valid
if (defaultModel) return true;
// If template has no provider or model, it requires a default model
return Boolean(template.provider && template.model);
};
return {
isSelectionValid,
hasDefaultModel: !!defaultModel,
defaultModel,
isTemplateValid,
};
}