Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

2.0 KB
Raw Blame
import { useStore } from "zustand";

import { applyFallbackDecisionModel } from "@/src/features/evals/v2/fns/evaluators/preferredDecisionModel";
import { prepareEvaluatorDraft } from "@/src/features/evals/v2/fns/evaluators/prepareEvaluatorDraft";
import { getScoreOutputValidation } from "@/src/features/evals/v2/fns/scoreOutput/getScoreOutputValidation";
import { useEvaluatorSetupSample } from "@/src/features/evals/v2/hooks/useEvaluatorSetupSample";
import { useFallbackDecisionModel } from "@/src/features/evals/v2/hooks/useFallbackDecisionModel";
import type { EvaluatorSetupStore } from "@/src/features/evals/v2/store/evaluatorSetupStore/evaluatorSetupStore";

export function useEvaluatorTestAvailability({
  projectId,
  store,
  hasValidModel,
}: {
  projectId: string;
  store: EvaluatorSetupStore;
  hasValidModel: boolean;
}) {
  const sampleObject = useEvaluatorSetupSample({ projectId, store });
  const selectedObservation = useStore(
    store,
    (state) => state.selectedObservation,
  );
  const evaluatorType = useStore(store, (state) => state.type);
  const fallbackDecisionModel = useFallbackDecisionModel(
    projectId,
    evaluatorType === "DECISION_MODEL",
  );
  const definitionAvailable = useStore(store, (state) =>
    Boolean(
      prepareEvaluatorDraft(
        applyFallbackDecisionModel(state, fallbackDecisionModel),
      ).definition,
    ),
  );
  const scoreOutputReason = useStore(store, (state) =>
    state.type === "LLM_AS_JUDGE"
      ? getScoreOutputValidation(state.scoreOutput).reason
      : null,
  );
  const modelReason = useStore(store, (state) =>
    state.type === "LLM_AS_JUDGE" && !hasValidModel
      ? "Select a model before running a test."
      : null,
  );

  if (scoreOutputReason) {
    return scoreOutputReason;
  }
  if (modelReason) {
    return modelReason;
  }
  if (!definitionAvailable) {
    return "Complete the evaluator before running a test.";
  }
  if (!selectedObservation) {
    return "Select a sample observation first.";
  }
  if (!sampleObject) {
    return "Loading the selected sample.";
  }
  return null;
}