Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

6.9 KB
Raw Blame
/* eslint-disable no-nested-ternary */
import { useStore } from "zustand";
import type { ReactNode } from "react";

import { Card, CardContent } from "@/src/components/ui/card";
import { InfoTooltip } from "@/src/components/ui/InfoTooltip/InfoTooltip";
import { Skeleton } from "@/src/components/ui/skeleton";
import { EvaluatorAssignmentsEditor } from "@/src/features/evals/v2/components/Rules/EvaluatorAssignmentsEditor/EvaluatorAssignmentsEditor";
import type {
  RuleEvaluatorOption,
  RuleSetupStore,
} from "@/src/features/evals/v2/types/rules";
import {
  useRuleCostEstimate,
  type RuleCostEstimate,
} from "@/src/features/evals/v2/hooks/useRuleCostEstimate";

import { usdFormatter } from "@/src/utils/numbers";

export function EvaluatorMappingStep({
  projectId,
  store,
  evaluatorOptions,
  isQueryLoading,
  isQueryError,
  queryErrorMessage,
  search,
  onSearchChange,
  sampleObject,
  costObservationCount,
}: {
  projectId: string;
  store: RuleSetupStore;
  evaluatorOptions: RuleEvaluatorOption[];
  isQueryLoading: boolean;
  isQueryError: boolean;
  queryErrorMessage: string | undefined;
  search: string;
  onSearchChange: (value: string) => void;
  sampleObject: Record<string, unknown> | null;
  costObservationCount: number | null;
}) {
  if (costObservationCount == null) {
    return (
      <EvaluatorMappingEditor
        store={store}
        evaluatorOptions={evaluatorOptions}
        isQueryLoading={isQueryLoading}
        isQueryError={isQueryError}
        queryErrorMessage={queryErrorMessage}
        search={search}
        onSearchChange={onSearchChange}
        sampleObject={sampleObject}
        costEstimates={[]}
        estimatingEvaluatorIds={[]}
        footerTrailing={
          <p className="text-muted-foreground max-w-64 text-right text-xs">
            Cost estimate unavailable for experiment-scoped evaluations
          </p>
        }
        costError={false}
      />
    );
  }

  return (
    <EvaluatorMappingStepWithCosts
      projectId={projectId}
      store={store}
      evaluatorOptions={evaluatorOptions}
      isQueryLoading={isQueryLoading}
      isQueryError={isQueryError}
      queryErrorMessage={queryErrorMessage}
      search={search}
      onSearchChange={onSearchChange}
      sampleObject={sampleObject}
      costObservationCount={costObservationCount}
    />
  );
}

function EvaluatorMappingStepWithCosts({
  projectId,
  store,
  evaluatorOptions,
  isQueryLoading,
  isQueryError,
  queryErrorMessage,
  search,
  onSearchChange,
  sampleObject,
  costObservationCount,
}: {
  projectId: string;
  store: RuleSetupStore;
  evaluatorOptions: RuleEvaluatorOption[];
  isQueryLoading: boolean;
  isQueryError: boolean;
  queryErrorMessage: string | undefined;
  search: string;
  onSearchChange: (value: string) => void;
  sampleObject: Record<string, unknown> | null;
  costObservationCount: number;
}) {
  const assignments = useStore(store, (state) => state.assignments);
  const costEstimate = useRuleCostEstimate({
    projectId,
    store,
    matchingObservations: costObservationCount,
  });
  const availableEstimates = costEstimate.estimates.filter(
    (estimate) => estimate.estimatedCostUsd !== null,
  );
  const totalCostUsd = availableEstimates.reduce(
    (total, estimate) => total + (estimate.estimatedCostUsd ?? 0),
    0,
  );
  const footerTrailing =
    assignments.length > 0 ? (
      <div className="w-48 border-t pt-2 text-right text-sm">
        {availableEstimates.length > 0 ? (
          <div>
            <div className="flex items-center justify-end gap-1.5">
              <p className="font-mono leading-none font-bold whitespace-nowrap tabular-nums">
                ≈ {usdFormatter(totalCostUsd, 2, 2)}
              </p>
              <InfoTooltip label="About total estimated LLM costs">
                Sum of the available LLM cost estimates for this run.
              </InfoTooltip>
            </div>
            <p className="text-muted-foreground text-xs whitespace-nowrap">
              estimated LLM costs for this run
            </p>
          </div>
        ) : costEstimate.status === "estimating" ? (
          <Skeleton className="ml-auto h-5 w-28" />
        ) : (
          <div>
            <p className="font-mono font-bold">Unavailable</p>
            <p className="text-muted-foreground text-xs whitespace-nowrap">
              estimated LLM costs for this run
            </p>
          </div>
        )}
      </div>
    ) : null;

  return (
    <EvaluatorMappingEditor
      store={store}
      evaluatorOptions={evaluatorOptions}
      isQueryLoading={isQueryLoading}
      isQueryError={isQueryError}
      queryErrorMessage={queryErrorMessage}
      search={search}
      onSearchChange={onSearchChange}
      sampleObject={sampleObject}
      costEstimates={costEstimate.estimates}
      estimatingEvaluatorIds={costEstimate.estimatingEvaluatorIds}
      footerTrailing={footerTrailing}
      costError={costEstimate.status === "error" && assignments.length > 0}
    />
  );
}

function EvaluatorMappingEditor({
  store,
  evaluatorOptions,
  isQueryLoading,
  isQueryError,
  queryErrorMessage,
  search,
  onSearchChange,
  sampleObject,
  costEstimates,
  estimatingEvaluatorIds,
  footerTrailing,
  costError,
}: {
  store: RuleSetupStore;
  evaluatorOptions: RuleEvaluatorOption[];
  isQueryLoading: boolean;
  isQueryError: boolean;
  queryErrorMessage: string | undefined;
  search: string;
  onSearchChange: (value: string) => void;
  sampleObject: Record<string, unknown> | null;
  costEstimates: RuleCostEstimate[];
  estimatingEvaluatorIds: string[];
  footerTrailing: ReactNode;
  costError: boolean;
}) {
  return (
    <div className="flex h-full min-h-0 flex-col gap-3">
      <div className="min-h-0 flex-1 overflow-y-auto">
        {isQueryLoading ? (
          <p className="text-muted-foreground text-sm">Loading evaluators...</p>
        ) : isQueryError ? (
          <Card>
            <CardContent className="text-destructive p-4 text-sm">
              Failed to load evaluators: {queryErrorMessage}
            </CardContent>
          </Card>
        ) : (
          <>
            <EvaluatorAssignmentsEditor
              evaluatorOptions={evaluatorOptions}
              store={store}
              search={search}
              onSearchChange={onSearchChange}
              sampleObject={sampleObject}
              costEstimates={costEstimates}
              estimatingEvaluatorIds={estimatingEvaluatorIds}
              footerTrailing={footerTrailing}
              emptyActionLabel="Select evaluator"
              additionalActionLabel="Select another evaluator"
              emptyDescription="Select an evaluator to score this selection."
              sourceUnavailableMessage="No observation is available to validate JSON paths."
            />
            {costError ? (
              <p className="text-muted-foreground mt-2 text-xs">
                Cost estimates are temporarily unavailable.
              </p>
            ) : null}
          </>
        )}
      </div>
    </div>
  );
}