Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

27.8 KB
Raw Blame
import { z } from "zod/v4";
import { randomUUID } from "crypto";
import { addDays } from "date-fns";
import {
  type ExperimentMetadata,
  createDatasetItemFilterState,
  ExperimentCreateQueue,
  getCategoricalScoresGroupedByName,
  getBooleanScoresGroupedByName,
  getDatasetItemsCount,
  countDatasetItemVariableMatches,
  getEventsGroupedByExperimentDatasetId,
  getExperimentsCountFromEvents,
  getExperimentsFromEvents,
  getExperimentItemsBatchIO,
  getExperimentItemsCountFromEvents,
  getExperimentItemsFromEvents,
  getExperimentMetricsFromEvents,
  getNumericScoresGroupedByName,
  getScoresForExperimentItems,
  getScoresForExperiments,
  PromptService,
  QueueJobs,
  QueueName,
  redis,
  ZodModelConfig,
  getScoresForObservations,
  getScoresForTraces,
  traceException,
  getExperimentNamesFromEvents,
  getExperimentItemsFilterOptions,
  getExperimentScoreOptions,
} from "@langfuse/shared/src/server";
import {
  createTRPCRouter,
  protectedProjectProcedure,
} from "@/src/server/api/trpc";
import {
  extractVariables,
  isBaseError,
  UnauthorizedError,
  PromptType,
  extractPlaceholderNames,
  type PromptMessage,
  singleFilterList,
  type FilterState,
  orderBy,
  paginationZod,
  timeFilter,
  AGGREGATABLE_SCORE_TYPES,
  filterAndValidateDbScoreList,
  hasPromptToolStructuredOutputConflict,
  InvalidRequestError,
  parsePromptToolConfig,
  PROMPT_TOOL_STRUCTURED_OUTPUT_CONFLICT_MESSAGE,
} from "@langfuse/shared";
import { throwIfNoProjectAccess } from "@/src/features/rbac/utils/checkProjectAccess";
import { aggregateScores } from "@/src/features/scores/lib/aggregateScores";
import { describeVariableMismatch } from "@/src/features/experiments/fns/describeVariableMismatch";

const ExperimentFilterOptions = z.object({
  projectId: z.string(),
  filter: singleFilterList.nullable(),
  orderBy: orderBy,
  ...paginationZod,
});

/**
 * Lookback for `mostRecent`: wide enough to cover a project whose last run is
 * months old, still bounded so the query prunes partitions instead of scanning
 * the whole project. Deliberately wider than any table date-range preset.
 */
const MOST_RECENT_LOOKBACK_DAYS = 365;

const ValidConfigResponse = z.object({
  isValid: z.literal(true),
  totalItems: z.number(),
  variablesMap: z.record(z.string(), z.number()),
});

const InvalidConfigResponse = z.object({
  isValid: z.literal(false),
  message: z.string(),
});

const ConfigResponse = z.discriminatedUnion("isValid", [
  ValidConfigResponse,
  InvalidConfigResponse,
]);

export const experimentsRouter = createTRPCRouter({
  validateConfig: protectedProjectProcedure
    .input(
      z.object({
        projectId: z.string(),
        datasetId: z.string(),
        promptId: z.string(),
        datasetVersion: z.coerce.date().optional(),
      }),
    )
    .output(ConfigResponse)
    .query(async ({ input, ctx }) => {
      throwIfNoProjectAccess({
        session: ctx.session,
        projectId: input.projectId,
        scope: "promptExperiments:CUD",
      });

      const prompt = await ctx.prisma.prompt.findFirst({
        where: {
          id: input.promptId,
          projectId: input.projectId,
        },
      });

      if (!prompt) {
        return {
          isValid: false,
          message: "Selected prompt not found.",
        };
      }

      const promptService = new PromptService(ctx.prisma, redis);
      let resolvedPrompt;
      try {
        resolvedPrompt = await promptService.resolvePrompt(prompt);
      } catch (error) {
        if (
          error instanceof SyntaxError ||
          (isBaseError(error) && error.isUserError())
        ) {
          return {
            isValid: false,
            message: isBaseError(error)
              ? error.message
              : "Selected prompt could not be resolved.",
          };
        }
        throw error;
      }

      if (!resolvedPrompt) {
        return {
          isValid: false,
          message: "Selected prompt not found.",
        };
      }

      const extractedVariables = extractVariables(
        resolvedPrompt.type === PromptType.Text
          ? (resolvedPrompt.prompt?.toString() ?? "")
          : JSON.stringify(resolvedPrompt.prompt ?? ""),
      );

      const promptMessages =
        resolvedPrompt?.type === PromptType.Chat &&
        Array.isArray(resolvedPrompt?.prompt)
          ? resolvedPrompt.prompt
          : [];
      const placeholderNames = extractPlaceholderNames(
        promptMessages as PromptMessage[],
      );

      const allVariables = [...extractedVariables, ...placeholderNames];

      if (!Boolean(allVariables.length)) {
        return {
          isValid: false,
          message: "Selected prompt has no variables or placeholders.",
        };
      }

      const filterState = createDatasetItemFilterState({
        datasetIds: [input.datasetId],
        status: "ACTIVE",
      });

      const totalItems = await getDatasetItemsCount({
        projectId: input.projectId,
        filterState,
        version: input.datasetVersion,
      });

      if (!Boolean(totalItems)) {
        return {
          isValid: false,
          message: "Selected dataset is empty or all items are inactive.",
        };
      }

      const variablesMap = await countDatasetItemVariableMatches({
        projectId: input.projectId,
        filterState,
        version: input.datasetVersion,
        variables: allVariables,
      });

      if (!Boolean(Object.keys(variablesMap).length)) {
        return {
          isValid: false,
          message: describeVariableMismatch(allVariables),
        };
      }

      return {
        isValid: true,
        totalItems,
        variablesMap,
      };
    }),

  createExperiment: protectedProjectProcedure
    .input(
      z.object({
        projectId: z.string(),
        name: z.string().min(1, "Please enter an experiment name"),
        runName: z.string().min(1, "Run name is required"),
        promptId: z.string().min(1, "Please select a prompt"),
        datasetId: z.string().min(1, "Please select a dataset"),
        datasetVersion: z.coerce.date().optional(),
        description: z.string().max(1000).optional(),
        modelConfig: z.object({
          provider: z.string().min(1, "Please select a provider"),
          model: z.string().min(1, "Please select a model"),
          modelParams: ZodModelConfig,
        }),
        structuredOutputSchema: z.record(z.string(), z.any()).optional(),
      }),
    )
    .mutation(async ({ input, ctx }) => {
      throwIfNoProjectAccess({
        session: ctx.session,
        projectId: input.projectId,
        scope: "promptExperiments:CUD",
      });

      if (input.structuredOutputSchema) {
        const prompt = await ctx.prisma.prompt.findUnique({
          where: { id: input.promptId, projectId: input.projectId },
          select: { config: true },
        });
        const toolConfig = parsePromptToolConfig(prompt?.config);
        if (hasPromptToolStructuredOutputConflict(toolConfig, true)) {
          throw new InvalidRequestError(
            PROMPT_TOOL_STRUCTURED_OUTPUT_CONFLICT_MESSAGE,
          );
        }
      }

      if (!redis) {
        throw new UnauthorizedError("Experiment creation failed");
      }

      const metadata: ExperimentMetadata = {
        prompt_id: input.promptId,
        provider: input.modelConfig.provider,
        model: input.modelConfig.model,
        model_params: input.modelConfig.modelParams,
        ...(input.structuredOutputSchema && {
          structured_output_schema: input.structuredOutputSchema,
        }),
        ...(input.datasetVersion && {
          dataset_version: input.datasetVersion,
        }),
      };

      const datasetRun = await ctx.prisma.datasetRuns.create({
        data: {
          name: input.runName,
          description: input.description,
          datasetId: input.datasetId,
          metadata: {
            ...metadata,
            experiment_name: input.name,
            experiment_run_name: input.runName,
          },
          projectId: input.projectId,
        },
      });

      const queue = ExperimentCreateQueue.getInstance();

      if (queue) {
        await queue.add(QueueName.ExperimentCreate, {
          name: QueueJobs.ExperimentCreateJob,
          id: randomUUID(),
          timestamp: new Date(),
          payload: {
            projectId: input.projectId,
            datasetId: input.datasetId,
            runId: datasetRun.id,
            description: input.description,
          },
          retryBaggage: {
            originalJobTimestamp: new Date(),
            attempt: 0,
          },
        });
      }

      return {
        success: true,
        datasetId: input.datasetId,
        runId: datasetRun.id,
        runName: input.runName,
      };
    }),
  all: protectedProjectProcedure
    .input(ExperimentFilterOptions)
    .query(async ({ input, ctx }) => {
      throwIfNoProjectAccess({
        session: ctx.session,
        projectId: input.projectId,
        scope: "promptExperiments:read",
      });

      const experiments = await getExperimentsFromEvents({
        projectId: input.projectId,
        filter: input.filter ?? [],
        orderBy: input.orderBy,
        page: input.page,
        limit: input.limit,
      });

      return {
        data: experiments,
      };
    }),

  /**
   * The most recent runs regardless of the selected time range, for the empty
   * window on the experiments list ("if there's nothing in the last X days, I
   * still want to see the last ones"). Same scoping and filters as `all`, only
   * the start-time bounds are replaced.
   */
  mostRecent: protectedProjectProcedure
    .input(
      z.object({
        projectId: z.string(),
        filter: singleFilterList.nullable(),
        limit: z.number().int().min(1).max(50),
      }),
    )
    .query(async ({ input, ctx }) => {
      throwIfNoProjectAccess({
        session: ctx.session,
        projectId: input.projectId,
        scope: "promptExperiments:read",
      });

      // Return the server-resolved window so widgets share it without adding
      // a render-time timestamp to the fallback query input.
      const to = new Date();
      const from = addDays(to, -MOST_RECENT_LOOKBACK_DAYS);
      const filter: FilterState = [
        // The selected window is the one that came back empty; every other
        // filter the user applied still holds.
        ...(input.filter ?? []).filter((f) => f.column !== "startTime"),
        {
          column: "startTime",
          type: "datetime",
          operator: ">=",
          value: from,
        },
      ];

      const experiments = await getExperimentsFromEvents({
        projectId: input.projectId,
        filter,
        orderBy: { column: "startTime", order: "DESC" },
        page: 0,
        limit: input.limit,
      });

      return { data: experiments, dateRange: { from, to } };
    }),

  byId: protectedProjectProcedure
    .input(
      z.object({
        projectId: z.string(),
        experimentId: z.string(),
      }),
    )
    .query(async ({ input, ctx }) => {
      throwIfNoProjectAccess({
        session: ctx.session,
        projectId: input.projectId,
        scope: "promptExperiments:read",
      });

      const experiments = await getExperimentsFromEvents({
        projectId: input.projectId,
        filter: [
          {
            type: "string",
            column: "id",
            operator: "=",
            value: input.experimentId,
          },
        ],
        orderBy: undefined,
        page: 0,
        limit: 1,
      });

      if (experiments.length === 0) {
        return null;
      }

      return experiments[0];
    }),

  countAll: protectedProjectProcedure
    .input(
      z.object({
        projectId: z.string(),
        filter: singleFilterList.nullable(),
      }),
    )
    .query(async ({ input, ctx }) => {
      throwIfNoProjectAccess({
        session: ctx.session,
        projectId: input.projectId,
        scope: "promptExperiments:read",
      });

      const count = await getExperimentsCountFromEvents({
        projectId: input.projectId,
        filter: input.filter ?? [],
      });

      return {
        count: count,
      };
    }),

  metrics: protectedProjectProcedure
    .input(
      z.object({
        projectId: z.string(),
        experimentIds: z.array(z.string()),
        filter: singleFilterList.nullable(),
      }),
    )
    .query(async ({ input, ctx }) => {
      throwIfNoProjectAccess({
        session: ctx.session,
        projectId: input.projectId,
        scope: "promptExperiments:read",
      });

      if (input.experimentIds.length === 0) {
        return [];
      }

      // Fetch metrics (cost, latency) and both score types in parallel
      const [metrics, itemScores, experimentScores] = await Promise.all([
        getExperimentMetricsFromEvents({
          projectId: input.projectId,
          experimentIds: input.experimentIds,
        }),
        getScoresForExperimentItems(input.projectId, input.experimentIds),
        getScoresForExperiments({
          projectId: input.projectId,
          runIds: input.experimentIds, // experiment_id === dataset_run_id
          excludeMetadata: true,
          includeHasMetadata: true,
        }),
      ]);

      return metrics.map((metric) => {
        // Filter item scores for this experiment
        const experimentItemScores = itemScores.filter(
          (s) => s.experimentId === metric.id,
        );

        return {
          id: metric.id,
          totalCost: metric.totalCost,
          latencyAvg: metric.latencyAvg,
          // Trace-level item scores (observation_id is null/empty)
          traceItemScores: aggregateScores(
            experimentItemScores.filter((s) => !s.observationId),
          ),
          // Observation-level item scores (observation_id is present)
          observationItemScores: aggregateScores(
            experimentItemScores.filter((s) => Boolean(s.observationId)),
          ),
          // Experiment-level scores (direct dataset_run_id match)
          experimentScores: aggregateScores(
            experimentScores.filter((s) => s.datasetRunId === metric.id),
          ),
        };
      });
    }),

  filterOptions: protectedProjectProcedure
    .input(
      z.object({
        projectId: z.string(),
        startTimeFilter: z.array(timeFilter).optional(),
      }),
    )
    .query(async ({ input, ctx }) => {
      throwIfNoProjectAccess({
        session: ctx.session,
        projectId: input.projectId,
        scope: "promptExperiments:read",
      });

      // Map startTimeFilter to Timestamp column for trace queries
      const traceTimestampFilters =
        input.startTimeFilter && input.startTimeFilter.length > 0
          ? input.startTimeFilter.map((f) => ({
              column: "Timestamp" as const,
              operator: f.operator,
              value: f.value,
              type: "datetime" as const,
            }))
          : [];

      // Offered-set == matchable-set: the level-agnostic score filters match a
      // score only if it rolls up into a trace, so discovery is scoped the same
      // way. Session- and dataset-run scores can never match, and offering them
      // would advertise a filter that cannot fire.
      const nullCondition = (
        column: "traceId" | "observationId",
        operator: "is null" | "is not null",
      ) => ({ type: "null" as const, column, operator, value: "" as const });
      const traceScopedFilter = [
        nullCondition("traceId", "is not null"),
        ...(traceTimestampFilters ?? []),
      ];
      // Discovered per level as well, to tag each offered name with the level(s)
      // it exists at rather than duplicating the facets per level.
      const observationScopedFilter = [
        ...traceScopedFilter,
        nullCondition("observationId", "is not null"),
      ];
      const traceLevelScopedFilter = [
        ...traceScopedFilter,
        nullCondition("observationId", "is null"),
      ];

      const [
        numericScoreNames,
        categoricalScoreNames,
        booleanScoreNames,
        observationLevelNumeric,
        observationLevelCategorical,
        observationLevelBoolean,
        traceLevelNumeric,
        traceLevelCategorical,
        traceLevelBoolean,
        experimentDatasetIds,
      ] = await Promise.all([
        getNumericScoresGroupedByName(input.projectId, traceScopedFilter),
        getCategoricalScoresGroupedByName(input.projectId, traceScopedFilter),
        getBooleanScoresGroupedByName(input.projectId, traceScopedFilter),
        getNumericScoresGroupedByName(input.projectId, observationScopedFilter),
        getCategoricalScoresGroupedByName(
          input.projectId,
          observationScopedFilter,
        ),
        getBooleanScoresGroupedByName(input.projectId, observationScopedFilter),
        getNumericScoresGroupedByName(input.projectId, traceLevelScopedFilter),
        getCategoricalScoresGroupedByName(
          input.projectId,
          traceLevelScopedFilter,
        ),
        getBooleanScoresGroupedByName(input.projectId, traceLevelScopedFilter),
        getEventsGroupedByExperimentDatasetId(
          input.projectId,
          input.startTimeFilter ?? [],
        ),
      ]);

      /**
       * Split per data type: a name can be reused across types at different
       * levels (a numeric observation-level "accuracy" beside an unrelated
       * categorical trace-level one), and a name-only map would mislabel both.
       */
      const levelsOf = (
        observationNames: string[],
        traceNames: string[],
      ): Record<string, ("observation" | "trace")[]> => {
        const out: Record<string, ("observation" | "trace")[]> = {};
        for (const name of observationNames) {
          if (out[name] === undefined) {
            out[name] = [];
          }
          out[name].push("observation");
        }
        for (const name of traceNames) {
          if (out[name] === undefined) {
            out[name] = [];
          }
          out[name].push("trace");
        }
        return out;
      };

      const experimentDatasetIdSet = new Set<string>();
      for (const { experimentDatasetId } of experimentDatasetIds) {
        if (experimentDatasetId !== null) {
          experimentDatasetIdSet.add(experimentDatasetId);
        }
      }

      // Return score options for both observation-level and trace-level filters
      // The same score names are available at both levels
      const numericScoreOptions = numericScoreNames.map((score) => score.name);
      const booleanScoreOptions = booleanScoreNames.map((score) => score.name);

      return {
        // What the three score facets offer, with the level(s) each name exists
        // at so the picker can tag them.
        scores_avg: numericScoreOptions,
        score_categories: categoricalScoreNames,
        score_booleans: booleanScoreOptions,
        score_name_levels_numeric: levelsOf(
          observationLevelNumeric.map((score) => score.name),
          traceLevelNumeric.map((score) => score.name),
        ),
        score_name_levels_categorical: levelsOf(
          observationLevelCategorical.map((score) => score.label),
          traceLevelCategorical.map((score) => score.label),
        ),
        score_name_levels_boolean: levelsOf(
          observationLevelBoolean.map((score) => score.name),
          traceLevelBoolean.map((score) => score.name),
        ),
        experimentDatasetIds: Array.from(experimentDatasetIdSet),
      };
    }),

  items: protectedProjectProcedure
    .input(
      z.object({
        projectId: z.string(),
        baseExperimentId: z.string().nullish(),
        compExperimentIds: z.array(z.string()),
        filterByExperiment: z
          .array(
            z.object({
              experimentId: z.string(),
              filters: singleFilterList,
            }),
          )
          .nullish(),
        orderBy: orderBy,
        itemVisibility: z
          .enum(["baseline-only", "all"])
          .optional()
          .default("baseline-only"),
        ...paginationZod,
      }),
    )
    .query(async ({ input, ctx }) => {
      throwIfNoProjectAccess({
        session: ctx.session,
        projectId: input.projectId,
        scope: "promptExperiments:read",
      });

      const items = await getExperimentItemsFromEvents({
        projectId: input.projectId,
        baseExperimentId: input.baseExperimentId ?? undefined,
        compExperimentIds: input.compExperimentIds,
        filterByExperiment: input.filterByExperiment ?? [],
        offset: input.page * input.limit,
        limit: input.limit,
        config: {
          requireBaselinePresence: input.itemVisibility === "baseline-only",
        },
      });

      const observationIds = Array.from(
        new Set(
          items.flatMap((item) => item.experiments.map((i) => i.observationId)),
        ),
      );
      const traceIds = Array.from(
        new Set(
          items.flatMap((item) => item.experiments.map((i) => i.traceId)),
        ),
      );

      const [observationScores, traceScores] = await Promise.all([
        getScoresForObservations({
          projectId: input.projectId,
          observationIds,
          excludeMetadata: true,
          includeHasMetadata: true,
          preferredClickhouseService: "ReadOnly",
        }),
        getScoresForTraces({
          projectId: input.projectId,
          traceIds,
          level: "trace",
          excludeMetadata: true,
          includeHasMetadata: true,
        }),
      ]);

      const validatedObservationScores = filterAndValidateDbScoreList({
        scores: observationScores,
        dataTypes: AGGREGATABLE_SCORE_TYPES,
        includeHasMetadata: true,
        onParseError: traceException,
      });
      const validatedTraceScores = filterAndValidateDbScoreList({
        scores: traceScores,
        dataTypes: AGGREGATABLE_SCORE_TYPES,
        includeHasMetadata: true,
        onParseError: traceException,
      });

      const scoresByObservationId = new Map<
        string,
        Array<(typeof validatedObservationScores)[number]>
      >();
      for (const score of validatedObservationScores) {
        if (!score.observationId) continue;
        const existingScores = scoresByObservationId.get(score.observationId);
        if (existingScores) {
          existingScores.push(score);
        } else {
          scoresByObservationId.set(score.observationId, [score]);
        }
      }

      const scoresByTraceId = new Map<
        string,
        Array<(typeof validatedTraceScores)[number]>
      >();
      for (const score of validatedTraceScores) {
        if (!score.traceId) continue;
        const existingScores = scoresByTraceId.get(score.traceId);
        if (existingScores) {
          existingScores.push(score);
        } else {
          scoresByTraceId.set(score.traceId, [score]);
        }
      }

      return {
        data: items.map(({ itemId, experiments }) => ({
          itemId,
          experiments: experiments.map(
            ({
              observationId,
              traceId,
              experimentId,
              level,
              startTime,
              totalCost,
              latencyMs,
            }) => ({
              observationId,
              traceId,
              experimentId,
              level,
              startTime,
              totalCost,
              latencyMs,
              observationScores: aggregateScores(
                scoresByObservationId.get(observationId) ?? [],
              ),
              traceScores: aggregateScores(scoresByTraceId.get(traceId) ?? []),
            }),
          ),
        })),
      };
    }),

  itemsCount: protectedProjectProcedure
    .input(
      z.object({
        projectId: z.string(),
        baseExperimentId: z.string().nullish(),
        compExperimentIds: z.array(z.string()),
        filterByExperiment: z
          .array(
            z.object({
              experimentId: z.string(),
              filters: singleFilterList,
            }),
          )
          .nullish(),
        itemVisibility: z
          .enum(["baseline-only", "all"])
          .optional()
          .default("baseline-only"),
      }),
    )
    .query(async ({ input, ctx }) => {
      throwIfNoProjectAccess({
        session: ctx.session,
        projectId: input.projectId,
        scope: "promptExperiments:read",
      });

      const count = await getExperimentItemsCountFromEvents({
        projectId: input.projectId,
        baseExperimentId: input.baseExperimentId ?? undefined,
        compExperimentIds: input.compExperimentIds,
        filterByExperiment: input.filterByExperiment ?? [],
        config: {
          requireBaselinePresence: input.itemVisibility === "baseline-only",
        },
      });

      return {
        count,
      };
    }),

  itemsFilterOptions: protectedProjectProcedure
    .input(
      z.object({
        projectId: z.string(),
        experimentIds: z.array(z.string()).min(1),
      }),
    )
    .query(async ({ input, ctx }) => {
      throwIfNoProjectAccess({
        session: ctx.session,
        projectId: input.projectId,
        scope: "promptExperiments:read",
      });

      return getExperimentItemsFilterOptions({
        projectId: input.projectId,
        experimentIds: input.experimentIds,
      });
    }),

  scoreOptions: protectedProjectProcedure
    .input(
      z.object({
        projectId: z.string(),
        experimentIds: z.array(z.string()).min(1),
      }),
    )
    .query(async ({ input, ctx }) => {
      throwIfNoProjectAccess({
        session: ctx.session,
        projectId: input.projectId,
        scope: "promptExperiments:read",
      });

      return getExperimentScoreOptions({
        projectId: input.projectId,
        experimentIds: input.experimentIds,
      });
    }),

  batchIO: protectedProjectProcedure
    .input(
      z.object({
        projectId: z.string(),
        itemIds: z.array(z.string()),
        baseExperimentId: z.string().nullish(),
        compExperimentIds: z.array(z.string()),
        ioCharLimit: z.number().int().positive().max(10_000).optional(),
      }),
    )
    .query(async ({ input, ctx }) => {
      throwIfNoProjectAccess({
        session: ctx.session,
        projectId: input.projectId,
        scope: "promptExperiments:read",
      });

      if (input.itemIds.length === 0) {
        return [];
      }

      const batchIO = await getExperimentItemsBatchIO({
        projectId: input.projectId,
        itemIds: input.itemIds,
        baseExperimentId: input.baseExperimentId ?? undefined,
        compExperimentIds: input.compExperimentIds,
        ioCharLimit: input.ioCharLimit,
      });

      return batchIO;
    }),

  byProjectId: protectedProjectProcedure
    .input(
      z.object({
        projectId: z.string(),
      }),
    )
    .query(async ({ input, ctx }) => {
      throwIfNoProjectAccess({
        session: ctx.session,
        projectId: input.projectId,
        scope: "promptExperiments:read",
      });

      const experiments = await getExperimentNamesFromEvents({
        projectId: input.projectId,
      });

      // Dataset names live in Postgres only — ClickHouse carries the id — so the
      // display name is resolved here rather than in the projection, and the
      // pickers never have to render a raw dataset id.
      const datasetIds = [
        ...new Set(
          experiments
            .map((experiment) => experiment.datasetId)
            .filter((datasetId): datasetId is string => Boolean(datasetId)),
        ),
      ];
      const datasets = datasetIds.length
        ? await ctx.prisma.dataset.findMany({
            where: { projectId: input.projectId, id: { in: datasetIds } },
            select: { id: true, name: true },
          })
        : [];
      const datasetNameById = new Map(datasets.map((d) => [d.id, d.name]));

      return {
        experimentNames: experiments.map((experiment) => ({
          experimentId: experiment.experimentId,
          experimentName: experiment.experimentName,
          startTime: experiment.startTime,
          datasetId: experiment.datasetId,
          datasetName: experiment.datasetId
            ? (datasetNameById.get(experiment.datasetId) ?? null)
            : null,
        })),
      };
    }),
});