Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

78.3 KB
Raw Blame
/* eslint-disable no-nested-ternary */
import { z } from "zod";
import { omit } from "lodash";
import {
  createTRPCRouter,
  protectedProjectProcedure,
} from "@/src/server/api/trpc";
import { Prisma, type Dataset } from "@langfuse/shared/src/db";
import { env as sharedEnv } from "@langfuse/shared/src/env";
import { throwIfNoProjectAccess } from "@/src/features/rbac/utils/checkProjectAccess";
import { auditLog } from "@/src/features/audit-logs/auditLog";
import { createMediaUploadUrl } from "@/src/features/media/server/mediaService";
import {
  datasetItemMediaReferenceKey,
  resolveDatasetItemMediaReferences,
} from "@/src/features/media/server/datasetItemMediaReferences";
import { MediaContentType } from "@/src/features/media/validation";
import {
  paginationZod,
  singleFilterList,
  StringNoHTML,
  StringNoHTMLNonEmpty,
  type FilterState,
  isPresent,
  TracingSearchType,
  timeFilter,
  isClickhouseFilterColumn,
  optionalPaginationZod,
  LangfuseConflictError,
  LangfuseNotFoundError,
  InvalidRequestError,
  datasetItemMediaFields,
  DatasetNameSchema,
  BatchActionQuerySchema,
  ActionId,
  BatchActionType,
  BatchExportTableName,
  type BulkDatasetItemValidationError,
} from "@langfuse/shared";
import { env } from "@/src/env.mjs";
import { TRPCError } from "@trpc/server";
import {
  datasetRunsTableSchema,
  datasetRunTableMetricsSchema,
  enrichAndMapToDatasetItemId,
  fetchDatasetItems,
  getRunItemsByRunIdOrItemId,
} from "@/src/features/datasets/server/service";
import {
  logger,
  addToDeleteDatasetQueue,
  getDatasetRunItemsByDatasetIdCh,
  getDatasetRunItemsCountByDatasetIdCh,
  getDatasetRunsTableMetricsCh,
  getScoresForExperiments,
  getTraceScoresForDatasetRuns,
  getDatasetRunItemsCountCh,
  getNumericScoresGroupedByName,
  getBooleanScoresGroupedByName,
  getCategoricalScoresGroupedByName,
  getDatasetRunsTableRowsCh,
  getDatasetRunsTableCountCh,
  validateWebhookURL,
  getDatasetRunItemsWithoutIOByItemIds,
  getDatasetItemsWithRunDataCount,
  getDatasetItemIdsWithRunData,
  createDatasetItem,
  upsertDatasetItem,
  deleteDatasetItem,
  createManyDatasetItems,
  linkDatasetItemMedia,
  validateDatasetItemMediaReferences,
  markDatasetMediaUploadComplete,
  validateAllDatasetItems,
  DatasetJSONSchema,
  type DatasetMutationResult,
  getDatasetItemById,
  getDatasetItems,
  getDatasetItemsCount,
  createDatasetItemFilterState,
  executeWithDatasetServiceStrategy,
  OperationType,
  Implementation,
  listDatasetVersions,
  getDatasetItemVersionHistory,
  getDatasetItemChangesSinceVersion,
  getDatasetItemsCountGrouped,
  getDatasetExperimentMetricsFromEvents,
  getDatasetVersionForRun,
  escapeSqlLikePattern,
  fetchWithSecureRedirects,
  whitelistFromEnv,
  WEBHOOK_URL_VALIDATION_LOG_CONTEXT,
  deleteDatasetsByIds,
  findDatasetsForDeletion,
} from "@langfuse/shared/src/server";
import { aggregateScores } from "@/src/features/scores/lib/aggregateScores";
import {
  updateDataset,
  upsertDataset,
} from "@/src/features/datasets/server/actions/createDataset";
import {
  buildRemoteExperimentRequest,
  ensureRemoteExperimentSecret,
  getRemoteExperimentConfig,
  getRemoteExperimentConfigWithSecrets,
  parseStoredRemoteExperimentHeaders,
  processRemoteExperimentHeaders,
  RemoteExperimentHeadersSchema,
} from "@/src/features/datasets/server/remoteExperimentHelpers";
import { v4 } from "uuid";
import { createBatchActionJob } from "@/src/features/table/server/createBatchActionJob";

// Batch size kept small (100) as items may have large input/output/metadata JSON
const DUPLICATE_DATASET_ITEMS_BATCH_SIZE = 100;
const REMOTE_EXPERIMENT_TIMEOUT_MS = 20_000;
const REMOTE_EXPERIMENT_MAX_REDIRECTS = 10;

const getAzureStorageVersionFromUploadUrl = (uploadUrl?: string | null) => {
  if (!uploadUrl) return null;

  try {
    return new URL(uploadUrl).searchParams.get("sv");
  } catch {
    return null;
  }
};

export const getItemMediaUploadHeaders = ({
  sha256Hash,
  uploadUrl,
  useAzureBlob = sharedEnv.LANGFUSE_USE_AZURE_BLOB,
  useGoogleCloudStorage = sharedEnv.LANGFUSE_USE_GOOGLE_CLOUD_STORAGE,
  useOciNativeObjectStorage = sharedEnv.LANGFUSE_USE_OCI_NATIVE_OBJECT_STORAGE,
}: {
  sha256Hash: string;
  uploadUrl?: string | null;
  useAzureBlob?: "true" | "false";
  useGoogleCloudStorage?: "true" | "false";
  useOciNativeObjectStorage?: "true" | "false";
}) => {
  if (
    useGoogleCloudStorage === "true" ||
    useOciNativeObjectStorage === "true"
  ) {
    return {};
  }

  if (useAzureBlob === "true") {
    const azureStorageVersion = getAzureStorageVersionFromUploadUrl(uploadUrl);

    return {
      "x-amz-checksum-sha256": sha256Hash,
      "x-ms-blob-type": "BlockBlob",
      ...(azureStorageVersion ? { "x-ms-version": azureStorageVersion } : {}),
    };
  }

  return {
    "x-amz-checksum-sha256": sha256Hash,
  };
};

/**
 * Adds a case-insensitive search condition to a query
 * @param searchQuery The search term (optional)
 * @returns The search condition
 */
const resolveSearchCondition = (searchQuery?: string | null) => {
  if (!searchQuery || searchQuery.trim() === "") return Prisma.empty;

  // Add case-insensitive search condition
  return Prisma.sql`AND d.name ILIKE ${`%${escapeSqlLikePattern(searchQuery)}%`}`;
};

const buildPathPrefixFilter = (pathPrefix?: string): Prisma.Sql => {
  if (!pathPrefix) {
    return Prisma.empty;
  }

  const escapedPathPrefix = escapeSqlLikePattern(pathPrefix);
  return Prisma.sql` AND (d.name LIKE ${`${escapedPathPrefix}/%`} ESCAPE '\\' OR d.name = ${pathPrefix})`;
};

/**
 * Determines whether the given filters require Dataset Run Items (DRI) metrics from ClickHouse.
 *
 * @param filters - Array of filter conditions to evaluate
 * @returns true if any filter requires DRI metrics, false if using basic dataset run data is sufficient
 */
const requiresClickhouseLookups = (filters: FilterState): boolean => {
  if (filters.length === 0) {
    return false;
  }

  return filters.some((filter) => {
    return isClickhouseFilterColumn(filter.column);
  });
};

const resolveMetadata = (metadata: string | null | undefined) => {
  if (metadata === "") return Prisma.DbNull;
  try {
    return !!metadata
      ? (JSON.parse(metadata) as Prisma.InputJsonObject)
      : undefined;
  } catch (e) {
    logger.info(
      "[trpc.datasets.resolveMetadata] failed to parse dataset metadata",
      e,
    );
    return undefined;
  }
};

type GenerateDatasetQueryInput = {
  select: Prisma.Sql;
  projectId: string;
  pathFilter: Prisma.Sql;
  searchFilter: Prisma.Sql;
  orderCondition?: Prisma.Sql;
  limit?: number;
  page?: number;
  pathPrefix?: string;
};

const generateDatasetQuery = ({
  select,
  projectId,
  pathFilter,
  orderCondition = Prisma.empty,
  searchFilter = Prisma.empty,
  pathPrefix = "",
  limit = 1,
  page = 0,
}: GenerateDatasetQueryInput) => {
  // CTE to get datasets for given project (same for root and folder queries)
  const datasetsCTE = Prisma.sql`
  filtered_datasets AS (
   SELECT
     d.id,
     d.name,
     d.description,
     d.metadata,
     d.project_id,
     d.updated_at,
     d.created_at,
     d.input_schema,
     d.expected_output_schema
   FROM datasets d
   WHERE d.project_id = ${projectId}
     ${pathFilter}
     ${searchFilter}
  )`;

  // Common ORDER BY and LIMIT clauses
  const orderAndLimit = Prisma.sql`
   ${orderCondition.sql ? Prisma.sql`ORDER BY d.sort_priority, ${Prisma.raw(orderCondition.sql.replace(/ORDER BY /i, ""))}` : Prisma.empty}
   LIMIT ${limit} OFFSET ${page * limit}`;

  if (pathPrefix) {
    // When we're inside a folder, show individual datasets within that folder
    // and folder representatives for subfolders

    return Prisma.sql`
    WITH ${datasetsCTE},
    individual_datasets_in_folder AS (
      /* Individual datasets exactly at this folder level (no deeper slashes) */
      SELECT
        d.id,
        SUBSTRING(d.name, CHAR_LENGTH(${pathPrefix}) + 2) as name, -- Remove prefix, show relative name
        d.description,
        d.metadata,
        d.project_id,
        d.updated_at,
        d.created_at,
        d.input_schema,
        d.expected_output_schema,
        2 as sort_priority, -- Individual datasets second
        'dataset'::text as row_type  -- Mark as individual dataset
      FROM filtered_datasets d
      WHERE SUBSTRING(d.name, CHAR_LENGTH(${pathPrefix}) + 2) NOT LIKE '%/%'
        AND SUBSTRING(d.name, CHAR_LENGTH(${pathPrefix}) + 2) != ''  -- Exclude datasets that match prefix exactly
        AND d.name != ${pathPrefix}  -- Additional safety check
    ),
    subfolder_representatives AS (
      /* Folder representatives for deeper nested datasets */
      SELECT
        d.id,
        SPLIT_PART(SUBSTRING(d.name, CHAR_LENGTH(${pathPrefix}) + 2), '/', 1) as name, -- First segment after prefix
        d.description,
        d.metadata,
        d.project_id,
        d.updated_at,
        d.created_at,
        d.input_schema,
        d.expected_output_schema,
        1 as sort_priority, -- Folders first
        'folder'::text as row_type, -- Mark as folder representative
        ROW_NUMBER() OVER (
          PARTITION BY SPLIT_PART(SUBSTRING(d.name, CHAR_LENGTH(${pathPrefix}) + 2), '/', 1)
          ORDER BY LENGTH(d.name) - LENGTH(REPLACE(d.name, '/', '')) ASC, d.created_at DESC
        ) AS rn
      FROM filtered_datasets d
      WHERE SUBSTRING(d.name, CHAR_LENGTH(${pathPrefix}) + 2) LIKE '%/%'
    ),
    combined AS (
      SELECT
        id, name, description, metadata, project_id, updated_at, created_at, sort_priority, row_type, input_schema, expected_output_schema
      FROM individual_datasets_in_folder
      UNION ALL
      SELECT
        id, name, description, metadata, project_id, updated_at, created_at, sort_priority, row_type, input_schema, expected_output_schema
      FROM subfolder_representatives WHERE rn = 1
    )
    SELECT
      ${select}
    FROM combined d
    ${orderAndLimit}
    `;
  }
  const baseColumns = Prisma.sql`id, name, description, metadata, project_id, updated_at, created_at, input_schema, expected_output_schema`;

  // When we're at the root level, show all individual datasets that don't have folders
  // and one representative per folder for datasets that do have folders
  return Prisma.sql`
    WITH ${datasetsCTE},
    individual_datasets AS (
      /* Individual datasets without folders */
      SELECT d.id, d.name, d.description, d.metadata, d.project_id, d.updated_at, d.created_at, d.input_schema, d.expected_output_schema, 'dataset'::text as row_type
      FROM filtered_datasets d
      WHERE d.name NOT LIKE '%/%'
    ),
    folder_representatives AS (
      /* One representative per folder - return folder name, not full dataset name */
      SELECT
        d.id,
        SPLIT_PART(d.name, '/', 1) as name,  -- Return folder segment name instead of full name
        d.description,
        d.metadata,
        d.project_id,
        d.updated_at,
        d.created_at,
        d.input_schema,
        d.expected_output_schema,
        'folder'::text as row_type, -- Mark as folder representative
        ROW_NUMBER() OVER (PARTITION BY SPLIT_PART(d.name, '/', 1) ORDER BY LENGTH(d.name) ASC, d.updated_at DESC) AS rn
      FROM filtered_datasets d
      WHERE d.name LIKE '%/%'
    ),
    combined AS (
      SELECT ${baseColumns}, row_type, 1 as sort_priority  -- Folders first
      FROM folder_representatives WHERE rn = 1
      UNION ALL
      SELECT ${baseColumns}, row_type, 2 as sort_priority  -- Individual datasets second
      FROM individual_datasets
    )
    SELECT
      ${select}
    FROM combined d
    ${orderAndLimit}
    `;
};

export const datasetRouter = createTRPCRouter({
  hasAny: protectedProjectProcedure
    .input(
      z.object({
        projectId: z.string(),
      }),
    )
    .query(async ({ input, ctx }) => {
      throwIfNoProjectAccess({
        session: ctx.session,
        projectId: input.projectId,
        scope: "datasets:read",
      });

      const dataset = await ctx.prisma.dataset.findFirst({
        where: {
          projectId: input.projectId,
        },
        select: { id: true },
        take: 1,
      });

      return dataset !== null;
    }),
  allDatasetMeta: protectedProjectProcedure
    .input(z.object({ projectId: z.string() }))
    .query(async ({ input, ctx }) => {
      throwIfNoProjectAccess({
        session: ctx.session,
        projectId: input.projectId,
        scope: "datasets:read",
      });

      return ctx.prisma.dataset.findMany({
        where: {
          projectId: input.projectId,
        },
        select: {
          id: true,
          name: true,
          inputSchema: true,
          expectedOutputSchema: true,
        },
      });
    }),
  allNamesAndIds: protectedProjectProcedure
    .input(
      z.object({
        projectId: z.string(),
      }),
    )
    .query(async ({ input, ctx }) => {
      throwIfNoProjectAccess({
        session: ctx.session,
        projectId: input.projectId,
        scope: "datasets:read",
      });

      return ctx.prisma.dataset.findMany({
        where: { projectId: input.projectId },
        select: { id: true, name: true },
        orderBy: { updatedAt: "desc" },
      });
    }),
  allDatasets: protectedProjectProcedure
    .input(
      z.object({
        projectId: z.string(),
        searchQuery: z.string().nullable(),
        pathPrefix: z.string().optional(),
        ...paginationZod,
      }),
    )
    .query(async ({ input, ctx }) => {
      throwIfNoProjectAccess({
        session: ctx.session,
        projectId: input.projectId,
        scope: "datasets:read",
      });

      // pathFilter: SQL WHERE clause to filter datasets by folder (e.g., "AND d.name LIKE 'folder/%'")
      const pathFilter = buildPathPrefixFilter(input.pathPrefix);

      const searchFilter = resolveSearchCondition(input.searchQuery);

      // Query for dataset and count
      const [datasets, datasetCount] = await Promise.all([
        // datasets
        ctx.prisma.$queryRaw<
          Array<
            Omit<
              Dataset,
              | "remoteExperimentUrl"
              | "remoteExperimentPayload"
              | "remoteExperimentEnabled"
            > & {
              row_type: "folder" | "dataset";
            }
          >
        >(
          generateDatasetQuery({
            select: Prisma.sql`
            d.id,
            d.name,
            d.description,
            d.project_id as "projectId",
            d.created_at as "createdAt",
            d.updated_at as "updatedAt",
            d.metadata,
            d.input_schema as "inputSchema",
            d.expected_output_schema as "expectedOutputSchema",
            d.row_type`,
            projectId: input.projectId,
            limit: input.limit,
            page: input.page,
            pathFilter, // SQL WHERE clause: filters DB to only datasets in current folder, derived from prefix.
            pathPrefix: input.pathPrefix, // Raw folder path: used for segment splitting & folder detection logic
            searchFilter,
            orderCondition: Prisma.sql`ORDER BY d.created_at DESC`,
          }),
        ),
        // datasetCount
        ctx.prisma.$queryRaw<Array<{ totalCount: bigint }>>(
          generateDatasetQuery({
            select: Prisma.sql`count(*) AS "totalCount"`,
            searchFilter,
            projectId: input.projectId,
            pathFilter,
            pathPrefix: input.pathPrefix,
          }),
        ),
      ]);

      return {
        datasets,
        totalDatasets:
          datasetCount.length > 0 ? Number(datasetCount[0]?.totalCount) : 0,
      };
    }),
  allDatasetsMetrics: protectedProjectProcedure
    .input(z.object({ projectId: z.string(), datasetIds: z.array(z.string()) }))
    .query(async ({ input, ctx }) => {
      throwIfNoProjectAccess({
        session: ctx.session,
        projectId: input.projectId,
        scope: "datasets:read",
      });

      if (input.datasetIds.length === 0) return { metrics: [] };

      const [runsMetrics, itemsCounts] = await Promise.all([
        ctx.session.user.v4BetaEnabled === true
          ? getDatasetExperimentMetricsFromEvents({
              projectId: input.projectId,
              datasetIds: input.datasetIds,
            })
          : ctx.prisma.$queryRaw<
              Array<{
                datasetId: string;
                countDatasetRuns: bigint;
                lastRunAt: Date | null;
              }>
            >`
                SELECT d.id AS "datasetId", COUNT(DISTINCT dr.id) AS "countDatasetRuns", MAX(dr.created_at) AS "lastRunAt"
                FROM datasets d
                LEFT JOIN dataset_runs dr ON d.id = dr.dataset_id AND dr.project_id = ${input.projectId}
                WHERE d.project_id = ${input.projectId}
                AND d.id IN (${Prisma.join(input.datasetIds)})
                GROUP BY d.id
              `.then((rows) =>
              rows.map((row) => ({
                ...row,
                countDatasetRuns: Number(row.countDatasetRuns),
              })),
            ),
        getDatasetItemsCountGrouped({
          projectId: input.projectId,
          datasetIds: input.datasetIds,
        }),
      ]);

      // Merge the metrics
      const metrics = input.datasetIds.map((datasetId) => {
        const runsMetric = runsMetrics.find((m) => m.datasetId === datasetId);
        const itemsCount = itemsCounts.find((m) => m.datasetId === datasetId);

        return {
          id: datasetId,
          countDatasetItems: itemsCount?.count ?? 0,
          countDatasetRuns: runsMetric?.countDatasetRuns ?? 0,
          lastRunAt: runsMetric?.lastRunAt ?? null,
        };
      });

      return { metrics };
    }),
  // counts all dataset run items that match the filter
  countAllDatasetItems: protectedProjectProcedure
    .input(
      z.object({
        projectId: z.string(), // Required for protectedProjectProcedure
        filter: singleFilterList.nullable(),
      }),
    )
    .query(async ({ input, ctx }) => {
      throwIfNoProjectAccess({
        session: ctx.session,
        projectId: input.projectId,
        scope: "datasets:read",
      });

      const count = await getDatasetRunItemsCountCh({
        projectId: input.projectId,
        filter: input.filter ?? [],
      });
      return { totalCount: count };
    }),
  byId: protectedProjectProcedure
    .input(
      z.object({
        projectId: z.string(),
        datasetId: z.string(),
      }),
    )
    .query(async ({ input, ctx }) => {
      throwIfNoProjectAccess({
        session: ctx.session,
        projectId: input.projectId,
        scope: "datasets:read",
      });

      return ctx.prisma.dataset.findUnique({
        where: {
          id_projectId: {
            id: input.datasetId,
            projectId: input.projectId,
          },
        },
        select: {
          id: true,
          projectId: true,
          name: true,
          description: true,
          metadata: true,
          inputSchema: true,
          expectedOutputSchema: true,
          createdAt: true,
          updatedAt: true,
        },
      });
    }),
  runById: protectedProjectProcedure
    .input(
      z.object({
        projectId: z.string(),
        datasetId: z.string(),
        runId: z.string(),
      }),
    )
    .query(async ({ input, ctx }) => {
      throwIfNoProjectAccess({
        session: ctx.session,
        projectId: input.projectId,
        scope: "datasets:read",
      });

      const run = await ctx.prisma.datasetRuns.findUnique({
        where: {
          id_projectId: {
            id: input.runId,
            projectId: input.projectId,
          },
          datasetId: input.datasetId,
        },
      });

      if (!run) return null;

      // Resolve dataset version from run items
      const datasetVersion = await getDatasetVersionForRun({
        projectId: input.projectId,
        datasetId: input.datasetId,
        runId: input.runId,
      });

      return {
        ...run,
        datasetVersion,
      };
    }),
  baseRunDataByDatasetId: protectedProjectProcedure
    .input(z.object({ projectId: z.string(), datasetId: z.string() }))
    .query(async ({ input, ctx }) => {
      throwIfNoProjectAccess({
        session: ctx.session,
        projectId: input.projectId,
        scope: "datasets:read",
      });

      return ctx.prisma.datasetRuns.findMany({
        where: { datasetId: input.datasetId, projectId: input.projectId },
        select: {
          name: true,
          id: true,
          metadata: true,
          description: true,
          createdAt: true,
        },
      });
    }),
  runsByDatasetId: protectedProjectProcedure
    .input(datasetRunsTableSchema)
    .query(async ({ input, ctx }) => {
      throwIfNoProjectAccess({
        session: ctx.session,
        projectId: input.projectId,
        scope: "datasets:read",
      });

      // Use helper function to determine if we need DRI metrics
      if (!requiresClickhouseLookups(input.filter ?? [])) {
        const [runs, totalRuns] = await Promise.all([
          await ctx.prisma.datasetRuns.findMany({
            where: {
              datasetId: input.datasetId,
              projectId: input.projectId,
            },
            orderBy: {
              createdAt: "desc",
            },
            take: input.limit,
            skip:
              isPresent(input.page) && isPresent(input.limit)
                ? input.page * input.limit
                : undefined,
          }),
          // dataset run items will continue to be stored in postgres
          await ctx.prisma.datasetRuns.count({
            where: {
              datasetId: input.datasetId,
              projectId: input.projectId,
            },
          }),
        ]);

        return {
          totalRuns,
          runs,
        };
      }
      const [runs, totalRuns] = await Promise.all([
        getDatasetRunsTableRowsCh({
          projectId: input.projectId,
          datasetId: input.datasetId,
          filter: input.filter ?? [],
          limit: isPresent(input.limit) ? input.limit : undefined,
          offset:
            isPresent(input.page) && isPresent(input.limit)
              ? input.page * input.limit
              : undefined,
        }),
        getDatasetRunsTableCountCh({
          projectId: input.projectId,
          datasetId: input.datasetId,
          filter: input.filter ?? [],
        }),
      ]);

      return {
        totalRuns,
        runs,
      };
    }),

  runsByDatasetIdMetrics: protectedProjectProcedure
    .input(datasetRunTableMetricsSchema)
    .query(async ({ input, ctx }) => {
      throwIfNoProjectAccess({
        session: ctx.session,
        projectId: input.projectId,
        scope: "datasets:read",
      });

      // Get runs that have metrics (only runs with dataset_run_items_rmt)
      const runsWithMetrics = await getDatasetRunsTableMetricsCh({
        projectId: input.projectId,
        datasetId: input.datasetId,
        runIds: input.runIds ?? [],
        filter: input.filter ?? [],
      });

      // Only fetch scores for runs that have metrics (runs without dataset_run_items_rmt won't have trace scores)
      const runsWithMetricsIds = runsWithMetrics.map((run) => run.id);
      const [traceScores, runScores] = await Promise.all([
        runsWithMetricsIds.length > 0
          ? getTraceScoresForDatasetRuns(input.projectId, runsWithMetricsIds)
          : [],
        getScoresForExperiments({
          projectId: input.projectId,
          runIds: runsWithMetrics.map((run) => run.id),
          includeHasMetadata: true,
          excludeMetadata: true,
        }),
      ]);

      // Merge all runs: use metrics where available, defaults otherwise
      const allRuns = runsWithMetrics.map((run) => {
        return {
          id: run.id,
          name: run.name,
          // Use ClickHouse metrics if available, otherwise use defaults for runs without dataset_run_items_rmt
          countRunItems: run.countRunItems ?? 0,
          avgTotalCost: run.avgTotalCost ?? null,
          totalCost: run.totalCost ?? null,
          avgLatency: run.avgLatency ?? null,
          scores: aggregateScores(
            traceScores.filter((s) => s.datasetRunId === run.id),
          ),
          runScores: aggregateScores(
            runScores.filter((s) => s.datasetRunId === run.id),
          ),
        };
      });

      return {
        runs: allRuns,
      };
    }),

  runFilterOptions: protectedProjectProcedure
    .input(
      z.object({
        projectId: z.string(),
        datasetId: z.string(),
        timestampFilter: timeFilter.optional(),
      }),
    )
    .query(async ({ input, ctx }) => {
      throwIfNoProjectAccess({
        session: ctx.session,
        projectId: input.projectId,
        scope: "datasets:read",
      });

      const { timestampFilter } = input;

      const [numericScoreNames, categoricalScoreNames, booleanScoreNames] =
        await Promise.all([
          getNumericScoresGroupedByName(
            input.projectId,
            timestampFilter ? [timestampFilter] : [],
          ),
          getCategoricalScoresGroupedByName(
            input.projectId,
            timestampFilter ? [timestampFilter] : [],
          ),
          getBooleanScoresGroupedByName(
            input.projectId,
            timestampFilter ? [timestampFilter] : [],
          ),
        ]);

      return {
        agg_scores_avg: numericScoreNames.map((s) => s.name),
        agg_score_categories: categoricalScoreNames,
        agg_score_booleans: booleanScoreNames.map((s) => s.name),
      };
    }),

  // TODO LFE-6512: only return score options present on the given dataset run
  runItemFilterOptions: protectedProjectProcedure
    .input(
      z.object({
        projectId: z.string(),
        datasetId: z.string(),
        datasetRunId: z.string().optional(),
        // TODO: make required
        datasetRunIds: z.array(z.string()).optional(),
        timestampFilter: timeFilter.optional(),
      }),
    )
    .query(async ({ input, ctx }) => {
      throwIfNoProjectAccess({
        session: ctx.session,
        projectId: input.projectId,
        scope: "datasets:read",
      });

      const { projectId, timestampFilter } = input;

      const [numericScoreNames, categoricalScoreNames, booleanScoreNames] =
        await Promise.all([
          getNumericScoresGroupedByName(
            projectId,
            timestampFilter ? [timestampFilter] : [],
          ),
          getCategoricalScoresGroupedByName(
            projectId,
            timestampFilter ? [timestampFilter] : [],
          ),
          getBooleanScoresGroupedByName(
            projectId,
            timestampFilter ? [timestampFilter] : [],
          ),
        ]);

      return {
        agg_scores_avg: numericScoreNames.map((s) => s.name),
        agg_score_categories: categoricalScoreNames,
        agg_score_booleans: booleanScoreNames.map((s) => s.name),
      };
    }),

  itemById: protectedProjectProcedure
    .input(
      z.object({
        projectId: z.string(),
        datasetId: z.string(),
        datasetItemId: z.string(),
        version: z.date().optional(),
      }),
    )
    .query(async ({ input, ctx }) => {
      throwIfNoProjectAccess({
        session: ctx.session,
        projectId: input.projectId,
        scope: "datasets:read",
      });

      const item = await getDatasetItemById({
        projectId: input.projectId,
        datasetItemId: input.datasetItemId,
        datasetId: input.datasetId,
        version: input.version,
      });
      if (!item) {
        throw new LangfuseNotFoundError("Dataset item not found");
      }
      return item;
    }),
  itemByIdAtVersion: protectedProjectProcedure
    .input(
      z.object({
        projectId: z.string(),
        datasetId: z.string(),
        datasetItemId: z.string(),
        version: z.date().optional(),
      }),
    )
    .query(async ({ input, ctx }) => {
      throwIfNoProjectAccess({
        session: ctx.session,
        projectId: input.projectId,
        scope: "datasets:read",
      });

      const item = await getDatasetItemById({
        projectId: input.projectId,
        datasetItemId: input.datasetItemId,
        datasetId: input.datasetId,
        version: input.version ?? new Date(),
      });
      // Return null if item doesn't exist at this version (not created yet or deleted)
      return item;
    }),
  // Issue a presigned upload URL for media attached to a dataset item, declaring
  // a pending association that is claimed when the item is written.
  getItemMediaUploadUrl: protectedProjectProcedure
    .input(
      z.object({
        projectId: z.string(),
        // Declares a pending association for this (item, field, media); the
        // item need not exist yet. The dataset is verified to belong to the
        // project in createMediaUploadUrl.
        datasetId: z.string(),
        datasetItemId: z.string(),
        field: z.enum(datasetItemMediaFields),
        contentType: z.enum(MediaContentType),
        contentLength: z.number().positive().int(),
        sha256Hash: z
          .string()
          .regex(
            /^[A-Za-z0-9+/=]{44}$/,
            "Must be a 44 character base64 encoded SHA-256 hash",
          ),
      }),
    )
    .mutation(async ({ input, ctx }) => {
      throwIfNoProjectAccess({
        session: ctx.session,
        projectId: input.projectId,
        scope: "datasets:CUD",
      });

      // Mirror the public media route: this is the only server-side size gate
      // before a direct-to-storage upload URL is signed, so enforce the media
      // size limit here too.
      if (input.contentLength > env.LANGFUSE_S3_MEDIA_MAX_CONTENT_LENGTH)
        throw new InvalidRequestError(
          `File size must be less than ${env.LANGFUSE_S3_MEDIA_MAX_CONTENT_LENGTH} bytes`,
        );

      const result = await createMediaUploadUrl({
        projectId: input.projectId,
        body: {
          contentType: input.contentType,
          contentLength: input.contentLength,
          sha256Hash: input.sha256Hash,
          datasetId: input.datasetId,
          datasetItemId: input.datasetItemId,
          field: input.field,
        },
      });

      return {
        ...result,
        uploadHeaders: getItemMediaUploadHeaders({
          sha256Hash: input.sha256Hash,
          uploadUrl: result.uploadUrl,
        }),
      };
    }),
  markItemMediaUploadComplete: protectedProjectProcedure
    .input(
      z.object({
        projectId: z.string(),
        mediaId: z.string(),
        uploadedAt: z.coerce.date(),
        uploadHttpStatus: z.number().positive().int(),
        uploadHttpError: z.string().nullish(),
        uploadTimeMs: z.number().nullish(),
      }),
    )
    .mutation(async ({ input, ctx }) => {
      throwIfNoProjectAccess({
        session: ctx.session,
        projectId: input.projectId,
        scope: "datasets:CUD",
      });

      await markDatasetMediaUploadComplete({
        projectId: input.projectId,
        mediaId: input.mediaId,
        uploadedAt: input.uploadedAt,
        uploadHttpStatus: input.uploadHttpStatus,
        uploadHttpError: input.uploadHttpError,
        uploadTimeMs: input.uploadTimeMs,
      });

      return { success: true as const };
    }),
  // Resolve a saved dataset item version's media from dataset_item_media.
  // The create/edit forms instead derive media from the live JSON.
  itemMediaByItemId: protectedProjectProcedure
    .input(
      z.object({
        projectId: z.string(),
        datasetItemId: z.string(),
        // The exact version to resolve media for. Media is linked per item
        // version (keyed by validFrom), so the historical-version view passes
        // the viewed version's validFrom to see that version's attachments.
        // Omitted for the latest view, where the current version is resolved.
        datasetItemValidFrom: z.date().optional(),
      }),
    )
    .query(async ({ input, ctx }) => {
      throwIfNoProjectAccess({
        session: ctx.session,
        projectId: input.projectId,
        scope: "datasets:read",
      });

      let validFrom = input.datasetItemValidFrom;
      if (!validFrom) {
        const item = await getDatasetItemById({
          projectId: input.projectId,
          datasetItemId: input.datasetItemId,
        });
        if (!item) return [];
        validFrom = item.validFrom;
      }

      const item = { id: input.datasetItemId, validFrom };
      const referencesByItem = await resolveDatasetItemMediaReferences({
        projectId: input.projectId,
        items: [item],
      });

      return referencesByItem.get(datasetItemMediaReferenceKey(item)) ?? [];
    }),
  countItemsByDatasetId: protectedProjectProcedure
    .input(z.object({ projectId: z.string(), datasetId: z.string() }))
    .query(async ({ input, ctx }) => {
      throwIfNoProjectAccess({
        session: ctx.session,
        projectId: input.projectId,
        scope: "datasets:read",
      });

      return await getDatasetItemsCount({
        projectId: input.projectId,
        filterState: createDatasetItemFilterState({
          datasetIds: [input.datasetId],
        }),
      });
    }),
  listDatasetVersions: protectedProjectProcedure
    .input(z.object({ projectId: z.string(), datasetId: z.string() }))
    .query(async ({ input, ctx }) => {
      throwIfNoProjectAccess({
        session: ctx.session,
        projectId: input.projectId,
        scope: "datasets:read",
      });

      return await listDatasetVersions({
        projectId: input.projectId,
        datasetId: input.datasetId,
      });
    }),
  itemVersionHistory: protectedProjectProcedure
    .input(
      z.object({
        projectId: z.string(),
        datasetId: z.string(),
        itemId: z.string(),
      }),
    )
    .query(async ({ input, ctx }) => {
      throwIfNoProjectAccess({
        session: ctx.session,
        projectId: input.projectId,
        scope: "datasets:read",
      });

      return await getDatasetItemVersionHistory({
        projectId: input.projectId,
        datasetId: input.datasetId,
        itemId: input.itemId,
      });
    }),
  countChangesSinceVersion: protectedProjectProcedure
    .input(
      z.object({
        projectId: z.string(),
        datasetId: z.string(),
        version: z.date(),
      }),
    )
    .query(async ({ input, ctx }) => {
      throwIfNoProjectAccess({
        session: ctx.session,
        projectId: input.projectId,
        scope: "datasets:read",
      });

      return await getDatasetItemChangesSinceVersion({
        projectId: input.projectId,
        datasetId: input.datasetId,
        sinceVersion: input.version,
      });
    }),
  itemsByDatasetId: protectedProjectProcedure
    .input(
      z.object({
        projectId: z.string(),
        datasetId: z.string(),
        filter: singleFilterList.nullish(),
        searchQuery: z.string().optional(),
        searchType: z.array(TracingSearchType).optional(),
        version: z.date().optional(),
        ...paginationZod,
      }),
    )
    .query(async ({ input, ctx }) => {
      throwIfNoProjectAccess({
        session: ctx.session,
        projectId: input.projectId,
        scope: "datasets:read",
      });

      return await fetchDatasetItems({
        projectId: input.projectId,
        datasetId: input.datasetId,
        filter: input.filter ?? [],
        version: input.version,
        limit: input.limit,
        page: input.page,
        prisma: ctx.prisma,
        searchQuery: input.searchQuery,
        searchType: input.searchType,
      });
    }),

  updateDatasetItem: protectedProjectProcedure
    .input(
      z.object({
        projectId: z.string(),
        datasetId: z.string(),
        datasetItemId: z.string(),
        input: z.string().optional(),
        expectedOutput: z.string().optional(),
        metadata: z.string().optional(),
        sourceTraceId: z.string().optional(),
        sourceObservationId: z.string().optional(),
        status: z.enum(["ACTIVE", "ARCHIVED"]).optional(),
      }),
    )
    .mutation(async ({ input, ctx }) => {
      throwIfNoProjectAccess({
        session: ctx.session,
        projectId: input.projectId,
        scope: "datasets:CUD",
      });

      const datasetItem = await upsertDatasetItem({
        projectId: input.projectId,
        datasetId: input.datasetId,
        datasetItemId: input.datasetItemId,
        input: input.input,
        expectedOutput: input.expectedOutput,
        metadata: input.metadata,
        sourceTraceId: input.sourceTraceId,
        sourceObservationId: input.sourceObservationId,
        status: input.status,
        validateOpts: { normalizeUndefinedToNull: false }, // For UPDATE, undefined means "don't update"
      });

      await auditLog({
        session: ctx.session,
        resourceType: "datasetItem",
        resourceId: input.datasetItemId,
        action: "update",
        after: omit(datasetItem, ["input", "expectedOutput", "metadata"]),
      });

      return datasetItem;
    }),
  createDataset: protectedProjectProcedure
    .input(
      z.object({
        projectId: z.string(),
        name: StringNoHTMLNonEmpty,
        description: StringNoHTML.nullish(),
        metadata: z.string().nullish(),
        inputSchema: DatasetJSONSchema.nullish(),
        expectedOutputSchema: DatasetJSONSchema.nullish(),
      }),
    )
    .mutation(async ({ input, ctx }): Promise<DatasetMutationResult> => {
      throwIfNoProjectAccess({
        session: ctx.session,
        projectId: input.projectId,
        scope: "datasets:CUD",
      });

      try {
        const dataset = await upsertDataset({
          input: {
            name: input.name,
            description: input.description ?? undefined,
            metadata: resolveMetadata(input.metadata),
            inputSchema: input.inputSchema,
            expectedOutputSchema: input.expectedOutputSchema,
          },
          projectId: input.projectId,
        });

        await auditLog({
          session: ctx.session,
          resourceType: "dataset",
          resourceId: dataset.id,
          action: "create",
          after: dataset,
        });

        return { success: true, dataset };
      } catch (error) {
        // Check if this is a validation error from upsertDataset
        if (
          error instanceof Error &&
          error.message.includes("Schema validation failed")
        ) {
          // Parse validation errors from message
          const match = error.message.match(/Details: (\[.*\])$/);
          if (match) {
            try {
              const validationErrors = JSON.parse(match[1]);
              return { success: false, validationErrors };
            } catch (_e) {
              // Failed to parse, rethrow original error
              throw error;
            }
          }
        }
        // Re-throw non-validation errors
        throw error;
      }
    }),
  updateDataset: protectedProjectProcedure
    .input(
      z.object({
        projectId: z.string(),
        datasetId: z.string(),
        name: StringNoHTMLNonEmpty.nullish(),
        description: StringNoHTML.nullish(),
        metadata: z.string().nullish(),
        inputSchema: DatasetJSONSchema.nullish(),
        expectedOutputSchema: DatasetJSONSchema.nullish(),
      }),
    )
    .mutation(async ({ input, ctx }): Promise<DatasetMutationResult> => {
      throwIfNoProjectAccess({
        session: ctx.session,
        projectId: input.projectId,
        scope: "datasets:CUD",
      });

      // If schemas are being updated, validate all existing items
      // Fast validation (10K items in <1s) means we can always validate
      if (
        input.inputSchema !== undefined ||
        input.expectedOutputSchema !== undefined
      ) {
        const existingDataset = await ctx.prisma.dataset.findUnique({
          where: {
            id_projectId: {
              id: input.datasetId,
              projectId: input.projectId,
            },
          },
          select: { inputSchema: true, expectedOutputSchema: true },
        });

        if (existingDataset) {
          // Determine the final schemas after update
          const finalInputSchema =
            input.inputSchema !== undefined
              ? input.inputSchema
              : existingDataset.inputSchema;
          const finalExpectedOutputSchema =
            input.expectedOutputSchema !== undefined
              ? input.expectedOutputSchema
              : existingDataset.expectedOutputSchema;

          // Validate all items if at least one schema is being set (not null)
          if (finalInputSchema !== null || finalExpectedOutputSchema !== null) {
            const validationResult = await validateAllDatasetItems({
              datasetId: input.datasetId,
              projectId: input.projectId,
              inputSchema: finalInputSchema as Record<string, unknown> | null,
              expectedOutputSchema: finalExpectedOutputSchema as Record<
                string,
                unknown
              > | null,
            });

            if (!validationResult.isValid) {
              // Return validation errors instead of throwing
              return {
                success: false,
                validationErrors: validationResult.errors,
              };
            }
          }
        }
      }

      const dataset = await updateDataset({
        input: {
          id: input.datasetId,
          name: input.name ?? undefined,
          description: input.description ?? undefined,
          metadata: resolveMetadata(input.metadata),
          inputSchema: input.inputSchema,
          expectedOutputSchema: input.expectedOutputSchema,
        },
        projectId: input.projectId,
      });

      await auditLog({
        session: ctx.session,
        resourceType: "dataset",
        resourceId: dataset.id,
        action: "update",
        after: dataset,
      });

      return { success: true, dataset };
    }),
  deleteDataset: protectedProjectProcedure
    .input(z.object({ projectId: z.string(), datasetId: z.string() }))
    .mutation(async ({ input, ctx }) => {
      throwIfNoProjectAccess({
        session: ctx.session,
        projectId: input.projectId,
        scope: "datasets:CUD",
      });

      try {
        const deletedDataset = await ctx.prisma.dataset.delete({
          where: {
            id_projectId: {
              id: input.datasetId,
              projectId: input.projectId,
            },
          },
        });

        await addToDeleteDatasetQueue({
          deletionType: "dataset",
          projectId: input.projectId,
          datasetId: deletedDataset.id,
        });

        await auditLog({
          session: ctx.session,
          resourceType: "dataset",
          resourceId: deletedDataset.id,
          action: "delete",
          before: deletedDataset,
        });

        return deletedDataset;
      } catch (error) {
        if (
          error instanceof Prisma.PrismaClientKnownRequestError &&
          error.code === "P2025"
        ) {
          throw new LangfuseConflictError(
            "The dataset you are trying to delete has likely been deleted",
          );
        }
        throw error;
      }
    }),
  deleteMany: protectedProjectProcedure
    .input(
      z.object({
        projectId: z.string(),
        datasetIds: z.array(z.string()).default([]),
        folderPaths: z.array(DatasetNameSchema).default([]),
        query: BatchActionQuerySchema.optional(),
        isBatchAction: z.boolean().default(false),
      }),
    )
    .mutation(async ({ input, ctx }) => {
      throwIfNoProjectAccess({
        session: ctx.session,
        projectId: input.projectId,
        scope: "datasets:CUD",
      });

      if (input.isBatchAction && input.query) {
        await createBatchActionJob({
          projectId: input.projectId,
          actionId: ActionId.DatasetDelete,
          actionType: BatchActionType.Delete,
          tableName: BatchExportTableName.Datasets,
          session: ctx.session,
          query: input.query,
        });

        return { deletedCount: null };
      }

      if (input.datasetIds.length === 0 && input.folderPaths.length === 0) {
        throw new TRPCError({
          code: "BAD_REQUEST",
          message:
            "Either datasetIds, folderPaths, or a batch action query must be provided to delete datasets.",
        });
      }

      const deletedDatasets = await ctx.prisma.$transaction(async (tx) => {
        const datasetsToDelete = await findDatasetsForDeletion({
          client: tx,
          projectId: input.projectId,
          datasetIds: input.datasetIds,
          folderPaths: input.folderPaths,
        });

        if (datasetsToDelete.length === 0) return [];

        await deleteDatasetsByIds({
          client: tx,
          projectId: input.projectId,
          datasetIds: datasetsToDelete.map((dataset) => dataset.id),
        });

        return datasetsToDelete;
      });

      await Promise.all(
        deletedDatasets.map((dataset) =>
          addToDeleteDatasetQueue({
            deletionType: "dataset",
            projectId: input.projectId,
            datasetId: dataset.id,
          }),
        ),
      );

      await Promise.all(
        deletedDatasets.map((dataset) =>
          auditLog({
            session: ctx.session,
            resourceType: "dataset",
            resourceId: dataset.id,
            action: "delete",
            before: dataset,
          }),
        ),
      );

      return {
        deletedCount: deletedDatasets.length,
      };
    }),

  deleteDatasetItem: protectedProjectProcedure
    .input(
      z.object({
        projectId: z.string(),
        datasetId: z.string(),
        datasetItemId: z.string(),
      }),
    )
    .mutation(async ({ input, ctx }) => {
      throwIfNoProjectAccess({
        session: ctx.session,
        projectId: input.projectId,
        scope: "datasets:CUD",
      });

      const result = await deleteDatasetItem({
        projectId: input.projectId,
        datasetId: input.datasetId,
        datasetItemId: input.datasetItemId,
      });

      await auditLog({
        session: ctx.session,
        resourceType: "datasetItem",
        resourceId: result.deletedItem.id,
        action: "delete",
        before: omit(result.deletedItem, [
          "input",
          "expectedOutput",
          "metadata",
        ]),
      });

      return result.deletedItem;
    }),
  duplicateDataset: protectedProjectProcedure
    .input(
      z.object({
        projectId: z.string(),
        datasetId: z.string(),
      }),
    )
    .mutation(async ({ input, ctx }) => {
      throwIfNoProjectAccess({
        session: ctx.session,
        projectId: input.projectId,
        scope: "datasets:CUD",
      });
      const dataset = await ctx.prisma.dataset.findUnique({
        where: {
          id_projectId: {
            id: input.datasetId,
            projectId: input.projectId,
          },
        },
      });
      if (!dataset) {
        throw new TRPCError({
          code: "NOT_FOUND",
          message: "Dataset not found",
        });
      }

      // find a unique name for the new dataset
      // by appending a counter to the name in case of the name already exists
      // e.g. "dataset (copy)" -> "dataset (copy 2)"
      const existingDatasetNames = (
        await ctx.prisma.dataset.findMany({
          select: {
            name: true,
          },
          where: {
            projectId: input.projectId,
            name: {
              startsWith: escapeSqlLikePattern(dataset.name) + " (copy",
            },
          },
        })
      ).map((d) => d.name);
      let counter = 0;
      const duplicateDatasetName = (pCounter: number) =>
        pCounter === 0
          ? `${dataset.name} (copy)`
          : `${dataset.name} (copy ${counter})`;
      while (true) {
        if (!existingDatasetNames.includes(duplicateDatasetName(counter))) {
          break;
        }
        counter++;
      }

      const newDataset = await upsertDataset({
        input: {
          name: duplicateDatasetName(counter),
          description: dataset.description ?? undefined,
          metadata: dataset.metadata ?? undefined,
          inputSchema: dataset.inputSchema,
          expectedOutputSchema: dataset.expectedOutputSchema,
        },
        projectId: input.projectId,
      });

      // Copy items in batches to avoid 256MB JSONB limit
      let page = 0;
      const validFrom = new Date();

      while (true) {
        const itemsBatch = await getDatasetItems({
          projectId: input.projectId,
          filterState: createDatasetItemFilterState({
            datasetIds: [input.datasetId],
          }),
          limit: DUPLICATE_DATASET_ITEMS_BATCH_SIZE,
          page,
        });

        if (itemsBatch.length === 0) break;

        const preparedItems = itemsBatch.map((item) => ({
          id: v4(),
          input: item.input ?? undefined,
          expectedOutput: item.expectedOutput ?? undefined,
          metadata: item.metadata ?? undefined,
          sourceTraceId: item.sourceTraceId,
          sourceObservationId: item.sourceObservationId,
          status: item.status,
          projectId: input.projectId,
          datasetId: newDataset.id,
          validFrom: validFrom,
        }));

        const mediaItems = preparedItems.map((item) => ({
          datasetId: item.datasetId,
          datasetItemId: item.id,
          datasetItemValidFrom: validFrom,
          input: item.input,
          expectedOutput: item.expectedOutput,
          metadata: item.metadata,
        }));

        await validateDatasetItemMediaReferences({
          projectId: input.projectId,
          items: mediaItems,
        });

        await executeWithDatasetServiceStrategy(OperationType.WRITE, {
          [Implementation.STATEFUL]: async () => {
            await ctx.prisma.$transaction(async (tx) => {
              await tx.datasetItem.createMany({
                data: preparedItems,
              });
              await linkDatasetItemMedia(tx, {
                projectId: input.projectId,
                items: mediaItems,
                replaceExisting: false,
              });
            });
          },
          [Implementation.VERSIONED]: async () => {
            // always creates new dataset; hence no need to invalidate old rows
            await ctx.prisma.$transaction(async (tx) => {
              await tx.datasetItem.createMany({
                data: preparedItems,
              });
              await linkDatasetItemMedia(tx, {
                projectId: input.projectId,
                items: mediaItems,
                replaceExisting: false,
              });
            });
          },
        });

        if (itemsBatch.length < DUPLICATE_DATASET_ITEMS_BATCH_SIZE) break; // Last batch
        page++;
      }

      await auditLog({
        session: ctx.session,
        resourceType: "dataset",
        resourceId: newDataset.id,
        action: "create",
        after: newDataset,
      });

      return { id: newDataset.id };
    }),

  createDatasetItem: protectedProjectProcedure
    .input(
      z.object({
        projectId: z.string(),
        datasetId: z.string(),
        input: z.string().nullish(),
        expectedOutput: z.string().nullish(),
        metadata: z.string().nullish(),
        sourceTraceId: z.string().optional(),
        sourceObservationId: z.string().optional(),
      }),
    )
    .mutation(async ({ input, ctx }) => {
      throwIfNoProjectAccess({
        session: ctx.session,
        projectId: input.projectId,
        scope: "datasets:CUD",
      });

      const result = await createDatasetItem({
        projectId: input.projectId,
        datasetId: input.datasetId,
        input: input.input,
        expectedOutput: input.expectedOutput,
        metadata: input.metadata,
        sourceTraceId: input.sourceTraceId,
        sourceObservationId: input.sourceObservationId,
        normalizeOpts: { sanitizeControlChars: true },
        validateOpts: {
          normalizeUndefinedToNull: true, // For CREATE, undefined becomes null in DB
        },
      });

      if (!result.success) {
        throw new TRPCError({
          code: "BAD_REQUEST",
          message: result.message,
          cause: result.cause,
        });
      }

      await auditLog({
        session: ctx.session,
        resourceType: "datasetItem",
        resourceId: result.datasetItem.id,
        action: "create",
        after: omit(result.datasetItem, [
          "input",
          "expectedOutput",
          "metadata",
        ]),
      });

      return result.datasetItem;
    }),

  createManyDatasetItems: protectedProjectProcedure
    .input(
      z.object({
        projectId: z.string(),
        items: z.array(
          z.object({
            // Optional client-generated id so the UI can declare media uploads
            // against the item before it exists; claimed on write.
            id: z.string().optional(),
            datasetId: z.string(),
            input: z.string().nullish(),
            expectedOutput: z.string().nullish(),
            metadata: z.string().nullish(),
            sourceTraceId: z.string().optional(),
            sourceObservationId: z.string().optional(),
          }),
        ),
      }),
    )
    .mutation(
      async ({
        input,
        ctx,
      }): Promise<
        | { success: true }
        | {
            success: false;
            validationErrors: BulkDatasetItemValidationError[];
          }
      > => {
        throwIfNoProjectAccess({
          session: ctx.session,
          projectId: input.projectId,
          scope: "datasets:CUD",
        });

        const result = await createManyDatasetItems({
          projectId: input.projectId,
          items: input.items,
          normalizeOpts: { sanitizeControlChars: true },
          validateOpts: { normalizeUndefinedToNull: true },
        });

        if (!result.success) return result;

        await Promise.all(
          result.datasetItems.map(async (item) =>
            auditLog({
              session: ctx.session,
              resourceType: "datasetItem",
              resourceId: item.id,
              action: "create",
              after: omit(item, ["input", "expectedOutput", "metadata"]),
            }),
          ),
        );

        return { success: true };
      },
    ),
  runItemsByItemId: protectedProjectProcedure
    .input(
      z.object({
        projectId: z.string(),
        datasetId: z.string(),
        datasetItemId: z.string(),
        datasetRunIds: z.array(z.string()).optional(),
        ...optionalPaginationZod,
      }),
    )
    .query(async ({ input, ctx }) => {
      throwIfNoProjectAccess({
        session: ctx.session,
        projectId: input.projectId,
        scope: "datasets:read",
      });

      const { datasetItemId, datasetId } = input;

      const filter = [
        {
          column: "datasetItemId",
          operator: "any of",
          value: [datasetItemId],
          type: "stringOptions" as const,
        },
        ...(input.datasetRunIds && input.datasetRunIds.length > 0
          ? [
              {
                column: "datasetRunId",
                operator: "any of",
                value: input.datasetRunIds,
                type: "stringOptions" as const,
              },
            ]
          : []),
      ] as FilterState;

      const datasetItem = await getDatasetItemById({
        projectId: input.projectId,
        datasetItemId: datasetItemId,
        datasetId: datasetId,
      });
      if (!datasetItem) {
        throw new TRPCError({
          code: "NOT_FOUND",
          message: "Dataset item not found",
        });
      }

      const [runItems, totalRunItems] = await Promise.all([
        getDatasetRunItemsByDatasetIdCh({
          projectId: input.projectId,
          datasetId: datasetId,
          filter,
          orderBy: [
            {
              column: "createdAt",
              order: "DESC",
            },
            { column: "datasetItemId", order: "DESC" },
          ],
          limit: input.limit ?? undefined,
          offset:
            input.page !== undefined && input.limit !== undefined
              ? input.page * input.limit
              : undefined,
        }),
        getDatasetRunItemsCountByDatasetIdCh({
          projectId: input.projectId,
          datasetId: datasetId,
          filter,
        }),
      ]);

      const enrichedRunItems = await getRunItemsByRunIdOrItemId(
        input.projectId,
        runItems,
      );

      // Note: We early return in case of no run items, when adding parameters here, make sure to update the early return above
      return {
        totalRunItems,
        runItems: enrichedRunItems,
      };
    }),

  runItemsByRunId: protectedProjectProcedure
    .input(
      z.object({
        projectId: z.string(),
        datasetId: z.string(),
        datasetRunId: z.string(),
        datasetItemIds: z.array(z.string()).optional(),
        filter: singleFilterList,
        ...optionalPaginationZod,
      }),
    )
    .query(async ({ input, ctx }) => {
      throwIfNoProjectAccess({
        session: ctx.session,
        projectId: input.projectId,
        scope: "datasets:read",
      });

      const {
        datasetRunId,
        datasetItemIds,
        datasetId,
        filter: userFilter,
      } = input;

      const datasetRun = await ctx.prisma.datasetRuns.findFirst({
        where: {
          id: datasetRunId,
          projectId: input.projectId,
        },
      });

      if (!datasetRun) {
        throw new TRPCError({
          code: "NOT_FOUND",
          message: "Dataset run not found",
        });
      }

      const combinedFilter = [
        ...userFilter,
        {
          column: "datasetRunId",
          operator: "any of",
          value: [datasetRunId],
          type: "stringOptions" as const,
        },
        ...(datasetItemIds && datasetItemIds.length > 0
          ? [
              {
                column: "datasetItemId",
                operator: "any of",
                value: datasetItemIds,
                type: "stringOptions" as const,
              },
            ]
          : []),
      ] as FilterState;

      const [runItems, totalRunItems] = await Promise.all([
        getDatasetRunItemsByDatasetIdCh({
          projectId: input.projectId,
          datasetId: datasetId,
          filter: combinedFilter,
          // ensure consistent ordering with datasets.baseDatasetItemByDatasetId
          // CH run items are created in reverse order as postgres execution path
          // can be refactored once we switch to CH only implementation
          orderBy: [
            {
              column: "createdAt",
              order: "DESC",
            },
            { column: "datasetItemId", order: "DESC" },
          ],
          limit: input.limit ?? undefined,
          offset:
            input.page !== undefined && input.limit !== undefined
              ? input.page * input.limit
              : undefined,
        }),
        getDatasetRunItemsCountByDatasetIdCh({
          projectId: input.projectId,
          datasetId: datasetId,
          filter: combinedFilter,
        }),
      ]);

      const enrichedRunItems = await getRunItemsByRunIdOrItemId(
        input.projectId,
        runItems,
      );

      // Note: We early return in case of no run items, when adding parameters here, make sure to update the early return above
      return {
        totalRunItems,
        runItems: enrichedRunItems,
      };
    }),

  datasetItemsWithRunData: protectedProjectProcedure
    .input(
      z.object({
        projectId: z.string(),
        datasetId: z.string(),
        runIds: z.array(z.string()),
        filterByRun: z
          .array(z.object({ runId: z.string(), filters: singleFilterList }))
          .nullish(),
        ...paginationZod,
      }),
    )
    .query(async ({ input, ctx }) => {
      throwIfNoProjectAccess({
        session: ctx.session,
        projectId: input.projectId,
        scope: "datasets:read",
      });

      const { filterByRun, datasetId, projectId, runIds, limit, page } = input;

      if (runIds.length === 0) {
        return {
          data: [],
          totalCount: 0,
        };
      }

      // Step 1: Return dataset item ids for which the run items match the filters
      const datasetItemIds = await getDatasetItemIdsWithRunData({
        projectId: input.projectId,
        datasetId: datasetId,
        runIds,
        filterByRun: filterByRun ?? [],
        limit: limit,
        offset: page * limit,
      });

      if (datasetItemIds.length === 0) {
        return { data: [] };
      }

      // Step 2: Given dataset item ids, lookup dataset run items in clickhouse
      // Note: for each unique dataset item id and dataset run id combination, we will retrieve a dataset run item
      const datasetRunItems = await getDatasetRunItemsWithoutIOByItemIds({
        projectId: input.projectId,
        datasetId: datasetId,
        runIds,
        datasetItemIds,
      });

      const [runData, items] = await Promise.all([
        enrichAndMapToDatasetItemId(projectId, datasetRunItems),
        getDatasetItems({
          projectId: input.projectId,
          filterState: createDatasetItemFilterState({
            datasetIds: [datasetId],
            itemIds: datasetItemIds,
            status: "ACTIVE",
          }),
        }),
      ]);

      return {
        data: items.map((item) => ({
          id: item.id,
          input: item.input,
          expectedOutput: item.expectedOutput,
          metadata: item.metadata,
          runData: runData.get(item.id) ?? {},
        })),
      };
    }),

  runItemCompareCount: protectedProjectProcedure
    .input(
      z.object({
        projectId: z.string(),
        datasetId: z.string(),
        runIds: z.array(z.string()),
        filterByRun: z
          .array(z.object({ runId: z.string(), filters: singleFilterList }))
          .nullish(),
      }),
    )
    .query(async ({ input, ctx }) => {
      throwIfNoProjectAccess({
        session: ctx.session,
        projectId: input.projectId,
        scope: "datasets:read",
      });

      const { filterByRun, datasetId, projectId, runIds } = input;

      // Rely on clickhouse to return only dataset item count that match the filters
      const datasetItemCount = await getDatasetItemsWithRunDataCount({
        projectId,
        datasetId,
        runIds,
        filterByRun: filterByRun ?? [],
      });

      return {
        totalCount: datasetItemCount,
      };
    }),

  datasetItemsBasedOnTraceOrObservation: protectedProjectProcedure
    .input(
      z.object({
        projectId: z.string(),
        traceId: z.string(),
        observationId: z.string().optional(),
      }),
    )
    .query(async ({ input, ctx }) => {
      throwIfNoProjectAccess({
        session: ctx.session,
        projectId: input.projectId,
        scope: "datasets:read",
      });

      const items = await getDatasetItems({
        projectId: input.projectId,
        filterState: createDatasetItemFilterState({
          sourceTraceId: input.traceId,
          // if no observationId is passed -> should not include observations from the same trace
          ...(input.observationId
            ? { sourceObservationId: input.observationId }
            : {
                sourceObservationIdIsNull: true,
              }),
        }),
        includeDatasetName: true,
      });

      return items;
    }),
  deleteDatasetRuns: protectedProjectProcedure
    .input(
      z.object({
        projectId: z.string(),
        // temporary: make optional to not break existing contracts
        datasetId: z.string().optional(),
        datasetRunIds: z.array(z.string()),
      }),
    )
    .mutation(async ({ input, ctx }) => {
      throwIfNoProjectAccess({
        session: ctx.session,
        projectId: input.projectId,
        scope: "datasets:CUD",
      });

      // Get all dataset runs first for audit logging
      const datasetRuns = await ctx.prisma.datasetRuns.findMany({
        where: {
          id: { in: input.datasetRunIds },
          projectId: input.projectId,
        },
      });

      // Delete all dataset runs
      await ctx.prisma.datasetRuns.deleteMany({
        where: {
          id: { in: input.datasetRunIds },
          projectId: input.projectId,
        },
      });

      // Trigger async delete of dataset run items
      await addToDeleteDatasetQueue({
        deletionType: "dataset-runs",
        projectId: input.projectId,
        // temporary: while dataset id is optional, we can pull it from the first run
        // users can only use this on pages in UI that are pre-filtered by dataset id
        datasetId: input.datasetId ?? datasetRuns[0].datasetId,
        datasetRunIds: input.datasetRunIds,
      });

      // Log audit entries for each deleted run
      await Promise.all(
        datasetRuns.map((run) =>
          auditLog({
            session: ctx.session,
            resourceType: "datasetRun",
            resourceId: run.id,
            action: "delete",
            before: run,
          }),
        ),
      );

      return datasetRuns;
    }),
  upsertRemoteExperiment: protectedProjectProcedure
    .input(
      z.object({
        projectId: z.string(),
        datasetId: z.string(),
        url: z.string(),
        defaultPayload: z.string(),
        enabled: z.boolean().optional(),
        signingEnabled: z.boolean().optional(),
        requestHeaders: RemoteExperimentHeadersSchema.optional(),
      }),
    )
    .mutation(async ({ input, ctx }) => {
      throwIfNoProjectAccess({
        session: ctx.session,
        projectId: input.projectId,
        scope: "datasets:CUD",
      });

      // Read encrypted custom headers to preserve masked values on update.
      // The signing secret itself is preserved by leaving its columns untouched.
      const dataset = await getRemoteExperimentConfig({
        projectId: input.projectId,
        datasetId: input.datasetId,
      });

      if (!dataset) {
        throw new TRPCError({
          code: "NOT_FOUND",
          message: "Dataset not found",
        });
      }

      try {
        await validateWebhookURL(input.url);
      } catch (error) {
        throw new TRPCError({
          code: "BAD_REQUEST",
          message: `Invalid remote run URL: ${error instanceof Error ? error.message : "Unknown error"}`,
        });
      }

      const existingHeaders = parseStoredRemoteExperimentHeaders(
        dataset.remoteExperimentRequestHeaders,
      );
      const submittedHeadersByLowerKey = input.requestHeaders
        ? Object.fromEntries(
            Object.entries(input.requestHeaders).map(([key, value]) => [
              key.trim().toLowerCase(),
              value,
            ]),
          )
        : undefined;
      const reusesSecretHeader = Object.entries(existingHeaders).some(
        ([key, existingHeader]) => {
          if (!existingHeader.secret) {
            return false;
          }

          // Omitting requestHeaders preserves every existing header.
          if (input.requestHeaders === undefined) {
            return true;
          }

          const normalizedKey = key.trim().toLowerCase();
          const submittedHeader = submittedHeadersByLowerKey?.[normalizedKey];

          // An empty submitted value preserves the existing secret.
          return submittedHeader?.value.trim() === "";
        },
      );

      if (input.url !== dataset.remoteExperimentUrl && reusesSecretHeader) {
        throw new TRPCError({
          code: "BAD_REQUEST",
          message:
            "Secret headers must be re-entered when changing the remote experiment URL",
        });
      }

      const { requestHeaders, displayHeaders } = processRemoteExperimentHeaders(
        input.requestHeaders,
        existingHeaders,
      );

      const signingSecret =
        input.signingEnabled === false
          ? {
              secretKey: undefined,
              displaySecretKey: undefined,
              unencryptedSecretKey: undefined,
            }
          : ensureRemoteExperimentSecret({
              displaySecretKey: dataset.remoteExperimentDisplaySecretKey,
            });

      // Result excludes the secret columns via the global Prisma omit, so it
      // is safe to audit-log and return.
      const updatedDataset = await ctx.prisma.dataset.update({
        where: {
          id_projectId: {
            id: input.datasetId,
            projectId: input.projectId,
          },
        },
        data: {
          remoteExperimentUrl: input.url,
          remoteExperimentPayload: input.defaultPayload ?? {},
          remoteExperimentEnabled: input.enabled,
          ...(input.signingEnabled === false
            ? {
                remoteExperimentSecretKey: null,
                remoteExperimentDisplaySecretKey: null,
              }
            : signingSecret.secretKey && signingSecret.displaySecretKey
              ? {
                  remoteExperimentSecretKey: signingSecret.secretKey,
                  remoteExperimentDisplaySecretKey:
                    signingSecret.displaySecretKey,
                }
              : {}),
          remoteExperimentRequestHeaders: requestHeaders,
          remoteExperimentDisplayHeaders: displayHeaders,
        },
      });

      await auditLog({
        session: ctx.session,
        resourceType: "dataset",
        resourceId: updatedDataset.id,
        action: "update",
        after: updatedDataset,
      });

      return {
        datasetId: updatedDataset.id,
        // Only present when a new secret was generated; shown once in the UI.
        unencryptedSecretKey: signingSecret.unencryptedSecretKey,
      };
    }),
  getRemoteExperiment: protectedProjectProcedure
    .input(z.object({ projectId: z.string(), datasetId: z.string() }))
    .query(async ({ input, ctx }) => {
      throwIfNoProjectAccess({
        session: ctx.session,
        projectId: input.projectId,
        scope: "datasets:CUD",
      });

      const dataset = await ctx.prisma.dataset.findUnique({
        where: {
          id_projectId: { id: input.datasetId, projectId: input.projectId },
        },
        select: {
          remoteExperimentUrl: true,
          remoteExperimentPayload: true,
          remoteExperimentEnabled: true,
          remoteExperimentDisplaySecretKey: true,
          remoteExperimentDisplayHeaders: true,
        },
      });

      if (!dataset || !dataset.remoteExperimentUrl) return null;

      return {
        url: dataset.remoteExperimentUrl,
        payload: dataset.remoteExperimentPayload,
        enabled: dataset.remoteExperimentEnabled,
        displaySecretKey: dataset.remoteExperimentDisplaySecretKey,
        displayHeaders: parseStoredRemoteExperimentHeaders(
          dataset.remoteExperimentDisplayHeaders,
        ),
      };
    }),
  triggerRemoteExperiment: protectedProjectProcedure
    .input(
      z.object({
        projectId: z.string(),
        datasetId: z.string(),
        payload: z.string().optional(),
      }),
    )
    .mutation(async ({ input, ctx }) => {
      throwIfNoProjectAccess({
        session: ctx.session,
        projectId: input.projectId,
        scope: "datasets:CUD",
      });

      // Delivery path: reads the secret-bearing columns through the
      // centralized helper to decrypt and sign the outbound request.
      const dataset = await getRemoteExperimentConfigWithSecrets({
        projectId: input.projectId,
        datasetId: input.datasetId,
      });

      if (!dataset) {
        throw new TRPCError({
          code: "NOT_FOUND",
          message: "Dataset not found",
        });
      }

      if (!dataset.remoteExperimentUrl) {
        throw new TRPCError({
          code: "BAD_REQUEST",
          message: "No remote run URL configured for this dataset",
        });
      }

      if (!dataset.remoteExperimentEnabled) {
        // Trigger is configured but intentionally disabled — skip the remote call
        // without surfacing an error toast to the user.
        return {
          success: true,
          skipped: true,
        };
      }

      const whitelist = whitelistFromEnv();

      try {
        await validateWebhookURL(dataset.remoteExperimentUrl, whitelist);
      } catch (error) {
        throw new TRPCError({
          code: "BAD_REQUEST",
          message: `Invalid remote run URL: ${error instanceof Error ? error.message : "Unknown error"}`,
        });
      }

      try {
        const { body, headers, sensitiveHeaderNames } =
          buildRemoteExperimentRequest({
            storedHeaders: dataset.remoteExperimentRequestHeaders,
            encryptedSecretKey: dataset.remoteExperimentSecretKey,
            bodyObject: {
              projectId: input.projectId,
              datasetId: input.datasetId,
              datasetName: dataset.name,
              payload: input.payload ?? dataset.remoteExperimentPayload,
            },
          });

        const { response, redirectChain, finalUrl } =
          await fetchWithSecureRedirects(
            dataset.remoteExperimentUrl,
            {
              method: "POST",
              headers,
              body,
              signal: AbortSignal.timeout(REMOTE_EXPERIMENT_TIMEOUT_MS),
            },
            {
              maxRedirects: REMOTE_EXPERIMENT_MAX_REDIRECTS,
              // Strip custom secret headers on cross-origin redirects
              additionalSensitiveHeaders: sensitiveHeaderNames,
              redirectValidation: {
                validateUrl: validateWebhookURL,
                whitelist,
                logContext: WEBHOOK_URL_VALIDATION_LOG_CONTEXT,
              },
            },
          );

        if (redirectChain.length > 0) {
          logger.info("Remote experiment trigger followed redirects", {
            datasetId: input.datasetId,
            projectId: input.projectId,
            initialUrl: dataset.remoteExperimentUrl,
            finalUrl,
            redirectCount: redirectChain.length,
            redirectChain,
          });
        }

        if (!response.ok) {
          logger.info(`Remote server returned error (${response.status})`);
          return {
            success: false,
            error: `Remote server returned error (${response.status})`,
          };
        }

        return {
          success: true,
        };
      } catch (error) {
        if (error instanceof Error) {
          logger.info(`Failed to trigger remote experiment: ${error.name}`);
          return {
            success: false,
            error:
              error.name === "AbortError"
                ? "Request timed out"
                : "Failed to connect to remote server",
          };
        }
        return {
          success: false,
        };
      }
    }),
  deleteRemoteExperiment: protectedProjectProcedure
    .input(z.object({ projectId: z.string(), datasetId: z.string() }))
    .mutation(async ({ input, ctx }) => {
      throwIfNoProjectAccess({
        session: ctx.session,
        projectId: input.projectId,
        scope: "datasets:CUD",
      });

      const dataset = await ctx.prisma.dataset.findUnique({
        where: {
          id_projectId: {
            id: input.datasetId,
            projectId: input.projectId,
          },
        },
      });

      if (!dataset) {
        throw new TRPCError({
          code: "NOT_FOUND",
          message: "Dataset not found",
        });
      }

      const updatedDataset = await ctx.prisma.dataset.update({
        where: {
          id_projectId: {
            id: input.datasetId,
            projectId: input.projectId,
          },
        },
        data: {
          remoteExperimentUrl: null,
          remoteExperimentPayload: Prisma.DbNull,
          // Reset to true so a future upsert doesn't inherit a stale disabled state
          remoteExperimentEnabled: true,
          remoteExperimentSecretKey: null,
          remoteExperimentDisplaySecretKey: null,
          remoteExperimentRequestHeaders: Prisma.DbNull,
          remoteExperimentDisplayHeaders: Prisma.DbNull,
        },
      });

      await auditLog({
        session: ctx.session,
        resourceType: "dataset",
        resourceId: updatedDataset.id,
        action: "update",
        after: updatedDataset,
      });

      return updatedDataset;
    }),

  validateDatasetSchema: protectedProjectProcedure
    .input(
      z.object({
        projectId: z.string(),
        datasetId: z.string(),
        inputSchema: DatasetJSONSchema.nullable(),
        expectedOutputSchema: DatasetJSONSchema.nullable(),
      }),
    )
    .mutation(async ({ input, ctx }) => {
      throwIfNoProjectAccess({
        session: ctx.session,
        projectId: input.projectId,
        scope: "datasets:CUD",
      });

      const dataset = await ctx.prisma.dataset.findUnique({
        where: {
          id_projectId: {
            id: input.datasetId,
            projectId: input.projectId,
          },
        },
      });

      if (!dataset) {
        throw new TRPCError({
          code: "NOT_FOUND",
          message: "Dataset not found",
        });
      }

      const validationResult = await validateAllDatasetItems({
        datasetId: input.datasetId,
        projectId: input.projectId,
        inputSchema: input.inputSchema,
        expectedOutputSchema: input.expectedOutputSchema,
      });

      return validationResult;
    }),

  setDatasetSchema: protectedProjectProcedure
    .input(
      z.object({
        projectId: z.string(),
        datasetId: z.string(),
        inputSchema: DatasetJSONSchema.nullable(),
        expectedOutputSchema: DatasetJSONSchema.nullable(),
      }),
    )
    .mutation(async ({ input, ctx }) => {
      throwIfNoProjectAccess({
        session: ctx.session,
        projectId: input.projectId,
        scope: "datasets:CUD",
      });

      const dataset = await ctx.prisma.dataset.findUnique({
        where: {
          id_projectId: {
            id: input.datasetId,
            projectId: input.projectId,
          },
        },
      });

      if (!dataset) {
        throw new TRPCError({
          code: "NOT_FOUND",
          message: "Dataset not found",
        });
      }

      // Validate all existing items before applying schema
      const validationResult = await validateAllDatasetItems({
        datasetId: input.datasetId,
        projectId: input.projectId,
        inputSchema: input.inputSchema,
        expectedOutputSchema: input.expectedOutputSchema,
      });

      if (!validationResult.isValid) {
        throw new TRPCError({
          code: "BAD_REQUEST",
          message: `Schema validation failed for ${validationResult.errors.length} item(s)`,
          cause: validationResult.errors,
        });
      }

      // Update dataset with new schemas
      const updatedDataset = await ctx.prisma.dataset.update({
        where: {
          id_projectId: {
            id: input.datasetId,
            projectId: input.projectId,
          },
        },
        data: {
          inputSchema: input.inputSchema ?? Prisma.DbNull,
          expectedOutputSchema: input.expectedOutputSchema ?? Prisma.DbNull,
        },
      });

      await auditLog({
        session: ctx.session,
        resourceType: "dataset",
        resourceId: input.datasetId,
        action: "updateSchema",
        before: {
          inputSchema: dataset.inputSchema,
          expectedOutputSchema: dataset.expectedOutputSchema,
        },
        after: {
          inputSchema: updatedDataset.inputSchema,
          expectedOutputSchema: updatedDataset.expectedOutputSchema,
        },
      });

      return updatedDataset;
    }),
});