🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
fix(experiments): validate config without loading all dataset item IO (#16396)
cursor[bot]2667e0c1ac13
diff --git a/packages/shared/src/server/repositories/dataset-items.ts b/packages/shared/src/server/repositories/dataset-items.tsindex c086975fa6..110ae723d2 100644--- a/packages/shared/src/server/repositories/dataset-items.ts+++ b/packages/shared/src/server/repositories/dataset-items.ts@@ -1233,10 +1233,13 @@ function buildStatefulDatasetItemsCountQuery(},});+ // Match VERSIONED / countDatasetItemVariableMatches: current, non-deleted rows only.return Prisma.sql`SELECT COUNT(*) as countFROM dataset_items diWHERE di.project_id = ${projectId}+ AND di.is_deleted = false+ AND di.valid_to IS NULL${filterCondition}${searchCondition}`;@@ -1787,9 +1790,12 @@ export async function getDatasetItemsCount(props: {return result.length > 0 ? Number(result[0].count) : 0;}- // Otherwise use Prisma+ // Otherwise use Prisma. Same current-row rules as+ // countDatasetItemVariableMatchesInternal (is_deleted / valid_to).const where = {projectId: props.projectId,+ isDeleted: false,+ validTo: null,...buildPrismaWhereFromFilterState(props.filterState),};@@ -1808,6 +1814,109 @@ export async function getDatasetItemsCount(props: {});}+/**+ * Counts dataset items whose input contains each prompt variable, without+ * transferring item IO to the Node process.+ *+ * Must stay aligned with `validateDatasetItem` in+ * `packages/shared/src/features/experiments/utils.ts`:+ * - object inputs: top-level key presence (`variable in input`)+ * - string inputs: counted only when there is exactly one variable+ * - arrays, null, empty string: not counted+ *+ * Used by `experiments.validateConfig`, which previously loaded every item's+ * input/output/metadata into memory and 500'd on large datasets.+ */+export async function countDatasetItemVariableMatches(props: {+ projectId: string;+ filterState: FilterState;+ variables: string[];+ version?: Date;+}): Promise<Record<string, number>> {+ if (props.variables.length === 0) {+ return {};+ }++ return executeWithDatasetServiceStrategy(OperationType.READ, {+ [Implementation.STATEFUL]: async () =>+ countDatasetItemVariableMatchesInternal({+ ...props,+ version: undefined,+ }),+ [Implementation.VERSIONED]: async () =>+ countDatasetItemVariableMatchesInternal(props),+ });+}++async function countDatasetItemVariableMatchesInternal(params: {+ projectId: string;+ filterState: FilterState;+ variables: string[];+ version?: Date;+}): Promise<Record<string, number>> {+ const filterCondition = tableColumnsToSqlFilterAndPrefix(+ params.filterState,+ datasetItemsFilterCols,+ "dataset_item_events",+ );++ const versionCondition = params.version+ ? Prisma.sql`+ AND di.valid_from <= ${params.version}+ AND (di.valid_to IS NULL OR di.valid_to > ${params.version})+ `+ : Prisma.sql`AND di.valid_to IS NULL`;++ const variableList = Prisma.join(+ params.variables.map((variable) => Prisma.sql`${variable}`),+ );++ const objectKeyCounts = await prisma.$queryRaw<+ Array<{ obj_key: string; count: bigint }>+ >(Prisma.sql`+ SELECT obj_key, COUNT(*)::bigint AS count+ FROM dataset_items di+ CROSS JOIN LATERAL jsonb_object_keys(di.input::jsonb) AS obj_key+ WHERE di.project_id = ${params.projectId}+ AND di.is_deleted = false+ AND di.input IS NOT NULL+ AND jsonb_typeof(di.input::jsonb) = 'object'+ AND obj_key IN (${variableList})+ ${versionCondition}+ ${filterCondition}+ GROUP BY obj_key+ `);++ const variablesMap: Record<string, number> = {};+ for (const row of objectKeyCounts) {+ variablesMap[row.obj_key] = Number(row.count);+ }++ if (params.variables.length === 1) {+ const variable = params.variables[0];+ const stringCounts = await prisma.$queryRaw<Array<{ count: bigint }>>(+ Prisma.sql`+ SELECT COUNT(*)::bigint AS count+ FROM dataset_items di+ WHERE di.project_id = ${params.projectId}+ AND di.is_deleted = false+ AND di.input IS NOT NULL+ AND jsonb_typeof(di.input::jsonb) = 'string'+ AND coalesce(di.input #>> '{}', '') <> ''+ ${versionCondition}+ ${filterCondition}+ `,+ );+ const stringCount =+ stringCounts.length > 0 ? Number(stringCounts[0].count) : 0;+ if (stringCount > 0) {+ variablesMap[variable] = (variablesMap[variable] ?? 0) + stringCount;+ }+ }++ return variablesMap;+}+export async function getDatasetItemsCountGrouped(props: {projectId: string;datasetIds: string[];diff --git a/web/src/__tests__/server/experiments-versioning.servertest.ts b/web/src/__tests__/server/experiments-versioning.servertest.tsindex 28a6a1059f..b9d3f04295 100644--- a/web/src/__tests__/server/experiments-versioning.servertest.ts+++ b/web/src/__tests__/server/experiments-versioning.servertest.ts@@ -236,6 +236,174 @@ describe("Experiments with Dataset Versioning", () => {expect(validation.totalItems).toBeGreaterThan(0);}});++ it("counts object-key matches per variable without excluding unmatched items from totalItems", async () => {+ const { project, caller } = await prepare();++ const datasetId = v4();+ await prisma.dataset.create({+ data: { id: datasetId, name: v4(), projectId: project.id },+ });++ const promptId = v4();+ await prisma.prompt.create({+ data: {+ id: promptId,+ name: v4(),+ version: 1,+ projectId: project.id,+ createdBy: "user-1",+ prompt: "Hello {{name}} from {{city}}",+ type: "text",+ },+ });++ await createDatasetItem({+ projectId: project.id,+ datasetId,+ input: { name: "Ada", city: "London" },+ expectedOutput: "ok",+ normalizeOpts: {},+ validateOpts: {},+ });+ await createDatasetItem({+ projectId: project.id,+ datasetId,+ input: { name: "Grace" },+ expectedOutput: "ok",+ normalizeOpts: {},+ validateOpts: {},+ });+ await createDatasetItem({+ projectId: project.id,+ datasetId,+ input: { other: "ignored" },+ expectedOutput: "ok",+ normalizeOpts: {},+ validateOpts: {},+ });+ await createDatasetItem({+ projectId: project.id,+ datasetId,+ input: ["not", "an", "object"],+ expectedOutput: "ok",+ normalizeOpts: {},+ validateOpts: {},+ });++ const validation = await caller.experiments.validateConfig({+ projectId: project.id,+ promptId,+ datasetId,+ });++ expect(validation.isValid).toBe(true);+ if (validation.isValid) {+ expect(validation.totalItems).toBe(4);+ expect(validation.variablesMap).toEqual({ name: 2, city: 1 });+ }+ });++ it("counts non-empty string inputs when the prompt has a single variable", async () => {+ const { project, caller } = await prepare();++ const datasetId = v4();+ await prisma.dataset.create({+ data: { id: datasetId, name: v4(), projectId: project.id },+ });++ const promptId = v4();+ await prisma.prompt.create({+ data: {+ id: promptId,+ name: v4(),+ version: 1,+ projectId: project.id,+ createdBy: "user-1",+ prompt: "Hello {{name}}",+ type: "text",+ },+ });++ await createDatasetItem({+ projectId: project.id,+ datasetId,+ input: "Ada",+ expectedOutput: "ok",+ normalizeOpts: {},+ validateOpts: {},+ });+ await createDatasetItem({+ projectId: project.id,+ datasetId,+ input: { name: "Grace" },+ expectedOutput: "ok",+ normalizeOpts: {},+ validateOpts: {},+ });+ await createDatasetItem({+ projectId: project.id,+ datasetId,+ input: "",+ expectedOutput: "ok",+ normalizeOpts: {},+ validateOpts: {},+ });++ const validation = await caller.experiments.validateConfig({+ projectId: project.id,+ promptId,+ datasetId,+ });++ expect(validation.isValid).toBe(true);+ if (validation.isValid) {+ expect(validation.totalItems).toBe(3);+ expect(validation.variablesMap).toEqual({ name: 2 });+ }+ });++ it("returns invalid instead of throwing when a composed prompt cannot be resolved", async () => {+ const { project, caller } = await prepare();++ const datasetId = v4();+ await prisma.dataset.create({+ data: { id: datasetId, name: v4(), projectId: project.id },+ });++ const promptId = v4();+ const missingChildName = v4();+ await prisma.prompt.create({+ data: {+ id: promptId,+ name: v4(),+ version: 1,+ projectId: project.id,+ createdBy: "user-1",+ prompt: `Hello {{name}} @@@langfusePrompt:name=${missingChildName}|version=1@@@`,+ type: "text",+ },+ });+ await prisma.promptDependency.create({+ data: {+ projectId: project.id,+ parentId: promptId,+ childName: missingChildName,+ childVersion: 1,+ },+ });++ const validation = await caller.experiments.validateConfig({+ projectId: project.id,+ promptId,+ datasetId,+ });++ expect(validation.isValid).toBe(false);+ if (!validation.isValid) {+ expect(validation.message).toContain("Prompt dependency not found");+ }+ });});describe("getDatasetItems temporal query behavior", () => {diff --git a/web/src/__tests__/server/repositories/dataset-items-stateful-count-alignment.servertest.ts b/web/src/__tests__/server/repositories/dataset-items-stateful-count-alignment.servertest.tsnew file mode 100644index 0000000000..9074edc891--- /dev/null+++ b/web/src/__tests__/server/repositories/dataset-items-stateful-count-alignment.servertest.ts@@ -0,0 +1,108 @@+vi.hoisted(() => {+ // STATEFUL reads with historical rows present (mixed-mode / versioned writes).+ process.env.LANGFUSE_DATASET_SERVICE_READ_FROM_VERSIONED_IMPLEMENTATION =+ "false";+ process.env.LANGFUSE_DATASET_SERVICE_WRITE_TO_VERSIONED_IMPLEMENTATION =+ "false";+});++import { prisma } from "@langfuse/shared/src/db";+import {+ countDatasetItemVariableMatches,+ createDatasetItemFilterState,+ getDatasetItemsCount,+} from "@langfuse/shared/src/server";+import { v4 } from "uuid";++const projectId = "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a";++describe("STATEFUL getDatasetItemsCount vs countDatasetItemVariableMatches", () => {+ it("excludes superseded and deleted rows from both aggregates", async () => {+ const datasetId = v4();+ await prisma.dataset.create({+ data: { id: datasetId, name: v4(), projectId },+ });++ const currentItemId = v4();+ const supersededItemId = v4();+ const deletedItemId = v4();+ const now = new Date();+ const earlier = new Date(now.getTime() - 60_000);++ // Current ACTIVE row — should count.+ await prisma.datasetItem.create({+ data: {+ id: currentItemId,+ projectId,+ datasetId,+ status: "ACTIVE",+ input: { name: "Ada" },+ expectedOutput: "ok",+ validFrom: now,+ validTo: null,+ isDeleted: false,+ },+ });++ // Superseded historical row for another logical item — must not inflate totals.+ await prisma.datasetItem.create({+ data: {+ id: supersededItemId,+ projectId,+ datasetId,+ status: "ACTIVE",+ input: { name: "Grace" },+ expectedOutput: "ok",+ validFrom: earlier,+ validTo: now,+ isDeleted: false,+ },+ });+ await prisma.datasetItem.create({+ data: {+ id: supersededItemId,+ projectId,+ datasetId,+ status: "ACTIVE",+ input: { name: "Grace", city: "London" },+ expectedOutput: "ok",+ validFrom: now,+ validTo: null,+ isDeleted: false,+ },+ });++ // Soft-deleted current row — must not count.+ await prisma.datasetItem.create({+ data: {+ id: deletedItemId,+ projectId,+ datasetId,+ status: "ACTIVE",+ input: { name: "Deleted" },+ expectedOutput: "ok",+ validFrom: now,+ validTo: null,+ isDeleted: true,+ },+ });++ const filterState = createDatasetItemFilterState({+ datasetIds: [datasetId],+ status: "ACTIVE",+ });++ const [totalItems, variablesMap] = await Promise.all([+ getDatasetItemsCount({ projectId, filterState }),+ countDatasetItemVariableMatches({+ projectId,+ filterState,+ variables: ["name", "city"],+ }),+ ]);++ // Two current non-deleted ACTIVE rows: currentItemId + supersededItemId latest.+ expect(totalItems).toBe(2);+ expect(variablesMap).toEqual({ name: 2, city: 1 });+ });+});diff --git a/web/src/features/experiments/server/router.ts b/web/src/features/experiments/server/router.tsindex 0e5d9ff6c0..0654d5386f 100644--- a/web/src/features/experiments/server/router.ts+++ b/web/src/features/experiments/server/router.ts@@ -7,7 +7,8 @@ import {ExperimentCreateQueue,getCategoricalScoresGroupedByName,getBooleanScoresGroupedByName,- getDatasetItems,+ getDatasetItemsCount,+ countDatasetItemVariableMatches,getEventsGroupedByExperimentDatasetId,getExperimentsCountFromEvents,getExperimentsFromEvents,@@ -36,13 +37,11 @@ import {} from "@/src/server/api/trpc";import {extractVariables,- validateDatasetItem,+ isBaseError,UnauthorizedError,PromptType,extractPlaceholderNames,type PromptMessage,- isPresent,- type DatasetItemDomain,singleFilter,type FilterState,orderBy,@@ -88,39 +87,6 @@ const ConfigResponse = z.discriminatedUnion("isValid", [InvalidConfigResponse,]);-const countValidDatasetItems = (- datasetItems: Omit<DatasetItemDomain, "status">[],- variables: string[],-): Record<string, number> => {- const variableMap: Record<string, number> = {};-- for (const { input } of datasetItems) {- // Step 1: Validate item- if (!isPresent(input) || !validateDatasetItem(input, variables)) {- continue;- }-- // Step 2: Count variable matches-- // String with single variable - count that variable- if (typeof input === "string" && variables.length === 1) {- variableMap[variables[0]] = (variableMap[variables[0]] || 0) + 1;- continue;- }-- // For object inputs, count each matching variable- if (typeof input === "object" && !Array.isArray(input)) {- for (const variable of variables) {- if (variable in input) {- variableMap[variable] = (variableMap[variable] || 0) + 1;- }- }- }- }-- return variableMap;-};-export const experimentsRouter = createTRPCRouter({validateConfig: protectedProjectProcedure.input(@@ -154,7 +120,23 @@ export const experimentsRouter = createTRPCRouter({}const promptService = new PromptService(ctx.prisma, redis);- const resolvedPrompt = await promptService.resolvePrompt(prompt);+ let resolvedPrompt;+ try {+ resolvedPrompt = await promptService.resolvePrompt(prompt);+ } catch (error) {+ if (+ error instanceof SyntaxError ||+ (isBaseError(error) && error.isUserError())+ ) {+ return {+ isValid: false,+ message: isBaseError(error)+ ? error.message+ : "Selected prompt could not be resolved.",+ };+ }+ throw error;+ }if (!resolvedPrompt) {return {@@ -164,9 +146,9 @@ export const experimentsRouter = createTRPCRouter({}const extractedVariables = extractVariables(- resolvedPrompt?.type === PromptType.Text+ resolvedPrompt.type === PromptType.Text? (resolvedPrompt.prompt?.toString() ?? "")- : JSON.stringify(resolvedPrompt?.prompt),+ : JSON.stringify(resolvedPrompt.prompt ?? ""),);const promptMessages =@@ -187,23 +169,30 @@ export const experimentsRouter = createTRPCRouter({};}- const items = await getDatasetItems({+ const filterState = createDatasetItemFilterState({+ datasetIds: [input.datasetId],+ status: "ACTIVE",+ });++ const totalItems = await getDatasetItemsCount({projectId: input.projectId,- filterState: createDatasetItemFilterState({- datasetIds: [input.datasetId],- status: "ACTIVE",- }),+ filterState,version: input.datasetVersion,});- if (!Boolean(items.length)) {+ if (!Boolean(totalItems)) {return {isValid: false,message: "Selected dataset is empty or all items are inactive.",};}- const variablesMap = countValidDatasetItems(items, allVariables);+ const variablesMap = await countDatasetItemVariableMatches({+ projectId: input.projectId,+ filterState,+ version: input.datasetVersion,+ variables: allVariables,+ });if (!Boolean(Object.keys(variablesMap).length)) {return {@@ -214,8 +203,8 @@ export const experimentsRouter = createTRPCRouter({return {isValid: true,- totalItems: items.length,- variablesMap: variablesMap,+ totalItems,+ variablesMap,};}),