Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

fix(experiments): validate config without loading all dataset item IO (#16396)

cursor[bot]2667e0c1ac13
diff --git a/packages/shared/src/server/repositories/dataset-items.ts b/packages/shared/src/server/repositories/dataset-items.ts
index c086975fa6..110ae723d2 100644
--- a/packages/shared/src/server/repositories/dataset-items.ts
+++ b/packages/shared/src/server/repositories/dataset-items.ts
@@ -1233,10 +1233,13 @@ function buildStatefulDatasetItemsCountQuery(
},
});
+ // Match VERSIONED / countDatasetItemVariableMatches: current, non-deleted rows only.
return Prisma.sql`
SELECT COUNT(*) as count
FROM dataset_items di
WHERE di.project_id = ${projectId}
+ AND di.is_deleted = false
+ AND di.valid_to IS NULL
${filterCondition}
${searchCondition}
`;
@@ -1787,9 +1790,12 @@ export async function getDatasetItemsCount(props: {
return result.length > 0 ? Number(result[0].count) : 0;
}
- // Otherwise use Prisma
+ // Otherwise use Prisma. Same current-row rules as
+ // countDatasetItemVariableMatchesInternal (is_deleted / valid_to).
const where = {
projectId: props.projectId,
+ isDeleted: false,
+ validTo: null,
...buildPrismaWhereFromFilterState(props.filterState),
};
@@ -1808,6 +1814,109 @@ export async function getDatasetItemsCount(props: {
});
}
+/**
+ * Counts dataset items whose input contains each prompt variable, without
+ * transferring item IO to the Node process.
+ *
+ * Must stay aligned with `validateDatasetItem` in
+ * `packages/shared/src/features/experiments/utils.ts`:
+ * - object inputs: top-level key presence (`variable in input`)
+ * - string inputs: counted only when there is exactly one variable
+ * - arrays, null, empty string: not counted
+ *
+ * Used by `experiments.validateConfig`, which previously loaded every item's
+ * input/output/metadata into memory and 500'd on large datasets.
+ */
+export async function countDatasetItemVariableMatches(props: {
+ projectId: string;
+ filterState: FilterState;
+ variables: string[];
+ version?: Date;
+}): Promise<Record<string, number>> {
+ if (props.variables.length === 0) {
+ return {};
+ }
+
+ return executeWithDatasetServiceStrategy(OperationType.READ, {
+ [Implementation.STATEFUL]: async () =>
+ countDatasetItemVariableMatchesInternal({
+ ...props,
+ version: undefined,
+ }),
+ [Implementation.VERSIONED]: async () =>
+ countDatasetItemVariableMatchesInternal(props),
+ });
+}
+
+async function countDatasetItemVariableMatchesInternal(params: {
+ projectId: string;
+ filterState: FilterState;
+ variables: string[];
+ version?: Date;
+}): Promise<Record<string, number>> {
+ const filterCondition = tableColumnsToSqlFilterAndPrefix(
+ params.filterState,
+ datasetItemsFilterCols,
+ "dataset_item_events",
+ );
+
+ const versionCondition = params.version
+ ? Prisma.sql`
+ AND di.valid_from <= ${params.version}
+ AND (di.valid_to IS NULL OR di.valid_to > ${params.version})
+ `
+ : Prisma.sql`AND di.valid_to IS NULL`;
+
+ const variableList = Prisma.join(
+ params.variables.map((variable) => Prisma.sql`${variable}`),
+ );
+
+ const objectKeyCounts = await prisma.$queryRaw<
+ Array<{ obj_key: string; count: bigint }>
+ >(Prisma.sql`
+ SELECT obj_key, COUNT(*)::bigint AS count
+ FROM dataset_items di
+ CROSS JOIN LATERAL jsonb_object_keys(di.input::jsonb) AS obj_key
+ WHERE di.project_id = ${params.projectId}
+ AND di.is_deleted = false
+ AND di.input IS NOT NULL
+ AND jsonb_typeof(di.input::jsonb) = 'object'
+ AND obj_key IN (${variableList})
+ ${versionCondition}
+ ${filterCondition}
+ GROUP BY obj_key
+ `);
+
+ const variablesMap: Record<string, number> = {};
+ for (const row of objectKeyCounts) {
+ variablesMap[row.obj_key] = Number(row.count);
+ }
+
+ if (params.variables.length === 1) {
+ const variable = params.variables[0];
+ const stringCounts = await prisma.$queryRaw<Array<{ count: bigint }>>(
+ Prisma.sql`
+ SELECT COUNT(*)::bigint AS count
+ FROM dataset_items di
+ WHERE di.project_id = ${params.projectId}
+ AND di.is_deleted = false
+ AND di.input IS NOT NULL
+ AND jsonb_typeof(di.input::jsonb) = 'string'
+ AND coalesce(di.input #>> '{}', '') <> ''
+ ${versionCondition}
+ ${filterCondition}
+ `,
+ );
+ const stringCount =
+ stringCounts.length > 0 ? Number(stringCounts[0].count) : 0;
+ if (stringCount > 0) {
+ variablesMap[variable] = (variablesMap[variable] ?? 0) + stringCount;
+ }
+ }
+
+ return variablesMap;
+}
+
export async function getDatasetItemsCountGrouped(props: {
projectId: string;
datasetIds: string[];
diff --git a/web/src/__tests__/server/experiments-versioning.servertest.ts b/web/src/__tests__/server/experiments-versioning.servertest.ts
index 28a6a1059f..b9d3f04295 100644
--- a/web/src/__tests__/server/experiments-versioning.servertest.ts
+++ b/web/src/__tests__/server/experiments-versioning.servertest.ts
@@ -236,6 +236,174 @@ describe("Experiments with Dataset Versioning", () => {
expect(validation.totalItems).toBeGreaterThan(0);
}
});
+
+ it("counts object-key matches per variable without excluding unmatched items from totalItems", async () => {
+ const { project, caller } = await prepare();
+
+ const datasetId = v4();
+ await prisma.dataset.create({
+ data: { id: datasetId, name: v4(), projectId: project.id },
+ });
+
+ const promptId = v4();
+ await prisma.prompt.create({
+ data: {
+ id: promptId,
+ name: v4(),
+ version: 1,
+ projectId: project.id,
+ createdBy: "user-1",
+ prompt: "Hello {{name}} from {{city}}",
+ type: "text",
+ },
+ });
+
+ await createDatasetItem({
+ projectId: project.id,
+ datasetId,
+ input: { name: "Ada", city: "London" },
+ expectedOutput: "ok",
+ normalizeOpts: {},
+ validateOpts: {},
+ });
+ await createDatasetItem({
+ projectId: project.id,
+ datasetId,
+ input: { name: "Grace" },
+ expectedOutput: "ok",
+ normalizeOpts: {},
+ validateOpts: {},
+ });
+ await createDatasetItem({
+ projectId: project.id,
+ datasetId,
+ input: { other: "ignored" },
+ expectedOutput: "ok",
+ normalizeOpts: {},
+ validateOpts: {},
+ });
+ await createDatasetItem({
+ projectId: project.id,
+ datasetId,
+ input: ["not", "an", "object"],
+ expectedOutput: "ok",
+ normalizeOpts: {},
+ validateOpts: {},
+ });
+
+ const validation = await caller.experiments.validateConfig({
+ projectId: project.id,
+ promptId,
+ datasetId,
+ });
+
+ expect(validation.isValid).toBe(true);
+ if (validation.isValid) {
+ expect(validation.totalItems).toBe(4);
+ expect(validation.variablesMap).toEqual({ name: 2, city: 1 });
+ }
+ });
+
+ it("counts non-empty string inputs when the prompt has a single variable", async () => {
+ const { project, caller } = await prepare();
+
+ const datasetId = v4();
+ await prisma.dataset.create({
+ data: { id: datasetId, name: v4(), projectId: project.id },
+ });
+
+ const promptId = v4();
+ await prisma.prompt.create({
+ data: {
+ id: promptId,
+ name: v4(),
+ version: 1,
+ projectId: project.id,
+ createdBy: "user-1",
+ prompt: "Hello {{name}}",
+ type: "text",
+ },
+ });
+
+ await createDatasetItem({
+ projectId: project.id,
+ datasetId,
+ input: "Ada",
+ expectedOutput: "ok",
+ normalizeOpts: {},
+ validateOpts: {},
+ });
+ await createDatasetItem({
+ projectId: project.id,
+ datasetId,
+ input: { name: "Grace" },
+ expectedOutput: "ok",
+ normalizeOpts: {},
+ validateOpts: {},
+ });
+ await createDatasetItem({
+ projectId: project.id,
+ datasetId,
+ input: "",
+ expectedOutput: "ok",
+ normalizeOpts: {},
+ validateOpts: {},
+ });
+
+ const validation = await caller.experiments.validateConfig({
+ projectId: project.id,
+ promptId,
+ datasetId,
+ });
+
+ expect(validation.isValid).toBe(true);
+ if (validation.isValid) {
+ expect(validation.totalItems).toBe(3);
+ expect(validation.variablesMap).toEqual({ name: 2 });
+ }
+ });
+
+ it("returns invalid instead of throwing when a composed prompt cannot be resolved", async () => {
+ const { project, caller } = await prepare();
+
+ const datasetId = v4();
+ await prisma.dataset.create({
+ data: { id: datasetId, name: v4(), projectId: project.id },
+ });
+
+ const promptId = v4();
+ const missingChildName = v4();
+ await prisma.prompt.create({
+ data: {
+ id: promptId,
+ name: v4(),
+ version: 1,
+ projectId: project.id,
+ createdBy: "user-1",
+ prompt: `Hello {{name}} @@@langfusePrompt:name=${missingChildName}|version=1@@@`,
+ type: "text",
+ },
+ });
+ await prisma.promptDependency.create({
+ data: {
+ projectId: project.id,
+ parentId: promptId,
+ childName: missingChildName,
+ childVersion: 1,
+ },
+ });
+
+ const validation = await caller.experiments.validateConfig({
+ projectId: project.id,
+ promptId,
+ datasetId,
+ });
+
+ expect(validation.isValid).toBe(false);
+ if (!validation.isValid) {
+ expect(validation.message).toContain("Prompt dependency not found");
+ }
+ });
});
describe("getDatasetItems temporal query behavior", () => {
diff --git a/web/src/__tests__/server/repositories/dataset-items-stateful-count-alignment.servertest.ts b/web/src/__tests__/server/repositories/dataset-items-stateful-count-alignment.servertest.ts
new file mode 100644
index 0000000000..9074edc891
--- /dev/null
+++ b/web/src/__tests__/server/repositories/dataset-items-stateful-count-alignment.servertest.ts
@@ -0,0 +1,108 @@
+vi.hoisted(() => {
+ // STATEFUL reads with historical rows present (mixed-mode / versioned writes).
+ process.env.LANGFUSE_DATASET_SERVICE_READ_FROM_VERSIONED_IMPLEMENTATION =
+ "false";
+ process.env.LANGFUSE_DATASET_SERVICE_WRITE_TO_VERSIONED_IMPLEMENTATION =
+ "false";
+});
+
+import { prisma } from "@langfuse/shared/src/db";
+import {
+ countDatasetItemVariableMatches,
+ createDatasetItemFilterState,
+ getDatasetItemsCount,
+} from "@langfuse/shared/src/server";
+import { v4 } from "uuid";
+
+const projectId = "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a";
+
+describe("STATEFUL getDatasetItemsCount vs countDatasetItemVariableMatches", () => {
+ it("excludes superseded and deleted rows from both aggregates", async () => {
+ const datasetId = v4();
+ await prisma.dataset.create({
+ data: { id: datasetId, name: v4(), projectId },
+ });
+
+ const currentItemId = v4();
+ const supersededItemId = v4();
+ const deletedItemId = v4();
+ const now = new Date();
+ const earlier = new Date(now.getTime() - 60_000);
+
+ // Current ACTIVE row — should count.
+ await prisma.datasetItem.create({
+ data: {
+ id: currentItemId,
+ projectId,
+ datasetId,
+ status: "ACTIVE",
+ input: { name: "Ada" },
+ expectedOutput: "ok",
+ validFrom: now,
+ validTo: null,
+ isDeleted: false,
+ },
+ });
+
+ // Superseded historical row for another logical item — must not inflate totals.
+ await prisma.datasetItem.create({
+ data: {
+ id: supersededItemId,
+ projectId,
+ datasetId,
+ status: "ACTIVE",
+ input: { name: "Grace" },
+ expectedOutput: "ok",
+ validFrom: earlier,
+ validTo: now,
+ isDeleted: false,
+ },
+ });
+ await prisma.datasetItem.create({
+ data: {
+ id: supersededItemId,
+ projectId,
+ datasetId,
+ status: "ACTIVE",
+ input: { name: "Grace", city: "London" },
+ expectedOutput: "ok",
+ validFrom: now,
+ validTo: null,
+ isDeleted: false,
+ },
+ });
+
+ // Soft-deleted current row — must not count.
+ await prisma.datasetItem.create({
+ data: {
+ id: deletedItemId,
+ projectId,
+ datasetId,
+ status: "ACTIVE",
+ input: { name: "Deleted" },
+ expectedOutput: "ok",
+ validFrom: now,
+ validTo: null,
+ isDeleted: true,
+ },
+ });
+
+ const filterState = createDatasetItemFilterState({
+ datasetIds: [datasetId],
+ status: "ACTIVE",
+ });
+
+ const [totalItems, variablesMap] = await Promise.all([
+ getDatasetItemsCount({ projectId, filterState }),
+ countDatasetItemVariableMatches({
+ projectId,
+ filterState,
+ variables: ["name", "city"],
+ }),
+ ]);
+
+ // Two current non-deleted ACTIVE rows: currentItemId + supersededItemId latest.
+ expect(totalItems).toBe(2);
+ expect(variablesMap).toEqual({ name: 2, city: 1 });
+ });
+});
diff --git a/web/src/features/experiments/server/router.ts b/web/src/features/experiments/server/router.ts
index 0e5d9ff6c0..0654d5386f 100644
--- a/web/src/features/experiments/server/router.ts
+++ b/web/src/features/experiments/server/router.ts
@@ -7,7 +7,8 @@ import {
ExperimentCreateQueue,
getCategoricalScoresGroupedByName,
getBooleanScoresGroupedByName,
- getDatasetItems,
+ getDatasetItemsCount,
+ countDatasetItemVariableMatches,
getEventsGroupedByExperimentDatasetId,
getExperimentsCountFromEvents,
getExperimentsFromEvents,
@@ -36,13 +37,11 @@ import {
} from "@/src/server/api/trpc";
import {
extractVariables,
- validateDatasetItem,
+ isBaseError,
UnauthorizedError,
PromptType,
extractPlaceholderNames,
type PromptMessage,
- isPresent,
- type DatasetItemDomain,
singleFilter,
type FilterState,
orderBy,
@@ -88,39 +87,6 @@ const ConfigResponse = z.discriminatedUnion("isValid", [
InvalidConfigResponse,
]);
-const countValidDatasetItems = (
- datasetItems: Omit<DatasetItemDomain, "status">[],
- variables: string[],
-): Record<string, number> => {
- const variableMap: Record<string, number> = {};
-
- for (const { input } of datasetItems) {
- // Step 1: Validate item
- if (!isPresent(input) || !validateDatasetItem(input, variables)) {
- continue;
- }
-
- // Step 2: Count variable matches
-
- // String with single variable - count that variable
- if (typeof input === "string" && variables.length === 1) {
- variableMap[variables[0]] = (variableMap[variables[0]] || 0) + 1;
- continue;
- }
-
- // For object inputs, count each matching variable
- if (typeof input === "object" && !Array.isArray(input)) {
- for (const variable of variables) {
- if (variable in input) {
- variableMap[variable] = (variableMap[variable] || 0) + 1;
- }
- }
- }
- }
-
- return variableMap;
-};
-
export const experimentsRouter = createTRPCRouter({
validateConfig: protectedProjectProcedure
.input(
@@ -154,7 +120,23 @@ export const experimentsRouter = createTRPCRouter({
}
const promptService = new PromptService(ctx.prisma, redis);
- const resolvedPrompt = await promptService.resolvePrompt(prompt);
+ let resolvedPrompt;
+ try {
+ resolvedPrompt = await promptService.resolvePrompt(prompt);
+ } catch (error) {
+ if (
+ error instanceof SyntaxError ||
+ (isBaseError(error) && error.isUserError())
+ ) {
+ return {
+ isValid: false,
+ message: isBaseError(error)
+ ? error.message
+ : "Selected prompt could not be resolved.",
+ };
+ }
+ throw error;
+ }
if (!resolvedPrompt) {
return {
@@ -164,9 +146,9 @@ export const experimentsRouter = createTRPCRouter({
}
const extractedVariables = extractVariables(
- resolvedPrompt?.type === PromptType.Text
+ resolvedPrompt.type === PromptType.Text
? (resolvedPrompt.prompt?.toString() ?? "")
- : JSON.stringify(resolvedPrompt?.prompt),
+ : JSON.stringify(resolvedPrompt.prompt ?? ""),
);
const promptMessages =
@@ -187,23 +169,30 @@ export const experimentsRouter = createTRPCRouter({
};
}
- const items = await getDatasetItems({
+ const filterState = createDatasetItemFilterState({
+ datasetIds: [input.datasetId],
+ status: "ACTIVE",
+ });
+
+ const totalItems = await getDatasetItemsCount({
projectId: input.projectId,
- filterState: createDatasetItemFilterState({
- datasetIds: [input.datasetId],
- status: "ACTIVE",
- }),
+ filterState,
version: input.datasetVersion,
});
- if (!Boolean(items.length)) {
+ if (!Boolean(totalItems)) {
return {
isValid: false,
message: "Selected dataset is empty or all items are inactive.",
};
}
- const variablesMap = countValidDatasetItems(items, allVariables);
+ const variablesMap = await countDatasetItemVariableMatches({
+ projectId: input.projectId,
+ filterState,
+ version: input.datasetVersion,
+ variables: allVariables,
+ });
if (!Boolean(Object.keys(variablesMap).length)) {
return {
@@ -214,8 +203,8 @@ export const experimentsRouter = createTRPCRouter({
return {
isValid: true,
- totalItems: items.length,
- variablesMap: variablesMap,
+ totalItems,
+ variablesMap,
};
}),