Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

feat(experiments): name the expected variables when a dataset does not match (#17275)

Nikita Kabardina8718648bd92
diff --git a/web/src/features/experiments/fns/describeVariableMismatch.clienttest.ts b/web/src/features/experiments/fns/describeVariableMismatch.clienttest.ts
new file mode 100644
index 0000000000..6f6c47c69f
--- /dev/null
+++ b/web/src/features/experiments/fns/describeVariableMismatch.clienttest.ts
@@ -0,0 +1,17 @@
+import { describe, expect, it } from "vitest";
+
+import { describeVariableMismatch } from "./describeVariableMismatch";
+
+describe("describeVariableMismatch", () => {
+ it("names the expected variables once each", () => {
+ expect(
+ describeVariableMismatch(["country", "language", "country"]),
+ ).toContain("this prompt expects: country, language.");
+ });
+
+ it("stays a sentence when there is nothing to name", () => {
+ expect(describeVariableMismatch([])).toBe(
+ "No dataset item contains any of the variables this prompt expects.",
+ );
+ });
+});
diff --git a/web/src/features/experiments/fns/describeVariableMismatch.ts b/web/src/features/experiments/fns/describeVariableMismatch.ts
new file mode 100644
index 0000000000..5ff3c1bf73
--- /dev/null
+++ b/web/src/features/experiments/fns/describeVariableMismatch.ts
@@ -0,0 +1,16 @@
+/**
+ * Message for the Dataset step's "Invalid configuration" banner when no dataset
+ * item provides any of the prompt's variables.
+ *
+ * Names the variables so the reader can compare them against their dataset
+ * without opening the "Expected columns" popover.
+ */
+export function describeVariableMismatch(variables: string[]): string {
+ // Placeholder names are not deduped upstream, and a placeholder may repeat a
+ // mustache variable.
+ const named = [...new Set(variables)].join(", ");
+
+ return named
+ ? `No dataset item contains any of the variables this prompt expects: ${named}. Add them as top-level input keys, or choose a dataset that has them.`
+ : "No dataset item contains any of the variables this prompt expects.";
+}
diff --git a/web/src/features/experiments/server/router.ts b/web/src/features/experiments/server/router.ts
index 0654d5386f..8cbdfc8bea 100644
--- a/web/src/features/experiments/server/router.ts
+++ b/web/src/features/experiments/server/router.ts
@@ -56,6 +56,7 @@ import {
} from "@langfuse/shared";
import { throwIfNoProjectAccess } from "@/src/features/rbac";
import { aggregateScores } from "@/src/features/scores/lib/aggregateScores";
+import { describeVariableMismatch } from "@/src/features/experiments/fns/describeVariableMismatch";
const ExperimentFilterOptions = z.object({
projectId: z.string(),
@@ -197,7 +198,7 @@ export const experimentsRouter = createTRPCRouter({
if (!Boolean(Object.keys(variablesMap).length)) {
return {
isValid: false,
- message: "No dataset item contains any variables.",
+ message: describeVariableMismatch(allVariables),
};
}