🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
feat(experiments): name the expected variables when a dataset does not match (#17275)
Nikita Kabardina8718648bd92
diff --git a/web/src/features/experiments/fns/describeVariableMismatch.clienttest.ts b/web/src/features/experiments/fns/describeVariableMismatch.clienttest.tsnew file mode 100644index 0000000000..6f6c47c69f--- /dev/null+++ b/web/src/features/experiments/fns/describeVariableMismatch.clienttest.ts@@ -0,0 +1,17 @@+import { describe, expect, it } from "vitest";++import { describeVariableMismatch } from "./describeVariableMismatch";++describe("describeVariableMismatch", () => {+ it("names the expected variables once each", () => {+ expect(+ describeVariableMismatch(["country", "language", "country"]),+ ).toContain("this prompt expects: country, language.");+ });++ it("stays a sentence when there is nothing to name", () => {+ expect(describeVariableMismatch([])).toBe(+ "No dataset item contains any of the variables this prompt expects.",+ );+ });+});diff --git a/web/src/features/experiments/fns/describeVariableMismatch.ts b/web/src/features/experiments/fns/describeVariableMismatch.tsnew file mode 100644index 0000000000..5ff3c1bf73--- /dev/null+++ b/web/src/features/experiments/fns/describeVariableMismatch.ts@@ -0,0 +1,16 @@+/**+ * Message for the Dataset step's "Invalid configuration" banner when no dataset+ * item provides any of the prompt's variables.+ *+ * Names the variables so the reader can compare them against their dataset+ * without opening the "Expected columns" popover.+ */+export function describeVariableMismatch(variables: string[]): string {+ // Placeholder names are not deduped upstream, and a placeholder may repeat a+ // mustache variable.+ const named = [...new Set(variables)].join(", ");++ return named+ ? `No dataset item contains any of the variables this prompt expects: ${named}. Add them as top-level input keys, or choose a dataset that has them.`+ : "No dataset item contains any of the variables this prompt expects.";+}diff --git a/web/src/features/experiments/server/router.ts b/web/src/features/experiments/server/router.tsindex 0654d5386f..8cbdfc8bea 100644--- a/web/src/features/experiments/server/router.ts+++ b/web/src/features/experiments/server/router.ts@@ -56,6 +56,7 @@ import {} from "@langfuse/shared";import { throwIfNoProjectAccess } from "@/src/features/rbac";import { aggregateScores } from "@/src/features/scores/lib/aggregateScores";+import { describeVariableMismatch } from "@/src/features/experiments/fns/describeVariableMismatch";const ExperimentFilterOptions = z.object({projectId: z.string(),@@ -197,7 +198,7 @@ export const experimentsRouter = createTRPCRouter({if (!Boolean(Object.keys(variablesMap).length)) {return {isValid: false,- message: "No dataset item contains any variables.",+ message: describeVariableMismatch(allVariables),};}