🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
feat(csv): enhance CSV parsing with error handling for large and malformed files in preview mode (#16476)
marliessophiee5164beb27e4
diff --git a/web/src/features/datasets/lib/csv/helpers.clienttest.ts b/web/src/features/datasets/lib/csv/helpers.clienttest.tsindex 7c6720ede4..f3d7bc66e8 100644--- a/web/src/features/datasets/lib/csv/helpers.clienttest.ts+++ b/web/src/features/datasets/lib/csv/helpers.clienttest.ts@@ -43,4 +43,25 @@ describe("CSV dataset parsing", () => {expect(parseValue("true")).toBe(true);expect(parseValue("FALSE")).toBe(false);});++ it("explains the preview truncation when a >2MB file has a quoted cell cut at the preview boundary", async () => {+ const hugeCell = "A".repeat(3 * 1024 * 1024);+ const file = new File([`input\n"${hugeCell}"\n`], "large.csv", {+ type: "text/csv",+ });++ await expect(+ parseCsvClient(file, { isPreview: true, collectSamples: true }),+ ).rejects.toThrow(/single dataset items are too large/);+ });++ it("keeps the raw parser error for genuinely malformed small files", async () => {+ const file = new File(['input\n"unclosed\n'], "malformed.csv", {+ type: "text/csv",+ });++ await expect(+ parseCsvClient(file, { isPreview: true, collectSamples: true }),+ ).rejects.toThrow(/Quote Not Closed/);+ });});diff --git a/web/src/features/datasets/lib/csv/helpers.ts b/web/src/features/datasets/lib/csv/helpers.tsindex 6dad8ce66b..844c838415 100644--- a/web/src/features/datasets/lib/csv/helpers.ts+++ b/web/src/features/datasets/lib/csv/helpers.ts@@ -108,6 +108,20 @@ export async function parseCsvClient(options: ParseOptions,): Promise<CsvPreviewResult> {return new Promise((resolve, reject) => {+ const isTruncatedPreview =+ Boolean(options.isPreview) && file.size > PREVIEW_FILE_SIZE_BYTES;+ const rejectWithTruncationHint = (error: Error) => {+ if (isTruncatedPreview && error.message.includes("Quote Not Closed")) {+ reject(+ new Error(+ "The file's single dataset items are too large. CSV imports are intended for text and structured JSON dataset items. Use the UI item editor or API/SDKs for multi-modal or very large dataset items: https://langfuse.com/docs/evaluation/experiments/datasets#multi-modal-dataset-items",+ ),+ );+ return;+ }+ reject(error);+ };+const fileToRead = options.isPreview? file.slice(0, PREVIEW_FILE_SIZE_BYTES): file;@@ -115,7 +129,12 @@ export async function parseCsvClient(const reader = new FileReader();reader.onload = async () => {- const parser = await createParser(options, resolve, reject, file.name);+ const parser = await createParser(+ options,+ resolve,+ rejectWithTruncationHint,+ file.name,+ );parser.write(reader.result as string);parser.end();};