Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.
Go agentagenticaichatbotdsh-pluginembeddingsevaluationgenerative-aigolangknowledge-basellmmulti-tenantollamaopenaiquestion-answeringragrerankingsemantic-searchvector-searchwiki
fix(docparser): budget the CSV table render before padding it out (#4702)
ydflow657a3ffc88c7
diff --git a/internal/infrastructure/docparser/builtin_converter.go b/internal/infrastructure/docparser/builtin_converter.goindex 318ff5dbd..0a676f0cb 100644--- a/internal/infrastructure/docparser/builtin_converter.go+++ b/internal/infrastructure/docparser/builtin_converter.go@@ -5,6 +5,7 @@ import ("context""encoding/csv""fmt"+ "io""net/http""path/filepath""strings"@@ -203,6 +204,19 @@ func csvToMarkdown(data []byte) (string, error) {width = max(width, len(record))}+ // Padding every row out to the widest row amplifies a ragged file+ // quadratically: a few wide rows among many narrow ones turn+ // rows × columns cells into output. Budget the exact rendered size+ // before building the table and reject the blowup with an error+ // instead of allocating it.+ if size := projectedCSVTableSize(records, width); size > maxCSVMarkdownBytes {+ return "", fmt.Errorf(+ "csv table would render %d bytes (padding %d rows to %d columns) from %d bytes of input; "++ "the conversion limit is %d bytes — flatten or split the file before uploading",+ size, len(records), width, len(data), maxCSVMarkdownBytes,+ )+ }+var sb strings.BuilderwriteRow := func(record []string) {cells := make([]string, width)@@ -232,6 +246,31 @@ func csvToMarkdown(data []byte) (string, error) {return sb.String(), nil}+// maxCSVMarkdownBytes caps the rendered size of one CSV upload's table.+// A legitimate conversion renders roughly the input size plus table+// syntax, and uploads default to 50 MB (MAX_FILE_SIZE_MB), so a table+// anywhere near 64 MB means widest-row padding is amplifying a+// pathological file rather than preserving its fields.+const maxCSVMarkdownBytes = 64 << 20++// projectedCSVTableSize returns the exact byte length of the GFM table+// csvToMarkdown renders for these records padded to width cells —+// including the escaping csvCellReplacer applies — without allocating+// the output.+func projectedCSVTableSize(records [][]string, width int) int64 {+ // A rendered row is "| " + width cells joined by " | " + " |\n";+ // the separator line is "|" + width times " --- |" + "\n".+ total := int64(len("|") + width*len(" --- |") + len("\n"))+ for _, record := range records {+ total += int64(len("| ") + len(" |\n") + 3*(width-1))+ for _, cell := range record {+ n, _ := csvCellReplacer.WriteString(io.Discard, cell)+ total += int64(n)+ }+ }+ return total+}+// csvCellReplacer escapes what would end a GFM table cell or row: a pipe, and// the line breaks of a quoted multi-line field. They become <br />, as the// html-to-markdown table plugin writes them.diff --git a/internal/infrastructure/docparser/csv_converter_test.go b/internal/infrastructure/docparser/csv_converter_test.goindex c44c7a809..870a55861 100644--- a/internal/infrastructure/docparser/csv_converter_test.go+++ b/internal/infrastructure/docparser/csv_converter_test.go@@ -1,6 +1,10 @@package docparser-import "testing"+import (+ "encoding/csv"+ "strings"+ "testing"+)func TestCSVToMarkdown(t *testing.T) {cases := []struct {@@ -67,3 +71,61 @@ func TestCSVToMarkdown(t *testing.T) {})}}++func TestCSVToMarkdownRejectsExcessivePadding(t *testing.T) {+ // 20,000 one-column rows plus one 20,000-column row: 80 KB of input+ // whose table would be padded into over a gigabyte of mostly-empty+ // cells. The budget must reject it before any of that is rendered.+ var in strings.Builder+ in.WriteString(strings.Repeat("x\n", 20000))+ in.WriteString(strings.Repeat(",", 19999))+ in.WriteString("x\n")++ md, err := csvToMarkdown([]byte(in.String()))+ if err == nil {+ t.Fatalf("csvToMarkdown rendered %d bytes, want a size-limit error", len(md))+ }+ if md != "" {+ t.Errorf("csvToMarkdown on rejection returned %d bytes of output, want none", len(md))+ }+ if !strings.Contains(err.Error(), "would render") || !strings.Contains(err.Error(), "limit") {+ t.Errorf("csvToMarkdown error %q does not name the rendered size and the limit", err)+ }+}++func TestProjectedCSVTableSizeMatchesRender(t *testing.T) {+ // Cells chosen to exercise every csvCellReplacer escape: a pipe, a+ // backslash before it, a quoted newline, and a lone carriage return.+ in := "a|b,\"x\\|y\nz\",w\r\n1,p|q,\"lone\rcr\"\n"++ records, width := parseCSVForBudget(t, in)+ projected := projectedCSVTableSize(records, width)++ md, err := csvToMarkdown([]byte(in))+ if err != nil {+ t.Fatalf("csvToMarkdown(%q) error: %v", in, err)+ }+ if int64(len(md)) != projected {+ t.Errorf("projectedCSVTableSize = %d, rendered table is %d bytes", projected, len(md))+ }+}++// parseCSVForBudget re-parses input with the same reader settings+// csvToMarkdown uses, so the projection can be checked against a real+// render.+func parseCSVForBudget(t *testing.T, in string) ([][]string, int) {+ t.Helper()+ reader := csv.NewReader(strings.NewReader(in))+ reader.LazyQuotes = true+ reader.TrimLeadingSpace = true+ reader.FieldsPerRecord = -1+ records, err := reader.ReadAll()+ if err != nil {+ t.Fatalf("parseCSVForBudget(%q) error: %v", in, err)+ }+ width := 0+ for _, record := range records {+ width = max(width, len(record))+ }+ return records, width+}