Skip to content

Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.

fix(docparser): budget the CSV table render before padding it out (#4702)

ydflow657a3ffc88c7
diff --git a/internal/infrastructure/docparser/builtin_converter.go b/internal/infrastructure/docparser/builtin_converter.go
index 318ff5dbd..0a676f0cb 100644
--- a/internal/infrastructure/docparser/builtin_converter.go
+++ b/internal/infrastructure/docparser/builtin_converter.go
@@ -5,6 +5,7 @@ import (
"context"
"encoding/csv"
"fmt"
+ "io"
"net/http"
"path/filepath"
"strings"
@@ -203,6 +204,19 @@ func csvToMarkdown(data []byte) (string, error) {
width = max(width, len(record))
}
+ // Padding every row out to the widest row amplifies a ragged file
+ // quadratically: a few wide rows among many narrow ones turn
+ // rows × columns cells into output. Budget the exact rendered size
+ // before building the table and reject the blowup with an error
+ // instead of allocating it.
+ if size := projectedCSVTableSize(records, width); size > maxCSVMarkdownBytes {
+ return "", fmt.Errorf(
+ "csv table would render %d bytes (padding %d rows to %d columns) from %d bytes of input; "+
+ "the conversion limit is %d bytes — flatten or split the file before uploading",
+ size, len(records), width, len(data), maxCSVMarkdownBytes,
+ )
+ }
+
var sb strings.Builder
writeRow := func(record []string) {
cells := make([]string, width)
@@ -232,6 +246,31 @@ func csvToMarkdown(data []byte) (string, error) {
return sb.String(), nil
}
+// maxCSVMarkdownBytes caps the rendered size of one CSV upload's table.
+// A legitimate conversion renders roughly the input size plus table
+// syntax, and uploads default to 50 MB (MAX_FILE_SIZE_MB), so a table
+// anywhere near 64 MB means widest-row padding is amplifying a
+// pathological file rather than preserving its fields.
+const maxCSVMarkdownBytes = 64 << 20
+
+// projectedCSVTableSize returns the exact byte length of the GFM table
+// csvToMarkdown renders for these records padded to width cells —
+// including the escaping csvCellReplacer applies — without allocating
+// the output.
+func projectedCSVTableSize(records [][]string, width int) int64 {
+ // A rendered row is "| " + width cells joined by " | " + " |\n";
+ // the separator line is "|" + width times " --- |" + "\n".
+ total := int64(len("|") + width*len(" --- |") + len("\n"))
+ for _, record := range records {
+ total += int64(len("| ") + len(" |\n") + 3*(width-1))
+ for _, cell := range record {
+ n, _ := csvCellReplacer.WriteString(io.Discard, cell)
+ total += int64(n)
+ }
+ }
+ return total
+}
+
// csvCellReplacer escapes what would end a GFM table cell or row: a pipe, and
// the line breaks of a quoted multi-line field. They become <br />, as the
// html-to-markdown table plugin writes them.
diff --git a/internal/infrastructure/docparser/csv_converter_test.go b/internal/infrastructure/docparser/csv_converter_test.go
index c44c7a809..870a55861 100644
--- a/internal/infrastructure/docparser/csv_converter_test.go
+++ b/internal/infrastructure/docparser/csv_converter_test.go
@@ -1,6 +1,10 @@
package docparser
-import "testing"
+import (
+ "encoding/csv"
+ "strings"
+ "testing"
+)
func TestCSVToMarkdown(t *testing.T) {
cases := []struct {
@@ -67,3 +71,61 @@ func TestCSVToMarkdown(t *testing.T) {
})
}
}
+
+func TestCSVToMarkdownRejectsExcessivePadding(t *testing.T) {
+ // 20,000 one-column rows plus one 20,000-column row: 80 KB of input
+ // whose table would be padded into over a gigabyte of mostly-empty
+ // cells. The budget must reject it before any of that is rendered.
+ var in strings.Builder
+ in.WriteString(strings.Repeat("x\n", 20000))
+ in.WriteString(strings.Repeat(",", 19999))
+ in.WriteString("x\n")
+
+ md, err := csvToMarkdown([]byte(in.String()))
+ if err == nil {
+ t.Fatalf("csvToMarkdown rendered %d bytes, want a size-limit error", len(md))
+ }
+ if md != "" {
+ t.Errorf("csvToMarkdown on rejection returned %d bytes of output, want none", len(md))
+ }
+ if !strings.Contains(err.Error(), "would render") || !strings.Contains(err.Error(), "limit") {
+ t.Errorf("csvToMarkdown error %q does not name the rendered size and the limit", err)
+ }
+}
+
+func TestProjectedCSVTableSizeMatchesRender(t *testing.T) {
+ // Cells chosen to exercise every csvCellReplacer escape: a pipe, a
+ // backslash before it, a quoted newline, and a lone carriage return.
+ in := "a|b,\"x\\|y\nz\",w\r\n1,p|q,\"lone\rcr\"\n"
+
+ records, width := parseCSVForBudget(t, in)
+ projected := projectedCSVTableSize(records, width)
+
+ md, err := csvToMarkdown([]byte(in))
+ if err != nil {
+ t.Fatalf("csvToMarkdown(%q) error: %v", in, err)
+ }
+ if int64(len(md)) != projected {
+ t.Errorf("projectedCSVTableSize = %d, rendered table is %d bytes", projected, len(md))
+ }
+}
+
+// parseCSVForBudget re-parses input with the same reader settings
+// csvToMarkdown uses, so the projection can be checked against a real
+// render.
+func parseCSVForBudget(t *testing.T, in string) ([][]string, int) {
+ t.Helper()
+ reader := csv.NewReader(strings.NewReader(in))
+ reader.LazyQuotes = true
+ reader.TrimLeadingSpace = true
+ reader.FieldsPerRecord = -1
+ records, err := reader.ReadAll()
+ if err != nil {
+ t.Fatalf("parseCSVForBudget(%q) error: %v", in, err)
+ }
+ width := 0
+ for _, record := range records {
+ width = max(width, len(record))
+ }
+ return records, width
+}