Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.
Go agentagenticaichatbotdsh-pluginembeddingsevaluationgenerative-aigolangknowledge-basellmmulti-tenantollamaopenaiquestion-answeringragrerankingsemantic-searchvector-searchwiki
fix(docparser): keep every row and cell of a CSV upload in its table (#4435)
Lukasda3317039bdb
diff --git a/internal/infrastructure/docparser/builtin_converter.go b/internal/infrastructure/docparser/builtin_converter.goindex 0cf3860a5..318ff5dbd 100644--- a/internal/infrastructure/docparser/builtin_converter.go+++ b/internal/infrastructure/docparser/builtin_converter.go@@ -1,6 +1,7 @@package docparserimport (+ "bytes""context""encoding/csv""fmt"@@ -178,9 +179,14 @@ func ensureOriginalImageRef(req *types.ReadRequest, mdContent string, imageRefs}func csvToMarkdown(data []byte) (string, error) {- reader := csv.NewReader(strings.NewReader(string(data)))+ // Excel's "CSV UTF-8" export starts the file with a byte order mark.+ data = bytes.TrimPrefix(data, []byte("\xef\xbb\xbf"))+ reader := csv.NewReader(bytes.NewReader(data))reader.LazyQuotes = truereader.TrimLeadingSpace = true+ // Rows may differ in length. The default, the first row's length, made+ // one short or long row fail the whole file.+ reader.FieldsPerRecord = -1records, err := reader.ReadAll()if err != nil {@@ -190,34 +196,45 @@ func csvToMarkdown(data []byte) (string, error) {return "", nil}+ // The table is as wide as the widest row, so no field is dropped, and+ // shorter rows are padded with empty cells.+ width := 0+ for _, record := range records {+ width = max(width, len(record))+ }+var sb strings.Builder+ writeRow := func(record []string) {+ cells := make([]string, width)+ for i, cell := range record {+ cells[i] = csvCellReplacer.Replace(cell)+ }+ sb.WriteString("| ")+ sb.WriteString(strings.Join(cells, " | "))+ sb.WriteString(" |\n")+ }// Header row- header := records[0]- sb.WriteString("| ")- sb.WriteString(strings.Join(header, " | "))- sb.WriteString(" |\n")+ writeRow(records[0])// Separatorsb.WriteString("|")- for range header {+ for range width {sb.WriteString(" --- |")}sb.WriteString("\n")// Data rowsfor _, row := range records[1:] {- sb.WriteString("| ")- // Pad row if shorter than header- cells := make([]string, len(header))- for i := range cells {- if i < len(row) {- cells[i] = row[i]- }- }- sb.WriteString(strings.Join(cells, " | "))- sb.WriteString(" |\n")+ writeRow(row)}return sb.String(), nil}++// csvCellReplacer escapes what would end a GFM table cell or row: a pipe, and+// the line breaks of a quoted multi-line field. They become <br />, as the+// html-to-markdown table plugin writes them.+// Escape existing backslashes too, so a literal backslash before a pipe does+// not consume the pipe's escape and turn it back into a column separator.+var csvCellReplacer = strings.NewReplacer(`\`, `\\`, "|", `\|`, "\r\n", "<br />", "\n", "<br />", "\r", "<br />")diff --git a/internal/infrastructure/docparser/csv_converter_test.go b/internal/infrastructure/docparser/csv_converter_test.gonew file mode 100644index 000000000..c44c7a809--- /dev/null+++ b/internal/infrastructure/docparser/csv_converter_test.go@@ -0,0 +1,69 @@+package docparser++import "testing"++func TestCSVToMarkdown(t *testing.T) {+ cases := []struct {+ name string+ in string+ want string+ }{+ {+ name: "rectangular",+ in: "name,qty\npen,2\n",+ want: "| name | qty |\n| --- | --- |\n| pen | 2 |\n",+ },+ {+ // A short row failed the whole file with "wrong number of fields".+ name: "short row is padded",+ in: "name,qty,note\npen,2\nink,1,blue\n",+ want: "| name | qty | note |\n| --- | --- | --- |\n| pen | 2 | |\n| ink | 1 | blue |\n",+ },+ {+ name: "long row widens the table",+ in: "name,qty\npen,2,spare\n",+ want: "| name | qty | |\n| --- | --- | --- |\n| pen | 2 | spare |\n",+ },+ {+ name: "pipe in a cell is escaped",+ in: "expr,means\n\"a|b\",a or b\n",+ want: "| expr | means |\n| --- | --- |\n| a\\|b | a or b |\n",+ },+ {+ name: "backslash before pipe preserves both literal characters",+ in: "expr,note\na\\|b,literal\n",+ want: "| expr | note |\n| --- | --- |\n| " + `a\\\|b` + " | literal |\n",+ },+ {+ name: "multiple and trailing backslashes stay inside the cell",+ in: "expr,note\na\\\\|b\\,literal\n",+ want: "| expr | note |\n| --- | --- |\n| " + `a\\\\\|b\\` + " | literal |\n",+ },+ {+ name: "multi-line field stays in its row",+ in: "name,address\nAnn,\"1 Main St\r\nSpringfield\"\n",+ want: "| name | address |\n| --- | --- |\n| Ann | 1 Main St<br />Springfield |\n",+ },+ {+ name: "byte order mark is not part of the first header",+ in: "\xef\xbb\xbfname,qty\npen,2\n",+ want: "| name | qty |\n| --- | --- |\n| pen | 2 |\n",+ },+ {+ name: "empty file",+ in: "",+ want: "",+ },+ }+ for _, tc := range cases {+ t.Run(tc.name, func(t *testing.T) {+ got, err := csvToMarkdown([]byte(tc.in))+ if err != nil {+ t.Fatalf("csvToMarkdown(%q) error: %v", tc.in, err)+ }+ if got != tc.want {+ t.Errorf("csvToMarkdown(%q)\n got: %q\nwant: %q", tc.in, got, tc.want)+ }+ })+ }+}