Skip to content

Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.

fix(docparser): keep every row and cell of a CSV upload in its table (#4435)

Lukasda3317039bdb
diff --git a/internal/infrastructure/docparser/builtin_converter.go b/internal/infrastructure/docparser/builtin_converter.go
index 0cf3860a5..318ff5dbd 100644
--- a/internal/infrastructure/docparser/builtin_converter.go
+++ b/internal/infrastructure/docparser/builtin_converter.go
@@ -1,6 +1,7 @@
package docparser
import (
+ "bytes"
"context"
"encoding/csv"
"fmt"
@@ -178,9 +179,14 @@ func ensureOriginalImageRef(req *types.ReadRequest, mdContent string, imageRefs
}
func csvToMarkdown(data []byte) (string, error) {
- reader := csv.NewReader(strings.NewReader(string(data)))
+ // Excel's "CSV UTF-8" export starts the file with a byte order mark.
+ data = bytes.TrimPrefix(data, []byte("\xef\xbb\xbf"))
+ reader := csv.NewReader(bytes.NewReader(data))
reader.LazyQuotes = true
reader.TrimLeadingSpace = true
+ // Rows may differ in length. The default, the first row's length, made
+ // one short or long row fail the whole file.
+ reader.FieldsPerRecord = -1
records, err := reader.ReadAll()
if err != nil {
@@ -190,34 +196,45 @@ func csvToMarkdown(data []byte) (string, error) {
return "", nil
}
+ // The table is as wide as the widest row, so no field is dropped, and
+ // shorter rows are padded with empty cells.
+ width := 0
+ for _, record := range records {
+ width = max(width, len(record))
+ }
+
var sb strings.Builder
+ writeRow := func(record []string) {
+ cells := make([]string, width)
+ for i, cell := range record {
+ cells[i] = csvCellReplacer.Replace(cell)
+ }
+ sb.WriteString("| ")
+ sb.WriteString(strings.Join(cells, " | "))
+ sb.WriteString(" |\n")
+ }
// Header row
- header := records[0]
- sb.WriteString("| ")
- sb.WriteString(strings.Join(header, " | "))
- sb.WriteString(" |\n")
+ writeRow(records[0])
// Separator
sb.WriteString("|")
- for range header {
+ for range width {
sb.WriteString(" --- |")
}
sb.WriteString("\n")
// Data rows
for _, row := range records[1:] {
- sb.WriteString("| ")
- // Pad row if shorter than header
- cells := make([]string, len(header))
- for i := range cells {
- if i < len(row) {
- cells[i] = row[i]
- }
- }
- sb.WriteString(strings.Join(cells, " | "))
- sb.WriteString(" |\n")
+ writeRow(row)
}
return sb.String(), nil
}
+
+// csvCellReplacer escapes what would end a GFM table cell or row: a pipe, and
+// the line breaks of a quoted multi-line field. They become <br />, as the
+// html-to-markdown table plugin writes them.
+// Escape existing backslashes too, so a literal backslash before a pipe does
+// not consume the pipe's escape and turn it back into a column separator.
+var csvCellReplacer = strings.NewReplacer(`\`, `\\`, "|", `\|`, "\r\n", "<br />", "\n", "<br />", "\r", "<br />")
diff --git a/internal/infrastructure/docparser/csv_converter_test.go b/internal/infrastructure/docparser/csv_converter_test.go
new file mode 100644
index 000000000..c44c7a809
--- /dev/null
+++ b/internal/infrastructure/docparser/csv_converter_test.go
@@ -0,0 +1,69 @@
+package docparser
+
+import "testing"
+
+func TestCSVToMarkdown(t *testing.T) {
+ cases := []struct {
+ name string
+ in string
+ want string
+ }{
+ {
+ name: "rectangular",
+ in: "name,qty\npen,2\n",
+ want: "| name | qty |\n| --- | --- |\n| pen | 2 |\n",
+ },
+ {
+ // A short row failed the whole file with "wrong number of fields".
+ name: "short row is padded",
+ in: "name,qty,note\npen,2\nink,1,blue\n",
+ want: "| name | qty | note |\n| --- | --- | --- |\n| pen | 2 | |\n| ink | 1 | blue |\n",
+ },
+ {
+ name: "long row widens the table",
+ in: "name,qty\npen,2,spare\n",
+ want: "| name | qty | |\n| --- | --- | --- |\n| pen | 2 | spare |\n",
+ },
+ {
+ name: "pipe in a cell is escaped",
+ in: "expr,means\n\"a|b\",a or b\n",
+ want: "| expr | means |\n| --- | --- |\n| a\\|b | a or b |\n",
+ },
+ {
+ name: "backslash before pipe preserves both literal characters",
+ in: "expr,note\na\\|b,literal\n",
+ want: "| expr | note |\n| --- | --- |\n| " + `a\\\|b` + " | literal |\n",
+ },
+ {
+ name: "multiple and trailing backslashes stay inside the cell",
+ in: "expr,note\na\\\\|b\\,literal\n",
+ want: "| expr | note |\n| --- | --- |\n| " + `a\\\\\|b\\` + " | literal |\n",
+ },
+ {
+ name: "multi-line field stays in its row",
+ in: "name,address\nAnn,\"1 Main St\r\nSpringfield\"\n",
+ want: "| name | address |\n| --- | --- |\n| Ann | 1 Main St<br />Springfield |\n",
+ },
+ {
+ name: "byte order mark is not part of the first header",
+ in: "\xef\xbb\xbfname,qty\npen,2\n",
+ want: "| name | qty |\n| --- | --- |\n| pen | 2 |\n",
+ },
+ {
+ name: "empty file",
+ in: "",
+ want: "",
+ },
+ }
+ for _, tc := range cases {
+ t.Run(tc.name, func(t *testing.T) {
+ got, err := csvToMarkdown([]byte(tc.in))
+ if err != nil {
+ t.Fatalf("csvToMarkdown(%q) error: %v", tc.in, err)
+ }
+ if got != tc.want {
+ t.Errorf("csvToMarkdown(%q)\n got: %q\nwant: %q", tc.in, got, tc.want)
+ }
+ })
+ }
+}