Skip to content

Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.

fix(docparser): 限制 CSV 转换膨胀并支持取消 (#4485)

Jarvisb595e874eaba
diff --git a/internal/application/service/knowledge_process.go b/internal/application/service/knowledge_process.go
index fa309944e..e0217a17d 100644
--- a/internal/application/service/knowledge_process.go
+++ b/internal/application/service/knowledge_process.go
@@ -4411,7 +4411,7 @@ func (s *knowledgeService) convert(
result, err := s.callDocReaderWithTimeout(ctx, reader, req)
// Parser resource limits are permanent document rejections, not transient reader failures.
- if errors.Is(err, anydoc.ErrResourceLimit) {
+ if errors.Is(err, docparser.ErrCSVMarkdownTooLarge) || errors.Is(err, anydoc.ErrResourceLimit) {
result, err = &types.ReadResult{Error: err.Error()}, nil
}
if err != nil {
diff --git a/internal/application/service/knowledge_process_span_retry_test.go b/internal/application/service/knowledge_process_span_retry_test.go
index b0a0be307..a5ea6d48b 100644
--- a/internal/application/service/knowledge_process_span_retry_test.go
+++ b/internal/application/service/knowledge_process_span_retry_test.go
@@ -10,6 +10,7 @@ import (
"testing/iotest"
werrors "github.com/Tencent/WeKnora/internal/errors"
+ "github.com/Tencent/WeKnora/internal/infrastructure/docparser"
"github.com/Tencent/WeKnora/internal/infrastructure/docparser/anydoc"
"github.com/Tencent/WeKnora/internal/types"
"github.com/Tencent/WeKnora/internal/types/interfaces"
@@ -61,6 +62,10 @@ func TestConvertFailureSpanMatchesTaskOutcome(t *testing.T) {
{name: "file fetch fails", getErr: transient, retryable: true, code: werrors.ErrCodeDocReaderParseFailed},
{name: "file read fails", readErr: transient, retryable: true, code: werrors.ErrCodeDocReaderParseFailed},
{name: "reader call fails", rpcErr: transient, retryable: true, code: werrors.ErrCodeDocReaderParseFailed},
+ {
+ name: "CSV expansion budget exceeded", rpcErr: docparser.ErrCSVMarkdownTooLarge,
+ code: werrors.ErrCodeDocReaderParseFailed,
+ },
} {
for _, retried := range []int{0, 3} {
name := tc.name + "/retry_available"
diff --git a/internal/application/service/temporary_document_csv_limits_test.go b/internal/application/service/temporary_document_csv_limits_test.go
new file mode 100644
index 000000000..1295e7f9b
--- /dev/null
+++ b/internal/application/service/temporary_document_csv_limits_test.go
@@ -0,0 +1,65 @@
+package service
+
+import (
+ "context"
+ "encoding/json"
+ "errors"
+ "strconv"
+ "strings"
+ "testing"
+ "time"
+
+ "github.com/Tencent/WeKnora/internal/types"
+ "github.com/Tencent/WeKnora/internal/types/interfaces"
+ "github.com/hibiken/asynq"
+ "github.com/stretchr/testify/require"
+)
+
+type csvLimitTemporaryRepo struct {
+ interfaces.TemporaryDocumentRepository
+ document *types.TemporaryDocument
+ message string
+ saveErr error
+}
+
+func (r *csvLimitTemporaryRepo) GetByID(context.Context, uint64, string) (*types.TemporaryDocument, error) {
+ return r.document, nil
+}
+
+func (r *csvLimitTemporaryRepo) MarkProcessing(context.Context, uint64, string, time.Time) error {
+ return nil
+}
+
+func (r *csvLimitTemporaryRepo) MarkFailed(_ context.Context, _ uint64, _ string, message string) error {
+ r.message = message
+ return r.saveErr
+}
+
+func TestTemporaryDocumentCSVExpansionFailure(t *testing.T) {
+ for _, saveErr := range []error{nil, errors.New("database unavailable")} {
+ t.Run("save_failure="+strconv.FormatBool(saveErr != nil), func(t *testing.T) {
+ options, err := json.Marshal(types.TemporaryDocumentCreateOptions{ParserEngine: "simple"})
+ require.NoError(t, err)
+ doc := &types.TemporaryDocument{
+ ID: "doc", TenantID: 1, FileType: ".csv", FileName: "wide.csv", ResourceRef: "local://wide.csv",
+ ProcessingOptions: options,
+ }
+ repo := &csvLimitTemporaryRepo{document: doc, saveErr: saveErr}
+ data := []byte(strings.Repeat("x\n", 10000) + strings.Repeat(",", 9999) + "x\n")
+ svc := &temporaryDocumentService{
+ repo: repo,
+ fileService: &stagingFileService{files: map[string][]byte{doc.ResourceRef: data}},
+ tenantService: &stubTenantServiceForModelDelete{},
+ }
+ raw, err := json.Marshal(types.TemporaryDocumentTaskPayload{TenantID: 1, DocumentID: "doc"})
+ require.NoError(t, err)
+ err = svc.Process(context.Background(), asynq.NewTask("temporary-document", raw))
+ if saveErr == nil {
+ require.NoError(t, err)
+ } else {
+ require.ErrorIs(t, err, saveErr)
+ }
+ require.Contains(t, repo.message, "CSV conversion exceeds maximum Markdown size")
+ })
+ }
+}
diff --git a/internal/application/service/temporary_document_parse.go b/internal/application/service/temporary_document_parse.go
index 1605136c7..e21750983 100644
--- a/internal/application/service/temporary_document_parse.go
+++ b/internal/application/service/temporary_document_parse.go
@@ -3,6 +3,7 @@ package service
import (
"context"
"encoding/json"
+ "errors"
"fmt"
"io"
"os"
@@ -103,7 +104,8 @@ func (s *temporaryDocumentService) Process(ctx context.Context, task *asynq.Task
if parseErr != nil {
retryCount, hasRetryCount := asynq.GetRetryCount(ctx)
maxRetry, hasMaxRetry := asynq.GetMaxRetry(ctx)
- if hasRetryCount && hasMaxRetry && retryCount < maxRetry {
+ permanent := errors.Is(parseErr, docparser.ErrCSVMarkdownTooLarge)
+ if !permanent && hasRetryCount && hasMaxRetry && retryCount < maxRetry {
logger.Warnf(ctx, "temporary document parse will retry: document_id=%s attempt=%d/%d err=%v",
payload.DocumentID, retryCount+1, maxRetry+1, parseErr)
return parseErr
@@ -112,13 +114,16 @@ func (s *temporaryDocumentService) Process(ctx context.Context, task *asynq.Task
if len(message) > 2000 {
message = common.TruncateUTF8Bytes(message, 2000)
}
- _ = s.repo.MarkFailed(ctx, payload.TenantID, payload.DocumentID, message)
+ if err := s.repo.MarkFailed(ctx, payload.TenantID, payload.DocumentID, message); err != nil {
+ return fmt.Errorf("mark temporary document failed: %w", err)
+ }
logger.Errorf(ctx, "temporary document parse failed: document_id=%s err=%v", payload.DocumentID, parseErr)
- if hasRetryCount && hasMaxRetry {
+ if !permanent && hasRetryCount && hasMaxRetry {
return parseErr
}
- // Lite mode doesn't expose Asynq retry metadata in the context. Surface
- // a terminal state immediately instead of leaving the UI spinning.
+ // A permanent rejection is complete once its failed state is saved.
+ // Lite mode also lacks Asynq retry metadata, so surface a terminal state
+ // there instead of leaving the UI spinning.
return nil
}
content = common.CleanInvalidUTF8(content)
diff --git a/internal/infrastructure/docparser/builtin_converter.go b/internal/infrastructure/docparser/builtin_converter.go
index 46939fc60..2f20bc591 100644
--- a/internal/infrastructure/docparser/builtin_converter.go
+++ b/internal/infrastructure/docparser/builtin_converter.go
@@ -1,11 +1,8 @@
package docparser
import (
- "bytes"
"context"
- "encoding/csv"
"fmt"
- "io"
"net/http"
"net/url"
"path/filepath"
@@ -50,7 +47,7 @@ func IsSimpleFormat(fileType string) bool {
type SimpleFormatReader struct{}
// Read reads simple format files and returns markdown.
-func (b *SimpleFormatReader) Read(_ context.Context, req *types.ReadRequest) (*types.ReadResult, error) {
+func (b *SimpleFormatReader) Read(ctx context.Context, req *types.ReadRequest) (*types.ReadResult, error) {
ft := strings.ToLower(strings.TrimPrefix(req.FileType, "."))
if ft == "" {
ft = strings.TrimPrefix(strings.ToLower(filepath.Ext(req.FileName)), ".")
@@ -62,7 +59,7 @@ func (b *SimpleFormatReader) Read(_ context.Context, req *types.ReadRequest) (*t
case ft == "txt" || ft == "text":
return &types.ReadResult{MarkdownContent: string(req.FileContent)}, nil
case ft == "csv":
- md, err := csvToMarkdown(req.FileContent)
+ md, err := csvToMarkdownContext(ctx, req.FileContent, maxCSVMarkdownBytes)
if err != nil {
return nil, fmt.Errorf("csv conversion failed: %w", err)
}
@@ -189,98 +186,6 @@ func ensureOriginalImageRef(req *types.ReadRequest, mdContent string, imageRefs
return mdContent, imageRefs
}
-func csvToMarkdown(data []byte) (string, error) {
- // Excel's "CSV UTF-8" export starts the file with a byte order mark.
- data = bytes.TrimPrefix(data, []byte("\xef\xbb\xbf"))
- reader := csv.NewReader(bytes.NewReader(data))
- reader.LazyQuotes = true
- reader.TrimLeadingSpace = true
- // Rows may differ in length. The default, the first row's length, made
- // one short or long row fail the whole file.
- reader.FieldsPerRecord = -1
-
- records, err := reader.ReadAll()
- if err != nil {
- return "", err
- }
- if len(records) == 0 {
- return "", nil
- }
-
- // The table is as wide as the widest row, so no field is dropped, and
- // shorter rows are padded with empty cells.
- width := 0
- for _, record := range records {
- width = max(width, len(record))
- }
-
- // Padding every row out to the widest row amplifies a ragged file
- // quadratically: a few wide rows among many narrow ones turn
- // rows × columns cells into output. Budget the exact rendered size
- // before building the table and reject the blowup with an error
- // instead of allocating it.
- if size := projectedCSVTableSize(records, width); size > maxCSVMarkdownBytes {
- return "", fmt.Errorf(
- "csv table would render %d bytes (padding %d rows to %d columns) from %d bytes of input; "+
- "the conversion limit is %d bytes — flatten or split the file before uploading",
- size, len(records), width, len(data), maxCSVMarkdownBytes,
- )
- }
-
- var sb strings.Builder
- writeRow := func(record []string) {
- cells := make([]string, width)
- for i, cell := range record {
- cells[i] = csvCellReplacer.Replace(cell)
- }
- sb.WriteString("| ")
- sb.WriteString(strings.Join(cells, " | "))
- sb.WriteString(" |\n")
- }
-
- // Header row
- writeRow(records[0])
-
- // Separator
- sb.WriteString("|")
- for range width {
- sb.WriteString(" --- |")
- }
- sb.WriteString("\n")
-
- // Data rows
- for _, row := range records[1:] {
- writeRow(row)
- }
-
- return sb.String(), nil
-}
-
-// maxCSVMarkdownBytes caps the rendered size of one CSV upload's table.
-// A legitimate conversion renders roughly the input size plus table
-// syntax, and uploads default to 50 MB (MAX_FILE_SIZE_MB), so a table
-// anywhere near 64 MB means widest-row padding is amplifying a
-// pathological file rather than preserving its fields.
-const maxCSVMarkdownBytes = 64 << 20
-
-// projectedCSVTableSize returns the exact byte length of the GFM table
-// csvToMarkdown renders for these records padded to width cells —
-// including the escaping csvCellReplacer applies — without allocating
-// the output.
-func projectedCSVTableSize(records [][]string, width int) int64 {
- // A rendered row is "| " + width cells joined by " | " + " |\n";
- // the separator line is "|" + width times " --- |" + "\n".
- total := int64(len("|") + width*len(" --- |") + len("\n"))
- for _, record := range records {
- total += int64(len("| ") + len(" |\n") + 3*(width-1))
- for _, cell := range record {
- n, _ := csvCellReplacer.WriteString(io.Discard, cell)
- total += int64(n)
- }
- }
- return total
-}
-
// csvCellReplacer escapes what would end a GFM table cell or row: a pipe, and
// the line breaks of a quoted multi-line field. They become <br />, as the
// html-to-markdown table plugin writes them.
diff --git a/internal/infrastructure/docparser/csv_converter.go b/internal/infrastructure/docparser/csv_converter.go
new file mode 100644
index 000000000..6b68cf481
--- /dev/null
+++ b/internal/infrastructure/docparser/csv_converter.go
@@ -0,0 +1,213 @@
+package docparser
+
+import (
+ "bytes"
+ "context"
+ "encoding/csv"
+ "errors"
+ "fmt"
+ "io"
+ "strings"
+)
+
+// Match the maximum size of a single Markdown entry extracted from a parser
+// result package. The upload cap alone cannot bound padding of ragged CSV rows.
+const maxCSVMarkdownBytes = maxZipEntryBytes
+
+// ErrCSVMarkdownTooLarge marks deterministic CSV expansion failures.
+// Callers should report the failure without retrying the same document.
+var ErrCSVMarkdownTooLarge = errors.New("CSV conversion exceeds maximum Markdown size")
+
+func csvToMarkdown(data []byte) (string, error) {
+ return csvToMarkdownContext(context.Background(), data, maxCSVMarkdownBytes)
+}
+
+func csvToMarkdownContext(ctx context.Context, data []byte, limit int) (string, error) {
+ if err := ctx.Err(); err != nil {
+ return "", err
+ }
+ // Excel's "CSV UTF-8" export starts with a byte order mark.
+ data = bytes.TrimPrefix(data, []byte("\xef\xbb\xbf"))
+ rows, width, contentBytes := 0, 0, 0
+ reader := newCSVReader(ctx, data)
+ for {
+ if err := ctx.Err(); err != nil {
+ return "", err
+ }
+ record, err := reader.Read()
+ if err == io.EOF {
+ break
+ }
+ if err != nil {
+ return "", err
+ }
+ rows++
+ width = max(width, len(record))
+ for _, cell := range record {
+ size, err := csvEscapedSize(ctx, cell, limit-contentBytes)
+ if errors.Is(err, ErrCSVMarkdownTooLarge) {
+ return "", csvSizeError(limit)
+ }
+ if err != nil {
+ return "", err
+ }
+ contentBytes += size
+ }
+ // Check after every row: a late wide row also widens all preceding rows.
+ if _, err := csvMarkdownSize(rows, width, contentBytes, limit); err != nil {
+ return "", err
+ }
+ }
+ if rows == 0 {
+ return "", nil
+ }
+ size, err := csvMarkdownSize(rows, width, contentBytes, limit)
+ if err != nil {
+ return "", err
+ }
+ if err := ctx.Err(); err != nil {
+ return "", err
+ }
+
+ // Re-read the input rather than retain every record. Grow only after the
+ // entire output has passed preflight; rejected inputs allocate no table.
+ var out strings.Builder
+ out.Grow(size)
+ reader = newCSVReader(ctx, data)
+ for row := 0; row < rows; row++ {
+ if err := ctx.Err(); err != nil {
+ return "", err
+ }
+ record, err := reader.Read()
+ if err != nil {
+ return "", err
+ }
+ out.WriteString("| ")
+ for col := 0; col < width; col++ {
+ if col%1024 == 0 {
+ if err := ctx.Err(); err != nil {
+ return "", err
+ }
+ }
+ if col > 0 {
+ out.WriteString(" | ")
+ }
+ if col < len(record) {
+ writer := csvContextWriter{ctx: ctx, out: &out}
+ if _, err := csvCellReplacer.WriteString(writer, record[col]); err != nil {
+ return "", err
+ }
+ }
+ }
+ out.WriteString(" |\n")
+ if row == 0 {
+ out.WriteByte('|')
+ for col := 0; col < width; col++ {
+ if col%1024 == 0 {
+ if err := ctx.Err(); err != nil {
+ return "", err
+ }
+ }
+ out.WriteString(" --- |")
+ }
+ out.WriteByte('\n')
+ }
+ }
+ if err := ctx.Err(); err != nil {
+ return "", err
+ }
+ return out.String(), nil
+}
+
+func csvSizeError(limit int) error {
+ return fmt.Errorf("%w: table would render beyond the limit (%d bytes); reduce the number of rows or columns",
+ ErrCSVMarkdownTooLarge, limit)
+}
+
+// A row needs 3*width+2 framing bytes; the separator needs 6*width+2.
+// Divide before multiplying so even extreme row/column counts cannot overflow.
+func csvMarkdownSize(rows, width, contentBytes, limit int) (int, error) {
+ if limit < 2 || width > (limit-2)/6 {
+ return 0, csvSizeError(limit)
+ }
+ size := 6*width + 2
+ if contentBytes > limit-size {
+ return 0, csvSizeError(limit)
+ }
+ size += contentBytes
+ rowBytes := 3*width + 2
+ if rows > (limit-size)/rowBytes {
+ return 0, csvSizeError(limit)
+ }
+ return size + rows*rowBytes, nil
+}
+
+// Count the exact bytes csvCellReplacer will emit without allocating the
+// escaped string. CRLF is replaced once, matching strings.Replacer's ordering.
+func csvEscapedSize(ctx context.Context, cell string, remaining int) (int, error) {
+ size := 0
+ for i := 0; i < len(cell); i++ {
+ if i%4096 == 0 {
+ if err := ctx.Err(); err != nil {
+ return 0, err
+ }
+ }
+ n := 1
+ switch cell[i] {
+ case '\\', '|':
+ n = 2
+ case '\r':
+ n = 6
+ if i+1 < len(cell) && cell[i+1] == '\n' {
+ i++
+ }
+ case '\n':
+ n = 6
+ }
+ if n > remaining-size {
+ return 0, ErrCSVMarkdownTooLarge
+ }
+ size += n
+ }
+ return size, nil
+}
+
+func newCSVReader(ctx context.Context, data []byte) *csv.Reader {
+ reader := csv.NewReader(csvContextReader{ctx: ctx, reader: bytes.NewReader(data)})
+ reader.LazyQuotes = true
+ reader.TrimLeadingSpace = true
+ reader.FieldsPerRecord = -1
+ reader.ReuseRecord = true
+ return reader
+}
+
+type csvContextReader struct {
+ ctx context.Context
+ reader io.Reader
+}
+
+func (r csvContextReader) Read(p []byte) (int, error) {
+ if err := r.ctx.Err(); err != nil {
+ return 0, err
+ }
+ return r.reader.Read(p)
+}
+
+type csvContextWriter struct {
+ ctx context.Context
+ out *strings.Builder
+}
+
+func (w csvContextWriter) Write(p []byte) (int, error) {
+ if err := w.ctx.Err(); err != nil {
+ return 0, err
+ }
+ return w.out.Write(p)
+}
+
+func (w csvContextWriter) WriteString(s string) (int, error) {
+ if err := w.ctx.Err(); err != nil {
+ return 0, err
+ }
+ return w.out.WriteString(s)
+}
diff --git a/internal/infrastructure/docparser/csv_converter_test.go b/internal/infrastructure/docparser/csv_converter_test.go
index 870a55861..025fa284b 100644
--- a/internal/infrastructure/docparser/csv_converter_test.go
+++ b/internal/infrastructure/docparser/csv_converter_test.go
@@ -93,20 +93,29 @@ func TestCSVToMarkdownRejectsExcessivePadding(t *testing.T) {
}
}
-func TestProjectedCSVTableSizeMatchesRender(t *testing.T) {
+func TestCSVMarkdownSizeMatchesRender(t *testing.T) {
// Cells chosen to exercise every csvCellReplacer escape: a pipe, a
// backslash before it, a quoted newline, and a lone carriage return.
in := "a|b,\"x\\|y\nz\",w\r\n1,p|q,\"lone\rcr\"\n"
records, width := parseCSVForBudget(t, in)
- projected := projectedCSVTableSize(records, width)
+ contentBytes := 0
+ for _, record := range records {
+ for _, cell := range record {
+ contentBytes += len(csvCellReplacer.Replace(cell))
+ }
+ }
+ projected, err := csvMarkdownSize(len(records), width, contentBytes, maxCSVMarkdownBytes)
+ if err != nil {
+ t.Fatalf("csvMarkdownSize: %v", err)
+ }
md, err := csvToMarkdown([]byte(in))
if err != nil {
t.Fatalf("csvToMarkdown(%q) error: %v", in, err)
}
- if int64(len(md)) != projected {
- t.Errorf("projectedCSVTableSize = %d, rendered table is %d bytes", projected, len(md))
+ if len(md) != projected {
+ t.Errorf("csvMarkdownSize = %d, rendered table is %d bytes", projected, len(md))
}
}
diff --git a/internal/infrastructure/docparser/csv_limits_test.go b/internal/infrastructure/docparser/csv_limits_test.go
new file mode 100644
index 000000000..1f3cff385
--- /dev/null
+++ b/internal/infrastructure/docparser/csv_limits_test.go
@@ -0,0 +1,94 @@
+package docparser
+
+import (
+ "context"
+ "errors"
+ "math"
+ "strings"
+ "testing"
+
+ "github.com/Tencent/WeKnora/internal/types"
+ "github.com/stretchr/testify/require"
+)
+
+func TestCSVMarkdownExactBudget(t *testing.T) {
+ for _, data := range []string{
+ "name,qty,note\npen,2\nink,1,blue\n",
+ "name,qty\npen,2,spare\n",
+ "expr,note\na\\|b,literal\n",
+ "name,address\nAnn,\"line1\r\nline2\"\n",
+ "\xef\xbb\xbfname,value\n中文,\"a|b\\c\rd\"\n",
+ } {
+ t.Run(data, func(t *testing.T) {
+ want, err := csvToMarkdown([]byte(data))
+ require.NoError(t, err)
+ got, err := csvToMarkdownContext(context.Background(), []byte(data), len(want))
+ require.NoError(t, err)
+ require.Equal(t, want, got)
+ got, err = csvToMarkdownContext(context.Background(), []byte(data), len(want)-1)
+ require.ErrorIs(t, err, ErrCSVMarkdownTooLarge)
+ require.Empty(t, got, "never return a partial table")
+ })
+ }
+}
+
+func TestCSVRejectsPaddingAmplification(t *testing.T) {
+ wide := strings.Repeat(",", 9999) + "x\n"
+ narrow := strings.Repeat("x\n", 10000)
+ for _, data := range []string{wide + narrow, narrow + wide} {
+ // About 30 KB would previously generate 300 MB of Markdown.
+ require.Less(t, len(data), 31000)
+ result, err := (&SimpleFormatReader{}).Read(context.Background(), &types.ReadRequest{
+ FileType: "csv", FileContent: []byte(data),
+ })
+ require.ErrorIs(t, err, ErrCSVMarkdownTooLarge)
+ require.Nil(t, result)
+ require.Contains(t, err.Error(), "reduce the number of rows or columns")
+ }
+}
+
+func TestCSVEscapedContentExceedsBudget(t *testing.T) {
+ got, err := csvToMarkdownContext(context.Background(), []byte(strings.Repeat("|", 20)+"\n"), 16)
+ require.ErrorIs(t, err, ErrCSVMarkdownTooLarge)
+ require.Contains(t, err.Error(), "16 bytes")
+ require.Empty(t, got)
+}
+
+func TestCSVSizeBudgetCannotOverflow(t *testing.T) {
+ for _, dimensions := range [][3]int{
+ {math.MaxInt, 1, 0}, {1, math.MaxInt, 0}, {1, 1, math.MaxInt},
+ } {
+ _, err := csvMarkdownSize(dimensions[0], dimensions[1], dimensions[2], maxCSVMarkdownBytes)
+ require.ErrorIs(t, err, ErrCSVMarkdownTooLarge)
+ }
+}
+
+func TestCSVConversionCancellation(t *testing.T) {
+ ctx, cancel := context.WithCancel(context.Background())
+ cancel()
+ result, err := (&SimpleFormatReader{}).Read(ctx, &types.ReadRequest{
+ FileType: "csv", FileContent: []byte("a,b\n1,2\n"),
+ })
+ require.ErrorIs(t, err, context.Canceled)
+ require.False(t, errors.Is(err, ErrCSVMarkdownTooLarge))
+ require.Nil(t, result)
+}
+
+func TestCSVEmptyInputAndZeroBudget(t *testing.T) {
+ got, err := csvToMarkdownContext(context.Background(), []byte("\xef\xbb\xbf\n"), 0)
+ require.NoError(t, err)
+ require.Empty(t, got)
+ got, err = csvToMarkdownContext(context.Background(), []byte("a,b\n1,2\n"), 0)
+ require.ErrorIs(t, err, ErrCSVMarkdownTooLarge)
+ require.Empty(t, got)
+}
+
+func BenchmarkCSVToMarkdownRagged(b *testing.B) {
+ data := []byte(strings.Repeat("x\n", 1000) + strings.Repeat(",", 999) + "x\n")
+ b.ReportAllocs()
+ for i := 0; i < b.N; i++ {
+ if _, err := csvToMarkdown(data); err != nil {
+ b.Fatal(err)
+ }
+ }
+}
diff --git a/website-docs/03-features/03-document-parsing.md b/website-docs/03-features/03-document-parsing.md
index 19f5bbeae..cf6c6e00c 100644
--- a/website-docs/03-features/03-document-parsing.md
+++ b/website-docs/03-features/03-document-parsing.md
@@ -75,6 +75,8 @@ Go 侧客户端在 `docreader/client/auth.go`(`LoadAuthConfigFromEnv` 读取
Go App 中 `internal/application/service/knowledge_process.go` 在文档入库流水线的 docreader stage 调用解析(超时由 `docreader_call_timeout` 配置控制,防止挂死的 docreader 长时间占用 worker)。注意:**md/markdown/txt/csv/json/图片/音频由 Go 侧 `SimpleFormatReader` 原生处理,不经过 docreader**(见 `internal/infrastructure/docparser/builtin_converter.go` 的 `simpleFormats`)。
+CSV 使用 Go `simple` 解析器时,转换后的 Markdown 上限为 **64 MiB**,与原始上传文件大小限制分别检查。解析器会在生成表格前核算全部行按最大列数补齐后的大小,以及单元格转义后的字节数;超过上限会明确失败,不返回截断内容,也不会自动重试同一份文件。可按行或列拆分 CSV 后重新上传。正常规模的长短行仍会补齐空单元格并保留全部字段。
+
```mermaid
sequenceDiagram
participant U as "用户 / 前端"