Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.
Go agentagenticaichatbotdsh-pluginembeddingsevaluationgenerative-aigolangknowledge-basellmmulti-tenantollamaopenaiquestion-answeringragrerankingsemantic-searchvector-searchwiki
fix(session): cap text attachments at 32 KiB instead of failing on long lines (#4148)
Zhengxu (Joshua) Jin2aef22b60451
diff --git a/internal/handler/session/attachment_processor.go b/internal/handler/session/attachment_processor.goindex 358b4219c..0808b2b50 100644--- a/internal/handler/session/attachment_processor.go+++ b/internal/handler/session/attachment_processor.go@@ -1,13 +1,13 @@package sessionimport (- "bufio""bytes""context""encoding/base64""fmt""path/filepath""strings"+ "unicode/utf8""github.com/Tencent/WeKnora/internal/common""github.com/Tencent/WeKnora/internal/infrastructure/docparser"@@ -21,6 +21,9 @@ import (const (// maxTextFileLines is the line limit for inline text content; excess lines are truncated.maxTextFileLines = 500+ // maxTextFileContentBytes bounds inline text content (same cap as IM attachments);+ // a single minified-JSON or log line can otherwise be as large as the upload limit.+ maxTextFileContentBytes = 32 << 10 // 32 KiB// textFileExtensions lists plain-text extensions handled by the line-based reader.textFileExtensions = ".txt,.md,.markdown,.json,.xml,.yaml,.yml,.csv,.log")@@ -97,10 +100,7 @@ func (p *AttachmentProcessor) ProcessAttachment(// Extract text content based on file type; errors are non-fatal.if p.isTextFile(ext) {- if err := p.processTextFile(ctx, data, attachment); err != nil {- logger.Warnf(ctx, "text file processing failed: %v", err)- attachment.Content = fmt.Sprintf("<error><message>Failed to process text file</message><details>%v</details></error>", err)- }+ p.processTextFile(ctx, data, attachment)} else if docparser.IsAudioFormat(ext) {if err := p.processAudioFile(ctx, data, baseName, attachment, asrModelID); err != nil {logger.Warnf(ctx, "audio transcription failed: %v, keeping placeholder", err)@@ -131,32 +131,38 @@ func (p *AttachmentProcessor) isTextFile(ext string) bool {return strings.Contains(textFileExtensions, ext)}-// processTextFile reads plain-text content line by line, truncating at maxTextFileLines.-func (p *AttachmentProcessor) processTextFile(ctx context.Context, data []byte, attachment *types.MessageAttachment) error {- scanner := bufio.NewScanner(bytes.NewReader(data))- var lines []string+// processTextFile reads plain-text content line by line, truncating at maxTextFileLines+// lines and maxTextFileContentBytes bytes.+func (p *AttachmentProcessor) processTextFile(ctx context.Context, data []byte, attachment *types.MessageAttachment) {+ var lines [][]bytelineCount := 0- for scanner.Scan() {+ for line := range bytes.Lines(data) {lineCount++if lineCount <= maxTextFileLines {- lines = append(lines, scanner.Text())+ line = bytes.TrimSuffix(line, []byte("\n"))+ lines = append(lines, bytes.TrimSuffix(line, []byte("\r")))}}- if err := scanner.Err(); err != nil {- return fmt.Errorf("failed to read file content: %w", err)+ content := bytes.Join(lines, []byte("\n"))+ truncated := lineCount > maxTextFileLines+ if len(content) > maxTextFileContentBytes {+ end := maxTextFileContentBytes+ for end > 0 && !utf8.RuneStart(content[end]) {+ end--+ }+ content = content[:end]+ truncated = true}attachment.LineCount = lineCount- attachment.Content = strings.Join(lines, "\n")+ attachment.Content = string(content)- if lineCount > maxTextFileLines {+ if truncated {attachment.IsTruncated = true- logger.Infof(ctx, "text file truncated: total=%d, kept=%d", lineCount, maxTextFileLines)+ logger.Infof(ctx, "text file truncated: total=%d lines, kept=%d bytes", lineCount, len(content))}-- return nil}// processWithDocParser extracts content via SimpleFormatReader (md, csv, json, images, etc.).diff --git a/internal/handler/session/attachment_processor_test.go b/internal/handler/session/attachment_processor_test.gonew file mode 100644index 000000000..33c734729--- /dev/null+++ b/internal/handler/session/attachment_processor_test.go@@ -0,0 +1,63 @@+package session++import (+ "bufio"+ "context"+ "fmt"+ "strings"+ "testing"+ "unicode/utf8"++ "github.com/Tencent/WeKnora/internal/types"+ "github.com/stretchr/testify/assert"+ "github.com/stretchr/testify/require"+)++func TestProcessTextFileKeepsSmallFilesWhole(t *testing.T) {+ attachment := &types.MessageAttachment{}+ (&AttachmentProcessor{}).processTextFile(context.Background(), []byte("header\r\nbody\nfooter\n"), attachment)++ assert.Equal(t, "header\nbody\nfooter", attachment.Content)+ assert.Equal(t, 3, attachment.LineCount)+ assert.False(t, attachment.IsTruncated)+}++func TestProcessTextFileCapsASingleLineOverTheByteLimit(t *testing.T) {+ // A minified JSON export is one line, here longer than bufio.MaxScanTokenSize:+ // it keeps a prefix of maxTextFileContentBytes and is marked truncated.+ longLine := `{"rows":[` + strings.Repeat(`{"id":1,"name":"item"},`, bufio.MaxScanTokenSize/20) + `{}]}`+ require.Greater(t, len(longLine), bufio.MaxScanTokenSize)++ attachment := &types.MessageAttachment{}+ (&AttachmentProcessor{}).processTextFile(context.Background(), []byte(longLine), attachment)++ assert.Equal(t, longLine[:maxTextFileContentBytes], attachment.Content)+ assert.Equal(t, 1, attachment.LineCount)+ assert.True(t, attachment.IsTruncated)+}++func TestProcessTextFileByteCapKeepsValidUTF8(t *testing.T) {+ data := []byte(strings.Repeat("中", maxTextFileContentBytes))++ attachment := &types.MessageAttachment{}+ (&AttachmentProcessor{}).processTextFile(context.Background(), data, attachment)++ assert.True(t, attachment.IsTruncated)+ assert.True(t, utf8.ValidString(attachment.Content))+ assert.LessOrEqual(t, len(attachment.Content), maxTextFileContentBytes)+ assert.Greater(t, len(attachment.Content), maxTextFileContentBytes-utf8.UTFMax)+}++func TestProcessTextFileStillTruncatesByLineCount(t *testing.T) {+ var sb strings.Builder+ for i := 1; i <= maxTextFileLines+1; i++ {+ fmt.Fprintf(&sb, "line %d\n", i)+ }++ attachment := &types.MessageAttachment{}+ (&AttachmentProcessor{}).processTextFile(context.Background(), []byte(sb.String()), attachment)++ assert.Equal(t, maxTextFileLines+1, attachment.LineCount)+ assert.True(t, attachment.IsTruncated)+ assert.True(t, strings.HasSuffix(attachment.Content, fmt.Sprintf("line %d", maxTextFileLines)))+}