Skip to content

Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.

fix(session): cap text attachments at 32 KiB instead of failing on long lines (#4148)

Zhengxu (Joshua) Jin2aef22b60451
diff --git a/internal/handler/session/attachment_processor.go b/internal/handler/session/attachment_processor.go
index 358b4219c..0808b2b50 100644
--- a/internal/handler/session/attachment_processor.go
+++ b/internal/handler/session/attachment_processor.go
@@ -1,13 +1,13 @@
package session
import (
- "bufio"
"bytes"
"context"
"encoding/base64"
"fmt"
"path/filepath"
"strings"
+ "unicode/utf8"
"github.com/Tencent/WeKnora/internal/common"
"github.com/Tencent/WeKnora/internal/infrastructure/docparser"
@@ -21,6 +21,9 @@ import (
const (
// maxTextFileLines is the line limit for inline text content; excess lines are truncated.
maxTextFileLines = 500
+ // maxTextFileContentBytes bounds inline text content (same cap as IM attachments);
+ // a single minified-JSON or log line can otherwise be as large as the upload limit.
+ maxTextFileContentBytes = 32 << 10 // 32 KiB
// textFileExtensions lists plain-text extensions handled by the line-based reader.
textFileExtensions = ".txt,.md,.markdown,.json,.xml,.yaml,.yml,.csv,.log"
)
@@ -97,10 +100,7 @@ func (p *AttachmentProcessor) ProcessAttachment(
// Extract text content based on file type; errors are non-fatal.
if p.isTextFile(ext) {
- if err := p.processTextFile(ctx, data, attachment); err != nil {
- logger.Warnf(ctx, "text file processing failed: %v", err)
- attachment.Content = fmt.Sprintf("<error><message>Failed to process text file</message><details>%v</details></error>", err)
- }
+ p.processTextFile(ctx, data, attachment)
} else if docparser.IsAudioFormat(ext) {
if err := p.processAudioFile(ctx, data, baseName, attachment, asrModelID); err != nil {
logger.Warnf(ctx, "audio transcription failed: %v, keeping placeholder", err)
@@ -131,32 +131,38 @@ func (p *AttachmentProcessor) isTextFile(ext string) bool {
return strings.Contains(textFileExtensions, ext)
}
-// processTextFile reads plain-text content line by line, truncating at maxTextFileLines.
-func (p *AttachmentProcessor) processTextFile(ctx context.Context, data []byte, attachment *types.MessageAttachment) error {
- scanner := bufio.NewScanner(bytes.NewReader(data))
- var lines []string
+// processTextFile reads plain-text content line by line, truncating at maxTextFileLines
+// lines and maxTextFileContentBytes bytes.
+func (p *AttachmentProcessor) processTextFile(ctx context.Context, data []byte, attachment *types.MessageAttachment) {
+ var lines [][]byte
lineCount := 0
- for scanner.Scan() {
+ for line := range bytes.Lines(data) {
lineCount++
if lineCount <= maxTextFileLines {
- lines = append(lines, scanner.Text())
+ line = bytes.TrimSuffix(line, []byte("\n"))
+ lines = append(lines, bytes.TrimSuffix(line, []byte("\r")))
}
}
- if err := scanner.Err(); err != nil {
- return fmt.Errorf("failed to read file content: %w", err)
+ content := bytes.Join(lines, []byte("\n"))
+ truncated := lineCount > maxTextFileLines
+ if len(content) > maxTextFileContentBytes {
+ end := maxTextFileContentBytes
+ for end > 0 && !utf8.RuneStart(content[end]) {
+ end--
+ }
+ content = content[:end]
+ truncated = true
}
attachment.LineCount = lineCount
- attachment.Content = strings.Join(lines, "\n")
+ attachment.Content = string(content)
- if lineCount > maxTextFileLines {
+ if truncated {
attachment.IsTruncated = true
- logger.Infof(ctx, "text file truncated: total=%d, kept=%d", lineCount, maxTextFileLines)
+ logger.Infof(ctx, "text file truncated: total=%d lines, kept=%d bytes", lineCount, len(content))
}
-
- return nil
}
// processWithDocParser extracts content via SimpleFormatReader (md, csv, json, images, etc.).
diff --git a/internal/handler/session/attachment_processor_test.go b/internal/handler/session/attachment_processor_test.go
new file mode 100644
index 000000000..33c734729
--- /dev/null
+++ b/internal/handler/session/attachment_processor_test.go
@@ -0,0 +1,63 @@
+package session
+
+import (
+ "bufio"
+ "context"
+ "fmt"
+ "strings"
+ "testing"
+ "unicode/utf8"
+
+ "github.com/Tencent/WeKnora/internal/types"
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+)
+
+func TestProcessTextFileKeepsSmallFilesWhole(t *testing.T) {
+ attachment := &types.MessageAttachment{}
+ (&AttachmentProcessor{}).processTextFile(context.Background(), []byte("header\r\nbody\nfooter\n"), attachment)
+
+ assert.Equal(t, "header\nbody\nfooter", attachment.Content)
+ assert.Equal(t, 3, attachment.LineCount)
+ assert.False(t, attachment.IsTruncated)
+}
+
+func TestProcessTextFileCapsASingleLineOverTheByteLimit(t *testing.T) {
+ // A minified JSON export is one line, here longer than bufio.MaxScanTokenSize:
+ // it keeps a prefix of maxTextFileContentBytes and is marked truncated.
+ longLine := `{"rows":[` + strings.Repeat(`{"id":1,"name":"item"},`, bufio.MaxScanTokenSize/20) + `{}]}`
+ require.Greater(t, len(longLine), bufio.MaxScanTokenSize)
+
+ attachment := &types.MessageAttachment{}
+ (&AttachmentProcessor{}).processTextFile(context.Background(), []byte(longLine), attachment)
+
+ assert.Equal(t, longLine[:maxTextFileContentBytes], attachment.Content)
+ assert.Equal(t, 1, attachment.LineCount)
+ assert.True(t, attachment.IsTruncated)
+}
+
+func TestProcessTextFileByteCapKeepsValidUTF8(t *testing.T) {
+ data := []byte(strings.Repeat("中", maxTextFileContentBytes))
+
+ attachment := &types.MessageAttachment{}
+ (&AttachmentProcessor{}).processTextFile(context.Background(), data, attachment)
+
+ assert.True(t, attachment.IsTruncated)
+ assert.True(t, utf8.ValidString(attachment.Content))
+ assert.LessOrEqual(t, len(attachment.Content), maxTextFileContentBytes)
+ assert.Greater(t, len(attachment.Content), maxTextFileContentBytes-utf8.UTFMax)
+}
+
+func TestProcessTextFileStillTruncatesByLineCount(t *testing.T) {
+ var sb strings.Builder
+ for i := 1; i <= maxTextFileLines+1; i++ {
+ fmt.Fprintf(&sb, "line %d\n", i)
+ }
+
+ attachment := &types.MessageAttachment{}
+ (&AttachmentProcessor{}).processTextFile(context.Background(), []byte(sb.String()), attachment)
+
+ assert.Equal(t, maxTextFileLines+1, attachment.LineCount)
+ assert.True(t, attachment.IsTruncated)
+ assert.True(t, strings.HasSuffix(attachment.Content, fmt.Sprintf("line %d", maxTextFileLines)))
+}