Skip to content

Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.

fix(knowledge): keep long non-ASCII folder names valid UTF-8

Max Freedom Pollard801005f736b5
diff --git a/internal/types/knowledge_folder.go b/internal/types/knowledge_folder.go
index 3d314db7a..c1968917a 100644
--- a/internal/types/knowledge_folder.go
+++ b/internal/types/knowledge_folder.go
@@ -3,6 +3,7 @@ package types
import (
"sort"
"strings"
+ "unicode/utf8"
)
const (
@@ -49,7 +50,15 @@ func NormalizeKnowledgeFolderPath(raw string) string {
continue
}
if len(segment) > MaxKnowledgeFolderSegmentLength {
- segment = strings.TrimSpace(segment[:MaxKnowledgeFolderSegmentLength])
+ // The cap is a byte budget, so back the cut off to the start of a
+ // rune. Slicing at the raw index leaves the leading bytes of a
+ // partial rune behind, and a folder named in Chinese hits that on
+ // the 43rd character.
+ cut := MaxKnowledgeFolderSegmentLength
+ for cut > 0 && !utf8.RuneStart(segment[cut]) {
+ cut--
+ }
+ segment = strings.TrimSpace(segment[:cut])
}
if segment == "" {
continue
diff --git a/internal/types/knowledge_folder_test.go b/internal/types/knowledge_folder_test.go
index f45ee62a7..552757509 100644
--- a/internal/types/knowledge_folder_test.go
+++ b/internal/types/knowledge_folder_test.go
@@ -3,6 +3,7 @@ package types
import (
"strings"
"testing"
+ "unicode/utf8"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
@@ -44,6 +45,30 @@ func TestNormalizeKnowledgeFolderPathCapsDepthAndLength(t *testing.T) {
assert.LessOrEqual(t, len(NormalizeKnowledgeFolderPath(wide)), MaxKnowledgeFolderPathLength)
}
+func TestNormalizeKnowledgeFolderPathKeepsSegmentsValidUTF8(t *testing.T) {
+ // 60 Chinese characters is 180 bytes, so the segment cap cuts inside the
+ // 43rd character. The stored folder_path has to stay valid UTF-8: an
+ // invalid one no longer survives a JSON round trip, so the path the client
+ // gets back never matches the row again on rename or move.
+ segment := strings.Repeat("中", 60)
+ normalized := NormalizeKnowledgeFolderPath(segment)
+
+ assert.True(t, utf8.ValidString(normalized),
+ "segment truncated mid-rune: % x", []byte(normalized))
+ assert.LessOrEqual(t, len(normalized), MaxKnowledgeFolderSegmentLength,
+ "the cap is a byte budget and must still hold")
+ assert.True(t, strings.HasPrefix(segment, normalized),
+ "truncation must only drop a suffix")
+ assert.Equal(t, normalized, NormalizeKnowledgeFolderPath(normalized),
+ "normalizing an already-normalized path must be a no-op")
+
+ // A multi-byte character straddling the cap in an otherwise ASCII name.
+ mixed := strings.Repeat("a", MaxKnowledgeFolderSegmentLength-1) + "é" + "tail"
+ assert.True(t, utf8.ValidString(NormalizeKnowledgeFolderPath(mixed)))
+ assert.Equal(t, strings.Repeat("a", MaxKnowledgeFolderSegmentLength-1),
+ NormalizeKnowledgeFolderPath(mixed))
+}
+
func TestSplitKnowledgeRelativePath(t *testing.T) {
cases := []struct {
in string