Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.
Go agentagenticaichatbotdsh-pluginembeddingsevaluationgenerative-aigolangknowledge-basellmmulti-tenantollamaopenaiquestion-answeringragrerankingsemantic-searchvector-searchwiki
fix(knowledge): keep long non-ASCII folder names valid UTF-8
Max Freedom Pollard801005f736b5
diff --git a/internal/types/knowledge_folder.go b/internal/types/knowledge_folder.goindex 3d314db7a..c1968917a 100644--- a/internal/types/knowledge_folder.go+++ b/internal/types/knowledge_folder.go@@ -3,6 +3,7 @@ package typesimport ("sort""strings"+ "unicode/utf8")const (@@ -49,7 +50,15 @@ func NormalizeKnowledgeFolderPath(raw string) string {continue}if len(segment) > MaxKnowledgeFolderSegmentLength {- segment = strings.TrimSpace(segment[:MaxKnowledgeFolderSegmentLength])+ // The cap is a byte budget, so back the cut off to the start of a+ // rune. Slicing at the raw index leaves the leading bytes of a+ // partial rune behind, and a folder named in Chinese hits that on+ // the 43rd character.+ cut := MaxKnowledgeFolderSegmentLength+ for cut > 0 && !utf8.RuneStart(segment[cut]) {+ cut--+ }+ segment = strings.TrimSpace(segment[:cut])}if segment == "" {continuediff --git a/internal/types/knowledge_folder_test.go b/internal/types/knowledge_folder_test.goindex f45ee62a7..552757509 100644--- a/internal/types/knowledge_folder_test.go+++ b/internal/types/knowledge_folder_test.go@@ -3,6 +3,7 @@ package typesimport ("strings""testing"+ "unicode/utf8""github.com/stretchr/testify/assert""github.com/stretchr/testify/require"@@ -44,6 +45,30 @@ func TestNormalizeKnowledgeFolderPathCapsDepthAndLength(t *testing.T) {assert.LessOrEqual(t, len(NormalizeKnowledgeFolderPath(wide)), MaxKnowledgeFolderPathLength)}+func TestNormalizeKnowledgeFolderPathKeepsSegmentsValidUTF8(t *testing.T) {+ // 60 Chinese characters is 180 bytes, so the segment cap cuts inside the+ // 43rd character. The stored folder_path has to stay valid UTF-8: an+ // invalid one no longer survives a JSON round trip, so the path the client+ // gets back never matches the row again on rename or move.+ segment := strings.Repeat("中", 60)+ normalized := NormalizeKnowledgeFolderPath(segment)++ assert.True(t, utf8.ValidString(normalized),+ "segment truncated mid-rune: % x", []byte(normalized))+ assert.LessOrEqual(t, len(normalized), MaxKnowledgeFolderSegmentLength,+ "the cap is a byte budget and must still hold")+ assert.True(t, strings.HasPrefix(segment, normalized),+ "truncation must only drop a suffix")+ assert.Equal(t, normalized, NormalizeKnowledgeFolderPath(normalized),+ "normalizing an already-normalized path must be a no-op")++ // A multi-byte character straddling the cap in an otherwise ASCII name.+ mixed := strings.Repeat("a", MaxKnowledgeFolderSegmentLength-1) + "é" + "tail"+ assert.True(t, utf8.ValidString(NormalizeKnowledgeFolderPath(mixed)))+ assert.Equal(t, strings.Repeat("a", MaxKnowledgeFolderSegmentLength-1),+ NormalizeKnowledgeFolderPath(mixed))+}+func TestSplitKnowledgeRelativePath(t *testing.T) {cases := []struct {in string