Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.
Go agentagenticaichatbotdsh-pluginembeddingsevaluationgenerative-aigolangknowledge-basellmmulti-tenantollamaopenaiquestion-answeringragrerankingsemantic-searchvector-searchwiki
refactor(datasource): share connector filename sanitization
wizardchen820a14dafa0e
diff --git a/docs/code-slimming-audit.md b/docs/code-slimming-audit.mdindex b560d3dab..abfa34d77 100644--- a/docs/code-slimming-audit.md+++ b/docs/code-slimming-audit.md@@ -98,6 +98,16 @@- 本批只删除无调用代码,复用现有包测试和编译检查,没有添加仅断言函数不存在的测试。- 推送检查暴露新合入的 fork 测试未隔离 Git hook 环境:继承的 `GIT_DIR` 等变量使临时仓库操作指向当前仓库。测试辅助函数现清除继承的 `GIT_*` 变量;真实仓库测试主动注入指向临时路径的仓库变量,验证旧实现失败、修复后通过。该测试修复单独提交,不改变生产逻辑。+## 第六批已实施:外部来源文件名清理++基于第五批合入后的 `10973ed32`,IMA、语雀、RSS 复用 `datasource.SanitizeFileName`,统一无效标点替换、空标题回退和 200 字节 UTF-8 边界截断。计入新增共享实现后,生产 Go 源码净减少 49 行。++- IMA 的媒体与笔记路径、语雀文档直接调用共享函数;RSS 仅保留换行/制表符转空格和首尾空白清理,再调用共享函数。+- 保留原有扩展名拼接、源标题、空白、控制字符和无效 UTF-8 输入处理行为。飞书的扩展名保留、钉钉的控制字符/尾部标点清理、Confluence 的按字符截断规则不同,本批未合并。+- 将两份旧 helper 测试收敛到共享函数测试,补充 16 个边界用例;在实际连接器读取/组装路径新增 15 个用例,覆盖 IMA 两类内容、语雀和 RSS 的最终文件名。+- 用重构前的三个实现对 30,000 组确定性生成输入执行差异校验,结果一致;输入包含中文、emoji、空白、非法字节和截断边界。临时对比代码未留在仓库,避免长期维护旧实现副本。+- 4 个受影响包测试通过:104 项顶层测试,包含既有同步、游标和错误处理测试。未修改前端,未运行浏览器检查或真实外部来源服务测试。+## 后续优先级| 优先级 | 证据 | 建议边界 | 验证重点 |@@ -113,7 +123,7 @@### 后端重复的具体判断- Qdrant 与 Weaviate 的 `tokenizeQuery` 函数体相同,但只有 Qdrant 的关键字检索调用它;Weaviate 使用 `WithQuery(params.Query)`。第五批已删除 Weaviate 的失效函数,保留 Qdrant 实现。-- IMA、语雀的 `sanitizeFileName` 函数体相同,可以共享;RSS 的近似实现另外处理首尾空白和换行,合并时必须保留这些差异。+- 第六批已合并 IMA、语雀、RSS 的文件名标点替换和 UTF-8 截断逻辑;RSS 的空白预处理保留在连接器内。飞书、钉钉、Confluence 的其他规则继续独立维护。- Milvus/Qdrant/Weaviate 的结果映射体相似,但接收不同后端类型。当前各约 17 行,为此引入通用接口或泛型转换层未必减少维护成本。- 终端和桌面 WebSocket ticket handler 重复检查会话、用户、租户和 token,但签发方式不同。后续可共享前置身份校验,保留 ticket 和连接生命周期各自的契约。- 资源权限已经有 `internal/application/access`;继续沿已有授权边界收敛,避免新建与其并行的权限工具层。@@ -152,4 +162,4 @@- 构建仍报告大 chunk 提示;本轮未以 bundle 体积为目标,也没有测量构建前后的体积差异。- 未修改后端,未运行后端测试;未做浏览器视觉验收。-下一批优先处理 IMA、语雀等外部来源的重复文件名清理逻辑,核实调用与差异后再决定共享边界。涉及聊天、任务重试、权限或共享状态的改动应单独成批,以可验证的行为等价为边界。+下一批优先检查终端与桌面 WebSocket ticket 的重复身份校验,沿现有授权边界确定共享范围。涉及聊天、任务重试、权限或共享状态的改动应单独成批,以可验证的行为等价为边界。diff --git a/internal/datasource/connector/ima/connector.go b/internal/datasource/connector/ima/connector.goindex 89c305a19..6111ffc69 100644--- a/internal/datasource/connector/ima/connector.go+++ b/internal/datasource/connector/ima/connector.go@@ -7,7 +7,6 @@ import ("sort""strings""time"- "unicode/utf8""github.com/Tencent/WeKnora/internal/datasource""github.com/Tencent/WeKnora/internal/logger"@@ -437,7 +436,7 @@ func fetchNote(return types.FetchedItem{}, fetchSkipped}- fileName := sanitizeFileName(f.Title)+ fileName := datasource.SanitizeFileName(f.Title)if !strings.HasSuffix(strings.ToLower(fileName), ".md") {fileName += ".md"}@@ -540,7 +539,7 @@ func fetchOneMedia(ct = mimeForExtension(ext)}- fileName := sanitizeFileName(f.Title)+ fileName := datasource.SanitizeFileName(f.Title)if !strings.HasSuffix(strings.ToLower(fileName), "."+ext) {fileName = fileName + "." + ext}@@ -585,28 +584,3 @@ func baseMetadata(}return m}--// sanitizeFileName removes filesystem-hostile characters and truncates to a-// safe UTF-8 boundary.-func sanitizeFileName(name string) string {- if name == "" {- return "untitled"- }- replacer := strings.NewReplacer(- "/", "_", "\\", "_", ":", "_", "*", "_",- "?", "_", "\"", "_", "<", "_", ">", "_", "|", "_",- )- result := replacer.Replace(name)- const maxBytes = 200- if len(result) > maxBytes {- result = result[:maxBytes]- for len(result) > 0 {- r, size := utf8.DecodeLastRuneInString(result)- if r != utf8.RuneError || size != 1 {- break- }- result = result[:len(result)-1]- }- }- return result-}diff --git a/internal/datasource/connector/ima/connector_test.go b/internal/datasource/connector/ima/connector_test.goindex d1a447e78..42970c8ca 100644--- a/internal/datasource/connector/ima/connector_test.go+++ b/internal/datasource/connector/ima/connector_test.go@@ -509,3 +509,35 @@ func TestConnectorIsRegisteredInMetadata(t *testing.T) {t.Errorf("metadata type = %q, want %q", meta.Type, types.ConnectorTypeIMA)}}++func TestFetchAll_FileNamePolicy(t *testing.T) {+ for _, media := range []struct {+ name string+ kind int32+ }{{"media", mediaTypeMarkdown}, {"note", mediaTypeNote}} {+ for _, tt := range []struct{ name, title, want string }{+ {"punctuation", "A/B:C", "A_B_C.md"},+ {"existing extension and whitespace", " \tA/B.MD", " \tA_B.MD"},+ {"long title", strings.Repeat("测", 100) + ".MD", strings.Repeat("测", 66) + ".md"},+ } {+ t.Run(media.name+"/"+tt.name, func(t *testing.T) {+ f := newFakeIMA(t)+ f.setKB("kb1", []fakeFile{{+ MediaID: "file", Title: tt.title, MediaType: media.kind,+ Body: "# content", NotebookID: "123", NoteBody: "# content",+ }})+ items, err := NewConnector().FetchAll(context.Background(), f.config("kb1"), []string{"kb1"})+ if err != nil {+ t.Fatal(err)+ }+ if len(items) != 1 {+ t.Fatalf("got %d items, want 1", len(items))+ }+ if items[0].Title != tt.title || items[0].FileName != tt.want {+ t.Fatalf("title = %q, filename = %q; want %q, %q",+ items[0].Title, items[0].FileName, tt.title, tt.want)+ }+ })+ }+ }+}diff --git a/internal/datasource/connector/ima/types_test.go b/internal/datasource/connector/ima/types_test.goindex d7859c04c..e52f6ff63 100644--- a/internal/datasource/connector/ima/types_test.go+++ b/internal/datasource/connector/ima/types_test.go@@ -131,25 +131,6 @@ func TestExtensionForContentType(t *testing.T) {}}-func TestSanitizeFileName(t *testing.T) {- if got := sanitizeFileName(""); got != "untitled" {- t.Errorf("empty name = %q, want untitled", got)- }- if got := sanitizeFileName(`a/b\c:d*e?f"g<h>i|j`); strings.ContainsAny(got, `/\:*?"<>|`) {- t.Errorf("filesystem-hostile characters survived: %q", got)- }-- // A long CJK title must be truncated on a rune boundary, not mid-sequence.- long := strings.Repeat("知识", 200)- got := sanitizeFileName(long)- if len(got) > 200 {- t.Errorf("len = %d bytes, want <= 200", len(got))- }- if !strings.HasPrefix(long, got) {- t.Errorf("truncation corrupted the prefix: %q", got)- }-}-func TestIsSkippableMediaType(t *testing.T) {for _, mt := range []int32{mediaTypeAISession, mediaTypeVideo} {if !isSkippableMediaType(mt) {diff --git a/internal/datasource/connector/rss/connector_test.go b/internal/datasource/connector/rss/connector_test.goindex ff74c7db2..c7771a0fd 100644--- a/internal/datasource/connector/rss/connector_test.go+++ b/internal/datasource/connector/rss/connector_test.go@@ -413,3 +413,28 @@ func TestConnector_ResolveResourceAncestors_Empty(t *testing.T) {t.Fatalf("expected empty ancestors, got %v", got)}}++func TestResolveItem_FileNamePolicy(t *testing.T) {+ for _, tt := range []struct{ name, title, want string }{+ {"empty", "", "untitled.md"},+ {"blank", " \n\t\u2003", "untitled.md"},+ {"line breaks and punctuation", "\u2003 A\nB\rC\tD/E \u2003", "A B C D_E.md"},+ {"existing extension", "Title.md", "Title.md.md"},+ {"long title", strings.Repeat("测", 100), strings.Repeat("测", 66) + ".md"},+ {"space at truncation boundary", strings.Repeat("a", 199) + "\nmore", strings.Repeat("a", 199) + " .md"},+ {"other controls retained", "a\x01b", "a\x01b.md"},+ } {+ t.Run(tt.name, func(t *testing.T) {+ result := NewConnector().resolveItem(+ context.Background(), nil, &gofeed.Feed{}, &gofeed.Item{Title: tt.title},+ "https://example.com/feed", "id", "<p>content</p>",+ )+ if result.item.FileName != tt.want {+ t.Fatalf("FileName = %q, want %q", result.item.FileName, tt.want)+ }+ if strings.TrimSpace(tt.title) != "" && result.item.Title != tt.title {+ t.Fatalf("source title changed: %q", result.item.Title)+ }+ })+ }+}diff --git a/internal/datasource/connector/rss/types.go b/internal/datasource/connector/rss/types.goindex b6f672646..9fff63b5d 100644--- a/internal/datasource/connector/rss/types.go+++ b/internal/datasource/connector/rss/types.go@@ -29,7 +29,6 @@ import ("maps""strings""time"- "unicode/utf8""github.com/Tencent/WeKnora/internal/datasource""github.com/Tencent/WeKnora/internal/types"@@ -218,32 +217,8 @@ func firstNonEmpty(values ...string) string {return ""}-// sanitizeFileName removes characters invalid in filenames and truncates to a-// safe length at a UTF-8 rune boundary (mirrors the Yuque connector).+// sanitizeFileName keeps feed titles on one line before applying shared limits.func sanitizeFileName(name string) string {- name = strings.TrimSpace(name)- if name == "" {- return "untitled"- }- replacer := strings.NewReplacer(- "/", "_", "\\", "_", ":", "_", "*", "_",- "?", "_", "\"", "_", "<", "_", ">", "_", "|", "_",- "\n", " ", "\r", " ", "\t", " ",- )- result := strings.TrimSpace(replacer.Replace(name))- if result == "" {- return "untitled"- }- const maxBytes = 200- if len(result) > maxBytes {- result = result[:maxBytes]- for len(result) > 0 {- r, size := utf8.DecodeLastRuneInString(result)- if r != utf8.RuneError || size != 1 {- break- }- result = result[:len(result)-1]- }- }- return result+ name = strings.NewReplacer("\n", " ", "\r", " ", "\t", " ").Replace(name)+ return datasource.SanitizeFileName(strings.TrimSpace(name))}diff --git a/internal/datasource/connector/yuque/connector.go b/internal/datasource/connector/yuque/connector.goindex d07fce0c5..485eef13b 100644--- a/internal/datasource/connector/yuque/connector.go+++ b/internal/datasource/connector/yuque/connector.go@@ -262,7 +262,7 @@ func (c *Connector) walk(Title: d.Title,Content: []byte(detail.Body),ContentType: "text/markdown",- FileName: sanitizeFileName(d.Title) + ".md",+ FileName: datasource.SanitizeFileName(d.Title) + ".md",URL: buildDocURL(cfg.GetBaseURL(), detail.Book.Namespace, d.Slug),UpdatedAt: parseContentUpdatedAt(d.ContentUpdatedAt),SourceResourceID: bookIDStr,diff --git a/internal/datasource/connector/yuque/connector_test.go b/internal/datasource/connector/yuque/connector_test.goindex 44f3ca049..8fc609002 100644--- a/internal/datasource/connector/yuque/connector_test.go+++ b/internal/datasource/connector/yuque/connector_test.go@@ -5,6 +5,7 @@ import ("net/http""net/http/httptest""os"+ "strings""testing""github.com/Tencent/WeKnora/internal/types"@@ -484,3 +485,31 @@ func TestConnector_ListResources_ContinuesOnGroupFailure(t *testing.T) {t.Errorf("expected Mine + OK Book, got %v", names)}}++func TestConnector_FetchAll_FileNamePolicy(t *testing.T) {+ for _, tt := range []struct{ name, title, want string }{+ {"punctuation and extension", " \tA/B.md ", " \tA_B.md .md"},+ {"long title", strings.Repeat("测", 100), strings.Repeat("测", 66) + ".md"},+ } {+ t.Run(tt.name, func(t *testing.T) {+ f := newFakeYuque()+ defer f.Close()+ f.handleJSON("/api/v2/repos/7/docs", 200, v2DocListResponse{Data: []v2Doc{{+ ID: 101, Type: "Doc", Status: "1", Title: tt.title, ContentUpdatedAt: "2026-04-20T10:00:00Z",+ }}})+ f.handleJSON("/api/v2/repos/docs/101", 200, v2DocDetailResponse{Data: v2DocDetail{+ ID: 101, Title: tt.title, Body: "# content", Format: "markdown", Status: "1",+ }})+ items, err := NewConnector().FetchAll(context.Background(), makeDSConfig(f, []string{"7"}), []string{"7"})+ if err != nil {+ t.Fatal(err)+ }+ if len(items) != 1 {+ t.Fatalf("got %d items, want 1", len(items))+ }+ if items[0].Title != tt.title || items[0].FileName != tt.want {+ t.Fatalf("title = %q, filename = %q; want %q, %q", items[0].Title, items[0].FileName, tt.title, tt.want)+ }+ })+ }+}diff --git a/internal/datasource/connector/yuque/types.go b/internal/datasource/connector/yuque/types.goindex 63d878d54..b57b7ad25 100644--- a/internal/datasource/connector/yuque/types.go+++ b/internal/datasource/connector/yuque/types.go@@ -23,7 +23,6 @@ import ("strconv""strings""time"- "unicode/utf8""github.com/Tencent/WeKnora/internal/datasource""github.com/Tencent/WeKnora/internal/types"@@ -225,37 +224,6 @@ func parseContentUpdatedAt(ts string) time.Time {return t}-// sanitizeFileName removes characters that are invalid in filenames and-// truncates to a safe length at a UTF-8 rune boundary. Raw byte truncation-// would split a multi-byte codepoint (Chinese characters are 3 bytes in UTF-8)-// and produce an invalid UTF-8 string, which downstream filename validation-// (utf8.ValidString) rejects with "文件名包含非法字符".-func sanitizeFileName(name string) string {- if name == "" {- return "untitled"- }- replacer := strings.NewReplacer(- "/", "_", "\\", "_", ":", "_", "*", "_",- "?", "_", "\"", "_", "<", "_", ">", "_", "|", "_",- )- result := replacer.Replace(name)- const maxBytes = 200- if len(result) > maxBytes {- result = result[:maxBytes]- // Peel any trailing bytes that no longer form a complete rune.- // DecodeLastRuneInString returns (RuneError, 1) for a partial UTF-8- // sequence — at most 3 bytes need peeling since a rune is ≤ 4 bytes.- for len(result) > 0 {- r, size := utf8.DecodeLastRuneInString(result)- if r != utf8.RuneError || size != 1 {- break- }- result = result[:len(result)-1]- }- }- return result-}-// redactToken returns a masked form of the token for logging (never log the full token).func redactToken(t string) string {if len(t) < 12 {diff --git a/internal/datasource/connector/yuque/types_test.go b/internal/datasource/connector/yuque/types_test.goindex 1cf6510cb..80ede4d49 100644--- a/internal/datasource/connector/yuque/types_test.go+++ b/internal/datasource/connector/yuque/types_test.go@@ -2,9 +2,7 @@ package yuqueimport ("encoding/json"- "strings""testing"- "unicode/utf8""github.com/Tencent/WeKnora/internal/types")@@ -157,20 +155,3 @@ func TestGetBaseURL_TrimsWhitespace(t *testing.T) {t.Errorf("GetBaseURL = %q, want https://x.yuque.com", got)}}--func TestSanitizeFileName_TruncatesAtRuneBoundary(t *testing.T) {- // Long Chinese title (each 测 is 3 bytes in UTF-8). Raw byte slicing at 200- // would split a rune and produce invalid UTF-8 that downstream filename- // validation rejects with "文件名包含非法字符".- long := strings.Repeat("测试", 100) // 600 bytes- got := sanitizeFileName(long)- if !utf8.ValidString(got) {- t.Fatalf("sanitizeFileName produced invalid UTF-8: %q", got)- }- if len(got) > 200 {- t.Errorf("len = %d, want ≤ 200", len(got))- }- if len(got) == 0 {- t.Error("result is empty")- }-}diff --git a/internal/datasource/filename.go b/internal/datasource/filename.gonew file mode 100644index 000000000..79e5953b4--- /dev/null+++ b/internal/datasource/filename.go@@ -0,0 +1,34 @@+package datasource++import (+ "strings"+ "unicode/utf8"+)++var fileNameReplacer = strings.NewReplacer(+ "/", "_", "\\", "_", ":", "_", "*", "_",+ "?", "_", "\"", "_", "<", "_", ">", "_", "|", "_",+)++// SanitizeFileName replaces filesystem-hostile punctuation and limits a source+// title to 200 bytes without splitting a UTF-8 rune. Empty titles use "untitled".+// Whitespace, control characters and extension handling remain the caller's policy;+// this helper does not validate filenames or repair malformed input.+func SanitizeFileName(name string) string {+ if name == "" {+ return "untitled"+ }+ result := fileNameReplacer.Replace(name)+ const maxBytes = 200+ if len(result) > maxBytes {+ result = result[:maxBytes]+ for len(result) > 0 {+ r, size := utf8.DecodeLastRuneInString(result)+ if r != utf8.RuneError || size != 1 {+ break+ }+ result = result[:len(result)-1]+ }+ }+ return result+}diff --git a/internal/datasource/filename_test.go b/internal/datasource/filename_test.gonew file mode 100644index 000000000..2974a7dba--- /dev/null+++ b/internal/datasource/filename_test.go@@ -0,0 +1,42 @@+package datasource++import (+ "strings"+ "testing"+ "unicode/utf8"+)++func TestSanitizeFileName(t *testing.T) {+ cases := []struct{ name, input, want string }{+ {"empty", "", "untitled"},+ {"punctuation", `a/b\c:d*e?f"g<h>i|j`, "a_b_c_d_e_f_g_h_i_j"},+ {"whitespace retained", " \t标题\n\r ", " \t标题\n\r "},+ {"dots retained", "..", ".."},+ {"controls retained", "a\x00b\x1fc", "a\x00b\x1fc"},+ {"ascii at limit", strings.Repeat("a", 200), strings.Repeat("a", 200)},+ {"ascii over limit", strings.Repeat("a", 201), strings.Repeat("a", 200)},+ {"CJK boundary", strings.Repeat("测", 67), strings.Repeat("测", 66)},+ {"two byte boundary", strings.Repeat("a", 199) + "é", strings.Repeat("a", 199)},+ {"emoji boundary", strings.Repeat("a", 198) + "🙂", strings.Repeat("a", 198)},+ {"complete emoji", strings.Repeat("🙂", 51), strings.Repeat("🙂", 50)},+ {"replacement rune", strings.Repeat("a", 197) + "�x", strings.Repeat("a", 197) + "�"},+ {"no extension reservation", strings.Repeat("a", 200) + ".pdf", strings.Repeat("a", 200)},+ {"malformed short input retained", "a\xff", "a\xff"},+ {"malformed interior retained", "\xff" + strings.Repeat("a", 200), "\xff" + strings.Repeat("a", 199)},+ {"malformed cut boundary removed", strings.Repeat("a", 199) + "\xffx", strings.Repeat("a", 199)},+ }+ for _, tt := range cases {+ t.Run(tt.name, func(t *testing.T) {+ got := SanitizeFileName(tt.input)+ if got != tt.want {+ t.Fatalf("SanitizeFileName(%q) = %q, want %q", tt.input, got, tt.want)+ }+ if len(got) > 200 {+ t.Fatalf("result exceeds 200 bytes: %d", len(got))+ }+ if utf8.ValidString(tt.input) && !utf8.ValidString(got) {+ t.Fatalf("split a UTF-8 rune: %q", got)+ }+ })+ }+}