Skip to content

Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.

refactor(datasource): share connector filename sanitization

wizardchen820a14dafa0e
diff --git a/docs/code-slimming-audit.md b/docs/code-slimming-audit.md
index b560d3dab..abfa34d77 100644
--- a/docs/code-slimming-audit.md
+++ b/docs/code-slimming-audit.md
@@ -98,6 +98,16 @@
- 本批只删除无调用代码,复用现有包测试和编译检查,没有添加仅断言函数不存在的测试。
- 推送检查暴露新合入的 fork 测试未隔离 Git hook 环境:继承的 `GIT_DIR` 等变量使临时仓库操作指向当前仓库。测试辅助函数现清除继承的 `GIT_*` 变量;真实仓库测试主动注入指向临时路径的仓库变量,验证旧实现失败、修复后通过。该测试修复单独提交,不改变生产逻辑。
+## 第六批已实施:外部来源文件名清理
+
+基于第五批合入后的 `10973ed32`,IMA、语雀、RSS 复用 `datasource.SanitizeFileName`,统一无效标点替换、空标题回退和 200 字节 UTF-8 边界截断。计入新增共享实现后,生产 Go 源码净减少 49 行。
+
+- IMA 的媒体与笔记路径、语雀文档直接调用共享函数;RSS 仅保留换行/制表符转空格和首尾空白清理,再调用共享函数。
+- 保留原有扩展名拼接、源标题、空白、控制字符和无效 UTF-8 输入处理行为。飞书的扩展名保留、钉钉的控制字符/尾部标点清理、Confluence 的按字符截断规则不同,本批未合并。
+- 将两份旧 helper 测试收敛到共享函数测试,补充 16 个边界用例;在实际连接器读取/组装路径新增 15 个用例,覆盖 IMA 两类内容、语雀和 RSS 的最终文件名。
+- 用重构前的三个实现对 30,000 组确定性生成输入执行差异校验,结果一致;输入包含中文、emoji、空白、非法字节和截断边界。临时对比代码未留在仓库,避免长期维护旧实现副本。
+- 4 个受影响包测试通过:104 项顶层测试,包含既有同步、游标和错误处理测试。未修改前端,未运行浏览器检查或真实外部来源服务测试。
+
## 后续优先级
| 优先级 | 证据 | 建议边界 | 验证重点 |
@@ -113,7 +123,7 @@
### 后端重复的具体判断
- Qdrant 与 Weaviate 的 `tokenizeQuery` 函数体相同,但只有 Qdrant 的关键字检索调用它;Weaviate 使用 `WithQuery(params.Query)`。第五批已删除 Weaviate 的失效函数,保留 Qdrant 实现。
-- IMA、语雀的 `sanitizeFileName` 函数体相同,可以共享;RSS 的近似实现另外处理首尾空白和换行,合并时必须保留这些差异。
+- 第六批已合并 IMA、语雀、RSS 的文件名标点替换和 UTF-8 截断逻辑;RSS 的空白预处理保留在连接器内。飞书、钉钉、Confluence 的其他规则继续独立维护。
- Milvus/Qdrant/Weaviate 的结果映射体相似,但接收不同后端类型。当前各约 17 行,为此引入通用接口或泛型转换层未必减少维护成本。
- 终端和桌面 WebSocket ticket handler 重复检查会话、用户、租户和 token,但签发方式不同。后续可共享前置身份校验,保留 ticket 和连接生命周期各自的契约。
- 资源权限已经有 `internal/application/access`;继续沿已有授权边界收敛,避免新建与其并行的权限工具层。
@@ -152,4 +162,4 @@
- 构建仍报告大 chunk 提示;本轮未以 bundle 体积为目标,也没有测量构建前后的体积差异。
- 未修改后端,未运行后端测试;未做浏览器视觉验收。
-下一批优先处理 IMA、语雀等外部来源的重复文件名清理逻辑,核实调用与差异后再决定共享边界。涉及聊天、任务重试、权限或共享状态的改动应单独成批,以可验证的行为等价为边界。
+下一批优先检查终端与桌面 WebSocket ticket 的重复身份校验,沿现有授权边界确定共享范围。涉及聊天、任务重试、权限或共享状态的改动应单独成批,以可验证的行为等价为边界。
diff --git a/internal/datasource/connector/ima/connector.go b/internal/datasource/connector/ima/connector.go
index 89c305a19..6111ffc69 100644
--- a/internal/datasource/connector/ima/connector.go
+++ b/internal/datasource/connector/ima/connector.go
@@ -7,7 +7,6 @@ import (
"sort"
"strings"
"time"
- "unicode/utf8"
"github.com/Tencent/WeKnora/internal/datasource"
"github.com/Tencent/WeKnora/internal/logger"
@@ -437,7 +436,7 @@ func fetchNote(
return types.FetchedItem{}, fetchSkipped
}
- fileName := sanitizeFileName(f.Title)
+ fileName := datasource.SanitizeFileName(f.Title)
if !strings.HasSuffix(strings.ToLower(fileName), ".md") {
fileName += ".md"
}
@@ -540,7 +539,7 @@ func fetchOneMedia(
ct = mimeForExtension(ext)
}
- fileName := sanitizeFileName(f.Title)
+ fileName := datasource.SanitizeFileName(f.Title)
if !strings.HasSuffix(strings.ToLower(fileName), "."+ext) {
fileName = fileName + "." + ext
}
@@ -585,28 +584,3 @@ func baseMetadata(
}
return m
}
-
-// sanitizeFileName removes filesystem-hostile characters and truncates to a
-// safe UTF-8 boundary.
-func sanitizeFileName(name string) string {
- if name == "" {
- return "untitled"
- }
- replacer := strings.NewReplacer(
- "/", "_", "\\", "_", ":", "_", "*", "_",
- "?", "_", "\"", "_", "<", "_", ">", "_", "|", "_",
- )
- result := replacer.Replace(name)
- const maxBytes = 200
- if len(result) > maxBytes {
- result = result[:maxBytes]
- for len(result) > 0 {
- r, size := utf8.DecodeLastRuneInString(result)
- if r != utf8.RuneError || size != 1 {
- break
- }
- result = result[:len(result)-1]
- }
- }
- return result
-}
diff --git a/internal/datasource/connector/ima/connector_test.go b/internal/datasource/connector/ima/connector_test.go
index d1a447e78..42970c8ca 100644
--- a/internal/datasource/connector/ima/connector_test.go
+++ b/internal/datasource/connector/ima/connector_test.go
@@ -509,3 +509,35 @@ func TestConnectorIsRegisteredInMetadata(t *testing.T) {
t.Errorf("metadata type = %q, want %q", meta.Type, types.ConnectorTypeIMA)
}
}
+
+func TestFetchAll_FileNamePolicy(t *testing.T) {
+ for _, media := range []struct {
+ name string
+ kind int32
+ }{{"media", mediaTypeMarkdown}, {"note", mediaTypeNote}} {
+ for _, tt := range []struct{ name, title, want string }{
+ {"punctuation", "A/B:C", "A_B_C.md"},
+ {"existing extension and whitespace", " \tA/B.MD", " \tA_B.MD"},
+ {"long title", strings.Repeat("测", 100) + ".MD", strings.Repeat("测", 66) + ".md"},
+ } {
+ t.Run(media.name+"/"+tt.name, func(t *testing.T) {
+ f := newFakeIMA(t)
+ f.setKB("kb1", []fakeFile{{
+ MediaID: "file", Title: tt.title, MediaType: media.kind,
+ Body: "# content", NotebookID: "123", NoteBody: "# content",
+ }})
+ items, err := NewConnector().FetchAll(context.Background(), f.config("kb1"), []string{"kb1"})
+ if err != nil {
+ t.Fatal(err)
+ }
+ if len(items) != 1 {
+ t.Fatalf("got %d items, want 1", len(items))
+ }
+ if items[0].Title != tt.title || items[0].FileName != tt.want {
+ t.Fatalf("title = %q, filename = %q; want %q, %q",
+ items[0].Title, items[0].FileName, tt.title, tt.want)
+ }
+ })
+ }
+ }
+}
diff --git a/internal/datasource/connector/ima/types_test.go b/internal/datasource/connector/ima/types_test.go
index d7859c04c..e52f6ff63 100644
--- a/internal/datasource/connector/ima/types_test.go
+++ b/internal/datasource/connector/ima/types_test.go
@@ -131,25 +131,6 @@ func TestExtensionForContentType(t *testing.T) {
}
}
-func TestSanitizeFileName(t *testing.T) {
- if got := sanitizeFileName(""); got != "untitled" {
- t.Errorf("empty name = %q, want untitled", got)
- }
- if got := sanitizeFileName(`a/b\c:d*e?f"g<h>i|j`); strings.ContainsAny(got, `/\:*?"<>|`) {
- t.Errorf("filesystem-hostile characters survived: %q", got)
- }
-
- // A long CJK title must be truncated on a rune boundary, not mid-sequence.
- long := strings.Repeat("知识", 200)
- got := sanitizeFileName(long)
- if len(got) > 200 {
- t.Errorf("len = %d bytes, want <= 200", len(got))
- }
- if !strings.HasPrefix(long, got) {
- t.Errorf("truncation corrupted the prefix: %q", got)
- }
-}
-
func TestIsSkippableMediaType(t *testing.T) {
for _, mt := range []int32{mediaTypeAISession, mediaTypeVideo} {
if !isSkippableMediaType(mt) {
diff --git a/internal/datasource/connector/rss/connector_test.go b/internal/datasource/connector/rss/connector_test.go
index ff74c7db2..c7771a0fd 100644
--- a/internal/datasource/connector/rss/connector_test.go
+++ b/internal/datasource/connector/rss/connector_test.go
@@ -413,3 +413,28 @@ func TestConnector_ResolveResourceAncestors_Empty(t *testing.T) {
t.Fatalf("expected empty ancestors, got %v", got)
}
}
+
+func TestResolveItem_FileNamePolicy(t *testing.T) {
+ for _, tt := range []struct{ name, title, want string }{
+ {"empty", "", "untitled.md"},
+ {"blank", " \n\t\u2003", "untitled.md"},
+ {"line breaks and punctuation", "\u2003 A\nB\rC\tD/E \u2003", "A B C D_E.md"},
+ {"existing extension", "Title.md", "Title.md.md"},
+ {"long title", strings.Repeat("测", 100), strings.Repeat("测", 66) + ".md"},
+ {"space at truncation boundary", strings.Repeat("a", 199) + "\nmore", strings.Repeat("a", 199) + " .md"},
+ {"other controls retained", "a\x01b", "a\x01b.md"},
+ } {
+ t.Run(tt.name, func(t *testing.T) {
+ result := NewConnector().resolveItem(
+ context.Background(), nil, &gofeed.Feed{}, &gofeed.Item{Title: tt.title},
+ "https://example.com/feed", "id", "<p>content</p>",
+ )
+ if result.item.FileName != tt.want {
+ t.Fatalf("FileName = %q, want %q", result.item.FileName, tt.want)
+ }
+ if strings.TrimSpace(tt.title) != "" && result.item.Title != tt.title {
+ t.Fatalf("source title changed: %q", result.item.Title)
+ }
+ })
+ }
+}
diff --git a/internal/datasource/connector/rss/types.go b/internal/datasource/connector/rss/types.go
index b6f672646..9fff63b5d 100644
--- a/internal/datasource/connector/rss/types.go
+++ b/internal/datasource/connector/rss/types.go
@@ -29,7 +29,6 @@ import (
"maps"
"strings"
"time"
- "unicode/utf8"
"github.com/Tencent/WeKnora/internal/datasource"
"github.com/Tencent/WeKnora/internal/types"
@@ -218,32 +217,8 @@ func firstNonEmpty(values ...string) string {
return ""
}
-// sanitizeFileName removes characters invalid in filenames and truncates to a
-// safe length at a UTF-8 rune boundary (mirrors the Yuque connector).
+// sanitizeFileName keeps feed titles on one line before applying shared limits.
func sanitizeFileName(name string) string {
- name = strings.TrimSpace(name)
- if name == "" {
- return "untitled"
- }
- replacer := strings.NewReplacer(
- "/", "_", "\\", "_", ":", "_", "*", "_",
- "?", "_", "\"", "_", "<", "_", ">", "_", "|", "_",
- "\n", " ", "\r", " ", "\t", " ",
- )
- result := strings.TrimSpace(replacer.Replace(name))
- if result == "" {
- return "untitled"
- }
- const maxBytes = 200
- if len(result) > maxBytes {
- result = result[:maxBytes]
- for len(result) > 0 {
- r, size := utf8.DecodeLastRuneInString(result)
- if r != utf8.RuneError || size != 1 {
- break
- }
- result = result[:len(result)-1]
- }
- }
- return result
+ name = strings.NewReplacer("\n", " ", "\r", " ", "\t", " ").Replace(name)
+ return datasource.SanitizeFileName(strings.TrimSpace(name))
}
diff --git a/internal/datasource/connector/yuque/connector.go b/internal/datasource/connector/yuque/connector.go
index d07fce0c5..485eef13b 100644
--- a/internal/datasource/connector/yuque/connector.go
+++ b/internal/datasource/connector/yuque/connector.go
@@ -262,7 +262,7 @@ func (c *Connector) walk(
Title: d.Title,
Content: []byte(detail.Body),
ContentType: "text/markdown",
- FileName: sanitizeFileName(d.Title) + ".md",
+ FileName: datasource.SanitizeFileName(d.Title) + ".md",
URL: buildDocURL(cfg.GetBaseURL(), detail.Book.Namespace, d.Slug),
UpdatedAt: parseContentUpdatedAt(d.ContentUpdatedAt),
SourceResourceID: bookIDStr,
diff --git a/internal/datasource/connector/yuque/connector_test.go b/internal/datasource/connector/yuque/connector_test.go
index 44f3ca049..8fc609002 100644
--- a/internal/datasource/connector/yuque/connector_test.go
+++ b/internal/datasource/connector/yuque/connector_test.go
@@ -5,6 +5,7 @@ import (
"net/http"
"net/http/httptest"
"os"
+ "strings"
"testing"
"github.com/Tencent/WeKnora/internal/types"
@@ -484,3 +485,31 @@ func TestConnector_ListResources_ContinuesOnGroupFailure(t *testing.T) {
t.Errorf("expected Mine + OK Book, got %v", names)
}
}
+
+func TestConnector_FetchAll_FileNamePolicy(t *testing.T) {
+ for _, tt := range []struct{ name, title, want string }{
+ {"punctuation and extension", " \tA/B.md ", " \tA_B.md .md"},
+ {"long title", strings.Repeat("测", 100), strings.Repeat("测", 66) + ".md"},
+ } {
+ t.Run(tt.name, func(t *testing.T) {
+ f := newFakeYuque()
+ defer f.Close()
+ f.handleJSON("/api/v2/repos/7/docs", 200, v2DocListResponse{Data: []v2Doc{{
+ ID: 101, Type: "Doc", Status: "1", Title: tt.title, ContentUpdatedAt: "2026-04-20T10:00:00Z",
+ }}})
+ f.handleJSON("/api/v2/repos/docs/101", 200, v2DocDetailResponse{Data: v2DocDetail{
+ ID: 101, Title: tt.title, Body: "# content", Format: "markdown", Status: "1",
+ }})
+ items, err := NewConnector().FetchAll(context.Background(), makeDSConfig(f, []string{"7"}), []string{"7"})
+ if err != nil {
+ t.Fatal(err)
+ }
+ if len(items) != 1 {
+ t.Fatalf("got %d items, want 1", len(items))
+ }
+ if items[0].Title != tt.title || items[0].FileName != tt.want {
+ t.Fatalf("title = %q, filename = %q; want %q, %q", items[0].Title, items[0].FileName, tt.title, tt.want)
+ }
+ })
+ }
+}
diff --git a/internal/datasource/connector/yuque/types.go b/internal/datasource/connector/yuque/types.go
index 63d878d54..b57b7ad25 100644
--- a/internal/datasource/connector/yuque/types.go
+++ b/internal/datasource/connector/yuque/types.go
@@ -23,7 +23,6 @@ import (
"strconv"
"strings"
"time"
- "unicode/utf8"
"github.com/Tencent/WeKnora/internal/datasource"
"github.com/Tencent/WeKnora/internal/types"
@@ -225,37 +224,6 @@ func parseContentUpdatedAt(ts string) time.Time {
return t
}
-// sanitizeFileName removes characters that are invalid in filenames and
-// truncates to a safe length at a UTF-8 rune boundary. Raw byte truncation
-// would split a multi-byte codepoint (Chinese characters are 3 bytes in UTF-8)
-// and produce an invalid UTF-8 string, which downstream filename validation
-// (utf8.ValidString) rejects with "文件名包含非法字符".
-func sanitizeFileName(name string) string {
- if name == "" {
- return "untitled"
- }
- replacer := strings.NewReplacer(
- "/", "_", "\\", "_", ":", "_", "*", "_",
- "?", "_", "\"", "_", "<", "_", ">", "_", "|", "_",
- )
- result := replacer.Replace(name)
- const maxBytes = 200
- if len(result) > maxBytes {
- result = result[:maxBytes]
- // Peel any trailing bytes that no longer form a complete rune.
- // DecodeLastRuneInString returns (RuneError, 1) for a partial UTF-8
- // sequence — at most 3 bytes need peeling since a rune is ≤ 4 bytes.
- for len(result) > 0 {
- r, size := utf8.DecodeLastRuneInString(result)
- if r != utf8.RuneError || size != 1 {
- break
- }
- result = result[:len(result)-1]
- }
- }
- return result
-}
-
// redactToken returns a masked form of the token for logging (never log the full token).
func redactToken(t string) string {
if len(t) < 12 {
diff --git a/internal/datasource/connector/yuque/types_test.go b/internal/datasource/connector/yuque/types_test.go
index 1cf6510cb..80ede4d49 100644
--- a/internal/datasource/connector/yuque/types_test.go
+++ b/internal/datasource/connector/yuque/types_test.go
@@ -2,9 +2,7 @@ package yuque
import (
"encoding/json"
- "strings"
"testing"
- "unicode/utf8"
"github.com/Tencent/WeKnora/internal/types"
)
@@ -157,20 +155,3 @@ func TestGetBaseURL_TrimsWhitespace(t *testing.T) {
t.Errorf("GetBaseURL = %q, want https://x.yuque.com", got)
}
}
-
-func TestSanitizeFileName_TruncatesAtRuneBoundary(t *testing.T) {
- // Long Chinese title (each 测 is 3 bytes in UTF-8). Raw byte slicing at 200
- // would split a rune and produce invalid UTF-8 that downstream filename
- // validation rejects with "文件名包含非法字符".
- long := strings.Repeat("测试", 100) // 600 bytes
- got := sanitizeFileName(long)
- if !utf8.ValidString(got) {
- t.Fatalf("sanitizeFileName produced invalid UTF-8: %q", got)
- }
- if len(got) > 200 {
- t.Errorf("len = %d, want ≤ 200", len(got))
- }
- if len(got) == 0 {
- t.Error("result is empty")
- }
-}
diff --git a/internal/datasource/filename.go b/internal/datasource/filename.go
new file mode 100644
index 000000000..79e5953b4
--- /dev/null
+++ b/internal/datasource/filename.go
@@ -0,0 +1,34 @@
+package datasource
+
+import (
+ "strings"
+ "unicode/utf8"
+)
+
+var fileNameReplacer = strings.NewReplacer(
+ "/", "_", "\\", "_", ":", "_", "*", "_",
+ "?", "_", "\"", "_", "<", "_", ">", "_", "|", "_",
+)
+
+// SanitizeFileName replaces filesystem-hostile punctuation and limits a source
+// title to 200 bytes without splitting a UTF-8 rune. Empty titles use "untitled".
+// Whitespace, control characters and extension handling remain the caller's policy;
+// this helper does not validate filenames or repair malformed input.
+func SanitizeFileName(name string) string {
+ if name == "" {
+ return "untitled"
+ }
+ result := fileNameReplacer.Replace(name)
+ const maxBytes = 200
+ if len(result) > maxBytes {
+ result = result[:maxBytes]
+ for len(result) > 0 {
+ r, size := utf8.DecodeLastRuneInString(result)
+ if r != utf8.RuneError || size != 1 {
+ break
+ }
+ result = result[:len(result)-1]
+ }
+ }
+ return result
+}
diff --git a/internal/datasource/filename_test.go b/internal/datasource/filename_test.go
new file mode 100644
index 000000000..2974a7dba
--- /dev/null
+++ b/internal/datasource/filename_test.go
@@ -0,0 +1,42 @@
+package datasource
+
+import (
+ "strings"
+ "testing"
+ "unicode/utf8"
+)
+
+func TestSanitizeFileName(t *testing.T) {
+ cases := []struct{ name, input, want string }{
+ {"empty", "", "untitled"},
+ {"punctuation", `a/b\c:d*e?f"g<h>i|j`, "a_b_c_d_e_f_g_h_i_j"},
+ {"whitespace retained", " \t标题\n\r ", " \t标题\n\r "},
+ {"dots retained", "..", ".."},
+ {"controls retained", "a\x00b\x1fc", "a\x00b\x1fc"},
+ {"ascii at limit", strings.Repeat("a", 200), strings.Repeat("a", 200)},
+ {"ascii over limit", strings.Repeat("a", 201), strings.Repeat("a", 200)},
+ {"CJK boundary", strings.Repeat("测", 67), strings.Repeat("测", 66)},
+ {"two byte boundary", strings.Repeat("a", 199) + "é", strings.Repeat("a", 199)},
+ {"emoji boundary", strings.Repeat("a", 198) + "🙂", strings.Repeat("a", 198)},
+ {"complete emoji", strings.Repeat("🙂", 51), strings.Repeat("🙂", 50)},
+ {"replacement rune", strings.Repeat("a", 197) + "�x", strings.Repeat("a", 197) + "�"},
+ {"no extension reservation", strings.Repeat("a", 200) + ".pdf", strings.Repeat("a", 200)},
+ {"malformed short input retained", "a\xff", "a\xff"},
+ {"malformed interior retained", "\xff" + strings.Repeat("a", 200), "\xff" + strings.Repeat("a", 199)},
+ {"malformed cut boundary removed", strings.Repeat("a", 199) + "\xffx", strings.Repeat("a", 199)},
+ }
+ for _, tt := range cases {
+ t.Run(tt.name, func(t *testing.T) {
+ got := SanitizeFileName(tt.input)
+ if got != tt.want {
+ t.Fatalf("SanitizeFileName(%q) = %q, want %q", tt.input, got, tt.want)
+ }
+ if len(got) > 200 {
+ t.Fatalf("result exceeds 200 bytes: %d", len(got))
+ }
+ if utf8.ValidString(tt.input) && !utf8.ValidString(got) {
+ t.Fatalf("split a UTF-8 rune: %q", got)
+ }
+ })
+ }
+}