Skip to content

PasteFerry (隔空贴): clipboard and file sync across macOS, Windows and Linux. Direct-first, end-to-end encrypted. Optional official access: no server to host. Independent self-hosting remains available.

feat(core): search Chinese text by pinyin

baiyuze23949fd53234
diff --git a/client-core/go.mod b/client-core/go.mod
index 5741888..0810451 100644
--- a/client-core/go.mod
+++ b/client-core/go.mod
@@ -11,6 +11,7 @@ require (
github.com/google/uuid v1.6.0 // indirect
github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect
github.com/mattn/go-isatty v0.0.24 // indirect
+ github.com/mozillazg/go-pinyin v0.21.0
github.com/ncruces/go-strftime v1.0.0 // indirect
github.com/pion/datachannel v1.6.3 // indirect
github.com/pion/dtls/v3 v3.1.10 // indirect
diff --git a/client-core/go.sum b/client-core/go.sum
index 5890e05..b6562ae 100644
--- a/client-core/go.sum
+++ b/client-core/go.sum
@@ -27,6 +27,8 @@ github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ
github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I=
github.com/mattn/go-isatty v0.0.24 h1:tGZZoVgT/KiqK1c8ocVLeDS8BSWMRd47J3Lbz7vsReI=
github.com/mattn/go-isatty v0.0.24/go.mod h1:nMCL3Zebbrt45jsMDgnfIwz6ydEQApk5oEI3HqDio6A=
+github.com/mozillazg/go-pinyin v0.21.0 h1:Wo8/NT45z7P3er/9YSLHA3/kjZzbLz5hR7i+jGeIGao=
+github.com/mozillazg/go-pinyin v0.21.0/go.mod h1:iR4EnMMRXkfpFVV5FMi4FNB6wGq9NV6uDWbUuPhP4Yc=
github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w=
github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls=
github.com/pion/datachannel v1.6.3 h1:HZAYLpunVI0xi057ZiANl558d0uSpomcWw4UEPz3qXs=
diff --git a/client-core/internal/pinyin/pinyin.go b/client-core/internal/pinyin/pinyin.go
new file mode 100644
index 0000000..4daba0d
--- /dev/null
+++ b/client-core/internal/pinyin/pinyin.go
@@ -0,0 +1,168 @@
+// Package pinyin 让中文内容可以按拼音搜:全拼(huiyi)、首字母(hy)或两者混着打(huiy、hyi)
+// 都能找到「会议」。多音字的每个读音都算;ü 写作 v,也可以写作 u。读音来自 go-pinyin 的字典。
+package pinyin
+
+import (
+ "strings"
+ "sync"
+ "unicode"
+ "unicode/utf8"
+
+ gopinyin "github.com/mozillazg/go-pinyin"
+)
+
+var (
+ mu sync.RWMutex
+ cache = map[rune][]string{}
+ args = func() gopinyin.Args {
+ a := gopinyin.NewArgs()
+ a.Heteronym = true
+ return a
+ }()
+)
+
+// readings 返回汉字 c 的所有读音(不带声调,小写);不是汉字或字典里没有时返回 nil。
+func readings(c rune) []string {
+ mu.RLock()
+ r, ok := cache[c]
+ mu.RUnlock()
+ if ok {
+ return r
+ }
+ for _, p := range gopinyin.SinglePinyin(c, args) {
+ r = append(r, p)
+ // 「绿」lv 也可以打成 lu:有的输入法和用户习惯这样写
+ if strings.Contains(p, "v") {
+ r = append(r, strings.ReplaceAll(p, "v", "u"))
+ }
+ }
+ mu.Lock()
+ cache[c] = r
+ mu.Unlock()
+ return r
+}
+
+// Index 返回 q 在 s 里第一次匹配的位置(按 rune 计),找不到返回 -1。
+//
+// q 由小写字母组成。s 里的汉字可以用它任一读音的全拼或声母(首字母,或 zh、ch、sh)匹配;
+// q 的最后一段还可以只是某个读音的开头,边打边搜时就能看到结果。其他字符原样匹配,不分大小写;
+// s 里的空白可以跳过(拼音里不打空格)。
+// s 里没有汉字时直接返回 -1:只按原文匹配的情况由调用方处理。
+func Index(s, q string) int {
+ if q == "" {
+ return 0
+ }
+ text := []rune(strings.Map(unicode.ToLower, s))
+ if !containsHan(text) {
+ return -1
+ }
+ m := len(q)
+ // done[i*(m+1)+j]:text[i:] 能否从开头匹配 q[j:],0 还没算、1 能、2 不能
+ done := make([]uint8, (len(text)+1)*(m+1))
+ var match func(i, j int) bool
+ match = func(i, j int) bool {
+ if j == m {
+ return true
+ }
+ if i == len(text) {
+ return false
+ }
+ key := i*(m+1) + j
+ if done[key] != 0 {
+ return done[key] == 1
+ }
+ c := text[i]
+ ok := c < utf8.RuneSelf && byte(c) == q[j] && match(i+1, j+1)
+ // 拼音里不打空格:「发 PDF 给我」用 fapdfgw 也能找到
+ if !ok && j > 0 && unicode.IsSpace(c) {
+ ok = match(i+1, j)
+ }
+ if !ok && unicode.Is(unicode.Han, c) {
+ for _, r := range readings(c) {
+ for _, l := range spans(r, q[j:]) {
+ if match(i+1, j+l) {
+ ok = true
+ break
+ }
+ }
+ if ok {
+ break
+ }
+ }
+ }
+ done[key] = 2
+ if ok {
+ done[key] = 1
+ }
+ return ok
+ }
+ for i, c := range text {
+ if canStart(c, q[0]) && match(i, 0) {
+ return i
+ }
+ }
+ return -1
+}
+
+// canStart:一次匹配能不能从字符 c 开始,即 c 就是 first,或者是有以 first 开头的读音的汉字。
+func canStart(c rune, first byte) bool {
+ if c < utf8.RuneSelf {
+ return byte(c) == first
+ }
+ if !unicode.Is(unicode.Han, c) {
+ return false
+ }
+ for _, r := range readings(c) {
+ if r[0] == first {
+ return true
+ }
+ }
+ return false
+}
+
+// spans 返回读音 r 能吃掉 rest 开头的几种长度:完整的读音、声母,或者 rest 剩下的全部(只打了读音的开头)。
+func spans(r, rest string) []int {
+ var out []int
+ add := func(l int) {
+ if l > 0 && l <= len(r) && l <= len(rest) && rest[:l] == r[:l] {
+ out = append(out, l)
+ }
+ }
+ add(len(r))
+ add(1)
+ if len(r) > 2 && (r[:2] == "zh" || r[:2] == "ch" || r[:2] == "sh") {
+ add(2)
+ }
+ if len(rest) < len(r) {
+ add(len(rest))
+ }
+ return out
+}
+
+func containsHan(text []rune) bool {
+ for _, c := range text {
+ if unicode.Is(unicode.Han, c) {
+ return true
+ }
+ }
+ return false
+}
+
+// Query 把用户输入的搜索词整理成拼音搜索用的形式:去掉空格和隔音符号('),转成小写。
+// 只有剩下的全是字母、至少两个时才按拼音搜,否则返回空串:一个字母会匹配到太多字。
+func Query(s string) string {
+ var b strings.Builder
+ for _, c := range strings.ToLower(s) {
+ switch {
+ case c == ' ' || c == '\'' || c == '’':
+ case c >= 'a' && c <= 'z':
+ b.WriteRune(c)
+ default:
+ return ""
+ }
+ }
+ if b.Len() < 2 {
+ return ""
+ }
+ return b.String()
+}
diff --git a/client-core/internal/pinyin/pinyin_test.go b/client-core/internal/pinyin/pinyin_test.go
new file mode 100644
index 0000000..2be7fc3
--- /dev/null
+++ b/client-core/internal/pinyin/pinyin_test.go
@@ -0,0 +1,74 @@
+package pinyin
+
+import "testing"
+
+func TestIndex(t *testing.T) {
+ cases := []struct {
+ s, q string
+ want int
+ }{
+ {"会议链接:https://meeting.example.com", "huiyi", 0},
+ {"会议链接", "hy", 0},
+ {"会议链接", "huiy", 0},
+ {"会议链接", "hyi", 0},
+ {"会议链接", "hyl", 0},
+ {"会议链接", "lianjie", 2},
+ {"会议链接", "lj", 2},
+ {"周会改到周四下午 3 点", "zhouhui", 0},
+ {"周会改到周四下午 3 点", "zhh", 0},
+ {"周会改到周四下午 3 点", "zh", 0},
+ {"明天的周报", "zhoubao", 3},
+ {"明天的周报", "zb", 3},
+ // 打到一半:最后一段只是读音的开头
+ {"快递单号", "kuaidid", 0},
+ {"快递单号", "kuaidi", 0},
+ // 多音字:「长」zhang / chang,「行」xing / hang
+ {"银行卡号", "yinhang", 0},
+ {"长期有效", "changqi", 0},
+ {"组长", "zuzhang", 0},
+ // ü 写作 v 或 u
+ {"绿色", "lvse", 0},
+ {"绿色", "luse", 0},
+ // 中英混排:英文按原样
+ {"发pdf给我", "fapdf", 0},
+ {"发 PDF 给我", "fapdfgw", 0}, // 拼音里不打空格
+ {"周会改到周四下午 3 点", "xiawu3dian", 6},
+ // 没有汉字的文本不在这里匹配
+ {"meeting link", "ml", -1},
+ // 声母之外的单个字母不能代表一个字的中间
+ {"会议", "ui", -1},
+ {"会议", "huiyix", -1},
+ }
+ for _, c := range cases {
+ if got := Index(c.s, c.q); got != c.want {
+ t.Errorf("Index(%q, %q) = %d,应为 %d", c.s, c.q, got, c.want)
+ }
+ }
+}
+
+func TestQuery(t *testing.T) {
+ cases := map[string]string{
+ "huiyi": "huiyi",
+ "Hui Yi": "huiyi",
+ "hui'yi": "huiyi",
+ "h": "",
+ "会议": "",
+ "docker 1": "",
+ "": "",
+ }
+ for in, want := range cases {
+ if got := Query(in); got != want {
+ t.Errorf("Query(%q) = %q,应为 %q", in, got, want)
+ }
+ }
+}
+
+func BenchmarkIndexLongText(b *testing.B) {
+ long := ""
+ for len([]rune(long)) < 4000 {
+ long += "这是一段很长的会议纪要,用来测试拼音搜索在长文本上的速度。"
+ }
+ for i := 0; i < b.N; i++ {
+ Index(long, "kuaididanhao")
+ }
+}
diff --git a/client-core/internal/sync/search.go b/client-core/internal/sync/search.go
index 2da6c80..7b5fce8 100644
--- a/client-core/internal/sync/search.go
+++ b/client-core/internal/sync/search.go
@@ -3,17 +3,29 @@ package sync
import (
"slices"
"strings"
+ "time"
"unicode"
+ "github.com/baiyuze/pasteferry/client-core/internal/pinyin"
"github.com/baiyuze/pasteferry/client-core/internal/store"
)
+// 按拼音搜时最多看最近这么多条记录:数据库没法按拼音粗筛,只能逐条核对
+const pinyinScan = 2000
+
+// 每条文字只看开头这么多个字有没有拼音匹配,超长的文字不拖慢搜索
+const pinyinText = 5000
+
// SearchHistory 搜索复制记录:文字内容或文件名包含 query 的,不分大小写,按时间倒序,最多 limit 条。
//
// 数据库只能粗筛(见 store.SearchClips),这里按用户看得到的文字再核对一遍:
// HTML 记录看纯文本,不看标签。匹配处不在预览片段里的文字记录,预览换成匹配处前后的一段,
// 列表里一眼能看出为什么搜到了它。
//
+// 搜索词是两个以上的字母时,再按拼音找一遍中文内容(全拼、首字母或混着打,见 pinyin.Index),
+// 比如 huiyi 或 hy 都能找到「会议」。按拼音找到的只在最近 pinyinScan 条里找,和按原文找到的合在一起,
+// 仍按时间排序。
+//
// f 选定在复制记录还是收藏里搜、只搜哪一类,见 store.Filter。
func (e *Engine) SearchHistory(query string, limit int, f store.Filter) ([]store.Clip, error) {
q := []rune(strings.Map(unicode.ToLower, strings.TrimSpace(query)))
@@ -45,6 +57,76 @@ func (e *Engine) SearchHistory(query string, limit int, f store.Filter) ([]store
}
out = append(out, c)
}
+ py := pinyin.Query(query)
+ if py == "" {
+ return out, nil
+ }
+ more, err := e.searchPinyin(py, f, out)
+ if err != nil {
+ return nil, err
+ }
+ if len(more) == 0 {
+ return out, nil
+ }
+ out = append(out, more...)
+ slices.SortStableFunc(out, func(a, b store.Clip) int {
+ if f.Favorites && !a.FavoritedAt.Equal(b.FavoritedAt) {
+ return b.FavoritedAt.Compare(a.FavoritedAt)
+ }
+ return b.CreatedAt.Compare(a.CreatedAt)
+ })
+ if limit > 0 && len(out) > limit {
+ out = out[:limit]
+ }
+ return out, nil
+}
+
+// searchPinyin 在最近 pinyinScan 条记录里,找按原文没搜到、按拼音 py 能匹配的。
+func (e *Engine) searchPinyin(py string, f store.Filter, found []store.Clip) ([]store.Clip, error) {
+ seen := make(map[string]bool, len(found))
+ for _, c := range found {
+ seen[c.ID] = true
+ }
+ var out []store.Clip
+ var before time.Time
+ for scanned := 0; scanned < pinyinScan; {
+ page, err := e.store.ListRecords(500, before, f)
+ if err != nil {
+ return nil, err
+ }
+ for _, c := range page {
+ if seen[c.ID] {
+ continue
+ }
+ if c.Kind != store.KindImage && slices.ContainsFunc(c.Items, func(it store.Item) bool {
+ return pinyin.Index(it.Name, py) >= 0
+ }) {
+ out = append(out, c)
+ continue
+ }
+ if c.Kind != store.KindText && c.Kind != store.KindHTML {
+ continue
+ }
+ text := plainText(c)
+ if r := []rune(text); len(r) > pinyinText {
+ text = string(r[:pinyinText])
+ }
+ at := pinyin.Index(text, py)
+ if at < 0 {
+ continue
+ }
+ if pinyin.Index(c.TextPreview, py) < 0 {
+ c.TextPreview = snippet(text, at)
+ }
+ out = append(out, c)
+ }
+ scanned += len(page)
+ // 收藏不分页,一次就全取到了
+ if len(page) < 500 || f.Favorites {
+ break
+ }
+ before = page[len(page)-1].CreatedAt
+ }
return out, nil
}
diff --git a/client-core/internal/sync/search_test.go b/client-core/internal/sync/search_test.go
index ae72994..1984554 100644
--- a/client-core/internal/sync/search_test.go
+++ b/client-core/internal/sync/search_test.go
@@ -60,6 +60,81 @@ func TestSearchHistory(t *testing.T) {
}
}
+func TestSearchHistoryByPinyin(t *testing.T) {
+ e := newBacklogEngine(t, &fakePeers{})
+ now := time.Now().Truncate(time.Second)
+ long := strings.Repeat("前面的内容很长,", 30) + "快递单号 7712 3456 0098"
+ clips := []store.Clip{
+ {ID: "meeting", Kind: store.KindText, TextContent: "会议链接:https://meeting.example.com/j/8823"},
+ {ID: "long", Kind: store.KindText, TextContent: long, TextPreview: preview(long)},
+ {ID: "file", Kind: store.KindFile, Items: []store.Item{{Name: "合同扫描件.pdf"}}},
+ // 原文里就有 hy(why),按原文也能找到
+ {ID: "english", Kind: store.KindText, TextContent: "why not"},
+ {ID: "fav", Kind: store.KindText, TextContent: "周报模板:本周完成", Favorite: true},
+ }
+ for i, c := range clips {
+ c.Status = store.StatusReady
+ c.CreatedAt = now.Add(time.Duration(i) * time.Minute)
+ c.ExpiresAt = c.CreatedAt.Add(time.Hour)
+ if c.TextPreview == "" {
+ c.TextPreview = preview(c.TextContent)
+ }
+ if c.Favorite {
+ c.FavoritedAt = c.CreatedAt
+ c.ExpiresAt = store.FavoriteExpiry
+ }
+ if err := e.store.PutClip(c); err != nil {
+ t.Fatal(err)
+ }
+ }
+ ids := func(query string, f store.Filter) []string {
+ got, err := e.SearchHistory(query, 50, f)
+ if err != nil {
+ t.Fatal(err)
+ }
+ var out []string
+ for _, c := range got {
+ out = append(out, c.ID)
+ }
+ return out
+ }
+ same := func(a, b []string) bool { return strings.Join(a, ",") == strings.Join(b, ",") }
+
+ for _, q := range []string{"huiyi", "hy", "Hui Yi", "hui'yi", "huiylj"} {
+ got := ids(q, store.Filter{})
+ if q == "hy" {
+ // 按原文找到的 why 和按拼音找到的会议,合在一起按时间倒序
+ if !same(got, []string{"english", "meeting"}) {
+ t.Errorf("%q:%v", q, got)
+ }
+ continue
+ }
+ if !same(got, []string{"meeting"}) {
+ t.Errorf("%q:%v", q, got)
+ }
+ }
+ if got := ids("hetong", store.Filter{}); !same(got, []string{"file"}) {
+ t.Errorf("文件名也按拼音找:%v", got)
+ }
+ got, err := e.SearchHistory("kuaididanhao", 50, store.Filter{})
+ if err != nil || len(got) != 1 || got[0].ID != "long" {
+ t.Fatalf("长文本后半段的拼音匹配:%v %v", got, err)
+ }
+ if p := got[0].TextPreview; !strings.HasPrefix(p, "…") || !strings.Contains(p, "快递单号") {
+ t.Errorf("预览没有换到匹配处:%q", p)
+ }
+ if got := ids("zbmb", store.Filter{Favorites: true}); !same(got, []string{"fav"}) {
+ t.Errorf("收藏里也按拼音找:%v", got)
+ }
+ if got := ids("zbmb", store.Filter{}); len(got) != 0 {
+ t.Errorf("复制记录里不该搜到收藏:%v", got)
+ }
+ // 一个字母不按拼音找,免得什么都匹配
+ if got := ids("y", store.Filter{}); len(got) != 1 || got[0] != "english" {
+ t.Errorf("一个字母只按原文找:%v", got)
+ }
+}
+
func TestIndexFold(t *testing.T) {
q := []rune("straße")
if i := indexFold("Die STRAßE ist lang", q); i != 4 {