PasteFerry (隔空贴): clipboard and file sync across macOS, Windows and Linux. Direct-first, end-to-end encrypted. Optional official access: no server to host. Independent self-hosting remains available.
Dart airdrop-alternativeclipboardclipboard-sharingclipboard-synccopy-pastecross-platformend-to-end-encryptionfile-syncfile-transferfluttergolanglinuxmacosnat-traversalp2pself-hostedtailscaleuniversal-clipboardwebrtcwindows
feat(core): search Chinese text by pinyin
baiyuze23949fd53234
diff --git a/client-core/go.mod b/client-core/go.modindex 5741888..0810451 100644--- a/client-core/go.mod+++ b/client-core/go.mod@@ -11,6 +11,7 @@ require (github.com/google/uuid v1.6.0 // indirectgithub.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirectgithub.com/mattn/go-isatty v0.0.24 // indirect+ github.com/mozillazg/go-pinyin v0.21.0github.com/ncruces/go-strftime v1.0.0 // indirectgithub.com/pion/datachannel v1.6.3 // indirectgithub.com/pion/dtls/v3 v3.1.10 // indirectdiff --git a/client-core/go.sum b/client-core/go.sumindex 5890e05..b6562ae 100644--- a/client-core/go.sum+++ b/client-core/go.sum@@ -27,6 +27,8 @@ github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZgithub.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I=github.com/mattn/go-isatty v0.0.24 h1:tGZZoVgT/KiqK1c8ocVLeDS8BSWMRd47J3Lbz7vsReI=github.com/mattn/go-isatty v0.0.24/go.mod h1:nMCL3Zebbrt45jsMDgnfIwz6ydEQApk5oEI3HqDio6A=+github.com/mozillazg/go-pinyin v0.21.0 h1:Wo8/NT45z7P3er/9YSLHA3/kjZzbLz5hR7i+jGeIGao=+github.com/mozillazg/go-pinyin v0.21.0/go.mod h1:iR4EnMMRXkfpFVV5FMi4FNB6wGq9NV6uDWbUuPhP4Yc=github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w=github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls=github.com/pion/datachannel v1.6.3 h1:HZAYLpunVI0xi057ZiANl558d0uSpomcWw4UEPz3qXs=diff --git a/client-core/internal/pinyin/pinyin.go b/client-core/internal/pinyin/pinyin.gonew file mode 100644index 0000000..4daba0d--- /dev/null+++ b/client-core/internal/pinyin/pinyin.go@@ -0,0 +1,168 @@+// Package pinyin 让中文内容可以按拼音搜:全拼(huiyi)、首字母(hy)或两者混着打(huiy、hyi)+// 都能找到「会议」。多音字的每个读音都算;ü 写作 v,也可以写作 u。读音来自 go-pinyin 的字典。+package pinyin++import (+ "strings"+ "sync"+ "unicode"+ "unicode/utf8"++ gopinyin "github.com/mozillazg/go-pinyin"+)++var (+ mu sync.RWMutex+ cache = map[rune][]string{}+ args = func() gopinyin.Args {+ a := gopinyin.NewArgs()+ a.Heteronym = true+ return a+ }()+)++// readings 返回汉字 c 的所有读音(不带声调,小写);不是汉字或字典里没有时返回 nil。+func readings(c rune) []string {+ mu.RLock()+ r, ok := cache[c]+ mu.RUnlock()+ if ok {+ return r+ }+ for _, p := range gopinyin.SinglePinyin(c, args) {+ r = append(r, p)+ // 「绿」lv 也可以打成 lu:有的输入法和用户习惯这样写+ if strings.Contains(p, "v") {+ r = append(r, strings.ReplaceAll(p, "v", "u"))+ }+ }+ mu.Lock()+ cache[c] = r+ mu.Unlock()+ return r+}++// Index 返回 q 在 s 里第一次匹配的位置(按 rune 计),找不到返回 -1。+//+// q 由小写字母组成。s 里的汉字可以用它任一读音的全拼或声母(首字母,或 zh、ch、sh)匹配;+// q 的最后一段还可以只是某个读音的开头,边打边搜时就能看到结果。其他字符原样匹配,不分大小写;+// s 里的空白可以跳过(拼音里不打空格)。+// s 里没有汉字时直接返回 -1:只按原文匹配的情况由调用方处理。+func Index(s, q string) int {+ if q == "" {+ return 0+ }+ text := []rune(strings.Map(unicode.ToLower, s))+ if !containsHan(text) {+ return -1+ }+ m := len(q)+ // done[i*(m+1)+j]:text[i:] 能否从开头匹配 q[j:],0 还没算、1 能、2 不能+ done := make([]uint8, (len(text)+1)*(m+1))+ var match func(i, j int) bool+ match = func(i, j int) bool {+ if j == m {+ return true+ }+ if i == len(text) {+ return false+ }+ key := i*(m+1) + j+ if done[key] != 0 {+ return done[key] == 1+ }+ c := text[i]+ ok := c < utf8.RuneSelf && byte(c) == q[j] && match(i+1, j+1)+ // 拼音里不打空格:「发 PDF 给我」用 fapdfgw 也能找到+ if !ok && j > 0 && unicode.IsSpace(c) {+ ok = match(i+1, j)+ }+ if !ok && unicode.Is(unicode.Han, c) {+ for _, r := range readings(c) {+ for _, l := range spans(r, q[j:]) {+ if match(i+1, j+l) {+ ok = true+ break+ }+ }+ if ok {+ break+ }+ }+ }+ done[key] = 2+ if ok {+ done[key] = 1+ }+ return ok+ }+ for i, c := range text {+ if canStart(c, q[0]) && match(i, 0) {+ return i+ }+ }+ return -1+}++// canStart:一次匹配能不能从字符 c 开始,即 c 就是 first,或者是有以 first 开头的读音的汉字。+func canStart(c rune, first byte) bool {+ if c < utf8.RuneSelf {+ return byte(c) == first+ }+ if !unicode.Is(unicode.Han, c) {+ return false+ }+ for _, r := range readings(c) {+ if r[0] == first {+ return true+ }+ }+ return false+}++// spans 返回读音 r 能吃掉 rest 开头的几种长度:完整的读音、声母,或者 rest 剩下的全部(只打了读音的开头)。+func spans(r, rest string) []int {+ var out []int+ add := func(l int) {+ if l > 0 && l <= len(r) && l <= len(rest) && rest[:l] == r[:l] {+ out = append(out, l)+ }+ }+ add(len(r))+ add(1)+ if len(r) > 2 && (r[:2] == "zh" || r[:2] == "ch" || r[:2] == "sh") {+ add(2)+ }+ if len(rest) < len(r) {+ add(len(rest))+ }+ return out+}++func containsHan(text []rune) bool {+ for _, c := range text {+ if unicode.Is(unicode.Han, c) {+ return true+ }+ }+ return false+}++// Query 把用户输入的搜索词整理成拼音搜索用的形式:去掉空格和隔音符号('),转成小写。+// 只有剩下的全是字母、至少两个时才按拼音搜,否则返回空串:一个字母会匹配到太多字。+func Query(s string) string {+ var b strings.Builder+ for _, c := range strings.ToLower(s) {+ switch {+ case c == ' ' || c == '\'' || c == '’':+ case c >= 'a' && c <= 'z':+ b.WriteRune(c)+ default:+ return ""+ }+ }+ if b.Len() < 2 {+ return ""+ }+ return b.String()+}diff --git a/client-core/internal/pinyin/pinyin_test.go b/client-core/internal/pinyin/pinyin_test.gonew file mode 100644index 0000000..2be7fc3--- /dev/null+++ b/client-core/internal/pinyin/pinyin_test.go@@ -0,0 +1,74 @@+package pinyin++import "testing"++func TestIndex(t *testing.T) {+ cases := []struct {+ s, q string+ want int+ }{+ {"会议链接:https://meeting.example.com", "huiyi", 0},+ {"会议链接", "hy", 0},+ {"会议链接", "huiy", 0},+ {"会议链接", "hyi", 0},+ {"会议链接", "hyl", 0},+ {"会议链接", "lianjie", 2},+ {"会议链接", "lj", 2},+ {"周会改到周四下午 3 点", "zhouhui", 0},+ {"周会改到周四下午 3 点", "zhh", 0},+ {"周会改到周四下午 3 点", "zh", 0},+ {"明天的周报", "zhoubao", 3},+ {"明天的周报", "zb", 3},+ // 打到一半:最后一段只是读音的开头+ {"快递单号", "kuaidid", 0},+ {"快递单号", "kuaidi", 0},+ // 多音字:「长」zhang / chang,「行」xing / hang+ {"银行卡号", "yinhang", 0},+ {"长期有效", "changqi", 0},+ {"组长", "zuzhang", 0},+ // ü 写作 v 或 u+ {"绿色", "lvse", 0},+ {"绿色", "luse", 0},+ // 中英混排:英文按原样+ {"发pdf给我", "fapdf", 0},+ {"发 PDF 给我", "fapdfgw", 0}, // 拼音里不打空格+ {"周会改到周四下午 3 点", "xiawu3dian", 6},+ // 没有汉字的文本不在这里匹配+ {"meeting link", "ml", -1},+ // 声母之外的单个字母不能代表一个字的中间+ {"会议", "ui", -1},+ {"会议", "huiyix", -1},+ }+ for _, c := range cases {+ if got := Index(c.s, c.q); got != c.want {+ t.Errorf("Index(%q, %q) = %d,应为 %d", c.s, c.q, got, c.want)+ }+ }+}++func TestQuery(t *testing.T) {+ cases := map[string]string{+ "huiyi": "huiyi",+ "Hui Yi": "huiyi",+ "hui'yi": "huiyi",+ "h": "",+ "会议": "",+ "docker 1": "",+ "": "",+ }+ for in, want := range cases {+ if got := Query(in); got != want {+ t.Errorf("Query(%q) = %q,应为 %q", in, got, want)+ }+ }+}++func BenchmarkIndexLongText(b *testing.B) {+ long := ""+ for len([]rune(long)) < 4000 {+ long += "这是一段很长的会议纪要,用来测试拼音搜索在长文本上的速度。"+ }+ for i := 0; i < b.N; i++ {+ Index(long, "kuaididanhao")+ }+}diff --git a/client-core/internal/sync/search.go b/client-core/internal/sync/search.goindex 2da6c80..7b5fce8 100644--- a/client-core/internal/sync/search.go+++ b/client-core/internal/sync/search.go@@ -3,17 +3,29 @@ package syncimport ("slices""strings"+ "time""unicode"+ "github.com/baiyuze/pasteferry/client-core/internal/pinyin""github.com/baiyuze/pasteferry/client-core/internal/store")+// 按拼音搜时最多看最近这么多条记录:数据库没法按拼音粗筛,只能逐条核对+const pinyinScan = 2000++// 每条文字只看开头这么多个字有没有拼音匹配,超长的文字不拖慢搜索+const pinyinText = 5000+// SearchHistory 搜索复制记录:文字内容或文件名包含 query 的,不分大小写,按时间倒序,最多 limit 条。//// 数据库只能粗筛(见 store.SearchClips),这里按用户看得到的文字再核对一遍:// HTML 记录看纯文本,不看标签。匹配处不在预览片段里的文字记录,预览换成匹配处前后的一段,// 列表里一眼能看出为什么搜到了它。//+// 搜索词是两个以上的字母时,再按拼音找一遍中文内容(全拼、首字母或混着打,见 pinyin.Index),+// 比如 huiyi 或 hy 都能找到「会议」。按拼音找到的只在最近 pinyinScan 条里找,和按原文找到的合在一起,+// 仍按时间排序。+//// f 选定在复制记录还是收藏里搜、只搜哪一类,见 store.Filter。func (e *Engine) SearchHistory(query string, limit int, f store.Filter) ([]store.Clip, error) {q := []rune(strings.Map(unicode.ToLower, strings.TrimSpace(query)))@@ -45,6 +57,76 @@ func (e *Engine) SearchHistory(query string, limit int, f store.Filter) ([]store}out = append(out, c)}+ py := pinyin.Query(query)+ if py == "" {+ return out, nil+ }+ more, err := e.searchPinyin(py, f, out)+ if err != nil {+ return nil, err+ }+ if len(more) == 0 {+ return out, nil+ }+ out = append(out, more...)+ slices.SortStableFunc(out, func(a, b store.Clip) int {+ if f.Favorites && !a.FavoritedAt.Equal(b.FavoritedAt) {+ return b.FavoritedAt.Compare(a.FavoritedAt)+ }+ return b.CreatedAt.Compare(a.CreatedAt)+ })+ if limit > 0 && len(out) > limit {+ out = out[:limit]+ }+ return out, nil+}++// searchPinyin 在最近 pinyinScan 条记录里,找按原文没搜到、按拼音 py 能匹配的。+func (e *Engine) searchPinyin(py string, f store.Filter, found []store.Clip) ([]store.Clip, error) {+ seen := make(map[string]bool, len(found))+ for _, c := range found {+ seen[c.ID] = true+ }+ var out []store.Clip+ var before time.Time+ for scanned := 0; scanned < pinyinScan; {+ page, err := e.store.ListRecords(500, before, f)+ if err != nil {+ return nil, err+ }+ for _, c := range page {+ if seen[c.ID] {+ continue+ }+ if c.Kind != store.KindImage && slices.ContainsFunc(c.Items, func(it store.Item) bool {+ return pinyin.Index(it.Name, py) >= 0+ }) {+ out = append(out, c)+ continue+ }+ if c.Kind != store.KindText && c.Kind != store.KindHTML {+ continue+ }+ text := plainText(c)+ if r := []rune(text); len(r) > pinyinText {+ text = string(r[:pinyinText])+ }+ at := pinyin.Index(text, py)+ if at < 0 {+ continue+ }+ if pinyin.Index(c.TextPreview, py) < 0 {+ c.TextPreview = snippet(text, at)+ }+ out = append(out, c)+ }+ scanned += len(page)+ // 收藏不分页,一次就全取到了+ if len(page) < 500 || f.Favorites {+ break+ }+ before = page[len(page)-1].CreatedAt+ }return out, nil}diff --git a/client-core/internal/sync/search_test.go b/client-core/internal/sync/search_test.goindex ae72994..1984554 100644--- a/client-core/internal/sync/search_test.go+++ b/client-core/internal/sync/search_test.go@@ -60,6 +60,81 @@ func TestSearchHistory(t *testing.T) {}}+func TestSearchHistoryByPinyin(t *testing.T) {+ e := newBacklogEngine(t, &fakePeers{})+ now := time.Now().Truncate(time.Second)+ long := strings.Repeat("前面的内容很长,", 30) + "快递单号 7712 3456 0098"+ clips := []store.Clip{+ {ID: "meeting", Kind: store.KindText, TextContent: "会议链接:https://meeting.example.com/j/8823"},+ {ID: "long", Kind: store.KindText, TextContent: long, TextPreview: preview(long)},+ {ID: "file", Kind: store.KindFile, Items: []store.Item{{Name: "合同扫描件.pdf"}}},+ // 原文里就有 hy(why),按原文也能找到+ {ID: "english", Kind: store.KindText, TextContent: "why not"},+ {ID: "fav", Kind: store.KindText, TextContent: "周报模板:本周完成", Favorite: true},+ }+ for i, c := range clips {+ c.Status = store.StatusReady+ c.CreatedAt = now.Add(time.Duration(i) * time.Minute)+ c.ExpiresAt = c.CreatedAt.Add(time.Hour)+ if c.TextPreview == "" {+ c.TextPreview = preview(c.TextContent)+ }+ if c.Favorite {+ c.FavoritedAt = c.CreatedAt+ c.ExpiresAt = store.FavoriteExpiry+ }+ if err := e.store.PutClip(c); err != nil {+ t.Fatal(err)+ }+ }+ ids := func(query string, f store.Filter) []string {+ got, err := e.SearchHistory(query, 50, f)+ if err != nil {+ t.Fatal(err)+ }+ var out []string+ for _, c := range got {+ out = append(out, c.ID)+ }+ return out+ }+ same := func(a, b []string) bool { return strings.Join(a, ",") == strings.Join(b, ",") }++ for _, q := range []string{"huiyi", "hy", "Hui Yi", "hui'yi", "huiylj"} {+ got := ids(q, store.Filter{})+ if q == "hy" {+ // 按原文找到的 why 和按拼音找到的会议,合在一起按时间倒序+ if !same(got, []string{"english", "meeting"}) {+ t.Errorf("%q:%v", q, got)+ }+ continue+ }+ if !same(got, []string{"meeting"}) {+ t.Errorf("%q:%v", q, got)+ }+ }+ if got := ids("hetong", store.Filter{}); !same(got, []string{"file"}) {+ t.Errorf("文件名也按拼音找:%v", got)+ }+ got, err := e.SearchHistory("kuaididanhao", 50, store.Filter{})+ if err != nil || len(got) != 1 || got[0].ID != "long" {+ t.Fatalf("长文本后半段的拼音匹配:%v %v", got, err)+ }+ if p := got[0].TextPreview; !strings.HasPrefix(p, "…") || !strings.Contains(p, "快递单号") {+ t.Errorf("预览没有换到匹配处:%q", p)+ }+ if got := ids("zbmb", store.Filter{Favorites: true}); !same(got, []string{"fav"}) {+ t.Errorf("收藏里也按拼音找:%v", got)+ }+ if got := ids("zbmb", store.Filter{}); len(got) != 0 {+ t.Errorf("复制记录里不该搜到收藏:%v", got)+ }+ // 一个字母不按拼音找,免得什么都匹配+ if got := ids("y", store.Filter{}); len(got) != 1 || got[0] != "english" {+ t.Errorf("一个字母只按原文找:%v", got)+ }+}+func TestIndexFold(t *testing.T) {q := []rune("straße")if i := indexFold("Die STRAßE ist lang", q); i != 4 {