Memos is an open-source, self-hosted note-taking app that brings your everyday thoughts together in a personal timeline. Your data stays yours, and you choose what to keep private or share.
Go dockerfossgomarkdownmemomicroblognote-takingnotesopen-sourceown-your-datareactself-hostedsqlitetimeline
fix: add Unicode case-insensitive search for SQLite (#5559)
Steven8770b186e47d
diff --git a/go.mod b/go.modindex fc85404e7..b9a86cab6 100644--- a/go.mod+++ b/go.mod@@ -131,7 +131,7 @@ require (github.com/valyala/bytebufferpool v1.0.0 // indirectgithub.com/valyala/fasttemplate v1.2.2 // indirectgolang.org/x/sys v0.37.0 // indirect- golang.org/x/text v0.30.0 // indirect+ golang.org/x/text v0.30.0golang.org/x/time v0.12.0 // indirectgoogle.golang.org/protobuf v1.36.9gopkg.in/yaml.v3 v3.0.1 // indirectdiff --git a/plugin/filter/render.go b/plugin/filter/render.goindex c00de417e..c91096a7b 100644--- a/plugin/filter/render.go+++ b/plugin/filter/render.go@@ -454,6 +454,11 @@ func (r *renderer) renderContainsCondition(cond *ContainsCondition) (renderResulcolumn := field.columnExpr(r.dialect)arg := fmt.Sprintf("%%%s%%", cond.Value)switch r.dialect {+ case DialectSQLite:+ // Use custom Unicode-aware case folding function for case-insensitive comparison.+ // This overcomes SQLite's ASCII-only LOWER() limitation.+ sql := fmt.Sprintf("memos_unicode_lower(%s) LIKE memos_unicode_lower(%s)", column, r.addArg(arg))+ return renderResult{sql: sql}, nilcase DialectPostgres:sql := fmt.Sprintf("%s ILIKE %s", column, r.addArg(arg))return renderResult{sql: sql}, nildiff --git a/store/db/sqlite/functions.go b/store/db/sqlite/functions.gonew file mode 100644index 000000000..6b3021ca7--- /dev/null+++ b/store/db/sqlite/functions.go@@ -0,0 +1,44 @@+// Package sqlite provides SQLite driver implementation with custom functions.+// Custom functions are registered globally on first use to extend SQLite's+// limited ASCII-only text operations with proper Unicode support.+package sqlite++import (+ "database/sql/driver"+ "sync"++ "golang.org/x/text/cases"+ msqlite "modernc.org/sqlite"+)++var (+ registerUnicodeLowerOnce sync.Once+ registerUnicodeLowerErr error+ // unicodeFold provides Unicode case folding for case-insensitive comparisons.+ // It's safe to use concurrently and reused across all function calls.+ unicodeFold = cases.Fold()+)++// ensureUnicodeLowerRegistered registers the memos_unicode_lower custom function+// with SQLite. This function provides proper Unicode case folding for case-insensitive+// text comparisons, overcoming modernc.org/sqlite's lack of ICU extension.+//+// The function is registered once globally and is safe to call multiple times.+func ensureUnicodeLowerRegistered() error {+ registerUnicodeLowerOnce.Do(func() {+ registerUnicodeLowerErr = msqlite.RegisterScalarFunction("memos_unicode_lower", 1, func(_ *msqlite.FunctionContext, args []driver.Value) (driver.Value, error) {+ if len(args) == 0 || args[0] == nil {+ return nil, nil+ }+ switch v := args[0].(type) {+ case string:+ return unicodeFold.String(v), nil+ case []byte:+ return unicodeFold.String(string(v)), nil+ default:+ return v, nil+ }+ })+ })+ return registerUnicodeLowerErr+}diff --git a/store/db/sqlite/sqlite.go b/store/db/sqlite/sqlite.goindex 642e728cf..dcc979832 100644--- a/store/db/sqlite/sqlite.go+++ b/store/db/sqlite/sqlite.go@@ -6,8 +6,8 @@ import ("github.com/pkg/errors"- // Import the SQLite driver.- _ "modernc.org/sqlite"+ // Note: modernc.org/sqlite driver is imported in functions.go where+ // RegisterScalarFunction is used. No blank import needed here."github.com/usememos/memos/internal/profile""github.com/usememos/memos/store"@@ -27,6 +27,10 @@ func NewDB(profile *profile.Profile) (store.Driver, error) {return nil, errors.New("dsn required")}+ if err := ensureUnicodeLowerRegistered(); err != nil {+ return nil, errors.Wrap(err, "failed to register sqlite unicode lower function")+ }+// Connect to the database with some sane settings:// - No shared-cache: it's obsolete; WAL journal mode is a better solution.// - No foreign key constraints: it's currently disabled by default, but it's adiff --git a/store/test/memo_filter_test.go b/store/test/memo_filter_test.goindex 572086e79..7df99a6b0 100644--- a/store/test/memo_filter_test.go+++ b/store/test/memo_filter_test.go@@ -61,6 +61,17 @@ func TestMemoFilterContentUnicode(t *testing.T) {require.Len(t, memos, 1)}+func TestMemoFilterContentUnicodeCaseFold(t *testing.T) {+ t.Parallel()+ tc := NewMemoFilterTestContext(t)+ defer tc.Close()++ tc.CreateMemo(NewMemoBuilder("memo-unicode-case", tc.User.ID).Content("Привет Мир"))++ memos := tc.ListWithFilter(`content.contains("привет")`)+ require.Len(t, memos, 1)+}+func TestMemoFilterContentCaseSensitivity(t *testing.T) {t.Parallel()tc := NewMemoFilterTestContext(t)