Skip to content

armgit / memos

Public Imported from GitHub
0

Memos is an open-source, self-hosted note-taking app that brings your everyday thoughts together in a personal timeline. Your data stays yours, and you choose what to keep private or share.

fix: add Unicode case-insensitive search for SQLite (#5559)

Steven8770b186e47d
diff --git a/go.mod b/go.mod
index fc85404e7..b9a86cab6 100644
--- a/go.mod
+++ b/go.mod
@@ -131,7 +131,7 @@ require (
github.com/valyala/bytebufferpool v1.0.0 // indirect
github.com/valyala/fasttemplate v1.2.2 // indirect
golang.org/x/sys v0.37.0 // indirect
- golang.org/x/text v0.30.0 // indirect
+ golang.org/x/text v0.30.0
golang.org/x/time v0.12.0 // indirect
google.golang.org/protobuf v1.36.9
gopkg.in/yaml.v3 v3.0.1 // indirect
diff --git a/plugin/filter/render.go b/plugin/filter/render.go
index c00de417e..c91096a7b 100644
--- a/plugin/filter/render.go
+++ b/plugin/filter/render.go
@@ -454,6 +454,11 @@ func (r *renderer) renderContainsCondition(cond *ContainsCondition) (renderResul
column := field.columnExpr(r.dialect)
arg := fmt.Sprintf("%%%s%%", cond.Value)
switch r.dialect {
+ case DialectSQLite:
+ // Use custom Unicode-aware case folding function for case-insensitive comparison.
+ // This overcomes SQLite's ASCII-only LOWER() limitation.
+ sql := fmt.Sprintf("memos_unicode_lower(%s) LIKE memos_unicode_lower(%s)", column, r.addArg(arg))
+ return renderResult{sql: sql}, nil
case DialectPostgres:
sql := fmt.Sprintf("%s ILIKE %s", column, r.addArg(arg))
return renderResult{sql: sql}, nil
diff --git a/store/db/sqlite/functions.go b/store/db/sqlite/functions.go
new file mode 100644
index 000000000..6b3021ca7
--- /dev/null
+++ b/store/db/sqlite/functions.go
@@ -0,0 +1,44 @@
+// Package sqlite provides SQLite driver implementation with custom functions.
+// Custom functions are registered globally on first use to extend SQLite's
+// limited ASCII-only text operations with proper Unicode support.
+package sqlite
+
+import (
+ "database/sql/driver"
+ "sync"
+
+ "golang.org/x/text/cases"
+ msqlite "modernc.org/sqlite"
+)
+
+var (
+ registerUnicodeLowerOnce sync.Once
+ registerUnicodeLowerErr error
+ // unicodeFold provides Unicode case folding for case-insensitive comparisons.
+ // It's safe to use concurrently and reused across all function calls.
+ unicodeFold = cases.Fold()
+)
+
+// ensureUnicodeLowerRegistered registers the memos_unicode_lower custom function
+// with SQLite. This function provides proper Unicode case folding for case-insensitive
+// text comparisons, overcoming modernc.org/sqlite's lack of ICU extension.
+//
+// The function is registered once globally and is safe to call multiple times.
+func ensureUnicodeLowerRegistered() error {
+ registerUnicodeLowerOnce.Do(func() {
+ registerUnicodeLowerErr = msqlite.RegisterScalarFunction("memos_unicode_lower", 1, func(_ *msqlite.FunctionContext, args []driver.Value) (driver.Value, error) {
+ if len(args) == 0 || args[0] == nil {
+ return nil, nil
+ }
+ switch v := args[0].(type) {
+ case string:
+ return unicodeFold.String(v), nil
+ case []byte:
+ return unicodeFold.String(string(v)), nil
+ default:
+ return v, nil
+ }
+ })
+ })
+ return registerUnicodeLowerErr
+}
diff --git a/store/db/sqlite/sqlite.go b/store/db/sqlite/sqlite.go
index 642e728cf..dcc979832 100644
--- a/store/db/sqlite/sqlite.go
+++ b/store/db/sqlite/sqlite.go
@@ -6,8 +6,8 @@ import (
"github.com/pkg/errors"
- // Import the SQLite driver.
- _ "modernc.org/sqlite"
+ // Note: modernc.org/sqlite driver is imported in functions.go where
+ // RegisterScalarFunction is used. No blank import needed here.
"github.com/usememos/memos/internal/profile"
"github.com/usememos/memos/store"
@@ -27,6 +27,10 @@ func NewDB(profile *profile.Profile) (store.Driver, error) {
return nil, errors.New("dsn required")
}
+ if err := ensureUnicodeLowerRegistered(); err != nil {
+ return nil, errors.Wrap(err, "failed to register sqlite unicode lower function")
+ }
+
// Connect to the database with some sane settings:
// - No shared-cache: it's obsolete; WAL journal mode is a better solution.
// - No foreign key constraints: it's currently disabled by default, but it's a
diff --git a/store/test/memo_filter_test.go b/store/test/memo_filter_test.go
index 572086e79..7df99a6b0 100644
--- a/store/test/memo_filter_test.go
+++ b/store/test/memo_filter_test.go
@@ -61,6 +61,17 @@ func TestMemoFilterContentUnicode(t *testing.T) {
require.Len(t, memos, 1)
}
+func TestMemoFilterContentUnicodeCaseFold(t *testing.T) {
+ t.Parallel()
+ tc := NewMemoFilterTestContext(t)
+ defer tc.Close()
+
+ tc.CreateMemo(NewMemoBuilder("memo-unicode-case", tc.User.ID).Content("Привет Мир"))
+
+ memos := tc.ListWithFilter(`content.contains("привет")`)
+ require.Len(t, memos, 1)
+}
+
func TestMemoFilterContentCaseSensitivity(t *testing.T) {
t.Parallel()
tc := NewMemoFilterTestContext(t)