Skip to content

A local-first personal finance app

fix: use Unicode-aware SQLite LIKE filtering (#2903)

dymanoid710d9ab8ac81
diff --git a/packages/loot-core/src/platform/server/sqlite/index.electron.ts b/packages/loot-core/src/platform/server/sqlite/index.electron.ts
index d12d133e7e..df723b854c 100644
--- a/packages/loot-core/src/platform/server/sqlite/index.electron.ts
+++ b/packages/loot-core/src/platform/server/sqlite/index.electron.ts
@@ -4,6 +4,8 @@ import { v4 as uuidv4 } from 'uuid';
import { removeFile, readFile } from '../fs';
+import { unicodeLike } from './unicodeLike';
+
function verifyParamTypes(sql, arr) {
arr.forEach(val => {
if (typeof val !== 'string' && typeof val !== 'number' && val !== null) {
@@ -101,7 +103,7 @@ function regexp(regex: string, text: string | null) {
export function openDatabase(pathOrBuffer: string | Buffer) {
const db = new SQL(pathOrBuffer);
- // Define Unicode-aware LOWER and UPPER implementation.
+ // Define Unicode-aware LOWER, UPPER, and LIKE implementation.
// This is necessary because better-sqlite3 uses SQLite build without ICU support.
// @ts-expect-error @types/better-sqlite3 does not support setting strict 3rd argument
db.function('UNICODE_LOWER', { deterministic: true }, (arg: string | null) =>
@@ -112,6 +114,8 @@ export function openDatabase(pathOrBuffer: string | Buffer) {
arg?.toUpperCase(),
);
// @ts-expect-error @types/better-sqlite3 does not support setting strict 3rd argument
+ db.function('UNICODE_LIKE', { deterministic: true }, unicodeLike);
+ // @ts-expect-error @types/better-sqlite3 does not support setting strict 3rd argument
db.function('REGEXP', { deterministic: true }, regexp);
return db;
}
diff --git a/packages/loot-core/src/platform/server/sqlite/index.web.ts b/packages/loot-core/src/platform/server/sqlite/index.web.ts
index 8930e7a3e4..73d3bff04d 100644
--- a/packages/loot-core/src/platform/server/sqlite/index.web.ts
+++ b/packages/loot-core/src/platform/server/sqlite/index.web.ts
@@ -1,6 +1,8 @@
// @ts-strict-ignore
import initSqlJS, { type SqlJsStatic, type Database } from '@jlongster/sql.js';
+import { unicodeLike } from './unicodeLike';
+
let SQL: SqlJsStatic | null = null;
export async function init() {
@@ -193,7 +195,7 @@ export async function openDatabase(pathOrBuffer?: string | Buffer) {
db = new SQL.Database();
}
- // Define Unicode-aware LOWER and UPPER implementation.
+ // Define Unicode-aware LOWER, UPPER, and LIKE implementation.
// This is necessary because sql.js uses SQLite build without ICU support.
//
// Note that this function should ideally be created with a deterministic flag
@@ -201,6 +203,7 @@ export async function openDatabase(pathOrBuffer?: string | Buffer) {
// but SQL.js does not support this: https://github.com/sql-js/sql.js/issues/551
db.create_function('UNICODE_LOWER', arg => arg?.toLowerCase());
db.create_function('UNICODE_UPPER', arg => arg?.toUpperCase());
+ db.create_function('UNICODE_LIKE', unicodeLike);
db.create_function('REGEXP', regexp);
return db;
}
diff --git a/packages/loot-core/src/platform/server/sqlite/unicodeLike.test.ts b/packages/loot-core/src/platform/server/sqlite/unicodeLike.test.ts
new file mode 100644
index 0000000000..04f75619f6
--- /dev/null
+++ b/packages/loot-core/src/platform/server/sqlite/unicodeLike.test.ts
@@ -0,0 +1,57 @@
+import { unicodeLike } from './unicodeLike';
+
+describe('unicode LIKE functionality', () => {
+ it('empty pattern should not match to a value', () => {
+ const result = unicodeLike(null, 'value');
+
+ expect(result).toBe(0);
+ });
+
+ it('empty pattern should not match to null', () => {
+ const result = unicodeLike(null, null);
+
+ expect(result).toBe(0);
+ });
+
+ it('should match special characters', () => {
+ const result = unicodeLike('.*+^${}()|[]\\', '.*+^${}()|[]\\');
+
+ expect(result).toBe(1);
+ });
+
+ it('should use ? as the single character placeholder', () => {
+ const result = unicodeLike('t?st', 'test');
+
+ expect(result).toBe(1);
+ });
+
+ it('should use % as the zero-or-more characters placeholder', () => {
+ const result = unicodeLike('t%st', 'te123st');
+
+ expect(result).toBe(1);
+ });
+
+ it('should ignore case for unicode', () => {
+ const result = unicodeLike('á', 'Ábcdefg');
+
+ expect(result).toBe(1);
+ });
+
+ it('should ignore case for ascii', () => {
+ const result = unicodeLike('a', 'Abcdefg');
+
+ expect(result).toBe(1);
+ });
+
+ it('should treat null value as empty string', () => {
+ const result = unicodeLike('%', null);
+
+ expect(result).toBe(1);
+ });
+
+ it('should not match null value to the string “null”', () => {
+ const result = unicodeLike('null', null);
+
+ expect(result).toBe(0);
+ });
+});
diff --git a/packages/loot-core/src/platform/server/sqlite/unicodeLike.ts b/packages/loot-core/src/platform/server/sqlite/unicodeLike.ts
new file mode 100644
index 0000000000..33e9365fd9
--- /dev/null
+++ b/packages/loot-core/src/platform/server/sqlite/unicodeLike.ts
@@ -0,0 +1,32 @@
+// @ts-strict-ignore
+import LRU from 'lru-cache';
+
+const likePatternCache = new LRU({ max: 500 });
+
+export function unicodeLike(
+ pattern: string | null,
+ value: string | null,
+): number {
+ if (!pattern) {
+ return 0;
+ }
+
+ if (!value) {
+ value = '';
+ }
+
+ let cachedRegExp = likePatternCache.get(pattern);
+ if (!cachedRegExp) {
+ // we don't escape ? and % because we don't know
+ // whether they originate from the user input or from our query compiler.
+ // Maybe improve the query compiler to correctly process these characters?
+ const processedPattern = pattern
+ .replace(/[.*+^${}()|[\]\\]/g, '\\$&')
+ .replaceAll('?', '.')
+ .replaceAll('%', '.*');
+ cachedRegExp = new RegExp(processedPattern, 'i');
+ likePatternCache.set(pattern, cachedRegExp);
+ }
+
+ return cachedRegExp.test(value) ? 1 : 0;
+}
diff --git a/packages/loot-core/src/server/aql/compiler.test.ts b/packages/loot-core/src/server/aql/compiler.test.ts
index 8e1927f5e5..4907efa5e6 100644
--- a/packages/loot-core/src/server/aql/compiler.test.ts
+++ b/packages/loot-core/src/server/aql/compiler.test.ts
@@ -123,6 +123,28 @@ describe('sheet language', () => {
);
});
+ it('`like` should use unicode function', () => {
+ const result = generateSQLWithState(
+ q('transactions')
+ .select('payee')
+ .filter({ 'payee.name': { $like: `%TEST%` } })
+ .serialize(),
+ schemaWithRefs,
+ );
+ expect(result.sql).toMatch(`UNICODE_LIKE('%TEST%', payees1.name)`);
+ });
+
+ it('`notlike` should use unicode function', () => {
+ const result = generateSQLWithState(
+ q('transactions')
+ .select('payee')
+ .filter({ 'payee.name': { $notlike: `%TEST%` } })
+ .serialize(),
+ schemaWithRefs,
+ );
+ expect(result.sql).toMatch(`NOT UNICODE_LIKE('%TEST%', payees1.name)`);
+ });
+
it('`select` allows nested functions', () => {
const result = generateSQLWithState(
q('transactions')
diff --git a/packages/loot-core/src/server/aql/compiler.ts b/packages/loot-core/src/server/aql/compiler.ts
index 5488efa3ed..a7baeb0134 100644
--- a/packages/loot-core/src/server/aql/compiler.ts
+++ b/packages/loot-core/src/server/aql/compiler.ts
@@ -720,7 +720,7 @@ const compileOp = saveStack('op', (state, fieldRef, opData) => {
}
case '$like': {
const [left, right] = valArray(state, [lhs, rhs], ['string', 'string']);
- return `${left} LIKE ${right}`;
+ return `UNICODE_LIKE(${right}, ${left})`;
}
case '$regexp': {
const [left, right] = valArray(state, [lhs, rhs], ['string', 'string']);
@@ -728,7 +728,7 @@ const compileOp = saveStack('op', (state, fieldRef, opData) => {
}
case '$notlike': {
const [left, right] = valArray(state, [lhs, rhs], ['string', 'string']);
- return `(${left} NOT LIKE ${right}\n OR ${left} IS NULL)`;
+ return `(NOT UNICODE_LIKE(${right}, ${left})\n OR ${left} IS NULL)`;
}
default:
throw new CompileError(`Unknown operator: ${op}`);
diff --git a/upcoming-release-notes/2903.md b/upcoming-release-notes/2903.md
new file mode 100644
index 0000000000..c8b4d4d240
--- /dev/null
+++ b/upcoming-release-notes/2903.md
@@ -0,0 +1,6 @@
+---
+category: Bugfix
+authors: [dymanoid]
+---
+
+Use Unicode-aware database queries for filtering and searching.