A local-first personal finance app
Enhance Autocomplete sorting, Payees tab filter, and Schedules tab filter to ignore characters with accents / diacritics (#3045)
Bruno Ribeiro4439bb6abe7a
diff --git a/packages/desktop-client/src/components/ManageRules.tsx b/packages/desktop-client/src/components/ManageRules.tsxindex 65131e41aa..79bc64ec41 100644--- a/packages/desktop-client/src/components/ManageRules.tsx+++ b/packages/desktop-client/src/components/ManageRules.tsx@@ -13,6 +13,7 @@ import { pushModal } from 'loot-core/src/client/actions/modals';import { initiallyLoadPayees } from 'loot-core/src/client/actions/queries';import { send } from 'loot-core/src/platform/client/fetch';import * as undo from 'loot-core/src/platform/client/undo';+import { getNormalisedString } from 'loot-core/src/shared/normalisation';import { mapField, friendlyOp } from 'loot-core/src/shared/rules';import { describeSchedule } from 'loot-core/src/shared/schedules';import { type NewRuleEntity } from 'loot-core/src/types/models';@@ -125,9 +126,9 @@ function ManageRulesContent({(filter === ''? allRules: allRules.filter(rule =>- ruleToString(rule, filterData)- .toLowerCase()- .includes(filter.toLowerCase()),+ getNormalisedString(ruleToString(rule, filterData)).includes(+ getNormalisedString(filter),+ ),)).slice(0, 100 + page * 50),[allRules, filter, filterData, page],diff --git a/packages/desktop-client/src/components/autocomplete/Autocomplete.tsx b/packages/desktop-client/src/components/autocomplete/Autocomplete.tsxindex e34652bdbe..8eb90dd5d7 100644--- a/packages/desktop-client/src/components/autocomplete/Autocomplete.tsx+++ b/packages/desktop-client/src/components/autocomplete/Autocomplete.tsx@@ -14,6 +14,8 @@ import React, {import Downshift, { type StateChangeTypes } from 'downshift';import { css } from 'glamor';+import { getNormalisedString } from 'loot-core/src/shared/normalisation';+import { SvgRemove } from '../../icons/v2';import { useResponsive } from '../../ResponsiveProvider';import { theme, styles } from '../../style';@@ -92,16 +94,8 @@ export function defaultFilterSuggestion<T extends Item>(suggestion: T,value: string,) {- return getItemName(suggestion)- .toLowerCase()- .normalize('NFD')- .replace(/\p{Diacritic}/gu, '')- .includes(- value- .toLowerCase()- .normalize('NFD')- .replace(/\p{Diacritic}/gu, ''),- );+ const name = getItemName(suggestion);+ return getNormalisedString(name).includes(getNormalisedString(value));}function defaultFilterSuggestions<T extends Item>(diff --git a/packages/desktop-client/src/components/autocomplete/CategoryAutocomplete.tsx b/packages/desktop-client/src/components/autocomplete/CategoryAutocomplete.tsxindex 3782d2f92a..24612aa832 100644--- a/packages/desktop-client/src/components/autocomplete/CategoryAutocomplete.tsx+++ b/packages/desktop-client/src/components/autocomplete/CategoryAutocomplete.tsx@@ -14,6 +14,7 @@ import { css } from 'glamor';import { reportBudget, rolloverBudget } from 'loot-core/client/queries';import { integerToCurrency } from 'loot-core/shared/util';+import { getNormalisedString } from 'loot-core/src/shared/normalisation';import {type CategoryEntity,type CategoryGroupEntity,@@ -137,8 +138,8 @@ function CategoryList({}function customSort(obj: CategoryAutocompleteItem, value: string): number {- const name = obj.name.toLowerCase();- const groupName = obj.group ? obj.group.name.toLowerCase() : '';+ const name = getNormalisedString(obj.name);+ const groupName = obj.group ? getNormalisedString(obj.group.name) : '';if (obj.id === 'split') {return -2;}@@ -206,21 +207,27 @@ export function CategoryAutocomplete({): CategoryAutocompleteItem[] => {return suggestions.filter(suggestion => {- return (- suggestion.id === 'split' ||- suggestion.group?.name- .toLowerCase()- .includes(value.toLowerCase()) ||- (suggestion.group?.name + ' ' + suggestion.name)- .toLowerCase()- .includes(value.toLowerCase()) ||- defaultFilterSuggestion(suggestion, value)- );+ if (suggestion.id === 'split') {+ return true;+ }++ if (suggestion.group) {+ return (+ getNormalisedString(suggestion.group.name).includes(+ getNormalisedString(value),+ ) ||+ getNormalisedString(+ suggestion.group.name + ' ' + suggestion.name,+ ).includes(getNormalisedString(value))+ );+ }++ return defaultFilterSuggestion(suggestion, value);}).sort((a, b) =>- customSort(a, value.toLowerCase()) -- customSort(b, value.toLowerCase()),+ customSort(a, getNormalisedString(value)) -+ customSort(b, getNormalisedString(value)),);},[],diff --git a/packages/desktop-client/src/components/autocomplete/PayeeAutocomplete.tsx b/packages/desktop-client/src/components/autocomplete/PayeeAutocomplete.tsxindex 36343bc046..4d30eb934f 100644--- a/packages/desktop-client/src/components/autocomplete/PayeeAutocomplete.tsx+++ b/packages/desktop-client/src/components/autocomplete/PayeeAutocomplete.tsx@@ -16,6 +16,7 @@ import { css } from 'glamor';import { createPayee } from 'loot-core/src/client/actions/queries';import { getActivePayees } from 'loot-core/src/client/reducers/queries';+import { getNormalisedString } from 'loot-core/src/shared/normalisation';import {type AccountEntity,type PayeeEntity,@@ -379,8 +380,12 @@ export function PayeeAutocomplete({});filtered.sort((p1, p2) => {- const r1 = p1.name.toLowerCase().startsWith(value.toLowerCase());- const r2 = p2.name.toLowerCase().startsWith(value.toLowerCase());+ const r1 = getNormalisedString(p1.name).startsWith(+ getNormalisedString(value),+ );+ const r2 = getNormalisedString(p2.name).startsWith(+ getNormalisedString(value),+ );const r1exact = p1.name.toLowerCase() === value.toLowerCase();const r2exact = p2.name.toLowerCase() === value.toLowerCase();diff --git a/packages/desktop-client/src/components/payees/ManagePayees.jsx b/packages/desktop-client/src/components/payees/ManagePayees.jsxindex 1ec7fd9b54..bc0ce9df81 100644--- a/packages/desktop-client/src/components/payees/ManagePayees.jsx+++ b/packages/desktop-client/src/components/payees/ManagePayees.jsx@@ -11,6 +11,7 @@ import {import memoizeOne from 'memoize-one';+import { getNormalisedString } from 'loot-core/src/shared/normalisation';import { groupById } from 'loot-core/src/shared/util';import {@@ -111,7 +112,7 @@ export const ManagePayees = forwardRef(let filtered = payees;if (filter) {filtered = filtered.filter(p =>- p.name.toLowerCase().includes(filter.toLowerCase()),+ getNormalisedString(p.name).includes(getNormalisedString(filter)),);}if (orphanedOnly) {diff --git a/packages/desktop-client/src/components/schedules/SchedulesTable.tsx b/packages/desktop-client/src/components/schedules/SchedulesTable.tsxindex dae97da2dd..66bc8fa826 100644--- a/packages/desktop-client/src/components/schedules/SchedulesTable.tsx+++ b/packages/desktop-client/src/components/schedules/SchedulesTable.tsx@@ -6,6 +6,7 @@ import {type ScheduleStatuses,} from 'loot-core/src/client/data-hooks/schedules';import { format as monthUtilFormat } from 'loot-core/src/shared/months';+import { getNormalisedString } from 'loot-core/src/shared/normalisation';import { getScheduledAmount } from 'loot-core/src/shared/schedules';import { integerToCurrency } from 'loot-core/src/shared/util';import { type ScheduleEntity } from 'loot-core/src/types/models';@@ -203,8 +204,8 @@ export function SchedulesTable({}const filterIncludes = (str: string) =>str- ? str.toLowerCase().includes(filter.toLowerCase()) ||- filter.toLowerCase().includes(str.toLowerCase())+ ? getNormalisedString(str).includes(getNormalisedString(filter)) ||+ getNormalisedString(filter).includes(getNormalisedString(str)): false;return schedules.filter(schedule => {diff --git a/packages/loot-core/src/platform/server/sqlite/index.electron.ts b/packages/loot-core/src/platform/server/sqlite/index.electron.tsindex df723b854c..d8a225f533 100644--- a/packages/loot-core/src/platform/server/sqlite/index.electron.ts+++ b/packages/loot-core/src/platform/server/sqlite/index.electron.ts@@ -4,6 +4,7 @@ import { v4 as uuidv4 } from 'uuid';import { removeFile, readFile } from '../fs';+import { normalise } from './normalise';import { unicodeLike } from './unicodeLike';function verifyParamTypes(sql, arr) {@@ -117,6 +118,8 @@ export function openDatabase(pathOrBuffer: string | Buffer) {db.function('UNICODE_LIKE', { deterministic: true }, unicodeLike);// @ts-expect-error @types/better-sqlite3 does not support setting strict 3rd argumentdb.function('REGEXP', { deterministic: true }, regexp);+ // @ts-expect-error @types/better-sqlite3 does not support setting strict 3rd argument+ db.function('NORMALISE', { deterministic: true }, normalise);return db;}diff --git a/packages/loot-core/src/platform/server/sqlite/index.web.ts b/packages/loot-core/src/platform/server/sqlite/index.web.tsindex 73d3bff04d..f8f91b7a41 100644--- a/packages/loot-core/src/platform/server/sqlite/index.web.ts+++ b/packages/loot-core/src/platform/server/sqlite/index.web.ts@@ -1,6 +1,7 @@// @ts-strict-ignoreimport initSqlJS, { type SqlJsStatic, type Database } from '@jlongster/sql.js';+import { normalise } from './normalise';import { unicodeLike } from './unicodeLike';let SQL: SqlJsStatic | null = null;@@ -205,6 +206,7 @@ export async function openDatabase(pathOrBuffer?: string | Buffer) {db.create_function('UNICODE_UPPER', arg => arg?.toUpperCase());db.create_function('UNICODE_LIKE', unicodeLike);db.create_function('REGEXP', regexp);+ db.create_function('NORMALISE', normalise);return db;}diff --git a/packages/loot-core/src/platform/server/sqlite/normalise.ts b/packages/loot-core/src/platform/server/sqlite/normalise.tsnew file mode 100644index 0000000000..2f8330dfb0--- /dev/null+++ b/packages/loot-core/src/platform/server/sqlite/normalise.ts@@ -0,0 +1,9 @@+import { getNormalisedString } from '../../../shared/normalisation';++export function normalise(value: string | null): string | null {+ if (!value) {+ return null;+ }++ return getNormalisedString(value);+}diff --git a/packages/loot-core/src/server/aql/compiler.test.ts b/packages/loot-core/src/server/aql/compiler.test.tsindex 4907efa5e6..cc2add5ddb 100644--- a/packages/loot-core/src/server/aql/compiler.test.ts+++ b/packages/loot-core/src/server/aql/compiler.test.ts@@ -123,7 +123,7 @@ describe('sheet language', () => {);});- it('`like` should use unicode function', () => {+ it('`like` should use unicode and normalise function', () => {const result = generateSQLWithState(q('transactions').select('payee')@@ -131,10 +131,12 @@ describe('sheet language', () => {.serialize(),schemaWithRefs,);- expect(result.sql).toMatch(`UNICODE_LIKE('%TEST%', payees1.name)`);+ expect(result.sql).toMatch(+ `UNICODE_LIKE('%test%', NORMALISE(payees1.name))`,+ );});- it('`notlike` should use unicode function', () => {+ it('`notlike` should use unicode and normalise function', () => {const result = generateSQLWithState(q('transactions').select('payee')@@ -142,7 +144,9 @@ describe('sheet language', () => {.serialize(),schemaWithRefs,);- expect(result.sql).toMatch(`NOT UNICODE_LIKE('%TEST%', payees1.name)`);+ expect(result.sql).toMatch(+ `NOT UNICODE_LIKE('%test%', NORMALISE(payees1.name))`,+ );});it('`select` allows nested functions', () => {diff --git a/packages/loot-core/src/server/aql/compiler.ts b/packages/loot-core/src/server/aql/compiler.tsindex a7baeb0134..b5c1d4a7a2 100644--- a/packages/loot-core/src/server/aql/compiler.ts+++ b/packages/loot-core/src/server/aql/compiler.ts@@ -1,3 +1,5 @@+import { getNormalisedString } from '../../shared/normalisation';+// @ts-strict-ignorelet _uid = 0;function resetUid() {@@ -720,7 +722,7 @@ const compileOp = saveStack('op', (state, fieldRef, opData) => {}case '$like': {const [left, right] = valArray(state, [lhs, rhs], ['string', 'string']);- return `UNICODE_LIKE(${right}, ${left})`;+ return `UNICODE_LIKE(${getNormalisedString(right)}, NORMALISE(${left}))`;}case '$regexp': {const [left, right] = valArray(state, [lhs, rhs], ['string', 'string']);@@ -728,7 +730,7 @@ const compileOp = saveStack('op', (state, fieldRef, opData) => {}case '$notlike': {const [left, right] = valArray(state, [lhs, rhs], ['string', 'string']);- return `(NOT UNICODE_LIKE(${right}, ${left})\n OR ${left} IS NULL)`;+ return `(NOT UNICODE_LIKE(${getNormalisedString(right)}, NORMALISE(${left}))\n OR ${left} IS NULL)`;}default:throw new CompileError(`Unknown operator: ${op}`);diff --git a/packages/loot-core/src/shared/normalisation.ts b/packages/loot-core/src/shared/normalisation.tsnew file mode 100644index 0000000000..ab196c20fc--- /dev/null+++ b/packages/loot-core/src/shared/normalisation.ts@@ -0,0 +1,6 @@+export function getNormalisedString(value: string) {+ return value+ .toLowerCase()+ .normalize('NFD')+ .replace(/\p{Diacritic}/gu, '');+}diff --git a/upcoming-release-notes/3045.md b/upcoming-release-notes/3045.mdnew file mode 100644index 0000000000..c5195fe31a--- /dev/null+++ b/upcoming-release-notes/3045.md@@ -0,0 +1,6 @@+---+category: Enhancements+authors: [nullscope]+---++Enhanced Autocomplete sorting, Payees tab filter, and Schedules tab filter for languages with accents / diacritics.