Skip to content

A local-first personal finance app

Enhance Autocomplete sorting, Payees tab filter, and Schedules tab filter to ignore characters with accents / diacritics (#3045)

Bruno Ribeiro4439bb6abe7a
diff --git a/packages/desktop-client/src/components/ManageRules.tsx b/packages/desktop-client/src/components/ManageRules.tsx
index 65131e41aa..79bc64ec41 100644
--- a/packages/desktop-client/src/components/ManageRules.tsx
+++ b/packages/desktop-client/src/components/ManageRules.tsx
@@ -13,6 +13,7 @@ import { pushModal } from 'loot-core/src/client/actions/modals';
import { initiallyLoadPayees } from 'loot-core/src/client/actions/queries';
import { send } from 'loot-core/src/platform/client/fetch';
import * as undo from 'loot-core/src/platform/client/undo';
+import { getNormalisedString } from 'loot-core/src/shared/normalisation';
import { mapField, friendlyOp } from 'loot-core/src/shared/rules';
import { describeSchedule } from 'loot-core/src/shared/schedules';
import { type NewRuleEntity } from 'loot-core/src/types/models';
@@ -125,9 +126,9 @@ function ManageRulesContent({
(filter === ''
? allRules
: allRules.filter(rule =>
- ruleToString(rule, filterData)
- .toLowerCase()
- .includes(filter.toLowerCase()),
+ getNormalisedString(ruleToString(rule, filterData)).includes(
+ getNormalisedString(filter),
+ ),
)
).slice(0, 100 + page * 50),
[allRules, filter, filterData, page],
diff --git a/packages/desktop-client/src/components/autocomplete/Autocomplete.tsx b/packages/desktop-client/src/components/autocomplete/Autocomplete.tsx
index e34652bdbe..8eb90dd5d7 100644
--- a/packages/desktop-client/src/components/autocomplete/Autocomplete.tsx
+++ b/packages/desktop-client/src/components/autocomplete/Autocomplete.tsx
@@ -14,6 +14,8 @@ import React, {
import Downshift, { type StateChangeTypes } from 'downshift';
import { css } from 'glamor';
+import { getNormalisedString } from 'loot-core/src/shared/normalisation';
+
import { SvgRemove } from '../../icons/v2';
import { useResponsive } from '../../ResponsiveProvider';
import { theme, styles } from '../../style';
@@ -92,16 +94,8 @@ export function defaultFilterSuggestion<T extends Item>(
suggestion: T,
value: string,
) {
- return getItemName(suggestion)
- .toLowerCase()
- .normalize('NFD')
- .replace(/\p{Diacritic}/gu, '')
- .includes(
- value
- .toLowerCase()
- .normalize('NFD')
- .replace(/\p{Diacritic}/gu, ''),
- );
+ const name = getItemName(suggestion);
+ return getNormalisedString(name).includes(getNormalisedString(value));
}
function defaultFilterSuggestions<T extends Item>(
diff --git a/packages/desktop-client/src/components/autocomplete/CategoryAutocomplete.tsx b/packages/desktop-client/src/components/autocomplete/CategoryAutocomplete.tsx
index 3782d2f92a..24612aa832 100644
--- a/packages/desktop-client/src/components/autocomplete/CategoryAutocomplete.tsx
+++ b/packages/desktop-client/src/components/autocomplete/CategoryAutocomplete.tsx
@@ -14,6 +14,7 @@ import { css } from 'glamor';
import { reportBudget, rolloverBudget } from 'loot-core/client/queries';
import { integerToCurrency } from 'loot-core/shared/util';
+import { getNormalisedString } from 'loot-core/src/shared/normalisation';
import {
type CategoryEntity,
type CategoryGroupEntity,
@@ -137,8 +138,8 @@ function CategoryList({
}
function customSort(obj: CategoryAutocompleteItem, value: string): number {
- const name = obj.name.toLowerCase();
- const groupName = obj.group ? obj.group.name.toLowerCase() : '';
+ const name = getNormalisedString(obj.name);
+ const groupName = obj.group ? getNormalisedString(obj.group.name) : '';
if (obj.id === 'split') {
return -2;
}
@@ -206,21 +207,27 @@ export function CategoryAutocomplete({
): CategoryAutocompleteItem[] => {
return suggestions
.filter(suggestion => {
- return (
- suggestion.id === 'split' ||
- suggestion.group?.name
- .toLowerCase()
- .includes(value.toLowerCase()) ||
- (suggestion.group?.name + ' ' + suggestion.name)
- .toLowerCase()
- .includes(value.toLowerCase()) ||
- defaultFilterSuggestion(suggestion, value)
- );
+ if (suggestion.id === 'split') {
+ return true;
+ }
+
+ if (suggestion.group) {
+ return (
+ getNormalisedString(suggestion.group.name).includes(
+ getNormalisedString(value),
+ ) ||
+ getNormalisedString(
+ suggestion.group.name + ' ' + suggestion.name,
+ ).includes(getNormalisedString(value))
+ );
+ }
+
+ return defaultFilterSuggestion(suggestion, value);
})
.sort(
(a, b) =>
- customSort(a, value.toLowerCase()) -
- customSort(b, value.toLowerCase()),
+ customSort(a, getNormalisedString(value)) -
+ customSort(b, getNormalisedString(value)),
);
},
[],
diff --git a/packages/desktop-client/src/components/autocomplete/PayeeAutocomplete.tsx b/packages/desktop-client/src/components/autocomplete/PayeeAutocomplete.tsx
index 36343bc046..4d30eb934f 100644
--- a/packages/desktop-client/src/components/autocomplete/PayeeAutocomplete.tsx
+++ b/packages/desktop-client/src/components/autocomplete/PayeeAutocomplete.tsx
@@ -16,6 +16,7 @@ import { css } from 'glamor';
import { createPayee } from 'loot-core/src/client/actions/queries';
import { getActivePayees } from 'loot-core/src/client/reducers/queries';
+import { getNormalisedString } from 'loot-core/src/shared/normalisation';
import {
type AccountEntity,
type PayeeEntity,
@@ -379,8 +380,12 @@ export function PayeeAutocomplete({
});
filtered.sort((p1, p2) => {
- const r1 = p1.name.toLowerCase().startsWith(value.toLowerCase());
- const r2 = p2.name.toLowerCase().startsWith(value.toLowerCase());
+ const r1 = getNormalisedString(p1.name).startsWith(
+ getNormalisedString(value),
+ );
+ const r2 = getNormalisedString(p2.name).startsWith(
+ getNormalisedString(value),
+ );
const r1exact = p1.name.toLowerCase() === value.toLowerCase();
const r2exact = p2.name.toLowerCase() === value.toLowerCase();
diff --git a/packages/desktop-client/src/components/payees/ManagePayees.jsx b/packages/desktop-client/src/components/payees/ManagePayees.jsx
index 1ec7fd9b54..bc0ce9df81 100644
--- a/packages/desktop-client/src/components/payees/ManagePayees.jsx
+++ b/packages/desktop-client/src/components/payees/ManagePayees.jsx
@@ -11,6 +11,7 @@ import {
import memoizeOne from 'memoize-one';
+import { getNormalisedString } from 'loot-core/src/shared/normalisation';
import { groupById } from 'loot-core/src/shared/util';
import {
@@ -111,7 +112,7 @@ export const ManagePayees = forwardRef(
let filtered = payees;
if (filter) {
filtered = filtered.filter(p =>
- p.name.toLowerCase().includes(filter.toLowerCase()),
+ getNormalisedString(p.name).includes(getNormalisedString(filter)),
);
}
if (orphanedOnly) {
diff --git a/packages/desktop-client/src/components/schedules/SchedulesTable.tsx b/packages/desktop-client/src/components/schedules/SchedulesTable.tsx
index dae97da2dd..66bc8fa826 100644
--- a/packages/desktop-client/src/components/schedules/SchedulesTable.tsx
+++ b/packages/desktop-client/src/components/schedules/SchedulesTable.tsx
@@ -6,6 +6,7 @@ import {
type ScheduleStatuses,
} from 'loot-core/src/client/data-hooks/schedules';
import { format as monthUtilFormat } from 'loot-core/src/shared/months';
+import { getNormalisedString } from 'loot-core/src/shared/normalisation';
import { getScheduledAmount } from 'loot-core/src/shared/schedules';
import { integerToCurrency } from 'loot-core/src/shared/util';
import { type ScheduleEntity } from 'loot-core/src/types/models';
@@ -203,8 +204,8 @@ export function SchedulesTable({
}
const filterIncludes = (str: string) =>
str
- ? str.toLowerCase().includes(filter.toLowerCase()) ||
- filter.toLowerCase().includes(str.toLowerCase())
+ ? getNormalisedString(str).includes(getNormalisedString(filter)) ||
+ getNormalisedString(filter).includes(getNormalisedString(str))
: false;
return schedules.filter(schedule => {
diff --git a/packages/loot-core/src/platform/server/sqlite/index.electron.ts b/packages/loot-core/src/platform/server/sqlite/index.electron.ts
index df723b854c..d8a225f533 100644
--- a/packages/loot-core/src/platform/server/sqlite/index.electron.ts
+++ b/packages/loot-core/src/platform/server/sqlite/index.electron.ts
@@ -4,6 +4,7 @@ import { v4 as uuidv4 } from 'uuid';
import { removeFile, readFile } from '../fs';
+import { normalise } from './normalise';
import { unicodeLike } from './unicodeLike';
function verifyParamTypes(sql, arr) {
@@ -117,6 +118,8 @@ export function openDatabase(pathOrBuffer: string | Buffer) {
db.function('UNICODE_LIKE', { deterministic: true }, unicodeLike);
// @ts-expect-error @types/better-sqlite3 does not support setting strict 3rd argument
db.function('REGEXP', { deterministic: true }, regexp);
+ // @ts-expect-error @types/better-sqlite3 does not support setting strict 3rd argument
+ db.function('NORMALISE', { deterministic: true }, normalise);
return db;
}
diff --git a/packages/loot-core/src/platform/server/sqlite/index.web.ts b/packages/loot-core/src/platform/server/sqlite/index.web.ts
index 73d3bff04d..f8f91b7a41 100644
--- a/packages/loot-core/src/platform/server/sqlite/index.web.ts
+++ b/packages/loot-core/src/platform/server/sqlite/index.web.ts
@@ -1,6 +1,7 @@
// @ts-strict-ignore
import initSqlJS, { type SqlJsStatic, type Database } from '@jlongster/sql.js';
+import { normalise } from './normalise';
import { unicodeLike } from './unicodeLike';
let SQL: SqlJsStatic | null = null;
@@ -205,6 +206,7 @@ export async function openDatabase(pathOrBuffer?: string | Buffer) {
db.create_function('UNICODE_UPPER', arg => arg?.toUpperCase());
db.create_function('UNICODE_LIKE', unicodeLike);
db.create_function('REGEXP', regexp);
+ db.create_function('NORMALISE', normalise);
return db;
}
diff --git a/packages/loot-core/src/platform/server/sqlite/normalise.ts b/packages/loot-core/src/platform/server/sqlite/normalise.ts
new file mode 100644
index 0000000000..2f8330dfb0
--- /dev/null
+++ b/packages/loot-core/src/platform/server/sqlite/normalise.ts
@@ -0,0 +1,9 @@
+import { getNormalisedString } from '../../../shared/normalisation';
+
+export function normalise(value: string | null): string | null {
+ if (!value) {
+ return null;
+ }
+
+ return getNormalisedString(value);
+}
diff --git a/packages/loot-core/src/server/aql/compiler.test.ts b/packages/loot-core/src/server/aql/compiler.test.ts
index 4907efa5e6..cc2add5ddb 100644
--- a/packages/loot-core/src/server/aql/compiler.test.ts
+++ b/packages/loot-core/src/server/aql/compiler.test.ts
@@ -123,7 +123,7 @@ describe('sheet language', () => {
);
});
- it('`like` should use unicode function', () => {
+ it('`like` should use unicode and normalise function', () => {
const result = generateSQLWithState(
q('transactions')
.select('payee')
@@ -131,10 +131,12 @@ describe('sheet language', () => {
.serialize(),
schemaWithRefs,
);
- expect(result.sql).toMatch(`UNICODE_LIKE('%TEST%', payees1.name)`);
+ expect(result.sql).toMatch(
+ `UNICODE_LIKE('%test%', NORMALISE(payees1.name))`,
+ );
});
- it('`notlike` should use unicode function', () => {
+ it('`notlike` should use unicode and normalise function', () => {
const result = generateSQLWithState(
q('transactions')
.select('payee')
@@ -142,7 +144,9 @@ describe('sheet language', () => {
.serialize(),
schemaWithRefs,
);
- expect(result.sql).toMatch(`NOT UNICODE_LIKE('%TEST%', payees1.name)`);
+ expect(result.sql).toMatch(
+ `NOT UNICODE_LIKE('%test%', NORMALISE(payees1.name))`,
+ );
});
it('`select` allows nested functions', () => {
diff --git a/packages/loot-core/src/server/aql/compiler.ts b/packages/loot-core/src/server/aql/compiler.ts
index a7baeb0134..b5c1d4a7a2 100644
--- a/packages/loot-core/src/server/aql/compiler.ts
+++ b/packages/loot-core/src/server/aql/compiler.ts
@@ -1,3 +1,5 @@
+import { getNormalisedString } from '../../shared/normalisation';
+
// @ts-strict-ignore
let _uid = 0;
function resetUid() {
@@ -720,7 +722,7 @@ const compileOp = saveStack('op', (state, fieldRef, opData) => {
}
case '$like': {
const [left, right] = valArray(state, [lhs, rhs], ['string', 'string']);
- return `UNICODE_LIKE(${right}, ${left})`;
+ return `UNICODE_LIKE(${getNormalisedString(right)}, NORMALISE(${left}))`;
}
case '$regexp': {
const [left, right] = valArray(state, [lhs, rhs], ['string', 'string']);
@@ -728,7 +730,7 @@ const compileOp = saveStack('op', (state, fieldRef, opData) => {
}
case '$notlike': {
const [left, right] = valArray(state, [lhs, rhs], ['string', 'string']);
- return `(NOT UNICODE_LIKE(${right}, ${left})\n OR ${left} IS NULL)`;
+ return `(NOT UNICODE_LIKE(${getNormalisedString(right)}, NORMALISE(${left}))\n OR ${left} IS NULL)`;
}
default:
throw new CompileError(`Unknown operator: ${op}`);
diff --git a/packages/loot-core/src/shared/normalisation.ts b/packages/loot-core/src/shared/normalisation.ts
new file mode 100644
index 0000000000..ab196c20fc
--- /dev/null
+++ b/packages/loot-core/src/shared/normalisation.ts
@@ -0,0 +1,6 @@
+export function getNormalisedString(value: string) {
+ return value
+ .toLowerCase()
+ .normalize('NFD')
+ .replace(/\p{Diacritic}/gu, '');
+}
diff --git a/upcoming-release-notes/3045.md b/upcoming-release-notes/3045.md
new file mode 100644
index 0000000000..c5195fe31a
--- /dev/null
+++ b/upcoming-release-notes/3045.md
@@ -0,0 +1,6 @@
+---
+category: Enhancements
+authors: [nullscope]
+---
+
+Enhanced Autocomplete sorting, Payees tab filter, and Schedules tab filter for languages with accents / diacritics.