A local-first personal finance app
[AI] Add payeeNameNormalization option to import transactions API (#8710)
Haim Gelfenbeynf78a6fa041b1
diff --git a/packages/docs/docs/api/reference.md b/packages/docs/docs/api/reference.mdindex 14b9269f6b..c4a47167f4 100644--- a/packages/docs/docs/api/reference.md+++ b/packages/docs/docs/api/reference.md@@ -279,6 +279,7 @@ This method has the following optional flags (passed as the `opts` object):- `defaultCleared`: whether imported transactions should be marked as cleared (defaults to `true`)- `dryRun`: if `true`, returns what would be added/updated without actually modifying the database (defaults to `false`)- `reimportDeleted`: if `true`, transactions that were previously imported and then deleted will be reimported; if `false`, they will be skipped (defaults to `true` for backward compatibility — note that the [file import UI](../transactions/importing.md#avoiding-duplicate-transactions) defaults to `false`)+- `payeeNameNormalization`: how `payee_name` is processed when creating a payee — `'title-case'` re-capitalizes each word, `'original'` keeps the name as given, apart from trimming surrounding whitespace (defaults to `'title-case'`)Example using opts:diff --git a/packages/loot-core/src/server/accounts/app.ts b/packages/loot-core/src/server/accounts/app.tsindex 03ec882efd..c8922134cc 100644--- a/packages/loot-core/src/server/accounts/app.ts+++ b/packages/loot-core/src/server/accounts/app.ts@@ -1646,16 +1646,25 @@ async function importTransactions({throw APIError('transactions-import: accountId must be an id');}+ const payeeNameNormalization = opts?.payeeNameNormalization ?? 'title-case';+ if (!bankSync.PAYEE_NAME_NORMALIZATIONS.includes(payeeNameNormalization)) {+ throw APIError(+ `transactions-import: payeeNameNormalization must be one of ${bankSync.PAYEE_NAME_NORMALIZATIONS.join(+ ', ',+ )}, got '${String(payeeNameNormalization)}'`,+ );+ }+try {const reconciled = await bankSync.reconcileTransactions(accountId,transactions,- false,- true,- isPreview,- opts?.defaultCleared,- false,- opts?.reimportDeleted,+ {+ isPreview,+ defaultCleared: opts?.defaultCleared,+ reimportDeleted: opts?.reimportDeleted,+ payeeNameNormalization,+ },);return {errors: [],diff --git a/packages/loot-core/src/server/accounts/sync.test.ts b/packages/loot-core/src/server/accounts/sync.test.tsindex 8f2c20a077..6db27f7d03 100644--- a/packages/loot-core/src/server/accounts/sync.test.ts+++ b/packages/loot-core/src/server/accounts/sync.test.ts@@ -8,6 +8,7 @@ import { setSyncingMode } from '#server/sync';import { handlers } from '#server/tests/mockSyncServer';import { insertRule, loadRules } from '#server/transactions/transaction-rules';import * as monthUtils from '#shared/months';+import type { ImportTransactionsOpts } from '#types/api-handlers';import type { SyncedPrefs } from '#types/prefs';import { app as accountsApp } from './app';@@ -103,6 +104,123 @@ describe('Account sync', () => {);});+ test('reconcile title-cases the payee name by default', async () => {+ const { id } = await prepareDatabase();++ await reconcileTransactions(id, [+ {+ date: '2020-01-02',+ payee_name: 'Nintendo Store New York NY',+ amount: 4133,+ },+ ]);++ const payees = await getAllPayees();+ expect(payees.length).toBe(1);+ expect(payees[0].name).toBe('Nintendo Store New York Ny');++ const transactions = await getAllTransactions();+ expect(transactions[0].imported_payee).toBe('Nintendo Store New York Ny');+ });++ test('transactions-import title-cases the payee name by default', async () => {+ const { id } = await prepareDatabase();++ await accountsApp.handlers['transactions-import']({+ accountId: id,+ transactions: [+ {+ account: id,+ date: '2020-01-02',+ payee_name: 'Nintendo Store New York NY',+ amount: 4133,+ },+ ],+ isPreview: false,+ });++ const payees = await getAllPayees();+ expect(payees.length).toBe(1);+ expect(payees[0].name).toBe('Nintendo Store New York Ny');++ const transactions = await getAllTransactions();+ expect(transactions[0].imported_payee).toBe('Nintendo Store New York Ny');+ });++ test("transactions-import keeps the payee name with payeeNameNormalization 'original'", async () => {+ const { id } = await prepareDatabase();++ await accountsApp.handlers['transactions-import']({+ accountId: id,+ transactions: [+ {+ account: id,+ date: '2020-01-02',+ payee_name: 'Nintendo Store New York NY',+ amount: 4133,+ },+ ],+ isPreview: false,+ opts: { payeeNameNormalization: 'original' },+ });++ const payees = await getAllPayees();+ expect(payees.length).toBe(1);+ expect(payees[0].name).toBe('Nintendo Store New York NY');++ const transactions = await getAllTransactions();+ expect(transactions[0].imported_payee).toBe('Nintendo Store New York NY');+ });++ test("transactions-import trims the payee name with payeeNameNormalization 'original'", async () => {+ const { id } = await prepareDatabase();++ await accountsApp.handlers['transactions-import']({+ accountId: id,+ transactions: [+ {+ account: id,+ date: '2020-01-02',+ payee_name: ' Nintendo Store New York NY ',+ amount: 4133,+ },+ ],+ isPreview: false,+ opts: { payeeNameNormalization: 'original' },+ });++ const payees = await getAllPayees();+ expect(payees.length).toBe(1);+ expect(payees[0].name).toBe('Nintendo Store New York NY');++ const transactions = await getAllTransactions();+ expect(transactions[0].imported_payee).toBe('Nintendo Store New York NY');+ });++ test('transactions-import rejects an unknown payeeNameNormalization', async () => {+ const { id } = await prepareDatabase();++ await expect(+ accountsApp.handlers['transactions-import']({+ accountId: id,+ transactions: [+ {+ account: id,+ date: '2020-01-02',+ payee_name: 'Nintendo Store New York NY',+ amount: 4133,+ },+ ],+ isPreview: false,+ opts: {+ payeeNameNormalization: 'titlecase',+ } as unknown as ImportTransactionsOpts,+ }),+ ).rejects.toThrow(/payeeNameNormalization/);++ expect(await getAllPayees()).toEqual([]);+ });+test('reconcile handles transactions with undefined fields', async () => {const { id: acctId } = await prepareDatabase();@@ -190,12 +308,7 @@ describe('Account sync', () => {await reconcileTransactions(acctId,[{ date: '2020-01-01', imported_id: 'finid-override' }],- false,- true,- false,- true,- false,- false,+ { reimportDeleted: false },);const transactions2 = await getAllTransactions();expect(transactions2.length).toBe(1);@@ -216,12 +329,7 @@ describe('Account sync', () => {await reconcileTransactions(acctId,[{ date: '2020-01-01', imported_id: 'finid-override2' }],- false,- true,- false,- true,- false,- true,+ { reimportDeleted: true },);const transactions2 = await getAllTransactions();expect(transactions2.length).toBe(2);@@ -246,12 +354,7 @@ describe('Account sync', () => {await reconcileTransactions(acctId,[{ date: '2020-01-01', imported_id: 'finid-precedence' }],- false,- true,- false,- true,- false,- false,+ { reimportDeleted: false },);const transactions2 = await getAllTransactions();expect(transactions2.length).toBe(1);@@ -622,8 +725,7 @@ describe('Account sync', () => {imported_id: 'something-else-entirely',},],- false,- false,+ { strictIdChecking: false },);payees = await getAllPayees();diff --git a/packages/loot-core/src/server/accounts/sync.ts b/packages/loot-core/src/server/accounts/sync.tsindex 93aca4a9da..efcf65e14e 100644--- a/packages/loot-core/src/server/accounts/sync.ts+++ b/packages/loot-core/src/server/accounts/sync.ts@@ -413,10 +413,34 @@ async function resolvePayee(trans, payeeName, payeesToCreate) {return trans.payee;}+export const PAYEE_NAME_NORMALIZATIONS = ['original', 'title-case'] as const;+export type PayeeNameNormalization = (typeof PAYEE_NAME_NORMALIZATIONS)[number];++function normalizePayeeName(+ payeeName: string,+ normalization: PayeeNameNormalization,+): string {+ switch (normalization) {+ case 'original':+ return payeeName;+ case 'title-case':+ return title(payeeName);+ default:+ normalization satisfies never;+ throw new Error(+ `Unknown payee name normalization: ${String(normalization)}`,+ );+ }+}+async function normalizeTransactions(transactions,acctId,- { rawPayeeName = false } = {},+ {+ payeeNameNormalization = 'title-case',+ }: {+ payeeNameNormalization?: PayeeNameNormalization;+ } = {},) {const payeesToCreate = new Map();@@ -454,7 +478,7 @@ async function normalizeTransactions(if (trimmed === '') {payee_name = null;} else {- payee_name = rawPayeeName ? trimmed : title(trimmed);+ payee_name = normalizePayeeName(trimmed, payeeNameNormalization);}}@@ -584,6 +608,19 @@ async function createNewPayees(payeesToCreate, addsAndUpdates) {});}+export type MatchTransactionsOptions = {+ isBankSyncAccount?: boolean;+ strictIdChecking?: boolean;+ reimportDeleted?: boolean;+ payeeNameNormalization?: PayeeNameNormalization;+};++export type ReconcileTransactionsOptions = MatchTransactionsOptions & {+ isPreview?: boolean;+ defaultCleared?: boolean;+ updateDates?: boolean;+};+export type ReconcileTransactionsResult = {added: string[];updated: string[];@@ -598,12 +635,15 @@ export type ReconcileTransactionsResult = {export async function reconcileTransactions(acctId,transactions,- isBankSyncAccount = false,- strictIdChecking = true,- isPreview = false,- defaultCleared = true,- updateDates = false,- reimportDeleted?: boolean,+ {+ isBankSyncAccount = false,+ strictIdChecking = true,+ isPreview = false,+ defaultCleared = true,+ updateDates = false,+ reimportDeleted,+ payeeNameNormalization,+ }: ReconcileTransactionsOptions = {},): Promise<ReconcileTransactionsResult> {logger.log('Performing transaction reconciliation');@@ -617,13 +657,12 @@ export async function reconcileTransactions(transactionsStep1,transactionsStep2,transactionsStep3,- } = await matchTransactions(- acctId,- transactions,+ } = await matchTransactions(acctId, transactions, {isBankSyncAccount,strictIdChecking,reimportDeleted,- );+ payeeNameNormalization,+ });// Finally, generate & commit the changesfor (const { trans, subtransactions, match } of transactionsStep3) {@@ -758,9 +797,12 @@ export async function reconcileTransactions(export async function matchTransactions(acctId,transactions,- isBankSyncAccount = false,- strictIdChecking = true,- reimportDeletedOverride?: boolean,+ {+ isBankSyncAccount = false,+ strictIdChecking = true,+ reimportDeleted: reimportDeletedOverride,+ payeeNameNormalization,+ }: MatchTransactionsOptions = {},) {logger.log('Performing transaction reconciliation matching');@@ -775,14 +817,11 @@ export async function matchTransactions(const hasMatched = new Set();- const transactionNormalization = isBankSyncAccount- ? normalizeBankSyncTransactions- : normalizeTransactions;-- const { normalized, payeesToCreate } = await transactionNormalization(- transactions,- acctId,- );+ const { normalized, payeesToCreate } = isBankSyncAccount+ ? await normalizeBankSyncTransactions(transactions, acctId)+ : await normalizeTransactions(transactions, acctId, {+ payeeNameNormalization,+ });// The first pass runs the rules, and preps data for fuzzy matchingconst accounts: db.DbAccount[] = await db.getAccounts();@@ -968,7 +1007,7 @@ export async function addTransactions(const { normalized, payeesToCreate } = await normalizeTransactions(transactions,acctId,- { rawPayeeName: true },+ { payeeNameNormalization: 'original' },);const accounts: db.DbAccount[] = await db.getAccounts();@@ -1125,15 +1164,11 @@ async function processBankSyncDownload(starting_balance_flag: true,});- const result = await reconcileTransactions(- id,- transactions,- true,- useStrictIdChecking,- false,- true,+ const result = await reconcileTransactions(id, transactions, {+ isBankSyncAccount: true,+ strictIdChecking: useStrictIdChecking,updateDates,- );+ });return {...result,added: [initialId, ...result.added],@@ -1150,11 +1185,11 @@ async function processBankSyncDownload(const result = await reconcileTransactions(id,importTransactions ? transactions : [],- true,- useStrictIdChecking,- false,- true,- updateDates,+ {+ isBankSyncAccount: true,+ strictIdChecking: useStrictIdChecking,+ updateDates,+ },);if (currentBalance != null) {diff --git a/packages/loot-core/src/types/api-handlers.ts b/packages/loot-core/src/types/api-handlers.tsindex 383e178801..2b89b3e292 100644--- a/packages/loot-core/src/types/api-handlers.ts+++ b/packages/loot-core/src/types/api-handlers.ts@@ -1,5 +1,6 @@// @ts-strict-ignoreimport type { ImportTransactionsResult } from '#server/accounts/app';+import type { PayeeNameNormalization } from '#server/accounts/sync';import type {APIAccountEntity,APIAccountGroupEntity,@@ -32,6 +33,7 @@ export type ImportTransactionsOpts = {defaultCleared?: boolean;dryRun?: boolean;reimportDeleted?: boolean;+ payeeNameNormalization?: PayeeNameNormalization;};export type ApiHandlers = {diff --git a/upcoming-release-notes/api-import-payee-name-normalization.md b/upcoming-release-notes/api-import-payee-name-normalization.mdnew file mode 100644index 0000000000..b3b6039e99--- /dev/null+++ b/upcoming-release-notes/api-import-payee-name-normalization.md@@ -0,0 +1,6 @@+---+category: Enhancements+authors: [haimgel]+---++Add a `payeeNameNormalization` option to the API's `importTransactions`, so payee names can be kept as supplied instead of always being title-cased.