Skip to content

A local-first personal finance app

[AI] Add payeeNameNormalization option to import transactions API (#8710)

Haim Gelfenbeynf78a6fa041b1
diff --git a/packages/docs/docs/api/reference.md b/packages/docs/docs/api/reference.md
index 14b9269f6b..c4a47167f4 100644
--- a/packages/docs/docs/api/reference.md
+++ b/packages/docs/docs/api/reference.md
@@ -279,6 +279,7 @@ This method has the following optional flags (passed as the `opts` object):
- `defaultCleared`: whether imported transactions should be marked as cleared (defaults to `true`)
- `dryRun`: if `true`, returns what would be added/updated without actually modifying the database (defaults to `false`)
- `reimportDeleted`: if `true`, transactions that were previously imported and then deleted will be reimported; if `false`, they will be skipped (defaults to `true` for backward compatibility — note that the [file import UI](../transactions/importing.md#avoiding-duplicate-transactions) defaults to `false`)
+- `payeeNameNormalization`: how `payee_name` is processed when creating a payee — `'title-case'` re-capitalizes each word, `'original'` keeps the name as given, apart from trimming surrounding whitespace (defaults to `'title-case'`)
Example using opts:
diff --git a/packages/loot-core/src/server/accounts/app.ts b/packages/loot-core/src/server/accounts/app.ts
index 03ec882efd..c8922134cc 100644
--- a/packages/loot-core/src/server/accounts/app.ts
+++ b/packages/loot-core/src/server/accounts/app.ts
@@ -1646,16 +1646,25 @@ async function importTransactions({
throw APIError('transactions-import: accountId must be an id');
}
+ const payeeNameNormalization = opts?.payeeNameNormalization ?? 'title-case';
+ if (!bankSync.PAYEE_NAME_NORMALIZATIONS.includes(payeeNameNormalization)) {
+ throw APIError(
+ `transactions-import: payeeNameNormalization must be one of ${bankSync.PAYEE_NAME_NORMALIZATIONS.join(
+ ', ',
+ )}, got '${String(payeeNameNormalization)}'`,
+ );
+ }
+
try {
const reconciled = await bankSync.reconcileTransactions(
accountId,
transactions,
- false,
- true,
- isPreview,
- opts?.defaultCleared,
- false,
- opts?.reimportDeleted,
+ {
+ isPreview,
+ defaultCleared: opts?.defaultCleared,
+ reimportDeleted: opts?.reimportDeleted,
+ payeeNameNormalization,
+ },
);
return {
errors: [],
diff --git a/packages/loot-core/src/server/accounts/sync.test.ts b/packages/loot-core/src/server/accounts/sync.test.ts
index 8f2c20a077..6db27f7d03 100644
--- a/packages/loot-core/src/server/accounts/sync.test.ts
+++ b/packages/loot-core/src/server/accounts/sync.test.ts
@@ -8,6 +8,7 @@ import { setSyncingMode } from '#server/sync';
import { handlers } from '#server/tests/mockSyncServer';
import { insertRule, loadRules } from '#server/transactions/transaction-rules';
import * as monthUtils from '#shared/months';
+import type { ImportTransactionsOpts } from '#types/api-handlers';
import type { SyncedPrefs } from '#types/prefs';
import { app as accountsApp } from './app';
@@ -103,6 +104,123 @@ describe('Account sync', () => {
);
});
+ test('reconcile title-cases the payee name by default', async () => {
+ const { id } = await prepareDatabase();
+
+ await reconcileTransactions(id, [
+ {
+ date: '2020-01-02',
+ payee_name: 'Nintendo Store New York NY',
+ amount: 4133,
+ },
+ ]);
+
+ const payees = await getAllPayees();
+ expect(payees.length).toBe(1);
+ expect(payees[0].name).toBe('Nintendo Store New York Ny');
+
+ const transactions = await getAllTransactions();
+ expect(transactions[0].imported_payee).toBe('Nintendo Store New York Ny');
+ });
+
+ test('transactions-import title-cases the payee name by default', async () => {
+ const { id } = await prepareDatabase();
+
+ await accountsApp.handlers['transactions-import']({
+ accountId: id,
+ transactions: [
+ {
+ account: id,
+ date: '2020-01-02',
+ payee_name: 'Nintendo Store New York NY',
+ amount: 4133,
+ },
+ ],
+ isPreview: false,
+ });
+
+ const payees = await getAllPayees();
+ expect(payees.length).toBe(1);
+ expect(payees[0].name).toBe('Nintendo Store New York Ny');
+
+ const transactions = await getAllTransactions();
+ expect(transactions[0].imported_payee).toBe('Nintendo Store New York Ny');
+ });
+
+ test("transactions-import keeps the payee name with payeeNameNormalization 'original'", async () => {
+ const { id } = await prepareDatabase();
+
+ await accountsApp.handlers['transactions-import']({
+ accountId: id,
+ transactions: [
+ {
+ account: id,
+ date: '2020-01-02',
+ payee_name: 'Nintendo Store New York NY',
+ amount: 4133,
+ },
+ ],
+ isPreview: false,
+ opts: { payeeNameNormalization: 'original' },
+ });
+
+ const payees = await getAllPayees();
+ expect(payees.length).toBe(1);
+ expect(payees[0].name).toBe('Nintendo Store New York NY');
+
+ const transactions = await getAllTransactions();
+ expect(transactions[0].imported_payee).toBe('Nintendo Store New York NY');
+ });
+
+ test("transactions-import trims the payee name with payeeNameNormalization 'original'", async () => {
+ const { id } = await prepareDatabase();
+
+ await accountsApp.handlers['transactions-import']({
+ accountId: id,
+ transactions: [
+ {
+ account: id,
+ date: '2020-01-02',
+ payee_name: ' Nintendo Store New York NY ',
+ amount: 4133,
+ },
+ ],
+ isPreview: false,
+ opts: { payeeNameNormalization: 'original' },
+ });
+
+ const payees = await getAllPayees();
+ expect(payees.length).toBe(1);
+ expect(payees[0].name).toBe('Nintendo Store New York NY');
+
+ const transactions = await getAllTransactions();
+ expect(transactions[0].imported_payee).toBe('Nintendo Store New York NY');
+ });
+
+ test('transactions-import rejects an unknown payeeNameNormalization', async () => {
+ const { id } = await prepareDatabase();
+
+ await expect(
+ accountsApp.handlers['transactions-import']({
+ accountId: id,
+ transactions: [
+ {
+ account: id,
+ date: '2020-01-02',
+ payee_name: 'Nintendo Store New York NY',
+ amount: 4133,
+ },
+ ],
+ isPreview: false,
+ opts: {
+ payeeNameNormalization: 'titlecase',
+ } as unknown as ImportTransactionsOpts,
+ }),
+ ).rejects.toThrow(/payeeNameNormalization/);
+
+ expect(await getAllPayees()).toEqual([]);
+ });
+
test('reconcile handles transactions with undefined fields', async () => {
const { id: acctId } = await prepareDatabase();
@@ -190,12 +308,7 @@ describe('Account sync', () => {
await reconcileTransactions(
acctId,
[{ date: '2020-01-01', imported_id: 'finid-override' }],
- false,
- true,
- false,
- true,
- false,
- false,
+ { reimportDeleted: false },
);
const transactions2 = await getAllTransactions();
expect(transactions2.length).toBe(1);
@@ -216,12 +329,7 @@ describe('Account sync', () => {
await reconcileTransactions(
acctId,
[{ date: '2020-01-01', imported_id: 'finid-override2' }],
- false,
- true,
- false,
- true,
- false,
- true,
+ { reimportDeleted: true },
);
const transactions2 = await getAllTransactions();
expect(transactions2.length).toBe(2);
@@ -246,12 +354,7 @@ describe('Account sync', () => {
await reconcileTransactions(
acctId,
[{ date: '2020-01-01', imported_id: 'finid-precedence' }],
- false,
- true,
- false,
- true,
- false,
- false,
+ { reimportDeleted: false },
);
const transactions2 = await getAllTransactions();
expect(transactions2.length).toBe(1);
@@ -622,8 +725,7 @@ describe('Account sync', () => {
imported_id: 'something-else-entirely',
},
],
- false,
- false,
+ { strictIdChecking: false },
);
payees = await getAllPayees();
diff --git a/packages/loot-core/src/server/accounts/sync.ts b/packages/loot-core/src/server/accounts/sync.ts
index 93aca4a9da..efcf65e14e 100644
--- a/packages/loot-core/src/server/accounts/sync.ts
+++ b/packages/loot-core/src/server/accounts/sync.ts
@@ -413,10 +413,34 @@ async function resolvePayee(trans, payeeName, payeesToCreate) {
return trans.payee;
}
+export const PAYEE_NAME_NORMALIZATIONS = ['original', 'title-case'] as const;
+export type PayeeNameNormalization = (typeof PAYEE_NAME_NORMALIZATIONS)[number];
+
+function normalizePayeeName(
+ payeeName: string,
+ normalization: PayeeNameNormalization,
+): string {
+ switch (normalization) {
+ case 'original':
+ return payeeName;
+ case 'title-case':
+ return title(payeeName);
+ default:
+ normalization satisfies never;
+ throw new Error(
+ `Unknown payee name normalization: ${String(normalization)}`,
+ );
+ }
+}
+
async function normalizeTransactions(
transactions,
acctId,
- { rawPayeeName = false } = {},
+ {
+ payeeNameNormalization = 'title-case',
+ }: {
+ payeeNameNormalization?: PayeeNameNormalization;
+ } = {},
) {
const payeesToCreate = new Map();
@@ -454,7 +478,7 @@ async function normalizeTransactions(
if (trimmed === '') {
payee_name = null;
} else {
- payee_name = rawPayeeName ? trimmed : title(trimmed);
+ payee_name = normalizePayeeName(trimmed, payeeNameNormalization);
}
}
@@ -584,6 +608,19 @@ async function createNewPayees(payeesToCreate, addsAndUpdates) {
});
}
+export type MatchTransactionsOptions = {
+ isBankSyncAccount?: boolean;
+ strictIdChecking?: boolean;
+ reimportDeleted?: boolean;
+ payeeNameNormalization?: PayeeNameNormalization;
+};
+
+export type ReconcileTransactionsOptions = MatchTransactionsOptions & {
+ isPreview?: boolean;
+ defaultCleared?: boolean;
+ updateDates?: boolean;
+};
+
export type ReconcileTransactionsResult = {
added: string[];
updated: string[];
@@ -598,12 +635,15 @@ export type ReconcileTransactionsResult = {
export async function reconcileTransactions(
acctId,
transactions,
- isBankSyncAccount = false,
- strictIdChecking = true,
- isPreview = false,
- defaultCleared = true,
- updateDates = false,
- reimportDeleted?: boolean,
+ {
+ isBankSyncAccount = false,
+ strictIdChecking = true,
+ isPreview = false,
+ defaultCleared = true,
+ updateDates = false,
+ reimportDeleted,
+ payeeNameNormalization,
+ }: ReconcileTransactionsOptions = {},
): Promise<ReconcileTransactionsResult> {
logger.log('Performing transaction reconciliation');
@@ -617,13 +657,12 @@ export async function reconcileTransactions(
transactionsStep1,
transactionsStep2,
transactionsStep3,
- } = await matchTransactions(
- acctId,
- transactions,
+ } = await matchTransactions(acctId, transactions, {
isBankSyncAccount,
strictIdChecking,
reimportDeleted,
- );
+ payeeNameNormalization,
+ });
// Finally, generate & commit the changes
for (const { trans, subtransactions, match } of transactionsStep3) {
@@ -758,9 +797,12 @@ export async function reconcileTransactions(
export async function matchTransactions(
acctId,
transactions,
- isBankSyncAccount = false,
- strictIdChecking = true,
- reimportDeletedOverride?: boolean,
+ {
+ isBankSyncAccount = false,
+ strictIdChecking = true,
+ reimportDeleted: reimportDeletedOverride,
+ payeeNameNormalization,
+ }: MatchTransactionsOptions = {},
) {
logger.log('Performing transaction reconciliation matching');
@@ -775,14 +817,11 @@ export async function matchTransactions(
const hasMatched = new Set();
- const transactionNormalization = isBankSyncAccount
- ? normalizeBankSyncTransactions
- : normalizeTransactions;
-
- const { normalized, payeesToCreate } = await transactionNormalization(
- transactions,
- acctId,
- );
+ const { normalized, payeesToCreate } = isBankSyncAccount
+ ? await normalizeBankSyncTransactions(transactions, acctId)
+ : await normalizeTransactions(transactions, acctId, {
+ payeeNameNormalization,
+ });
// The first pass runs the rules, and preps data for fuzzy matching
const accounts: db.DbAccount[] = await db.getAccounts();
@@ -968,7 +1007,7 @@ export async function addTransactions(
const { normalized, payeesToCreate } = await normalizeTransactions(
transactions,
acctId,
- { rawPayeeName: true },
+ { payeeNameNormalization: 'original' },
);
const accounts: db.DbAccount[] = await db.getAccounts();
@@ -1125,15 +1164,11 @@ async function processBankSyncDownload(
starting_balance_flag: true,
});
- const result = await reconcileTransactions(
- id,
- transactions,
- true,
- useStrictIdChecking,
- false,
- true,
+ const result = await reconcileTransactions(id, transactions, {
+ isBankSyncAccount: true,
+ strictIdChecking: useStrictIdChecking,
updateDates,
- );
+ });
return {
...result,
added: [initialId, ...result.added],
@@ -1150,11 +1185,11 @@ async function processBankSyncDownload(
const result = await reconcileTransactions(
id,
importTransactions ? transactions : [],
- true,
- useStrictIdChecking,
- false,
- true,
- updateDates,
+ {
+ isBankSyncAccount: true,
+ strictIdChecking: useStrictIdChecking,
+ updateDates,
+ },
);
if (currentBalance != null) {
diff --git a/packages/loot-core/src/types/api-handlers.ts b/packages/loot-core/src/types/api-handlers.ts
index 383e178801..2b89b3e292 100644
--- a/packages/loot-core/src/types/api-handlers.ts
+++ b/packages/loot-core/src/types/api-handlers.ts
@@ -1,5 +1,6 @@
// @ts-strict-ignore
import type { ImportTransactionsResult } from '#server/accounts/app';
+import type { PayeeNameNormalization } from '#server/accounts/sync';
import type {
APIAccountEntity,
APIAccountGroupEntity,
@@ -32,6 +33,7 @@ export type ImportTransactionsOpts = {
defaultCleared?: boolean;
dryRun?: boolean;
reimportDeleted?: boolean;
+ payeeNameNormalization?: PayeeNameNormalization;
};
export type ApiHandlers = {
diff --git a/upcoming-release-notes/api-import-payee-name-normalization.md b/upcoming-release-notes/api-import-payee-name-normalization.md
new file mode 100644
index 0000000000..b3b6039e99
--- /dev/null
+++ b/upcoming-release-notes/api-import-payee-name-normalization.md
@@ -0,0 +1,6 @@
+---
+category: Enhancements
+authors: [haimgel]
+---
+
+Add a `payeeNameNormalization` option to the API's `importTransactions`, so payee names can be kept as supplied instead of always being title-cased.