Skip to content

A local-first personal finance app

[AI] prefer unlinked transactions when assigning fuzzy matches (#9001)

Amy Hawkinsa3330d2f5be2
diff --git a/packages/loot-core/src/server/accounts/sync.test.ts b/packages/loot-core/src/server/accounts/sync.test.ts
index 73896ed1d2..ec6ef8c803 100644
--- a/packages/loot-core/src/server/accounts/sync.test.ts
+++ b/packages/loot-core/src/server/accounts/sync.test.ts
@@ -14,6 +14,7 @@ import type { SyncedPrefs } from '#types/prefs';
import { app as accountsApp } from './app';
import {
addTransactions,
+ compareFuzzyMatchCandidates,
reconcileTransactions,
simpleFinBatchSync,
} from './sync';
@@ -638,6 +639,109 @@ describe('Account sync', () => {
).toBe(-1239);
});
+ describe('compareFuzzyMatchCandidates', () => {
+ const transDate = '2024-04-05';
+ const alreadyImported = {
+ date: db.toDateRepr('2024-04-05'),
+ imported_id: 'existing-import-id',
+ };
+ const notYetImported = {
+ date: db.toDateRepr('2024-04-05'),
+ imported_id: null,
+ };
+
+ test(
+ 'breaks a same-distance tie by preferring the unlinked candidate, ' +
+ 'regardless of which candidate is passed in first',
+ () => {
+ // Feed the comparator the already-imported candidate first -- the
+ // pre-fix comparator (`aDistance - bDistance`, which never returns 0
+ // for equal distances) would have kept it first. The sort must
+ // still put the unlinked candidate ahead of it either way.
+ expect(
+ [alreadyImported, notYetImported].sort((a, b) =>
+ compareFuzzyMatchCandidates(transDate, a, b),
+ ),
+ ).toEqual([notYetImported, alreadyImported]);
+
+ expect(
+ [notYetImported, alreadyImported].sort((a, b) =>
+ compareFuzzyMatchCandidates(transDate, a, b),
+ ),
+ ).toEqual([notYetImported, alreadyImported]);
+ },
+ );
+
+ test('prefers the closer date over imported_id status', () => {
+ const fartherButUnlinked = {
+ date: db.toDateRepr('2024-04-08'),
+ imported_id: null,
+ };
+
+ expect(
+ [fartherButUnlinked, alreadyImported].sort((a, b) =>
+ compareFuzzyMatchCandidates(transDate, a, b),
+ ),
+ ).toEqual([alreadyImported, fartherButUnlinked]);
+ });
+ });
+
+ test(
+ 'given two equally-dated candidates, an unlinked one is preferred over ' +
+ 'one that already has its own imported_id',
+ async () => {
+ const { id } = await prepareDatabase();
+
+ // Both candidates are tied on date-distance to the incoming
+ // transaction, so which one wins depends on the comparator's
+ // imported_id tie-break, verified directly (independent of DB read
+ // order) in the compareFuzzyMatchCandidates tests above. This test
+ // covers the same scenario end-to-end through reconcileTransactions.
+ await db.insertTransaction({
+ id: 'already-imported',
+ account: id,
+ amount: -1239,
+ date: '2024-04-05',
+ imported_id: 'existing-import-id',
+ });
+ await db.insertTransaction({
+ id: 'not-yet-imported',
+ account: id,
+ amount: -1239,
+ date: '2024-04-05',
+ });
+
+ // Neither candidate's date matches the incoming transaction's date
+ // any more closely than the other, so the two are tied on distance --
+ // the tie should be broken in favor of the row that isn't already
+ // linked to some other bank transaction.
+ await reconcileTransactions(
+ id,
+ [
+ {
+ date: '2024-04-05',
+ amount: -1239,
+ payee_name: 'Acme Inc.',
+ imported_id: 'new-import-id',
+ },
+ ],
+ { strictIdChecking: false },
+ );
+
+ const transactions = await getAllTransactions();
+ expect(transactions.length).toBe(2);
+
+ const alreadyImported = transactions.find(
+ t => t.id === 'already-imported',
+ );
+ expect(alreadyImported.imported_id).toBe('existing-import-id');
+
+ const notYetImported = transactions.find(
+ t => t.id === 'not-yet-imported',
+ );
+ expect(notYetImported.imported_id).toBe('new-import-id');
+ },
+ );
test('bank sync does not fuzzy-match children of an exactly matched split', async () => {
const { id } = await prepareDatabase();
diff --git a/packages/loot-core/src/server/accounts/sync.ts b/packages/loot-core/src/server/accounts/sync.ts
index 29eeb2861f..6adca336a6 100644
--- a/packages/loot-core/src/server/accounts/sync.ts
+++ b/packages/loot-core/src/server/accounts/sync.ts
@@ -794,6 +794,33 @@ export async function reconcileTransactions(
};
}
+// Ranks fuzzy-match candidates by distance from transaction date, nearest first.
+// On a same-distance tie, a candidate that already carries its own imported_id
+// (from a previous, unrelated sync) ranks after one that doesn't. Without this
+// tie-break, picking the already-imported one would lead the fuzzy match merge
+// to silently overwrite imported_id/payee/notes with this transaction's data.
+export function compareFuzzyMatchCandidates(
+ transactionDate: string,
+ a: Pick<db.DbViewTransaction, 'date' | 'imported_id'>,
+ b: Pick<db.DbViewTransaction, 'date' | 'imported_id'>,
+): number {
+ const aDistance = Math.abs(
+ dateFns.differenceInMilliseconds(
+ dateFns.parseISO(transactionDate),
+ dateFns.parseISO(db.fromDateRepr(a.date)),
+ ),
+ );
+ const bDistance = Math.abs(
+ dateFns.differenceInMilliseconds(
+ dateFns.parseISO(transactionDate),
+ dateFns.parseISO(db.fromDateRepr(b.date)),
+ ),
+ );
+ const aHasImportedId = Number(a.imported_id != null);
+ const bHasImportedId = Number(b.imported_id != null);
+ return aDistance - bDistance || aHasImportedId - bHasImportedId;
+}
+
export async function matchTransactions(
acctId,
transactions,
@@ -933,21 +960,9 @@ export async function matchTransactions(
// transactions date. i.e. if the original transaction is in 21-02-2024 and
// the matched transactions are: 20-02-2024, 21-02-2024, 29-02-2024 then
// the resulting data-set should be: 21-02-2024, 20-02-2024, 29-02-2024.
- fuzzyDataset = fuzzyDataset.sort((a, b) => {
- const aDistance = Math.abs(
- dateFns.differenceInMilliseconds(
- dateFns.parseISO(trans.date),
- dateFns.parseISO(db.fromDateRepr(a.date)),
- ),
- );
- const bDistance = Math.abs(
- dateFns.differenceInMilliseconds(
- dateFns.parseISO(trans.date),
- dateFns.parseISO(db.fromDateRepr(b.date)),
- ),
- );
- return aDistance > bDistance ? 1 : -1;
- });
+ fuzzyDataset = fuzzyDataset.sort((a, b) =>
+ compareFuzzyMatchCandidates(trans.date, a, b),
+ );
}
transactionsStep1.push({
diff --git a/upcoming-release-notes/fix-fuzzy-match-prefer-unlinked.md b/upcoming-release-notes/fix-fuzzy-match-prefer-unlinked.md
new file mode 100644
index 0000000000..208a778491
--- /dev/null
+++ b/upcoming-release-notes/fix-fuzzy-match-prefer-unlinked.md
@@ -0,0 +1,6 @@
+---
+category: Bugfixes
+authors: [ahawkins318]
+---
+
+When fuzzy matching bank candidates, prefer a candidate that has not already been linked in a previous sync