Open Source Wealth Management Software. Angular + NestJS + Prisma + Nx + TypeScript 🤍
TypeScript angularetffinancefintechghostfoliohacktoberfestinvestingnestjsosspersonal-financeportfolioprismasoftwarestocktrackertradingtypescriptwealth-managementweb
Task/improve check for duplicates in preview step of activities import (#7541)
Thomas Kaul74211cd68599
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 123c444108..0fe9d8de59 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -5,6 +5,18 @@ All notable changes to this project will be documented in this file.The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).+## Unreleased++### Changed++- Improved the check for duplicates in the preview step of the activities import (regardless of the account)+- Improved the check for duplicates in the preview step of the import dividends dialog (regardless of the account)+- Extended the activities import to reuse an existing account of the user by name and currency++### Fixed++- Fixed the check for duplicates in the preview step of the activities import for activities without a comment+## 3.44.0 - 2026-08-07### Addeddiff --git a/apps/api/src/app/import/import.service.ts b/apps/api/src/app/import/import.service.tsindex 9d1e899c3e..a9cde1544a 100644--- a/apps/api/src/app/import/import.service.ts+++ b/apps/api/src/app/import/import.service.ts@@ -15,7 +15,11 @@ import {NON_INVESTMENT_ACTIVITY_TYPES,TAG_ID_EXCLUDE_FROM_ANALYSIS} from '@ghostfolio/common/config';-import { CreateAssetProfileDto, CreateOrderDto } from '@ghostfolio/common/dtos';+import {+ CreateAccountWithBalancesDto,+ CreateAssetProfileDto,+ CreateOrderDto+} from '@ghostfolio/common/dtos';import {getAssetProfileIdentifier,isValidCustomAssetProfileSymbol,@@ -34,7 +38,7 @@ import {} from '@ghostfolio/common/types';import { Injectable } from '@nestjs/common';-import { DataSource, Prisma } from '@prisma/client';+import { Account, DataSource, Prisma } from '@prisma/client';import { Big } from 'big.js';import { endOfToday, isAfter, isSameSecond, parseISO } from 'date-fns';import { omit, uniqBy } from 'lodash';@@ -96,7 +100,9 @@ export class ImportService {filters,userCurrency,userId,- startDate: parseDate(dateOfFirstActivity)+ includeDrafts: true,+ startDate: parseDate(dateOfFirstActivity),+ withExcludedAccountsAndActivities: true}),this.symbolProfileService.getSymbolProfiles([{@@ -117,6 +123,8 @@ export class ImportService {return await Promise.all(Object.entries(dividends).map(([dateString, { marketPrice }]) => {+ const date = parseDate(dateString);+const quantity =historicalData.find((historicalDataItem) => {return historicalDataItem.date === dateString;@@ -124,10 +132,8 @@ export class ImportService {const value = new Big(quantity).mul(marketPrice).toNumber();- const date = parseDate(dateString);const isDuplicate = activities.some((activity) => {return (- activity.accountId === account?.id &&activity.assetProfile.currency === assetProfile.currency &&activity.assetProfile.dataSource === assetProfile.dataSource &&isSameSecond(activity.date, date) &&@@ -326,17 +332,29 @@ export class ImportService {}}- if (!isDryRun && accountsWithBalancesDto?.length) {- const [existingAccounts, existingPlatforms] = await Promise.all([+ if (accountsWithBalancesDto?.length) {+ const [+ existingAccountsOfOtherUsers,+ existingAccountsOfUser,+ existingPlatforms+ ] = await Promise.all([this.accountService.accounts({where: {id: {- in: accountsWithBalancesDto.map(({ id }) => {- return id;- })- }+ in: accountsWithBalancesDto+ .filter(({ id }) => {+ return Boolean(id);+ })+ .map(({ id }) => {+ return id;+ })+ },+ userId: { not: user.id }}}),+ this.accountService.accounts({+ where: { userId: user.id }+ }),this.platformService.getPlatforms()]);@@ -347,79 +365,116 @@ export class ImportService {);for (const accountWithBalances of accountsWithBalancesDto) {- // Check if there is any existing account with the same ID- const accountWithSameId = existingAccounts.find((existingAccount) => {- return existingAccount.id === accountWithBalances.id;+ // Skip the account if it already belongs to the user+ if (+ existingAccountsOfUser.some(({ id }) => {+ return id === accountWithBalances.id;+ })+ ) {+ continue;+ }++ // If there is no account or if the account belongs to a different+ // user, then reuse an existing account of the user with the same name+ // and currency or create a new account+ const accountToReuse = this.getAccountToReuse({+ accountWithBalances,+ accountsWithBalancesDto,+ existingAccountsOfUser});- // If there is no account or if the account belongs to a different user then create a new account- if (!accountWithSameId || accountWithSameId.userId !== user.id) {- const account = omit(accountWithBalances, [- 'balance',- 'balances',- 'isExcluded',- 'tags'- ]);+ if (accountToReuse) {+ // Reuse the account of the user instead of creating a duplicate. The+ // balances, the platform and the tags of the import are deliberately+ // not applied to leave the existing account of the user untouched.+ if (+ accountWithBalances.id &&+ accountWithBalances.id !== accountToReuse.id+ ) {+ // Store the new to old account ID mappings for updating activities+ accountIdMapping[accountWithBalances.id] = accountToReuse.id;+ }- let oldAccountId: string;- const platformId =- platformIdMapping[account.platformId] ?? account.platformId;+ continue;+ }- delete account.platformId;+ if (isDryRun) {+ continue;+ }- if (accountWithSameId) {- oldAccountId = account.id;- delete account.id;+ // Check if there is any existing account of a different user with the+ // same ID, since the ID cannot be reused in this case+ const accountWithSameIdOfOtherUser = existingAccountsOfOtherUsers.find(+ ({ id }) => {+ return id === accountWithBalances.id;}+ );- const tagIds = (accountWithBalances.tags ?? [])- .map((tagId) => {- return tagIdMapping[tagId] ?? tagId;- })- .filter((tagId) => {- return existingTagIds.has(tagId);- });+ const account = omit(accountWithBalances, [+ 'balance',+ 'balances',+ 'isExcluded',+ 'tags'+ ]);- // Map the legacy isExcluded attribute of old export files to- // the "Exclude from Analysis" tag- if (- accountWithBalances.isExcluded &&- existingTagIds.has(TAG_ID_EXCLUDE_FROM_ANALYSIS) &&- !tagIds.includes(TAG_ID_EXCLUDE_FROM_ANALYSIS)- ) {- tagIds.push(TAG_ID_EXCLUDE_FROM_ANALYSIS);- }+ let oldAccountId: string;+ const platformId =+ platformIdMapping[account.platformId] ?? account.platformId;- let accountObject: Prisma.AccountCreateInput = {- ...account,- balances: {- create: accountWithBalances.balances ?? []- },- user: { connect: { id: user.id } }- };+ delete account.platformId;- if (- existingPlatforms.some(({ id }) => {- return id === platformId;- })- ) {- accountObject = {- ...accountObject,- platform: { connect: { id: platformId } }- };- }+ if (accountWithSameIdOfOtherUser) {+ oldAccountId = account.id;+ delete account.id;+ }- const newAccount = await this.accountService.createAccount({- tagIds,- balance: accountWithBalances.balance,- data: accountObject,- userId: user.id+ const tagIds = (accountWithBalances.tags ?? [])+ .map((tagId) => {+ return tagIdMapping[tagId] ?? tagId;+ })+ .filter((tagId) => {+ return existingTagIds.has(tagId);});- // Store the new to old account ID mappings for updating activities- if (accountWithSameId && oldAccountId) {- accountIdMapping[oldAccountId] = newAccount.id;- }+ // Map the legacy isExcluded attribute of old export files to+ // the "Exclude from Analysis" tag+ if (+ accountWithBalances.isExcluded &&+ existingTagIds.has(TAG_ID_EXCLUDE_FROM_ANALYSIS) &&+ !tagIds.includes(TAG_ID_EXCLUDE_FROM_ANALYSIS)+ ) {+ tagIds.push(TAG_ID_EXCLUDE_FROM_ANALYSIS);+ }++ let accountObject: Prisma.AccountCreateInput = {+ ...account,+ balances: {+ create: accountWithBalances.balances ?? []+ },+ user: { connect: { id: user.id } }+ };++ if (+ existingPlatforms.some(({ id }) => {+ return id === platformId;+ })+ ) {+ accountObject = {+ ...accountObject,+ platform: { connect: { id: platformId } }+ };+ }++ const newAccount = await this.accountService.createAccount({+ tagIds,+ balance: accountWithBalances.balance,+ data: accountObject,+ userId: user.id+ });++ // Store the new to old account ID mappings for updating activities+ if (accountWithSameIdOfOtherUser && oldAccountId) {+ accountIdMapping[oldAccountId] = newAccount.id;}}}@@ -537,12 +592,12 @@ export class ImportService {activity.symbol = assetProfileSymbolMapping[activity.symbol];}- if (!isDryRun) {- // If a new account is created, then update the accountId in all activities- if (accountIdMapping[activity.accountId]) {- activity.accountId = accountIdMapping[activity.accountId];- }+ // If an account is created or reused, then update the accountId in all activities+ if (accountIdMapping[activity.accountId]) {+ activity.accountId = accountIdMapping[activity.accountId];+ }+ if (!isDryRun) {// If a new tag is created, then update the tag ID in all activitiesactivity.tags = (activity.tags ?? []).map((tagId) => {return tagIdMapping[tagId] ?? tagId;@@ -570,9 +625,20 @@ export class ImportService {);if (isDryRun) {- accountsWithBalancesDto.forEach(({ id, name }) => {- accounts.push({ id, name });- });+ accountsWithBalancesDto+ .filter(({ id }) => {+ // Skip the accounts which are reused or which already belong to the+ // user, since they are part of the accounts of the user above+ return (+ !accountIdMapping[id] &&+ !accounts.some(({ id: accountId }) => {+ return accountId === id;+ })+ );+ })+ .forEach(({ id, name }) => {+ accounts.push({ id, name });+ });}const tags = (await this.tagService.getTagsForUser(user.id)).map(@@ -826,10 +892,10 @@ export class ImportService {unitPrice}) => {const date = parseISO(dateString);+const isDuplicate = existingActivities.some((activity) => {return (- activity.accountId === accountId &&- activity.comment === comment &&+ (activity.comment || null) === (comment || null) &&(activity.currency === currency ||activity.assetProfile.currency === currency) &&activity.assetProfile.dataSource === dataSource &&@@ -877,6 +943,52 @@ export class ImportService {);}+ /**+ * Returns the account of the user to reuse for the given account of the+ * import, based on the name and the currency. The currency is considered+ * because the activities of the import would otherwise end up in an account+ * of a different currency. The name is only considered if it is unambiguous,+ * both in the accounts of the user and in the accounts of the import, since+ * it is not unique. Otherwise, distinct accounts would be merged into a+ * single one.+ */+ private getAccountToReuse({+ accountWithBalances,+ accountsWithBalancesDto,+ existingAccountsOfUser+ }: {+ accountWithBalances: CreateAccountWithBalancesDto;+ accountsWithBalancesDto: ImportDataDto['accounts'];+ existingAccountsOfUser: Account[];+ }): Account {+ const matchingAccountsOfUser = existingAccountsOfUser.filter(+ ({ currency, name }) => {+ return (+ currency === accountWithBalances.currency &&+ name === accountWithBalances.name+ );+ }+ );++ const matchingAccountsToImport = accountsWithBalancesDto.filter(+ ({ currency, name }) => {+ return (+ currency === accountWithBalances.currency &&+ name === accountWithBalances.name+ );+ }+ );++ if (+ matchingAccountsOfUser.length !== 1 ||+ matchingAccountsToImport.length !== 1+ ) {+ return undefined;+ }++ return matchingAccountsOfUser[0];+ }+private isUniqueAccount(accounts: AccountWithValue[]) {const uniqueAccountIds = new Set<string>();