Open Source Wealth Management Software. Angular + NestJS + Prisma + Nx + TypeScript 🤍
TypeScript angularetffinancefintechghostfoliohacktoberfestinvestingnestjsosspersonal-financeportfolioprismasoftwarestocktrackertradingtypescriptwealth-managementweb
Feature/optimize import validation by reducing to unique asset profiles (#2198)
Thomas Kaul286e41eb216f
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 80987a8119..33fa8fd898 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -5,6 +5,13 @@ All notable changes to this project will be documented in this file.The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).+## Unreleased++### Changed++- Optimized the validation in the activities import by reducing the list to unique asset profiles+- Optimized the data gathering in the activities import+## 1.295.0 - 2023-07-30### Addeddiff --git a/apps/api/src/app/admin/admin.controller.ts b/apps/api/src/app/admin/admin.controller.tsindex 54d36d8c55..8e3fb28713 100644--- a/apps/api/src/app/admin/admin.controller.ts+++ b/apps/api/src/app/admin/admin.controller.ts@@ -7,6 +7,7 @@ import {GATHER_ASSET_PROFILE_PROCESS,GATHER_ASSET_PROFILE_PROCESS_OPTIONS} from '@ghostfolio/common/config';+import { getAssetProfileIdentifier } from '@ghostfolio/common/helper';import {AdminData,AdminMarketData,@@ -116,7 +117,7 @@ export class AdminController {name: GATHER_ASSET_PROFILE_PROCESS,opts: {...GATHER_ASSET_PROFILE_PROCESS_OPTIONS,- jobId: `${dataSource}-${symbol}`+ jobId: getAssetProfileIdentifier({ dataSource, symbol })}};})@@ -152,7 +153,7 @@ export class AdminController {name: GATHER_ASSET_PROFILE_PROCESS,opts: {...GATHER_ASSET_PROFILE_PROCESS_OPTIONS,- jobId: `${dataSource}-${symbol}`+ jobId: getAssetProfileIdentifier({ dataSource, symbol })}};})@@ -185,7 +186,7 @@ export class AdminController {name: GATHER_ASSET_PROFILE_PROCESS,opts: {...GATHER_ASSET_PROFILE_PROCESS_OPTIONS,- jobId: `${dataSource}-${symbol}`+ jobId: getAssetProfileIdentifier({ dataSource, symbol })}});}diff --git a/apps/api/src/app/import/import.service.ts b/apps/api/src/app/import/import.service.tsindex 65611ce0dc..3d9c2999d6 100644--- a/apps/api/src/app/import/import.service.ts+++ b/apps/api/src/app/import/import.service.ts@@ -8,10 +8,14 @@ import {import { OrderService } from '@ghostfolio/api/app/order/order.service';import { PlatformService } from '@ghostfolio/api/app/platform/platform.service';import { PortfolioService } from '@ghostfolio/api/app/portfolio/portfolio.service';+import { DataGatheringService } from '@ghostfolio/api/services/data-gathering/data-gathering.service';import { DataProviderService } from '@ghostfolio/api/services/data-provider/data-provider.service';import { ExchangeRateDataService } from '@ghostfolio/api/services/exchange-rate-data/exchange-rate-data.service';import { SymbolProfileService } from '@ghostfolio/api/services/symbol-profile/symbol-profile.service';-import { parseDate } from '@ghostfolio/common/helper';+import {+ getAssetProfileIdentifier,+ parseDate+} from '@ghostfolio/common/helper';import { UniqueAsset } from '@ghostfolio/common/interfaces';import {AccountWithPlatform,@@ -21,12 +25,14 @@ import { Injectable } from '@nestjs/common';import { DataSource, Prisma, SymbolProfile } from '@prisma/client';import Big from 'big.js';import { endOfToday, isAfter, isSameDay, parseISO } from 'date-fns';+import { uniqBy } from 'lodash';import { v4 as uuidv4 } from 'uuid';@Injectable()export class ImportService {public constructor(private readonly accountService: AccountService,+ private readonly dataGatheringService: DataGatheringService,private readonly dataProviderService: DataProviderService,private readonly exchangeRateDataService: ExchangeRateDataService,private readonly orderService: OrderService,@@ -220,8 +226,7 @@ export class ImportService {const assetProfiles = await this.validateActivities({activitiesDto,- maxActivitiesToImport,- userId+ maxActivitiesToImport});const activitiesExtendedWithErrors = await this.extendActivitiesWithErrors({@@ -250,10 +255,37 @@ export class ImportService {error,fee,quantity,- SymbolProfile: assetProfile,+ SymbolProfile,type,unitPrice} of activitiesExtendedWithErrors) {+ const assetProfile = assetProfiles[+ getAssetProfileIdentifier({+ dataSource: SymbolProfile.dataSource,+ symbol: SymbolProfile.symbol+ })+ ] ?? {+ currency: SymbolProfile.currency,+ dataSource: SymbolProfile.dataSource,+ symbol: SymbolProfile.symbol+ };+ const {+ assetClass,+ assetSubClass,+ countries,+ createdAt,+ currency,+ dataSource,+ id,+ isin,+ name,+ scraperConfiguration,+ sectors,+ symbol,+ symbolMapping,+ url,+ updatedAt+ } = assetProfile;const validatedAccount = accounts.find(({ id }) => {return id === accountId;});@@ -279,23 +311,22 @@ export class ImportService {id: uuidv4(),isDraft: isAfter(date, endOfToday()),SymbolProfile: {- assetClass: assetProfile.assetClass,- assetSubClass: assetProfile.assetSubClass,- comment: assetProfile.comment,- countries: assetProfile.countries,- createdAt: assetProfile.createdAt,- currency: assetProfile.currency,- dataSource: assetProfile.dataSource,- id: assetProfile.id,- isin: assetProfile.isin,- name: assetProfile.name,- scraperConfiguration: assetProfile.scraperConfiguration,- sectors: assetProfile.sectors,- symbol: assetProfile.currency,- symbolMapping: assetProfile.symbolMapping,- updatedAt: assetProfile.updatedAt,- url: assetProfile.url,- ...assetProfiles[assetProfile.symbol]+ assetClass,+ assetSubClass,+ countries,+ createdAt,+ currency,+ dataSource,+ id,+ isin,+ name,+ scraperConfiguration,+ sectors,+ symbol,+ symbolMapping,+ updatedAt,+ url,+ comment: assetProfile.comment},Account: validatedAccount,symbolProfileId: undefined,@@ -318,14 +349,14 @@ export class ImportService {SymbolProfile: {connectOrCreate: {create: {- currency: assetProfile.currency,- dataSource: assetProfile.dataSource,- symbol: assetProfile.symbol+ currency,+ dataSource,+ symbol},where: {dataSource_symbol: {- dataSource: assetProfile.dataSource,- symbol: assetProfile.symbol+ dataSource,+ symbol}}}@@ -337,24 +368,49 @@ export class ImportService {const value = new Big(quantity).mul(unitPrice).toNumber();- //@ts-ignoreactivities.push({...order,error,value,feeInBaseCurrency: this.exchangeRateDataService.toCurrency(fee,- assetProfile.currency,+ currency,userCurrency),+ //@ts-ignore+ SymbolProfile: assetProfile,valueInBaseCurrency: this.exchangeRateDataService.toCurrency(value,- assetProfile.currency,+ currency,userCurrency)});}+ activities.sort((activity1, activity2) => {+ return Number(activity1.date) - Number(activity2.date);+ });++ if (!isDryRun) {+ // Gather symbol data in the background, if not dry run+ const uniqueActivities = uniqBy(activities, ({ SymbolProfile }) => {+ return getAssetProfileIdentifier({+ dataSource: SymbolProfile.dataSource,+ symbol: SymbolProfile.symbol+ });+ });++ this.dataGatheringService.gatherSymbols(+ uniqueActivities.map(({ date, SymbolProfile }) => {+ return {+ date,+ dataSource: SymbolProfile.dataSource,+ symbol: SymbolProfile.symbol+ };+ })+ );+ }+return activities;}@@ -446,25 +502,30 @@ export class ImportService {private async validateActivities({activitiesDto,- maxActivitiesToImport,- userId+ maxActivitiesToImport}: {activitiesDto: Partial<CreateOrderDto>[];maxActivitiesToImport: number;- userId: string;}) {if (activitiesDto?.length > maxActivitiesToImport) {throw new Error(`Too many activities (${maxActivitiesToImport} at most)`);}const assetProfiles: {- [symbol: string]: Partial<SymbolProfile>;+ [assetProfileIdentifier: string]: Partial<SymbolProfile>;} = {};+ const uniqueActivitiesDto = uniqBy(+ activitiesDto,+ ({ dataSource, symbol }) => {+ return getAssetProfileIdentifier({ dataSource, symbol });+ }+ );+for (const [index,{ currency, dataSource, symbol }- ] of activitiesDto.entries()) {+ ] of uniqueActivitiesDto.entries()) {if (dataSource !== 'MANUAL') {const assetProfile = (await this.dataProviderService.getAssetProfiles([@@ -484,7 +545,8 @@ export class ImportService {);}- assetProfiles[symbol] = assetProfile;+ assetProfiles[getAssetProfileIdentifier({ dataSource, symbol })] =+ assetProfile;}}diff --git a/apps/api/src/app/order/order.controller.ts b/apps/api/src/app/order/order.controller.tsindex c478860d24..0e617462e1 100644--- a/apps/api/src/app/order/order.controller.ts+++ b/apps/api/src/app/order/order.controller.ts@@ -2,6 +2,7 @@ import { RedactValuesInResponseInterceptor } from '@ghostfolio/api/interceptors/import { TransformDataSourceInRequestInterceptor } from '@ghostfolio/api/interceptors/transform-data-source-in-request.interceptor';import { TransformDataSourceInResponseInterceptor } from '@ghostfolio/api/interceptors/transform-data-source-in-response.interceptor';import { ApiService } from '@ghostfolio/api/services/api/api.service';+import { DataGatheringService } from '@ghostfolio/api/services/data-gathering/data-gathering.service';import { ImpersonationService } from '@ghostfolio/api/services/impersonation/impersonation.service';import { HEADER_KEY_IMPERSONATION } from '@ghostfolio/common/config';import { hasPermission, permissions } from '@ghostfolio/common/permissions';@@ -36,6 +37,7 @@ import { UpdateOrderDto } from './update-order.dto';export class OrderController {public constructor(private readonly apiService: ApiService,+ private readonly dataGatheringService: DataGatheringService,private readonly impersonationService: ImpersonationService,private readonly orderService: OrderService,@Inject(REQUEST) private readonly request: RequestWithUser@@ -123,7 +125,7 @@ export class OrderController {);}- return this.orderService.createOrder({+ const order = await this.orderService.createOrder({...data,date: parseISO(data.date),SymbolProfile: {@@ -144,6 +146,19 @@ export class OrderController {User: { connect: { id: this.request.user.id } },userId: this.request.user.id});++ if (!order.isDraft) {+ // Gather symbol data in the background, if not draft+ this.dataGatheringService.gatherSymbols([+ {+ dataSource: data.dataSource,+ date: order.date,+ symbol: data.symbol+ }+ ]);+ }++ return order;}@Put(':id')diff --git a/apps/api/src/app/order/order.service.ts b/apps/api/src/app/order/order.service.tsindex be5708a904..0f7da5da37 100644--- a/apps/api/src/app/order/order.service.ts+++ b/apps/api/src/app/order/order.service.ts@@ -7,6 +7,7 @@ import {GATHER_ASSET_PROFILE_PROCESS,GATHER_ASSET_PROFILE_PROCESS_OPTIONS} from '@ghostfolio/common/config';+import { getAssetProfileIdentifier } from '@ghostfolio/common/helper';import { Filter } from '@ghostfolio/common/interfaces';import { OrderWithAccount } from '@ghostfolio/common/types';import { Injectable } from '@nestjs/common';@@ -117,7 +118,7 @@ export class OrderService {};}- await this.dataGatheringService.addJobToQueue({+ this.dataGatheringService.addJobToQueue({data: {dataSource: data.SymbolProfile.connectOrCreate.create.dataSource,symbol: data.SymbolProfile.connectOrCreate.create.symbol@@ -125,25 +126,12 @@ export class OrderService {name: GATHER_ASSET_PROFILE_PROCESS,opts: {...GATHER_ASSET_PROFILE_PROCESS_OPTIONS,- jobId: `${data.SymbolProfile.connectOrCreate.create.dataSource}-${data.SymbolProfile.connectOrCreate.create.symbol}`- }- });-- const isDraft =- data.type === 'LIABILITY'- ? false- : isAfter(data.date as Date, endOfToday());-- if (!isDraft) {- // Gather symbol data of order in the background, if not draft- this.dataGatheringService.gatherSymbols([- {+ jobId: getAssetProfileIdentifier({dataSource: data.SymbolProfile.connectOrCreate.create.dataSource,- date: <Date>data.date,symbol: data.SymbolProfile.connectOrCreate.create.symbol- }- ]);- }+ })+ }+ });delete data.accountId;delete data.assetClass;@@ -162,6 +150,11 @@ export class OrderService {const orderData: Prisma.OrderCreateInput = data;+ const isDraft =+ data.type === 'LIABILITY'+ ? false+ : isAfter(data.date as Date, endOfToday());+const order = await this.prismaService.order.create({data: {...orderData,diff --git a/apps/api/src/app/redis-cache/redis-cache.service.ts b/apps/api/src/app/redis-cache/redis-cache.service.tsindex 865a23aead..aa0a2314e4 100644--- a/apps/api/src/app/redis-cache/redis-cache.service.ts+++ b/apps/api/src/app/redis-cache/redis-cache.service.ts@@ -1,4 +1,5 @@import { ConfigurationService } from '@ghostfolio/api/services/configuration/configuration.service';+import { getAssetProfileIdentifier } from '@ghostfolio/common/helper';import { UniqueAsset } from '@ghostfolio/common/interfaces';import { CACHE_MANAGER, Inject, Injectable, Logger } from '@nestjs/common';@@ -22,7 +23,7 @@ export class RedisCacheService {}public getQuoteKey({ dataSource, symbol }: UniqueAsset) {- return `quote-${dataSource}-${symbol}`;+ return `quote-${getAssetProfileIdentifier({ dataSource, symbol })}`;}public async remove(key: string) {diff --git a/apps/api/src/services/cron.service.ts b/apps/api/src/services/cron.service.tsindex 72043b36b3..e3597f0495 100644--- a/apps/api/src/services/cron.service.ts+++ b/apps/api/src/services/cron.service.ts@@ -2,6 +2,7 @@ import {GATHER_ASSET_PROFILE_PROCESS,GATHER_ASSET_PROFILE_PROCESS_OPTIONS} from '@ghostfolio/common/config';+import { getAssetProfileIdentifier } from '@ghostfolio/common/helper';import { Injectable } from '@nestjs/common';import { Cron, CronExpression } from '@nestjs/schedule';@@ -48,7 +49,7 @@ export class CronService {name: GATHER_ASSET_PROFILE_PROCESS,opts: {...GATHER_ASSET_PROFILE_PROCESS_OPTIONS,- jobId: `${dataSource}-${symbol}`+ jobId: getAssetProfileIdentifier({ dataSource, symbol })}};})diff --git a/apps/api/src/services/data-gathering/data-gathering.service.ts b/apps/api/src/services/data-gathering/data-gathering.service.tsindex fc77bdc609..20da098ff0 100644--- a/apps/api/src/services/data-gathering/data-gathering.service.ts+++ b/apps/api/src/services/data-gathering/data-gathering.service.ts@@ -10,7 +10,11 @@ import {GATHER_HISTORICAL_MARKET_DATA_PROCESS,GATHER_HISTORICAL_MARKET_DATA_PROCESS_OPTIONS} from '@ghostfolio/common/config';-import { DATE_FORMAT, resetHours } from '@ghostfolio/common/helper';+import {+ DATE_FORMAT,+ getAssetProfileIdentifier,+ resetHours+} from '@ghostfolio/common/helper';import { UniqueAsset } from '@ghostfolio/common/interfaces';import { InjectQueue } from '@nestjs/bull';import { Inject, Injectable, Logger } from '@nestjs/common';@@ -221,7 +225,10 @@ export class DataGatheringService {name: GATHER_HISTORICAL_MARKET_DATA_PROCESS,opts: {...GATHER_HISTORICAL_MARKET_DATA_PROCESS_OPTIONS,- jobId: `${dataSource}-${symbol}-${format(date, DATE_FORMAT)}`+ jobId: `${getAssetProfileIdentifier({+ dataSource,+ symbol+ })}-${format(date, DATE_FORMAT)}`}};})diff --git a/libs/common/src/lib/helper.ts b/libs/common/src/lib/helper.tsindex bc02acbe63..02b0cc08dc 100644--- a/libs/common/src/lib/helper.ts+++ b/libs/common/src/lib/helper.ts@@ -5,7 +5,7 @@ import { getDate, getMonth, getYear, parse, subDays } from 'date-fns';import { de, es, fr, it, nl, pt } from 'date-fns/locale';import { ghostfolioScraperApiSymbolPrefix, locale } from './config';-import { Benchmark } from './interfaces';+import { Benchmark, UniqueAsset } from './interfaces';import { ColorScheme } from './types';const NUMERIC_REGEXP = /[-]{0,1}[\d]*[.,]{0,1}[\d]+/g;@@ -64,6 +64,10 @@ export function extractNumberFromString(aString: string): number {}}+export function getAssetProfileIdentifier({ dataSource, symbol }: UniqueAsset) {+ return `${dataSource}-${symbol}`;+}+export function getBackgroundColor(aColorScheme: ColorScheme) {return getCssVariable(aColorScheme === 'DARK' ||