Skip to content

Open Source Wealth Management Software. Angular + NestJS + Prisma + Nx + TypeScript 🤍

Feature/optimize import validation by reducing to unique asset profiles (#2198)

Thomas Kaul286e41eb216f
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 80987a8119..33fa8fd898 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -5,6 +5,13 @@ All notable changes to this project will be documented in this file.
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+## Unreleased
+
+### Changed
+
+- Optimized the validation in the activities import by reducing the list to unique asset profiles
+- Optimized the data gathering in the activities import
+
## 1.295.0 - 2023-07-30
### Added
diff --git a/apps/api/src/app/admin/admin.controller.ts b/apps/api/src/app/admin/admin.controller.ts
index 54d36d8c55..8e3fb28713 100644
--- a/apps/api/src/app/admin/admin.controller.ts
+++ b/apps/api/src/app/admin/admin.controller.ts
@@ -7,6 +7,7 @@ import {
GATHER_ASSET_PROFILE_PROCESS,
GATHER_ASSET_PROFILE_PROCESS_OPTIONS
} from '@ghostfolio/common/config';
+import { getAssetProfileIdentifier } from '@ghostfolio/common/helper';
import {
AdminData,
AdminMarketData,
@@ -116,7 +117,7 @@ export class AdminController {
name: GATHER_ASSET_PROFILE_PROCESS,
opts: {
...GATHER_ASSET_PROFILE_PROCESS_OPTIONS,
- jobId: `${dataSource}-${symbol}`
+ jobId: getAssetProfileIdentifier({ dataSource, symbol })
}
};
})
@@ -152,7 +153,7 @@ export class AdminController {
name: GATHER_ASSET_PROFILE_PROCESS,
opts: {
...GATHER_ASSET_PROFILE_PROCESS_OPTIONS,
- jobId: `${dataSource}-${symbol}`
+ jobId: getAssetProfileIdentifier({ dataSource, symbol })
}
};
})
@@ -185,7 +186,7 @@ export class AdminController {
name: GATHER_ASSET_PROFILE_PROCESS,
opts: {
...GATHER_ASSET_PROFILE_PROCESS_OPTIONS,
- jobId: `${dataSource}-${symbol}`
+ jobId: getAssetProfileIdentifier({ dataSource, symbol })
}
});
}
diff --git a/apps/api/src/app/import/import.service.ts b/apps/api/src/app/import/import.service.ts
index 65611ce0dc..3d9c2999d6 100644
--- a/apps/api/src/app/import/import.service.ts
+++ b/apps/api/src/app/import/import.service.ts
@@ -8,10 +8,14 @@ import {
import { OrderService } from '@ghostfolio/api/app/order/order.service';
import { PlatformService } from '@ghostfolio/api/app/platform/platform.service';
import { PortfolioService } from '@ghostfolio/api/app/portfolio/portfolio.service';
+import { DataGatheringService } from '@ghostfolio/api/services/data-gathering/data-gathering.service';
import { DataProviderService } from '@ghostfolio/api/services/data-provider/data-provider.service';
import { ExchangeRateDataService } from '@ghostfolio/api/services/exchange-rate-data/exchange-rate-data.service';
import { SymbolProfileService } from '@ghostfolio/api/services/symbol-profile/symbol-profile.service';
-import { parseDate } from '@ghostfolio/common/helper';
+import {
+ getAssetProfileIdentifier,
+ parseDate
+} from '@ghostfolio/common/helper';
import { UniqueAsset } from '@ghostfolio/common/interfaces';
import {
AccountWithPlatform,
@@ -21,12 +25,14 @@ import { Injectable } from '@nestjs/common';
import { DataSource, Prisma, SymbolProfile } from '@prisma/client';
import Big from 'big.js';
import { endOfToday, isAfter, isSameDay, parseISO } from 'date-fns';
+import { uniqBy } from 'lodash';
import { v4 as uuidv4 } from 'uuid';
@Injectable()
export class ImportService {
public constructor(
private readonly accountService: AccountService,
+ private readonly dataGatheringService: DataGatheringService,
private readonly dataProviderService: DataProviderService,
private readonly exchangeRateDataService: ExchangeRateDataService,
private readonly orderService: OrderService,
@@ -220,8 +226,7 @@ export class ImportService {
const assetProfiles = await this.validateActivities({
activitiesDto,
- maxActivitiesToImport,
- userId
+ maxActivitiesToImport
});
const activitiesExtendedWithErrors = await this.extendActivitiesWithErrors({
@@ -250,10 +255,37 @@ export class ImportService {
error,
fee,
quantity,
- SymbolProfile: assetProfile,
+ SymbolProfile,
type,
unitPrice
} of activitiesExtendedWithErrors) {
+ const assetProfile = assetProfiles[
+ getAssetProfileIdentifier({
+ dataSource: SymbolProfile.dataSource,
+ symbol: SymbolProfile.symbol
+ })
+ ] ?? {
+ currency: SymbolProfile.currency,
+ dataSource: SymbolProfile.dataSource,
+ symbol: SymbolProfile.symbol
+ };
+ const {
+ assetClass,
+ assetSubClass,
+ countries,
+ createdAt,
+ currency,
+ dataSource,
+ id,
+ isin,
+ name,
+ scraperConfiguration,
+ sectors,
+ symbol,
+ symbolMapping,
+ url,
+ updatedAt
+ } = assetProfile;
const validatedAccount = accounts.find(({ id }) => {
return id === accountId;
});
@@ -279,23 +311,22 @@ export class ImportService {
id: uuidv4(),
isDraft: isAfter(date, endOfToday()),
SymbolProfile: {
- assetClass: assetProfile.assetClass,
- assetSubClass: assetProfile.assetSubClass,
- comment: assetProfile.comment,
- countries: assetProfile.countries,
- createdAt: assetProfile.createdAt,
- currency: assetProfile.currency,
- dataSource: assetProfile.dataSource,
- id: assetProfile.id,
- isin: assetProfile.isin,
- name: assetProfile.name,
- scraperConfiguration: assetProfile.scraperConfiguration,
- sectors: assetProfile.sectors,
- symbol: assetProfile.currency,
- symbolMapping: assetProfile.symbolMapping,
- updatedAt: assetProfile.updatedAt,
- url: assetProfile.url,
- ...assetProfiles[assetProfile.symbol]
+ assetClass,
+ assetSubClass,
+ countries,
+ createdAt,
+ currency,
+ dataSource,
+ id,
+ isin,
+ name,
+ scraperConfiguration,
+ sectors,
+ symbol,
+ symbolMapping,
+ updatedAt,
+ url,
+ comment: assetProfile.comment
},
Account: validatedAccount,
symbolProfileId: undefined,
@@ -318,14 +349,14 @@ export class ImportService {
SymbolProfile: {
connectOrCreate: {
create: {
- currency: assetProfile.currency,
- dataSource: assetProfile.dataSource,
- symbol: assetProfile.symbol
+ currency,
+ dataSource,
+ symbol
},
where: {
dataSource_symbol: {
- dataSource: assetProfile.dataSource,
- symbol: assetProfile.symbol
+ dataSource,
+ symbol
}
}
}
@@ -337,24 +368,49 @@ export class ImportService {
const value = new Big(quantity).mul(unitPrice).toNumber();
- //@ts-ignore
activities.push({
...order,
error,
value,
feeInBaseCurrency: this.exchangeRateDataService.toCurrency(
fee,
- assetProfile.currency,
+ currency,
userCurrency
),
+ //@ts-ignore
+ SymbolProfile: assetProfile,
valueInBaseCurrency: this.exchangeRateDataService.toCurrency(
value,
- assetProfile.currency,
+ currency,
userCurrency
)
});
}
+ activities.sort((activity1, activity2) => {
+ return Number(activity1.date) - Number(activity2.date);
+ });
+
+ if (!isDryRun) {
+ // Gather symbol data in the background, if not dry run
+ const uniqueActivities = uniqBy(activities, ({ SymbolProfile }) => {
+ return getAssetProfileIdentifier({
+ dataSource: SymbolProfile.dataSource,
+ symbol: SymbolProfile.symbol
+ });
+ });
+
+ this.dataGatheringService.gatherSymbols(
+ uniqueActivities.map(({ date, SymbolProfile }) => {
+ return {
+ date,
+ dataSource: SymbolProfile.dataSource,
+ symbol: SymbolProfile.symbol
+ };
+ })
+ );
+ }
+
return activities;
}
@@ -446,25 +502,30 @@ export class ImportService {
private async validateActivities({
activitiesDto,
- maxActivitiesToImport,
- userId
+ maxActivitiesToImport
}: {
activitiesDto: Partial<CreateOrderDto>[];
maxActivitiesToImport: number;
- userId: string;
}) {
if (activitiesDto?.length > maxActivitiesToImport) {
throw new Error(`Too many activities (${maxActivitiesToImport} at most)`);
}
const assetProfiles: {
- [symbol: string]: Partial<SymbolProfile>;
+ [assetProfileIdentifier: string]: Partial<SymbolProfile>;
} = {};
+ const uniqueActivitiesDto = uniqBy(
+ activitiesDto,
+ ({ dataSource, symbol }) => {
+ return getAssetProfileIdentifier({ dataSource, symbol });
+ }
+ );
+
for (const [
index,
{ currency, dataSource, symbol }
- ] of activitiesDto.entries()) {
+ ] of uniqueActivitiesDto.entries()) {
if (dataSource !== 'MANUAL') {
const assetProfile = (
await this.dataProviderService.getAssetProfiles([
@@ -484,7 +545,8 @@ export class ImportService {
);
}
- assetProfiles[symbol] = assetProfile;
+ assetProfiles[getAssetProfileIdentifier({ dataSource, symbol })] =
+ assetProfile;
}
}
diff --git a/apps/api/src/app/order/order.controller.ts b/apps/api/src/app/order/order.controller.ts
index c478860d24..0e617462e1 100644
--- a/apps/api/src/app/order/order.controller.ts
+++ b/apps/api/src/app/order/order.controller.ts
@@ -2,6 +2,7 @@ import { RedactValuesInResponseInterceptor } from '@ghostfolio/api/interceptors/
import { TransformDataSourceInRequestInterceptor } from '@ghostfolio/api/interceptors/transform-data-source-in-request.interceptor';
import { TransformDataSourceInResponseInterceptor } from '@ghostfolio/api/interceptors/transform-data-source-in-response.interceptor';
import { ApiService } from '@ghostfolio/api/services/api/api.service';
+import { DataGatheringService } from '@ghostfolio/api/services/data-gathering/data-gathering.service';
import { ImpersonationService } from '@ghostfolio/api/services/impersonation/impersonation.service';
import { HEADER_KEY_IMPERSONATION } from '@ghostfolio/common/config';
import { hasPermission, permissions } from '@ghostfolio/common/permissions';
@@ -36,6 +37,7 @@ import { UpdateOrderDto } from './update-order.dto';
export class OrderController {
public constructor(
private readonly apiService: ApiService,
+ private readonly dataGatheringService: DataGatheringService,
private readonly impersonationService: ImpersonationService,
private readonly orderService: OrderService,
@Inject(REQUEST) private readonly request: RequestWithUser
@@ -123,7 +125,7 @@ export class OrderController {
);
}
- return this.orderService.createOrder({
+ const order = await this.orderService.createOrder({
...data,
date: parseISO(data.date),
SymbolProfile: {
@@ -144,6 +146,19 @@ export class OrderController {
User: { connect: { id: this.request.user.id } },
userId: this.request.user.id
});
+
+ if (!order.isDraft) {
+ // Gather symbol data in the background, if not draft
+ this.dataGatheringService.gatherSymbols([
+ {
+ dataSource: data.dataSource,
+ date: order.date,
+ symbol: data.symbol
+ }
+ ]);
+ }
+
+ return order;
}
@Put(':id')
diff --git a/apps/api/src/app/order/order.service.ts b/apps/api/src/app/order/order.service.ts
index be5708a904..0f7da5da37 100644
--- a/apps/api/src/app/order/order.service.ts
+++ b/apps/api/src/app/order/order.service.ts
@@ -7,6 +7,7 @@ import {
GATHER_ASSET_PROFILE_PROCESS,
GATHER_ASSET_PROFILE_PROCESS_OPTIONS
} from '@ghostfolio/common/config';
+import { getAssetProfileIdentifier } from '@ghostfolio/common/helper';
import { Filter } from '@ghostfolio/common/interfaces';
import { OrderWithAccount } from '@ghostfolio/common/types';
import { Injectable } from '@nestjs/common';
@@ -117,7 +118,7 @@ export class OrderService {
};
}
- await this.dataGatheringService.addJobToQueue({
+ this.dataGatheringService.addJobToQueue({
data: {
dataSource: data.SymbolProfile.connectOrCreate.create.dataSource,
symbol: data.SymbolProfile.connectOrCreate.create.symbol
@@ -125,25 +126,12 @@ export class OrderService {
name: GATHER_ASSET_PROFILE_PROCESS,
opts: {
...GATHER_ASSET_PROFILE_PROCESS_OPTIONS,
- jobId: `${data.SymbolProfile.connectOrCreate.create.dataSource}-${data.SymbolProfile.connectOrCreate.create.symbol}`
- }
- });
-
- const isDraft =
- data.type === 'LIABILITY'
- ? false
- : isAfter(data.date as Date, endOfToday());
-
- if (!isDraft) {
- // Gather symbol data of order in the background, if not draft
- this.dataGatheringService.gatherSymbols([
- {
+ jobId: getAssetProfileIdentifier({
dataSource: data.SymbolProfile.connectOrCreate.create.dataSource,
- date: <Date>data.date,
symbol: data.SymbolProfile.connectOrCreate.create.symbol
- }
- ]);
- }
+ })
+ }
+ });
delete data.accountId;
delete data.assetClass;
@@ -162,6 +150,11 @@ export class OrderService {
const orderData: Prisma.OrderCreateInput = data;
+ const isDraft =
+ data.type === 'LIABILITY'
+ ? false
+ : isAfter(data.date as Date, endOfToday());
+
const order = await this.prismaService.order.create({
data: {
...orderData,
diff --git a/apps/api/src/app/redis-cache/redis-cache.service.ts b/apps/api/src/app/redis-cache/redis-cache.service.ts
index 865a23aead..aa0a2314e4 100644
--- a/apps/api/src/app/redis-cache/redis-cache.service.ts
+++ b/apps/api/src/app/redis-cache/redis-cache.service.ts
@@ -1,4 +1,5 @@
import { ConfigurationService } from '@ghostfolio/api/services/configuration/configuration.service';
+import { getAssetProfileIdentifier } from '@ghostfolio/common/helper';
import { UniqueAsset } from '@ghostfolio/common/interfaces';
import { CACHE_MANAGER, Inject, Injectable, Logger } from '@nestjs/common';
@@ -22,7 +23,7 @@ export class RedisCacheService {
}
public getQuoteKey({ dataSource, symbol }: UniqueAsset) {
- return `quote-${dataSource}-${symbol}`;
+ return `quote-${getAssetProfileIdentifier({ dataSource, symbol })}`;
}
public async remove(key: string) {
diff --git a/apps/api/src/services/cron.service.ts b/apps/api/src/services/cron.service.ts
index 72043b36b3..e3597f0495 100644
--- a/apps/api/src/services/cron.service.ts
+++ b/apps/api/src/services/cron.service.ts
@@ -2,6 +2,7 @@ import {
GATHER_ASSET_PROFILE_PROCESS,
GATHER_ASSET_PROFILE_PROCESS_OPTIONS
} from '@ghostfolio/common/config';
+import { getAssetProfileIdentifier } from '@ghostfolio/common/helper';
import { Injectable } from '@nestjs/common';
import { Cron, CronExpression } from '@nestjs/schedule';
@@ -48,7 +49,7 @@ export class CronService {
name: GATHER_ASSET_PROFILE_PROCESS,
opts: {
...GATHER_ASSET_PROFILE_PROCESS_OPTIONS,
- jobId: `${dataSource}-${symbol}`
+ jobId: getAssetProfileIdentifier({ dataSource, symbol })
}
};
})
diff --git a/apps/api/src/services/data-gathering/data-gathering.service.ts b/apps/api/src/services/data-gathering/data-gathering.service.ts
index fc77bdc609..20da098ff0 100644
--- a/apps/api/src/services/data-gathering/data-gathering.service.ts
+++ b/apps/api/src/services/data-gathering/data-gathering.service.ts
@@ -10,7 +10,11 @@ import {
GATHER_HISTORICAL_MARKET_DATA_PROCESS,
GATHER_HISTORICAL_MARKET_DATA_PROCESS_OPTIONS
} from '@ghostfolio/common/config';
-import { DATE_FORMAT, resetHours } from '@ghostfolio/common/helper';
+import {
+ DATE_FORMAT,
+ getAssetProfileIdentifier,
+ resetHours
+} from '@ghostfolio/common/helper';
import { UniqueAsset } from '@ghostfolio/common/interfaces';
import { InjectQueue } from '@nestjs/bull';
import { Inject, Injectable, Logger } from '@nestjs/common';
@@ -221,7 +225,10 @@ export class DataGatheringService {
name: GATHER_HISTORICAL_MARKET_DATA_PROCESS,
opts: {
...GATHER_HISTORICAL_MARKET_DATA_PROCESS_OPTIONS,
- jobId: `${dataSource}-${symbol}-${format(date, DATE_FORMAT)}`
+ jobId: `${getAssetProfileIdentifier({
+ dataSource,
+ symbol
+ })}-${format(date, DATE_FORMAT)}`
}
};
})
diff --git a/libs/common/src/lib/helper.ts b/libs/common/src/lib/helper.ts
index bc02acbe63..02b0cc08dc 100644
--- a/libs/common/src/lib/helper.ts
+++ b/libs/common/src/lib/helper.ts
@@ -5,7 +5,7 @@ import { getDate, getMonth, getYear, parse, subDays } from 'date-fns';
import { de, es, fr, it, nl, pt } from 'date-fns/locale';
import { ghostfolioScraperApiSymbolPrefix, locale } from './config';
-import { Benchmark } from './interfaces';
+import { Benchmark, UniqueAsset } from './interfaces';
import { ColorScheme } from './types';
const NUMERIC_REGEXP = /[-]{0,1}[\d]*[.,]{0,1}[\d]+/g;
@@ -64,6 +64,10 @@ export function extractNumberFromString(aString: string): number {
}
}
+export function getAssetProfileIdentifier({ dataSource, symbol }: UniqueAsset) {
+ return `${dataSource}-${symbol}`;
+}
+
export function getBackgroundColor(aColorScheme: ColorScheme) {
return getCssVariable(
aColorScheme === 'DARK' ||