Open Source Wealth Management Software. Angular + NestJS + Prisma + Nx + TypeScript π€
TypeScript angularetffinancefintechghostfoliohacktoberfestinvestingnestjsosspersonal-financeportfolioprismasoftwarestocktrackertradingtypescriptwealth-managementweb
Feature/extend activities import by custom asset profiles (#5243)
Attila Cseh3b92558e0390
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex bca053c0c5..a1048c6b20 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -14,6 +14,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0### Changed+- Extended the import functionality by custom asset profiles- Migrated the get country and sector weightings, dividends, ETF holdings, ETF info, historical price, profile, quote and symbol search functionalities of the _Financial Modeling Prep_ service to its stable API version- Refactored the toggle component to standalone- Improved the language localization for Dutch (`nl`)diff --git a/apps/api/src/app/admin/create-asset-profile.dto.ts b/apps/api/src/app/admin/create-asset-profile.dto.tsnew file mode 100644index 0000000000..8041b0f0e5--- /dev/null+++ b/apps/api/src/app/admin/create-asset-profile.dto.ts@@ -0,0 +1,92 @@+import { IsCurrencyCode } from '@ghostfolio/api/validators/is-currency-code';++import { AssetClass, AssetSubClass, DataSource, Prisma } from '@prisma/client';+import {+ IsArray,+ IsBoolean,+ IsEnum,+ IsObject,+ IsOptional,+ IsString,+ IsUrl+} from 'class-validator';++export class CreateAssetProfileDto {+ @IsEnum(AssetClass, { each: true })+ @IsOptional()+ assetClass?: AssetClass;++ @IsEnum(AssetSubClass, { each: true })+ @IsOptional()+ assetSubClass?: AssetSubClass;++ @IsOptional()+ @IsString()+ comment?: string;++ @IsArray()+ @IsOptional()+ countries?: Prisma.InputJsonArray;++ @IsCurrencyCode()+ currency: string;++ @IsOptional()+ @IsString()+ cusip?: string;++ @IsEnum(DataSource)+ dataSource: DataSource;++ @IsOptional()+ @IsString()+ figi?: string;++ @IsOptional()+ @IsString()+ figiComposite?: string;++ @IsOptional()+ @IsString()+ figiShareClass?: string;++ @IsArray()+ @IsOptional()+ holdings?: Prisma.InputJsonArray;++ @IsBoolean()+ @IsOptional()+ isActive?: boolean;++ @IsOptional()+ @IsString()+ isin?: string;++ @IsOptional()+ @IsString()+ name?: string;++ @IsObject()+ @IsOptional()+ scraperConfiguration?: Prisma.InputJsonObject;++ @IsArray()+ @IsOptional()+ sectors?: Prisma.InputJsonArray;++ @IsString()+ symbol: string;++ @IsObject()+ @IsOptional()+ symbolMapping?: {+ [dataProvider: string]: string;+ };++ @IsOptional()+ @IsUrl({+ protocols: ['https'],+ require_protocol: true+ })+ url?: string;+}diff --git a/apps/api/src/app/export/export.service.ts b/apps/api/src/app/export/export.service.tsindex 9c9d20682d..54fd0763db 100644--- a/apps/api/src/app/export/export.service.ts+++ b/apps/api/src/app/export/export.service.ts@@ -195,7 +195,6 @@ export class ExportService {figiComposite,figiShareClass,holdings: holdings as unknown as Prisma.JsonArray,- id,isActive,isin,marketData: marketDataByAssetProfile[id],diff --git a/apps/api/src/app/import/create-asset-profile-with-market-data.dto.ts b/apps/api/src/app/import/create-asset-profile-with-market-data.dto.tsnew file mode 100644index 0000000000..fd90ab1afa--- /dev/null+++ b/apps/api/src/app/import/create-asset-profile-with-market-data.dto.ts@@ -0,0 +1,17 @@+import { MarketData } from '@ghostfolio/common/interfaces';++import { DataSource } from '@prisma/client';+import { IsArray, IsEnum, IsOptional } from 'class-validator';++import { CreateAssetProfileDto } from '../admin/create-asset-profile.dto';++export class CreateAssetProfileWithMarketDataDto extends CreateAssetProfileDto {+ @IsEnum([DataSource.MANUAL], {+ message: `dataSource must be '${DataSource.MANUAL}'`+ })+ dataSource: DataSource;++ @IsArray()+ @IsOptional()+ marketData?: MarketData[];+}diff --git a/apps/api/src/app/import/import-data.dto.ts b/apps/api/src/app/import/import-data.dto.tsindex 207c8152b3..138d169612 100644--- a/apps/api/src/app/import/import-data.dto.ts+++ b/apps/api/src/app/import/import-data.dto.ts@@ -4,16 +4,23 @@ import { Type } from 'class-transformer';import { IsArray, IsOptional, ValidateNested } from 'class-validator';import { CreateAccountWithBalancesDto } from './create-account-with-balances.dto';+import { CreateAssetProfileWithMarketDataDto } from './create-asset-profile-with-market-data.dto';export class ImportDataDto {- @IsOptional()@IsArray()+ @IsOptional()@Type(() => CreateAccountWithBalancesDto)@ValidateNested({ each: true })- accounts: CreateAccountWithBalancesDto[];+ accounts?: CreateAccountWithBalancesDto[];@IsArray()@Type(() => CreateOrderDto)@ValidateNested({ each: true })activities: CreateOrderDto[];++ @IsArray()+ @IsOptional()+ @Type(() => CreateAssetProfileWithMarketDataDto)+ @ValidateNested({ each: true })+ assetProfiles?: CreateAssetProfileWithMarketDataDto[];}diff --git a/apps/api/src/app/import/import.controller.ts b/apps/api/src/app/import/import.controller.tsindex 081541fee0..8c06bdc6d6 100644--- a/apps/api/src/app/import/import.controller.ts+++ b/apps/api/src/app/import/import.controller.ts@@ -73,6 +73,7 @@ export class ImportController {maxActivitiesToImport,accountsWithBalancesDto: importData.accounts ?? [],activitiesDto: importData.activities,+ assetProfilesWithMarketDataDto: importData.assetProfiles ?? [],user: this.request.user});diff --git a/apps/api/src/app/import/import.module.ts b/apps/api/src/app/import/import.module.tsindex 142a939a69..fb6c29fc59 100644--- a/apps/api/src/app/import/import.module.ts+++ b/apps/api/src/app/import/import.module.ts@@ -9,6 +9,7 @@ import { TransformDataSourceInResponseModule } from '@ghostfolio/api/interceptorimport { ConfigurationModule } from '@ghostfolio/api/services/configuration/configuration.module';import { DataProviderModule } from '@ghostfolio/api/services/data-provider/data-provider.module';import { ExchangeRateDataModule } from '@ghostfolio/api/services/exchange-rate-data/exchange-rate-data.module';+import { MarketDataModule } from '@ghostfolio/api/services/market-data/market-data.module';import { PrismaModule } from '@ghostfolio/api/services/prisma/prisma.module';import { DataGatheringModule } from '@ghostfolio/api/services/queues/data-gathering/data-gathering.module';import { SymbolProfileModule } from '@ghostfolio/api/services/symbol-profile/symbol-profile.module';@@ -27,6 +28,7 @@ import { ImportService } from './import.service';DataGatheringModule,DataProviderModule,ExchangeRateDataModule,+ MarketDataModule,OrderModule,PlatformModule,PortfolioModule,diff --git a/apps/api/src/app/import/import.service.ts b/apps/api/src/app/import/import.service.tsindex e41dcd8190..d23427616e 100644--- a/apps/api/src/app/import/import.service.ts+++ b/apps/api/src/app/import/import.service.ts@@ -10,6 +10,7 @@ import { PlatformService } from '@ghostfolio/api/app/platform/platform.service';import { PortfolioService } from '@ghostfolio/api/app/portfolio/portfolio.service';import { ConfigurationService } from '@ghostfolio/api/services/configuration/configuration.service';import { DataProviderService } from '@ghostfolio/api/services/data-provider/data-provider.service';+import { MarketDataService } from '@ghostfolio/api/services/market-data/market-data.service';import { DataGatheringService } from '@ghostfolio/api/services/queues/data-gathering/data-gathering.service';import { SymbolProfileService } from '@ghostfolio/api/services/symbol-profile/symbol-profile.service';import { DATA_GATHERING_QUEUE_PRIORITY_HIGH } from '@ghostfolio/common/config';@@ -31,6 +32,7 @@ import { endOfToday, isAfter, isSameSecond, parseISO } from 'date-fns';import { omit, uniqBy } from 'lodash';import { v4 as uuidv4 } from 'uuid';+import { CreateAssetProfileDto } from '../admin/create-asset-profile.dto';import { ImportDataDto } from './import-data.dto';@Injectable()@@ -40,6 +42,7 @@ export class ImportService {private readonly configurationService: ConfigurationService,private readonly dataGatheringService: DataGatheringService,private readonly dataProviderService: DataProviderService,+ private readonly marketDataService: MarketDataService,private readonly orderService: OrderService,private readonly platformService: PlatformService,private readonly portfolioService: PortfolioService,@@ -148,17 +151,20 @@ export class ImportService {public async import({accountsWithBalancesDto,activitiesDto,+ assetProfilesWithMarketDataDto,isDryRun = false,maxActivitiesToImport,user}: {accountsWithBalancesDto: ImportDataDto['accounts'];activitiesDto: ImportDataDto['activities'];+ assetProfilesWithMarketDataDto: ImportDataDto['assetProfiles'];isDryRun?: boolean;maxActivitiesToImport: number;user: UserWithSettings;}): Promise<Activity[]> {const accountIdMapping: { [oldAccountId: string]: string } = {};+ const assetProfileSymbolMapping: { [oldSymbol: string]: string } = {};const userCurrency = user.settings.settings.baseCurrency;if (!isDryRun && accountsWithBalancesDto?.length) {@@ -230,6 +236,63 @@ export class ImportService {}}+ if (!isDryRun && assetProfilesWithMarketDataDto?.length) {+ const existingAssetProfiles =+ await this.symbolProfileService.getSymbolProfiles(+ assetProfilesWithMarketDataDto.map(({ dataSource, symbol }) => {+ return { dataSource, symbol };+ })+ );++ for (const assetProfileWithMarketData of assetProfilesWithMarketDataDto) {+ // Check if there is any existing asset profile+ const existingAssetProfile = existingAssetProfiles.find(+ ({ dataSource, symbol }) => {+ return (+ dataSource === assetProfileWithMarketData.dataSource &&+ symbol === assetProfileWithMarketData.symbol+ );+ }+ );++ // If there is no asset profile or if the asset profile belongs to a different user, then create a new asset profile+ if (!existingAssetProfile || existingAssetProfile.userId !== user.id) {+ const assetProfile: CreateAssetProfileDto = omit(+ assetProfileWithMarketData,+ 'marketData'+ );++ // Asset profile belongs to a different user+ if (existingAssetProfile) {+ const symbol = uuidv4();+ assetProfileSymbolMapping[assetProfile.symbol] = symbol;+ assetProfile.symbol = symbol;+ }++ // Create a new asset profile+ const assetProfileObject: Prisma.SymbolProfileCreateInput = {+ ...assetProfile,+ user: { connect: { id: user.id } }+ };++ await this.symbolProfileService.add(assetProfileObject);+ }++ // Insert or update market data+ const marketDataObjects = assetProfileWithMarketData.marketData.map(+ (marketData) => {+ return {+ ...marketData,+ dataSource: assetProfileWithMarketData.dataSource,+ symbol: assetProfileWithMarketData.symbol+ } as Prisma.MarketDataUpdateInput;+ }+ );++ await this.marketDataService.updateMany({ data: marketDataObjects });+ }+ }+for (const activity of activitiesDto) {if (!activity.dataSource) {if (['FEE', 'INTEREST', 'LIABILITY'].includes(activity.type)) {@@ -240,11 +303,16 @@ export class ImportService {}}- // If a new account is created, then update the accountId in all activitiesif (!isDryRun) {- if (Object.keys(accountIdMapping).includes(activity.accountId)) {+ // If a new account is created, then update the accountId in all activities+ if (accountIdMapping[activity.accountId]) {activity.accountId = accountIdMapping[activity.accountId];}++ // If a new asset profile is created, then update the symbol in all activities+ if (assetProfileSymbolMapping[activity.symbol]) {+ activity.symbol = assetProfileSymbolMapping[activity.symbol];+ }}}diff --git a/apps/client/src/app/pages/portfolio/activities/import-activities-dialog/import-activities-dialog.component.ts b/apps/client/src/app/pages/portfolio/activities/import-activities-dialog/import-activities-dialog.component.tsindex b9f0986ac2..660e7265ea 100644--- a/apps/client/src/app/pages/portfolio/activities/import-activities-dialog/import-activities-dialog.component.ts+++ b/apps/client/src/app/pages/portfolio/activities/import-activities-dialog/import-activities-dialog.component.ts@@ -1,4 +1,5 @@-import { CreateAccountDto } from '@ghostfolio/api/app/account/create-account.dto';+import { CreateAccountWithBalancesDto } from '@ghostfolio/api/app/import/create-account-with-balances.dto';+import { CreateAssetProfileWithMarketDataDto } from '@ghostfolio/api/app/import/create-asset-profile-with-market-data.dto';import { Activity } from '@ghostfolio/api/app/order/interfaces/activities.interface';import { GfDialogFooterModule } from '@ghostfolio/client/components/dialog-footer/dialog-footer.module';import { GfDialogHeaderModule } from '@ghostfolio/client/components/dialog-header/dialog-header.module';@@ -75,9 +76,10 @@ import { ImportActivitiesDialogParams } from './interfaces/interfaces';templateUrl: 'import-activities-dialog.html'})export class GfImportActivitiesDialog implements OnDestroy {- public accounts: CreateAccountDto[] = [];+ public accounts: CreateAccountWithBalancesDto[] = [];public activities: Activity[] = [];public assetProfileForm: FormGroup;+ public assetProfiles: CreateAssetProfileWithMarketDataDto[] = [];public dataSource: MatTableDataSource<Activity>;public details: any[] = [];public deviceType: string;@@ -166,7 +168,8 @@ export class GfImportActivitiesDialog implements OnDestroy {await this.importActivitiesService.importSelectedActivities({accounts: this.accounts,- activities: this.selectedActivities+ activities: this.selectedActivities,+ assetProfiles: this.assetProfiles});this.snackBar.open(@@ -293,6 +296,7 @@ export class GfImportActivitiesDialog implements OnDestroy {const content = JSON.parse(fileContent);this.accounts = content.accounts;+ this.assetProfiles = content.assetProfiles;if (!isArray(content.activities)) {if (isArray(content.orders)) {@@ -323,6 +327,7 @@ export class GfImportActivitiesDialog implements OnDestroy {await this.importActivitiesService.importJson({accounts: content.accounts,activities: content.activities,+ assetProfiles: content.assetProfiles,isDryRun: true});this.activities = activities;diff --git a/apps/client/src/app/services/import-activities.service.ts b/apps/client/src/app/services/import-activities.service.tsindex a4ffa7ec4c..033ae7e242 100644--- a/apps/client/src/app/services/import-activities.service.ts+++ b/apps/client/src/app/services/import-activities.service.ts@@ -1,4 +1,5 @@-import { CreateAccountDto } from '@ghostfolio/api/app/account/create-account.dto';+import { CreateAccountWithBalancesDto } from '@ghostfolio/api/app/import/create-account-with-balances.dto';+import { CreateAssetProfileWithMarketDataDto } from '@ghostfolio/api/app/import/create-asset-profile-with-market-data.dto';import { CreateOrderDto } from '@ghostfolio/api/app/order/create-order.dto';import { Activity } from '@ghostfolio/api/app/order/interfaces/activities.interface';import { parseDate as parseDateHelper } from '@ghostfolio/common/helper';@@ -73,20 +74,22 @@ export class ImportActivitiesService {public importJson({accounts,activities,+ assetProfiles,isDryRun = false}: {activities: CreateOrderDto[];- accounts?: CreateAccountDto[];+ accounts?: CreateAccountWithBalancesDto[];+ assetProfiles?: CreateAssetProfileWithMarketDataDto[];isDryRun?: boolean;}): Promise<{activities: Activity[];- accounts?: CreateAccountDto[];}> {return new Promise((resolve, reject) => {this.postImport({accounts,- activities+ activities,+ assetProfiles},isDryRun)@@ -106,13 +109,14 @@ export class ImportActivitiesService {public importSelectedActivities({accounts,- activities+ activities,+ assetProfiles}: {- accounts: CreateAccountDto[];+ accounts?: CreateAccountWithBalancesDto[];activities: Activity[];+ assetProfiles?: CreateAssetProfileWithMarketDataDto[];}): Promise<{activities: Activity[];- accounts?: CreateAccountDto[];}> {const importData: CreateOrderDto[] = [];@@ -120,7 +124,7 @@ export class ImportActivitiesService {importData.push(this.convertToCreateOrderDto(activity));}- return this.importJson({ accounts, activities: importData });+ return this.importJson({ accounts, assetProfiles, activities: importData });}private convertToCreateOrderDto({@@ -383,7 +387,11 @@ export class ImportActivitiesService {}private postImport(- aImportData: { accounts: CreateAccountDto[]; activities: CreateOrderDto[] },+ aImportData: {+ accounts?: CreateAccountWithBalancesDto[];+ activities: CreateOrderDto[];+ assetProfiles?: CreateAssetProfileWithMarketDataDto[];+ },aIsDryRun = false) {return this.http.post<{ activities: Activity[] }>(diff --git a/libs/common/src/lib/interfaces/export.interface.ts b/libs/common/src/lib/interfaces/export.interface.tsindex 905e33b060..16a49b0ef0 100644--- a/libs/common/src/lib/interfaces/export.interface.ts+++ b/libs/common/src/lib/interfaces/export.interface.ts@@ -8,6 +8,7 @@ import {} from '@prisma/client';import { AccountBalance } from './account-balance.interface';+import { MarketData } from './market-data.interface';export interface Export {accounts: (Omit<Account, 'createdAt' | 'updatedAt' | 'userId'> & {@@ -23,8 +24,11 @@ export interface Export {| 'updatedAt'| 'userId'> & { dataSource: DataSource; date: string; symbol: string })[];- assetProfiles: (Omit<SymbolProfile, 'createdAt' | 'updatedAt' | 'userId'> & {- marketData: { date: string; marketPrice: number }[];+ assetProfiles: (Omit<+ SymbolProfile,+ 'createdAt' | 'id' | 'updatedAt' | 'userId'+ > & {+ marketData: MarketData[];})[];meta: {date: string;diff --git a/libs/common/src/lib/interfaces/index.ts b/libs/common/src/lib/interfaces/index.tsindex 611a5c9630..52ca76b3a8 100644--- a/libs/common/src/lib/interfaces/index.ts+++ b/libs/common/src/lib/interfaces/index.ts@@ -25,6 +25,7 @@ import type { InfoItem } from './info-item.interface';import type { InvestmentItem } from './investment-item.interface';import type { LineChartItem } from './line-chart-item.interface';import type { LookupItem } from './lookup-item.interface';+import type { MarketData } from './market-data.interface';import type { PortfolioChart } from './portfolio-chart.interface';import type { PortfolioDetails } from './portfolio-details.interface';import type { PortfolioDividends } from './portfolio-dividends.interface';@@ -111,6 +112,7 @@ export {LineChartItem,LookupItem,LookupResponse,+ MarketData,MarketDataDetailsResponse,MarketDataOfMarketsResponse,OAuthResponse,diff --git a/libs/common/src/lib/interfaces/market-data.interface.ts b/libs/common/src/lib/interfaces/market-data.interface.tsnew file mode 100644index 0000000000..b7a410cba4--- /dev/null+++ b/libs/common/src/lib/interfaces/market-data.interface.ts@@ -0,0 +1,4 @@+export interface MarketData {+ date: string;+ marketPrice: number;+}diff --git a/test/import/ok/sample.json b/test/import/ok/sample.jsonindex 01bcc60d9a..4ebb323ded 100644--- a/test/import/ok/sample.json+++ b/test/import/ok/sample.json@@ -16,6 +16,7 @@"value": 1000}],+ "comment": null,"currency": "USD","id": "b2d3fe1d-d6a8-41a3-be39-07ef5e9480f0","isExcluded": false,@@ -23,30 +24,80 @@"platformId": null}],+ "assetProfiles": [+ {+ "assetClass": null,+ "assetSubClass": null,+ "comment": null,+ "countries": [],+ "currency": "USD",+ "cusip": null,+ "dataSource": "MANUAL",+ "figi": null,+ "figiComposite": null,+ "figiShareClass": null,+ "holdings": [],+ "isActive": true,+ "isin": null,+ "marketData": [],+ "name": "Account Opening Fee",+ "scraperConfiguration": null,+ "sectors": [],+ "symbol": "14a69cb9-1e31-43fa-b320-83703d8ed74b",+ "symbolMapping": {},+ "url": null+ },+ {+ "assetClass": null,+ "assetSubClass": null,+ "comment": null,+ "countries": [],+ "currency": "USD",+ "cusip": null,+ "dataSource": "MANUAL",+ "figi": null,+ "figiComposite": null,+ "figiShareClass": null,+ "holdings": [],+ "isActive": true,+ "isin": null,+ "marketData": [],+ "name": "Penthouse Apartment",+ "scraperConfiguration": null,+ "sectors": [],+ "symbol": "7e91b7d4-1430-4212-8380-289a06c9bbc1",+ "symbolMapping": {},+ "url": null+ }+ ],+ "platforms": [],+ "tags": [],"activities": [{"accountId": "b2d3fe1d-d6a8-41a3-be39-07ef5e9480f0","comment": null,- "fee": 0,+ "fee": 49,"quantity": 0,- "type": "BUY",+ "type": "FEE","unitPrice": 0,"currency": "USD",- "dataSource": "YAHOO",- "date": "2050-06-06T00:00:00.000Z",- "symbol": "US5949181045"+ "dataSource": "MANUAL",+ "date": "2021-09-01T00:00:00.000Z",+ "symbol": "14a69cb9-1e31-43fa-b320-83703d8ed74b",+ "tags": []},{- "accountId": null,- "comment": null,- "fee": 0,- "quantity": 1,+ "accountId": "b2d3fe1d-d6a8-41a3-be39-07ef5e9480f0",+ "comment": "My first order π€",+ "fee": 19,+ "quantity": 5,"type": "BUY",- "unitPrice": 500000,+ "unitPrice": 298.58,"currency": "USD",- "dataSource": "MANUAL",- "date": "2022-01-01T00:00:00.000Z",- "symbol": "Penthouse Apartment"+ "dataSource": "YAHOO",+ "date": "2021-09-16T00:00:00.000Z",+ "symbol": "MSFT",+ "tags": []},{"accountId": "b2d3fe1d-d6a8-41a3-be39-07ef5e9480f0",@@ -58,31 +109,39 @@"currency": "USD","dataSource": "YAHOO","date": "2021-11-17T00:00:00.000Z",- "symbol": "MSFT"+ "symbol": "MSFT",+ "tags": []},{- "accountId": "b2d3fe1d-d6a8-41a3-be39-07ef5e9480f0",- "comment": "My first order π€",- "fee": 19,- "quantity": 5,+ "accountId": null,+ "comment": null,+ "fee": 0,+ "quantity": 1,"type": "BUY",- "unitPrice": 298.58,+ "unitPrice": 500000,"currency": "USD",- "dataSource": "YAHOO",- "date": "2021-09-16T00:00:00.000Z",- "symbol": "MSFT"+ "dataSource": "MANUAL",+ "date": "2022-01-01T00:00:00.000Z",+ "symbol": "7e91b7d4-1430-4212-8380-289a06c9bbc1",+ "tags": []},{"accountId": "b2d3fe1d-d6a8-41a3-be39-07ef5e9480f0","comment": null,- "fee": 49,+ "fee": 0,"quantity": 0,- "type": "FEE",+ "type": "BUY","unitPrice": 0,"currency": "USD",- "dataSource": "MANUAL",- "date": "2021-09-01T00:00:00.000Z",- "symbol": "Account Opening Fee"+ "dataSource": "YAHOO",+ "date": "2050-06-06T00:00:00.000Z",+ "symbol": "MSFT",+ "tags": []+ }+ ],+ "user": {+ "settings": {+ "currency": "USD"}- ]+ }}