Open Source Wealth Management Software. Angular + NestJS + Prisma + Nx + TypeScript 🤍
TypeScript angularetffinancefintechghostfoliohacktoberfestinvestingnestjsosspersonal-financeportfolioprismasoftwarestocktrackertradingtypescriptwealth-managementweb
Feature/extend data source eod historical data by asset class and isin (#1791)
Thomas Kaulc8ca82b8037d
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex d39c31058e..a5c8890c6b 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -7,6 +7,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0## Unreleased+### Added++- Added support for asset and asset sub class to the `EOD_HISTORICAL_DATA` data source type+- Added `isin` to the asset profile model+### Changed- Improved the language localization for _Gather Data_diff --git a/apps/api/src/app/import/import.service.ts b/apps/api/src/app/import/import.service.tsindex 37693f1607..5b10e381d8 100644--- a/apps/api/src/app/import/import.service.ts+++ b/apps/api/src/app/import/import.service.ts@@ -254,6 +254,7 @@ export class ImportService {countries: null,createdAt: undefined,id: undefined,+ isin: null,name: null,scraperConfiguration: null,sectors: null,diff --git a/apps/api/src/app/symbol/symbol.controller.ts b/apps/api/src/app/symbol/symbol.controller.tsindex 249284ea80..6003c50d3f 100644--- a/apps/api/src/app/symbol/symbol.controller.ts+++ b/apps/api/src/app/symbol/symbol.controller.ts@@ -1,15 +1,18 @@import { TransformDataSourceInRequestInterceptor } from '@ghostfolio/api/interceptors/transform-data-source-in-request.interceptor';import { TransformDataSourceInResponseInterceptor } from '@ghostfolio/api/interceptors/transform-data-source-in-response.interceptor';import { IDataProviderHistoricalResponse } from '@ghostfolio/api/services/interfaces/interfaces';+import type { RequestWithUser } from '@ghostfolio/common/types';import {Controller,Get,HttpException,+ Inject,Param,Query,UseGuards,UseInterceptors} from '@nestjs/common';+import { REQUEST } from '@nestjs/core';import { AuthGuard } from '@nestjs/passport';import { DataSource } from '@prisma/client';import { StatusCodes, getReasonPhrase } from 'http-status-codes';@@ -21,7 +24,10 @@ import { SymbolService } from './symbol.service';@Controller('symbol')export class SymbolController {- public constructor(private readonly symbolService: SymbolService) {}+ public constructor(+ @Inject(REQUEST) private readonly request: RequestWithUser,+ private readonly symbolService: SymbolService+ ) {}/*** Must be before /:symbol@@ -33,7 +39,10 @@ export class SymbolController {@Query() { query = '' }): Promise<{ items: LookupItem[] }> {try {- return this.symbolService.lookup(query.toLowerCase());+ return this.symbolService.lookup({+ query: query.toLowerCase(),+ user: this.request.user+ });} catch {throw new HttpException(getReasonPhrase(StatusCodes.INTERNAL_SERVER_ERROR),diff --git a/apps/api/src/app/symbol/symbol.service.ts b/apps/api/src/app/symbol/symbol.service.tsindex 7a5f5586dc..718387b3b1 100644--- a/apps/api/src/app/symbol/symbol.service.ts+++ b/apps/api/src/app/symbol/symbol.service.ts@@ -5,7 +5,10 @@ import {} from '@ghostfolio/api/services/interfaces/interfaces';import { MarketDataService } from '@ghostfolio/api/services/market-data.service';import { DATE_FORMAT } from '@ghostfolio/common/helper';-import { HistoricalDataItem } from '@ghostfolio/common/interfaces';+import {+ HistoricalDataItem,+ UserWithSettings+} from '@ghostfolio/common/interfaces';import { Injectable, Logger } from '@nestjs/common';import { format, subDays } from 'date-fns';@@ -79,15 +82,24 @@ export class SymbolService {};}- public async lookup(aQuery: string): Promise<{ items: LookupItem[] }> {+ public async lookup({+ query,+ user+ }: {+ query: string;+ user: UserWithSettings;+ }): Promise<{ items: LookupItem[] }> {const results: { items: LookupItem[] } = { items: [] };- if (!aQuery) {+ if (!query) {return results;}try {- const { items } = await this.dataProviderService.search(aQuery);+ const { items } = await this.dataProviderService.search({+ query,+ user+ });results.items = items;return results;} catch (error) {diff --git a/apps/api/src/services/data-gathering.service.ts b/apps/api/src/services/data-gathering.service.tsindex d3843456d7..aef198e5b9 100644--- a/apps/api/src/services/data-gathering.service.ts+++ b/apps/api/src/services/data-gathering.service.ts@@ -152,10 +152,11 @@ export class DataGatheringService {countries,currency,dataSource,+ isin,name,sectors,url- } = assetProfiles[symbol];+ } = assetProfile;try {await this.prismaService.symbolProfile.upsert({@@ -165,6 +166,7 @@ export class DataGatheringService {countries,currency,dataSource,+ isin,name,sectors,symbol,@@ -175,6 +177,7 @@ export class DataGatheringService {assetSubClass,countries,currency,+ isin,name,sectors,urldiff --git a/apps/api/src/services/data-provider/data-provider.service.ts b/apps/api/src/services/data-provider/data-provider.service.tsindex 9edeedc1c6..ad8929eee6 100644--- a/apps/api/src/services/data-provider/data-provider.service.ts+++ b/apps/api/src/services/data-provider/data-provider.service.ts@@ -8,6 +8,7 @@ import {} from '@ghostfolio/api/services/interfaces/interfaces';import { PrismaService } from '@ghostfolio/api/services/prisma.service';import { DATE_FORMAT } from '@ghostfolio/common/helper';+import { UserWithSettings } from '@ghostfolio/common/interfaces';import { Granularity } from '@ghostfolio/common/types';import { Inject, Injectable, Logger } from '@nestjs/common';import { DataSource, MarketData, SymbolProfile } from '@prisma/client';@@ -260,18 +261,33 @@ export class DataProviderService {return response;}- public async search(aQuery: string): Promise<{ items: LookupItem[] }> {+ public async search({+ query,+ user+ }: {+ query: string;+ user: UserWithSettings;+ }): Promise<{ items: LookupItem[] }> {const promises: Promise<{ items: LookupItem[] }>[] = [];let lookupItems: LookupItem[] = [];- if (aQuery?.length < 2) {+ if (query?.length < 2) {return { items: lookupItems };}- for (const dataSource of this.configurationService.get('DATA_SOURCES')) {- promises.push(- this.getDataProvider(DataSource[dataSource]).search(aQuery)- );+ let dataSources = this.configurationService.get('DATA_SOURCES');++ if (+ this.configurationService.get('ENABLE_FEATURE_SUBSCRIPTION') &&+ user.subscription.type === 'Basic'+ ) {+ dataSources = dataSources.filter((dataSource) => {+ return !this.isPremiumDataSource(DataSource[dataSource]);+ });+ }++ for (const dataSource of dataSources) {+ promises.push(this.getDataProvider(DataSource[dataSource]).search(query));}const searchResults = await Promise.all(promises);@@ -305,4 +321,9 @@ export class DataProviderService {throw new Error('No data provider has been found.');}++ private isPremiumDataSource(aDataSource: DataSource) {+ const premiumDataSources: DataSource[] = [DataSource.EOD_HISTORICAL_DATA];+ return premiumDataSources.includes(aDataSource);+ }}diff --git a/apps/api/src/services/data-provider/eod-historical-data/eod-historical-data.service.ts b/apps/api/src/services/data-provider/eod-historical-data/eod-historical-data.service.tsindex a39100d1fb..e2d1efd95a 100644--- a/apps/api/src/services/data-provider/eod-historical-data/eod-historical-data.service.ts+++ b/apps/api/src/services/data-provider/eod-historical-data/eod-historical-data.service.ts@@ -8,7 +8,12 @@ import {import { DATE_FORMAT } from '@ghostfolio/common/helper';import { Granularity } from '@ghostfolio/common/types';import { Injectable, Logger } from '@nestjs/common';-import { DataSource, SymbolProfile } from '@prisma/client';+import {+ AssetClass,+ AssetSubClass,+ DataSource,+ SymbolProfile+} from '@prisma/client';import bent from 'bent';import { format, isToday } from 'date-fns';@@ -30,12 +35,15 @@ export class EodHistoricalDataService implements DataProviderInterface {public async getAssetProfile(aSymbol: string): Promise<Partial<SymbolProfile>> {- const { items } = await this.search(aSymbol);+ const [searchResult] = await this.getSearchResult(aSymbol);return {- currency: items[0]?.currency,+ assetClass: searchResult?.assetClass,+ assetSubClass: searchResult?.assetSubClass,+ currency: searchResult?.currency,dataSource: this.getName(),- name: items[0]?.name+ isin: searchResult?.isin,+ name: searchResult?.name};}@@ -156,7 +164,27 @@ export class EodHistoricalDataService implements DataProviderInterface {}public async search(aQuery: string): Promise<{ items: LookupItem[] }> {- let items: LookupItem[] = [];+ const searchResult = await this.getSearchResult(aQuery);++ return {+ items: searchResult+ .filter(({ symbol }) => {+ return !symbol.toLowerCase().endsWith('forex');+ })+ .map(({ currency, dataSource, name, symbol }) => {+ return { currency, dataSource, name, symbol };+ })+ };+ }++ private async getSearchResult(aQuery: string): Promise<+ (LookupItem & {+ assetClass: AssetClass;+ assetSubClass: AssetSubClass;+ isin: string;+ })[]+ > {+ let searchResult = [];try {const get = bent(@@ -167,10 +195,25 @@ export class EodHistoricalDataService implements DataProviderInterface {);const response = await get();- items = response.map(- ({ Code, Currency: currency, Exchange, Name: name }) => {+ searchResult = response.map(+ ({+ Code,+ Currency: currency,+ Exchange,+ ISIN: isin,+ Name: name,+ Type+ }) => {+ const { assetClass, assetSubClass } = this.parseAssetClass({+ Exchange,+ Type+ });+return {+ assetClass,+ assetSubClass,currency,+ isin,name,dataSource: this.getName(),symbol: `${Code}.${Exchange}`@@ -181,6 +224,41 @@ export class EodHistoricalDataService implements DataProviderInterface {Logger.error(error, 'EodHistoricalDataService');}- return { items };+ return searchResult;+ }++ private parseAssetClass({+ Exchange,+ Type+ }: {+ Exchange: string;+ Type: string;+ }): {+ assetClass: AssetClass;+ assetSubClass: AssetSubClass;+ } {+ let assetClass: AssetClass;+ let assetSubClass: AssetSubClass;++ switch (Type?.toLowerCase()) {+ case 'common stock':+ assetClass = AssetClass.EQUITY;+ assetSubClass = AssetSubClass.STOCK;+ break;+ case 'currency':+ assetClass = AssetClass.CASH;++ if (Exchange?.toLowerCase() === 'cc') {+ assetSubClass = AssetSubClass.CRYPTOCURRENCY;+ }++ break;+ case 'etf':+ assetClass = AssetClass.EQUITY;+ assetSubClass = AssetSubClass.ETF;+ break;+ }++ return { assetClass, assetSubClass };}}diff --git a/apps/client/src/app/components/subscription-interstitial-dialog/subscription-interstitial-dialog.html b/apps/client/src/app/components/subscription-interstitial-dialog/subscription-interstitial-dialog.htmlindex 061b43e3d0..321ab1e135 100644--- a/apps/client/src/app/components/subscription-interstitial-dialog/subscription-interstitial-dialog.html+++ b/apps/client/src/app/components/subscription-interstitial-dialog/subscription-interstitial-dialog.html@@ -19,16 +19,20 @@</li><li class="align-items-center d-flex mb-1"><ion-icon class="mr-1" name="checkmark-circle-outline"></ion-icon>- <span i18n>Performance Benchmarks</span>+ <span i18n>Portfolio Allocations</span></li><li class="align-items-center d-flex mb-1"><ion-icon class="mr-1" name="checkmark-circle-outline"></ion-icon>- <span i18n>Allocations</span>+ <span i18n>Performance Benchmarks</span></li><li class="align-items-center d-flex mb-1"><ion-icon class="mr-1" name="checkmark-circle-outline"></ion-icon><span i18n>FIRE Calculator</span></li>+ <li class="align-items-center d-flex mb-1">+ <ion-icon class="mr-1" name="checkmark-circle-outline"></ion-icon>+ <span i18n>Professional Data Provider</span>+ </li><li class="align-items-center d-flex mb-1"><ion-icon class="mr-1" name="checkmark-circle-outline"></ion-icon><span i18n>and more Features...</span>diff --git a/apps/client/src/app/pages/pricing/pricing-page.html b/apps/client/src/app/pages/pricing/pricing-page.htmlindex 8140822a64..1284cda005 100644--- a/apps/client/src/app/pages/pricing/pricing-page.html+++ b/apps/client/src/app/pages/pricing/pricing-page.html@@ -280,6 +280,13 @@<ion-icon name="information-circle-outline"></ion-icon></span></li>+ <li class="align-items-center d-flex mb-1">+ <ion-icon+ class="mr-1"+ name="checkmark-circle-outline"+ ></ion-icon>+ <span i18n>Professional Data Provider</span>+ </li><li class="align-items-center d-flex mb-1"><ion-iconclass="mr-1"diff --git a/prisma/migrations/20230318081658_added_isin_to_symbol_profile/migration.sql b/prisma/migrations/20230318081658_added_isin_to_symbol_profile/migration.sqlnew file mode 100644index 0000000000..f7081354cf--- /dev/null+++ b/prisma/migrations/20230318081658_added_isin_to_symbol_profile/migration.sql@@ -0,0 +1,2 @@+-- AlterTable+ALTER TABLE "SymbolProfile" ADD COLUMN "isin" TEXT;diff --git a/prisma/schema.prisma b/prisma/schema.prismaindex dc4c1302da..dd46e52b67 100644--- a/prisma/schema.prisma+++ b/prisma/schema.prisma@@ -119,6 +119,7 @@ model SymbolProfile {currency StringdataSource DataSourceid String @id @default(uuid())+ isin String?name String?updatedAt DateTime @updatedAtscraperConfiguration Json?