Open Source Wealth Management Software. Angular + NestJS + Prisma + Nx + TypeScript 🤍
TypeScript angularetffinancefintechghostfoliohacktoberfestinvestingnestjsosspersonal-financeportfolioprismasoftwarestocktrackertradingtypescriptwealth-managementweb
Feature/introduce env variable data source exchange rates and data source import (#1910)
Thomas Kaule500ccb61b10
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 6f93e48dee..6974ddf5fb 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -5,6 +5,12 @@ All notable changes to this project will be documented in this file.The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).+## Unreleased++### Changed++- Split the environment variable `DATA_SOURCE_PRIMARY` in `DATA_SOURCE_EXCHANGE_RATES` and `DATA_SOURCE_IMPORT`+## 1.262.0 - 2023-04-29### Addeddiff --git a/apps/api/src/app/import/import.service.ts b/apps/api/src/app/import/import.service.tsindex 7d0fda0925..c3b8f63b39 100644--- a/apps/api/src/app/import/import.service.ts+++ b/apps/api/src/app/import/import.service.ts@@ -15,7 +15,7 @@ import {OrderWithAccount} from '@ghostfolio/common/types';import { Injectable } from '@nestjs/common';-import { Prisma, SymbolProfile } from '@prisma/client';+import { DataSource, Prisma, SymbolProfile } from '@prisma/client';import Big from 'big.js';import { endOfToday, isAfter, isSameDay, parseISO } from 'date-fns';import { v4 as uuidv4 } from 'uuid';@@ -183,9 +183,10 @@ export class ImportService {for (const activity of activitiesDto) {if (!activity.dataSource) {if (activity.type === 'ITEM') {- activity.dataSource = 'MANUAL';+ activity.dataSource = DataSource.MANUAL;} else {- activity.dataSource = this.dataProviderService.getPrimaryDataSource();+ activity.dataSource =+ this.dataProviderService.getDataSourceForImport();}}diff --git a/apps/api/src/services/configuration/configuration.service.ts b/apps/api/src/services/configuration/configuration.service.tsindex df9d6fa24f..eed51d1a88 100644--- a/apps/api/src/services/configuration/configuration.service.ts+++ b/apps/api/src/services/configuration/configuration.service.ts@@ -16,7 +16,8 @@ export class ConfigurationService {default: 'USD'}),CACHE_TTL: num({ default: 1 }),- DATA_SOURCE_PRIMARY: str({ default: DataSource.YAHOO }),+ DATA_SOURCE_EXCHANGE_RATES: str({ default: DataSource.YAHOO }),+ DATA_SOURCE_IMPORT: str({ default: DataSource.YAHOO }),DATA_SOURCES: json({default: [DataSource.COINGECKO, DataSource.MANUAL, DataSource.YAHOO]}),diff --git a/apps/api/src/services/data-provider/data-provider.service.ts b/apps/api/src/services/data-provider/data-provider.service.tsindex db53d63418..472001db51 100644--- a/apps/api/src/services/data-provider/data-provider.service.ts+++ b/apps/api/src/services/data-provider/data-provider.service.ts@@ -58,6 +58,52 @@ export class DataProviderService {return false;}+ public async getAssetProfiles(items: IDataGatheringItem[]): Promise<{+ [symbol: string]: Partial<SymbolProfile>;+ }> {+ const response: {+ [symbol: string]: Partial<SymbolProfile>;+ } = {};++ const itemsGroupedByDataSource = groupBy(items, (item) => item.dataSource);++ const promises = [];++ for (const [dataSource, dataGatheringItems] of Object.entries(+ itemsGroupedByDataSource+ )) {+ const symbols = dataGatheringItems.map((dataGatheringItem) => {+ return dataGatheringItem.symbol;+ });++ for (const symbol of symbols) {+ const promise = Promise.resolve(+ this.getDataProvider(DataSource[dataSource]).getAssetProfile(symbol)+ );++ promises.push(+ promise.then((symbolProfile) => {+ response[symbol] = symbolProfile;+ })+ );+ }+ }++ await Promise.all(promises);++ return response;+ }++ public getDataSourceForExchangeRates(): DataSource {+ return DataSource[+ this.configurationService.get('DATA_SOURCE_EXCHANGE_RATES')+ ];+ }++ public getDataSourceForImport(): DataSource {+ return DataSource[this.configurationService.get('DATA_SOURCE_IMPORT')];+ }+public async getDividends({dataSource,from,@@ -182,46 +228,6 @@ export class DataProviderService {return result;}- public getPrimaryDataSource(): DataSource {- return DataSource[this.configurationService.get('DATA_SOURCE_PRIMARY')];- }-- public async getAssetProfiles(items: IDataGatheringItem[]): Promise<{- [symbol: string]: Partial<SymbolProfile>;- }> {- const response: {- [symbol: string]: Partial<SymbolProfile>;- } = {};-- const itemsGroupedByDataSource = groupBy(items, (item) => item.dataSource);-- const promises = [];-- for (const [dataSource, dataGatheringItems] of Object.entries(- itemsGroupedByDataSource- )) {- const symbols = dataGatheringItems.map((dataGatheringItem) => {- return dataGatheringItem.symbol;- });-- for (const symbol of symbols) {- const promise = Promise.resolve(- this.getDataProvider(DataSource[dataSource]).getAssetProfile(symbol)- );-- promises.push(- promise.then((symbolProfile) => {- response[symbol] = symbolProfile;- })- );- }- }-- await Promise.all(promises);-- return response;- }-public async getQuotes(items: IDataGatheringItem[]): Promise<{[symbol: string]: IDataProviderResponse;}> {diff --git a/apps/api/src/services/data-provider/eod-historical-data/eod-historical-data.service.ts b/apps/api/src/services/data-provider/eod-historical-data/eod-historical-data.service.tsindex 614ec46a8d..f1b56f659f 100644--- a/apps/api/src/services/data-provider/eod-historical-data/eod-historical-data.service.ts+++ b/apps/api/src/services/data-provider/eod-historical-data/eod-historical-data.service.ts@@ -5,7 +5,7 @@ import {IDataProviderHistoricalResponse,IDataProviderResponse} from '@ghostfolio/api/services/interfaces/interfaces';-import { DATE_FORMAT } from '@ghostfolio/common/helper';+import { DATE_FORMAT, isCurrency } from '@ghostfolio/common/helper';import { Granularity } from '@ghostfolio/common/types';import { Injectable, Logger } from '@nestjs/common';import {@@ -15,17 +15,20 @@ import {SymbolProfile} from '@prisma/client';import bent from 'bent';+import Big from 'big.js';import { format, isToday } from 'date-fns';@Injectable()export class EodHistoricalDataService implements DataProviderInterface {private apiKey: string;+ private baseCurrency: string;private readonly URL = 'https://eodhistoricaldata.com/api';public constructor(private readonly configurationService: ConfigurationService) {this.apiKey = this.configurationService.get('EOD_HISTORICAL_DATA_API_KEY');+ this.baseCurrency = this.configurationService.get('BASE_CURRENCY');}public canHandle(symbol: string) {@@ -70,9 +73,11 @@ export class EodHistoricalDataService implements DataProviderInterface {): Promise<{[symbol: string]: { [date: string]: IDataProviderHistoricalResponse };}> {+ const symbol = this.convertToEodSymbol(aSymbol);+try {const get = bent(- `${this.URL}/eod/${aSymbol}?api_token=${+ `${this.URL}/eod/${symbol}?api_token=${this.apiKey}&fmt=json&from=${format(from, DATE_FORMAT)}&to=${format(to,@@ -87,14 +92,17 @@ export class EodHistoricalDataService implements DataProviderInterface {return response.reduce((result, historicalItem, index, array) => {- result[aSymbol][historicalItem.date] = {- marketPrice: historicalItem.close,+ result[this.convertFromEodSymbol(symbol)][historicalItem.date] = {+ marketPrice: this.getConvertedValue({+ symbol: aSymbol,+ value: historicalItem.close+ }),performance: historicalItem.open - historicalItem.close};return result;},- { [aSymbol]: {} }+ { [this.convertFromEodSymbol(symbol)]: {} });} catch (error) {throw new Error(@@ -119,52 +127,87 @@ export class EodHistoricalDataService implements DataProviderInterface {public async getQuotes(aSymbols: string[]): Promise<{ [symbol: string]: IDataProviderResponse }> {- if (aSymbols.length <= 0) {+ const symbols = aSymbols.map((symbol) => {+ return this.convertToEodSymbol(symbol);+ });++ if (symbols.length <= 0) {return {};}try {const get = bent(- `${this.URL}/real-time/${aSymbols[0]}?api_token=${+ `${this.URL}/real-time/${symbols[0]}?api_token=${this.apiKey- }&fmt=json&s=${aSymbols.join(',')}`,+ }&fmt=json&s=${symbols.join(',')}`,'GET','json',200);- const [realTimeResponse, searchResponse] = await Promise.all([- get(),- this.search(aSymbols[0])- ]);+ const realTimeResponse = await get();const quotes =- aSymbols.length === 1 ? [realTimeResponse] : realTimeResponse;+ symbols.length === 1 ? [realTimeResponse] : realTimeResponse;++ const searchResponse = await Promise.all(+ symbols+ .filter((symbol) => {+ return !symbol.endsWith('.FOREX');+ })+ .map((symbol) => {+ return this.search(symbol);+ })+ );++ const lookupItems = searchResponse.flat().map(({ items }) => {+ return items[0];+ });- return quotes.reduce(+ const response = quotes.reduce((result: { [symbol: string]: IDataProviderResponse },{ close, code, timestamp }) => {- const currency = this.convertCurrency(- searchResponse?.items[0]?.currency- );-- if (currency) {- result[code] = {- currency,- dataSource: DataSource.EOD_HISTORICAL_DATA,- marketPrice: close,- marketState: isToday(new Date(timestamp * 1000))- ? 'open'- : 'closed'- };- }+ const currency = lookupItems.find((lookupItem) => {+ return lookupItem.symbol === code;+ })?.currency;++ result[this.convertFromEodSymbol(code)] = {+ currency: currency ?? this.baseCurrency,+ dataSource: DataSource.EOD_HISTORICAL_DATA,+ marketPrice: close,+ marketState: isToday(new Date(timestamp * 1000)) ? 'open' : 'closed'+ };return result;},{});++ if (response[`${this.baseCurrency}GBP`]) {+ response[`${this.baseCurrency}GBp`] = {+ ...response[`${this.baseCurrency}GBP`],+ currency: `${this.baseCurrency}GBp`,+ marketPrice: this.getConvertedValue({+ symbol: `${this.baseCurrency}GBp`,+ value: response[`${this.baseCurrency}GBP`].marketPrice+ })+ };+ }++ if (response[`${this.baseCurrency}ILS`]) {+ response[`${this.baseCurrency}ILA`] = {+ ...response[`${this.baseCurrency}ILS`],+ currency: `${this.baseCurrency}ILA`,+ marketPrice: this.getConvertedValue({+ symbol: `${this.baseCurrency}ILA`,+ value: response[`${this.baseCurrency}ILS`].marketPrice+ })+ };+ }++ return response;} catch (error) {Logger.error(error, 'EodHistoricalDataService');}@@ -182,7 +225,7 @@ export class EodHistoricalDataService implements DataProviderInterface {return {items: searchResult.filter(({ symbol }) => {- return !symbol.toLowerCase().endsWith('forex');+ return !symbol.endsWith('.FOREX');}).map(({@@ -216,6 +259,60 @@ export class EodHistoricalDataService implements DataProviderInterface {return currency;}+ private convertFromEodSymbol(aEodSymbol: string) {+ let symbol = aEodSymbol;++ if (symbol.endsWith('.FOREX')) {+ symbol = symbol.replace('GBX', 'GBp');+ symbol = symbol.replace('.FOREX', '');+ symbol = `${this.baseCurrency}${symbol}`;+ }++ return symbol;+ }++ /**+ * Converts a symbol to a EOD symbol+ *+ * Currency: USDCHF -> CHF.FOREX+ */+ private convertToEodSymbol(aSymbol: string) {+ if (+ aSymbol.startsWith(this.baseCurrency) &&+ aSymbol.length > this.baseCurrency.length+ ) {+ if (+ isCurrency(+ aSymbol.substring(0, aSymbol.length - this.baseCurrency.length)+ )+ ) {+ return `${aSymbol+ .replace('GBp', 'GBX')+ .replace(this.baseCurrency, '')}.FOREX`;+ }+ }++ return aSymbol;+ }++ private getConvertedValue({+ symbol,+ value+ }: {+ symbol: string;+ value: number;+ }) {+ if (symbol === `${this.baseCurrency}GBp`) {+ // Convert GPB to GBp (pence)+ return new Big(value).mul(100).toNumber();+ } else if (symbol === `${this.baseCurrency}ILA`) {+ // Convert ILS to ILA+ return new Big(value).mul(100).toNumber();+ }++ return value;+ }+private async getSearchResult(aQuery: string): Promise<(LookupItem & {assetClass: AssetClass;diff --git a/apps/api/src/services/data-provider/yahoo-finance/yahoo-finance.service.ts b/apps/api/src/services/data-provider/yahoo-finance/yahoo-finance.service.tsindex 9d20d94936..0671cf8c84 100644--- a/apps/api/src/services/data-provider/yahoo-finance/yahoo-finance.service.ts+++ b/apps/api/src/services/data-provider/yahoo-finance/yahoo-finance.service.ts@@ -203,9 +203,10 @@ export class YahooFinanceService implements DataProviderInterface {response[`${this.baseCurrency}GBp`] = {...response[symbol],currency: 'GBp',- marketPrice: new Big(response[symbol].marketPrice)- .mul(100)- .toNumber()+ marketPrice: this.getConvertedValue({+ symbol: `${this.baseCurrency}GBp`,+ value: response[symbol].marketPrice+ })};} else if (symbol === `${this.baseCurrency}ILS` &&@@ -215,9 +216,10 @@ export class YahooFinanceService implements DataProviderInterface {response[`${this.baseCurrency}ILA`] = {...response[symbol],currency: 'ILA',- marketPrice: new Big(response[symbol].marketPrice)- .mul(100)- .toNumber()+ marketPrice: this.getConvertedValue({+ symbol: `${this.baseCurrency}ILA`,+ value: response[symbol].marketPrice+ })};} else if (symbol === `${this.baseCurrency}ZAR` &&@@ -227,9 +229,10 @@ export class YahooFinanceService implements DataProviderInterface {response[`${this.baseCurrency}ZAc`] = {...response[symbol],currency: 'ZAc',- marketPrice: new Big(response[symbol].marketPrice)- .mul(100)- .toNumber()+ marketPrice: this.getConvertedValue({+ symbol: `${this.baseCurrency}ZAc`,+ value: response[symbol].marketPrice+ })};}}diff --git a/apps/api/src/services/exchange-rate-data/exchange-rate-data.service.ts b/apps/api/src/services/exchange-rate-data/exchange-rate-data.service.tsindex 10aef9411e..e39e926a91 100644--- a/apps/api/src/services/exchange-rate-data/exchange-rate-data.service.ts+++ b/apps/api/src/services/exchange-rate-data/exchange-rate-data.service.ts@@ -61,42 +61,41 @@ export class ExchangeRateDataService {getYesterday());- // TODO: add fallback- /*if (Object.keys(result).length !== this.currencyPairs.length) {+ if (Object.keys(result).length !== this.currencyPairs.length) {// Load currencies directly from data provider as a fallback// if historical data is not fully available- const historicalData = await this.dataProviderService.getQuotes(+ const quotes = await this.dataProviderService.getQuotes(this.currencyPairs.map(({ dataSource, symbol }) => {return { dataSource, symbol };}));- Object.keys(historicalData).forEach((key) => {- if (isNumber(historicalData[key].marketPrice)) {- result[key] = {+ for (const symbol of Object.keys(quotes)) {+ if (isNumber(quotes[symbol].marketPrice)) {+ result[symbol] = {[format(getYesterday(), DATE_FORMAT)]: {- marketPrice: historicalData[key].marketPrice+ marketPrice: quotes[symbol].marketPrice}};}- });- }*/+ }+ }const resultExtended = result;- Object.keys(result).forEach((pair) => {- const [currency1, currency2] = pair.match(/.{1,3}/g);- const [date] = Object.keys(result[pair]);+ for (const symbol of Object.keys(result)) {+ const [currency1, currency2] = symbol.match(/.{1,3}/g);+ const [date] = Object.keys(result[symbol]);// Calculate the opposite directionresultExtended[`${currency2}${currency1}`] = {[date]: {- marketPrice: 1 / result[pair][date].marketPrice+ marketPrice: 1 / result[symbol][date].marketPrice}};- });+ }- Object.keys(resultExtended).forEach((symbol) => {+ for (const symbol of Object.keys(resultExtended)) {const [currency1, currency2] = symbol.match(/.{1,3}/g);const date = format(getYesterday(), DATE_FORMAT);@@ -114,7 +113,7 @@ export class ExchangeRateDataService {this.exchangeRates[`${currency2}${currency1}`] =1 / this.exchangeRates[symbol];}- });+ }}public toCurrency(@@ -173,7 +172,8 @@ export class ExchangeRateDataService {let factor: number;if (aFromCurrency !== aToCurrency) {- const dataSource = this.dataProviderService.getPrimaryDataSource();+ const dataSource =+ this.dataProviderService.getDataSourceForExchangeRates();const symbol = `${aFromCurrency}${aToCurrency}`;const marketData = await this.marketDataService.get({@@ -274,7 +274,7 @@ export class ExchangeRateDataService {return {currency1: this.baseCurrency,currency2: currency,- dataSource: this.dataProviderService.getPrimaryDataSource(),+ dataSource: this.dataProviderService.getDataSourceForExchangeRates(),symbol: `${this.baseCurrency}${currency}`};});diff --git a/apps/api/src/services/interfaces/environment.interface.ts b/apps/api/src/services/interfaces/environment.interface.tsindex b8f05e98c5..586f199977 100644--- a/apps/api/src/services/interfaces/environment.interface.ts+++ b/apps/api/src/services/interfaces/environment.interface.ts@@ -5,7 +5,8 @@ export interface Environment extends CleanedEnvAccessors {ALPHA_VANTAGE_API_KEY: string;BASE_CURRENCY: string;CACHE_TTL: number;- DATA_SOURCE_PRIMARY: string;+ DATA_SOURCE_EXCHANGE_RATES: string;+ DATA_SOURCE_IMPORT: string;DATA_SOURCES: string[];ENABLE_FEATURE_BLOG: boolean;ENABLE_FEATURE_FEAR_AND_GREED_INDEX: boolean;