Skip to content

Open Source Wealth Management Software. Angular + NestJS + Prisma + Nx + TypeScript 🤍

Feature/introduce env variable data source exchange rates and data source import (#1910)

Thomas Kaule500ccb61b10
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 6f93e48dee..6974ddf5fb 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -5,6 +5,12 @@ All notable changes to this project will be documented in this file.
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+## Unreleased
+
+### Changed
+
+- Split the environment variable `DATA_SOURCE_PRIMARY` in `DATA_SOURCE_EXCHANGE_RATES` and `DATA_SOURCE_IMPORT`
+
## 1.262.0 - 2023-04-29
### Added
diff --git a/apps/api/src/app/import/import.service.ts b/apps/api/src/app/import/import.service.ts
index 7d0fda0925..c3b8f63b39 100644
--- a/apps/api/src/app/import/import.service.ts
+++ b/apps/api/src/app/import/import.service.ts
@@ -15,7 +15,7 @@ import {
OrderWithAccount
} from '@ghostfolio/common/types';
import { Injectable } from '@nestjs/common';
-import { Prisma, SymbolProfile } from '@prisma/client';
+import { DataSource, Prisma, SymbolProfile } from '@prisma/client';
import Big from 'big.js';
import { endOfToday, isAfter, isSameDay, parseISO } from 'date-fns';
import { v4 as uuidv4 } from 'uuid';
@@ -183,9 +183,10 @@ export class ImportService {
for (const activity of activitiesDto) {
if (!activity.dataSource) {
if (activity.type === 'ITEM') {
- activity.dataSource = 'MANUAL';
+ activity.dataSource = DataSource.MANUAL;
} else {
- activity.dataSource = this.dataProviderService.getPrimaryDataSource();
+ activity.dataSource =
+ this.dataProviderService.getDataSourceForImport();
}
}
diff --git a/apps/api/src/services/configuration/configuration.service.ts b/apps/api/src/services/configuration/configuration.service.ts
index df9d6fa24f..eed51d1a88 100644
--- a/apps/api/src/services/configuration/configuration.service.ts
+++ b/apps/api/src/services/configuration/configuration.service.ts
@@ -16,7 +16,8 @@ export class ConfigurationService {
default: 'USD'
}),
CACHE_TTL: num({ default: 1 }),
- DATA_SOURCE_PRIMARY: str({ default: DataSource.YAHOO }),
+ DATA_SOURCE_EXCHANGE_RATES: str({ default: DataSource.YAHOO }),
+ DATA_SOURCE_IMPORT: str({ default: DataSource.YAHOO }),
DATA_SOURCES: json({
default: [DataSource.COINGECKO, DataSource.MANUAL, DataSource.YAHOO]
}),
diff --git a/apps/api/src/services/data-provider/data-provider.service.ts b/apps/api/src/services/data-provider/data-provider.service.ts
index db53d63418..472001db51 100644
--- a/apps/api/src/services/data-provider/data-provider.service.ts
+++ b/apps/api/src/services/data-provider/data-provider.service.ts
@@ -58,6 +58,52 @@ export class DataProviderService {
return false;
}
+ public async getAssetProfiles(items: IDataGatheringItem[]): Promise<{
+ [symbol: string]: Partial<SymbolProfile>;
+ }> {
+ const response: {
+ [symbol: string]: Partial<SymbolProfile>;
+ } = {};
+
+ const itemsGroupedByDataSource = groupBy(items, (item) => item.dataSource);
+
+ const promises = [];
+
+ for (const [dataSource, dataGatheringItems] of Object.entries(
+ itemsGroupedByDataSource
+ )) {
+ const symbols = dataGatheringItems.map((dataGatheringItem) => {
+ return dataGatheringItem.symbol;
+ });
+
+ for (const symbol of symbols) {
+ const promise = Promise.resolve(
+ this.getDataProvider(DataSource[dataSource]).getAssetProfile(symbol)
+ );
+
+ promises.push(
+ promise.then((symbolProfile) => {
+ response[symbol] = symbolProfile;
+ })
+ );
+ }
+ }
+
+ await Promise.all(promises);
+
+ return response;
+ }
+
+ public getDataSourceForExchangeRates(): DataSource {
+ return DataSource[
+ this.configurationService.get('DATA_SOURCE_EXCHANGE_RATES')
+ ];
+ }
+
+ public getDataSourceForImport(): DataSource {
+ return DataSource[this.configurationService.get('DATA_SOURCE_IMPORT')];
+ }
+
public async getDividends({
dataSource,
from,
@@ -182,46 +228,6 @@ export class DataProviderService {
return result;
}
- public getPrimaryDataSource(): DataSource {
- return DataSource[this.configurationService.get('DATA_SOURCE_PRIMARY')];
- }
-
- public async getAssetProfiles(items: IDataGatheringItem[]): Promise<{
- [symbol: string]: Partial<SymbolProfile>;
- }> {
- const response: {
- [symbol: string]: Partial<SymbolProfile>;
- } = {};
-
- const itemsGroupedByDataSource = groupBy(items, (item) => item.dataSource);
-
- const promises = [];
-
- for (const [dataSource, dataGatheringItems] of Object.entries(
- itemsGroupedByDataSource
- )) {
- const symbols = dataGatheringItems.map((dataGatheringItem) => {
- return dataGatheringItem.symbol;
- });
-
- for (const symbol of symbols) {
- const promise = Promise.resolve(
- this.getDataProvider(DataSource[dataSource]).getAssetProfile(symbol)
- );
-
- promises.push(
- promise.then((symbolProfile) => {
- response[symbol] = symbolProfile;
- })
- );
- }
- }
-
- await Promise.all(promises);
-
- return response;
- }
-
public async getQuotes(items: IDataGatheringItem[]): Promise<{
[symbol: string]: IDataProviderResponse;
}> {
diff --git a/apps/api/src/services/data-provider/eod-historical-data/eod-historical-data.service.ts b/apps/api/src/services/data-provider/eod-historical-data/eod-historical-data.service.ts
index 614ec46a8d..f1b56f659f 100644
--- a/apps/api/src/services/data-provider/eod-historical-data/eod-historical-data.service.ts
+++ b/apps/api/src/services/data-provider/eod-historical-data/eod-historical-data.service.ts
@@ -5,7 +5,7 @@ import {
IDataProviderHistoricalResponse,
IDataProviderResponse
} from '@ghostfolio/api/services/interfaces/interfaces';
-import { DATE_FORMAT } from '@ghostfolio/common/helper';
+import { DATE_FORMAT, isCurrency } from '@ghostfolio/common/helper';
import { Granularity } from '@ghostfolio/common/types';
import { Injectable, Logger } from '@nestjs/common';
import {
@@ -15,17 +15,20 @@ import {
SymbolProfile
} from '@prisma/client';
import bent from 'bent';
+import Big from 'big.js';
import { format, isToday } from 'date-fns';
@Injectable()
export class EodHistoricalDataService implements DataProviderInterface {
private apiKey: string;
+ private baseCurrency: string;
private readonly URL = 'https://eodhistoricaldata.com/api';
public constructor(
private readonly configurationService: ConfigurationService
) {
this.apiKey = this.configurationService.get('EOD_HISTORICAL_DATA_API_KEY');
+ this.baseCurrency = this.configurationService.get('BASE_CURRENCY');
}
public canHandle(symbol: string) {
@@ -70,9 +73,11 @@ export class EodHistoricalDataService implements DataProviderInterface {
): Promise<{
[symbol: string]: { [date: string]: IDataProviderHistoricalResponse };
}> {
+ const symbol = this.convertToEodSymbol(aSymbol);
+
try {
const get = bent(
- `${this.URL}/eod/${aSymbol}?api_token=${
+ `${this.URL}/eod/${symbol}?api_token=${
this.apiKey
}&fmt=json&from=${format(from, DATE_FORMAT)}&to=${format(
to,
@@ -87,14 +92,17 @@ export class EodHistoricalDataService implements DataProviderInterface {
return response.reduce(
(result, historicalItem, index, array) => {
- result[aSymbol][historicalItem.date] = {
- marketPrice: historicalItem.close,
+ result[this.convertFromEodSymbol(symbol)][historicalItem.date] = {
+ marketPrice: this.getConvertedValue({
+ symbol: aSymbol,
+ value: historicalItem.close
+ }),
performance: historicalItem.open - historicalItem.close
};
return result;
},
- { [aSymbol]: {} }
+ { [this.convertFromEodSymbol(symbol)]: {} }
);
} catch (error) {
throw new Error(
@@ -119,52 +127,87 @@ export class EodHistoricalDataService implements DataProviderInterface {
public async getQuotes(
aSymbols: string[]
): Promise<{ [symbol: string]: IDataProviderResponse }> {
- if (aSymbols.length <= 0) {
+ const symbols = aSymbols.map((symbol) => {
+ return this.convertToEodSymbol(symbol);
+ });
+
+ if (symbols.length <= 0) {
return {};
}
try {
const get = bent(
- `${this.URL}/real-time/${aSymbols[0]}?api_token=${
+ `${this.URL}/real-time/${symbols[0]}?api_token=${
this.apiKey
- }&fmt=json&s=${aSymbols.join(',')}`,
+ }&fmt=json&s=${symbols.join(',')}`,
'GET',
'json',
200
);
- const [realTimeResponse, searchResponse] = await Promise.all([
- get(),
- this.search(aSymbols[0])
- ]);
+ const realTimeResponse = await get();
const quotes =
- aSymbols.length === 1 ? [realTimeResponse] : realTimeResponse;
+ symbols.length === 1 ? [realTimeResponse] : realTimeResponse;
+
+ const searchResponse = await Promise.all(
+ symbols
+ .filter((symbol) => {
+ return !symbol.endsWith('.FOREX');
+ })
+ .map((symbol) => {
+ return this.search(symbol);
+ })
+ );
+
+ const lookupItems = searchResponse.flat().map(({ items }) => {
+ return items[0];
+ });
- return quotes.reduce(
+ const response = quotes.reduce(
(
result: { [symbol: string]: IDataProviderResponse },
{ close, code, timestamp }
) => {
- const currency = this.convertCurrency(
- searchResponse?.items[0]?.currency
- );
-
- if (currency) {
- result[code] = {
- currency,
- dataSource: DataSource.EOD_HISTORICAL_DATA,
- marketPrice: close,
- marketState: isToday(new Date(timestamp * 1000))
- ? 'open'
- : 'closed'
- };
- }
+ const currency = lookupItems.find((lookupItem) => {
+ return lookupItem.symbol === code;
+ })?.currency;
+
+ result[this.convertFromEodSymbol(code)] = {
+ currency: currency ?? this.baseCurrency,
+ dataSource: DataSource.EOD_HISTORICAL_DATA,
+ marketPrice: close,
+ marketState: isToday(new Date(timestamp * 1000)) ? 'open' : 'closed'
+ };
return result;
},
{}
);
+
+ if (response[`${this.baseCurrency}GBP`]) {
+ response[`${this.baseCurrency}GBp`] = {
+ ...response[`${this.baseCurrency}GBP`],
+ currency: `${this.baseCurrency}GBp`,
+ marketPrice: this.getConvertedValue({
+ symbol: `${this.baseCurrency}GBp`,
+ value: response[`${this.baseCurrency}GBP`].marketPrice
+ })
+ };
+ }
+
+ if (response[`${this.baseCurrency}ILS`]) {
+ response[`${this.baseCurrency}ILA`] = {
+ ...response[`${this.baseCurrency}ILS`],
+ currency: `${this.baseCurrency}ILA`,
+ marketPrice: this.getConvertedValue({
+ symbol: `${this.baseCurrency}ILA`,
+ value: response[`${this.baseCurrency}ILS`].marketPrice
+ })
+ };
+ }
+
+ return response;
} catch (error) {
Logger.error(error, 'EodHistoricalDataService');
}
@@ -182,7 +225,7 @@ export class EodHistoricalDataService implements DataProviderInterface {
return {
items: searchResult
.filter(({ symbol }) => {
- return !symbol.toLowerCase().endsWith('forex');
+ return !symbol.endsWith('.FOREX');
})
.map(
({
@@ -216,6 +259,60 @@ export class EodHistoricalDataService implements DataProviderInterface {
return currency;
}
+ private convertFromEodSymbol(aEodSymbol: string) {
+ let symbol = aEodSymbol;
+
+ if (symbol.endsWith('.FOREX')) {
+ symbol = symbol.replace('GBX', 'GBp');
+ symbol = symbol.replace('.FOREX', '');
+ symbol = `${this.baseCurrency}${symbol}`;
+ }
+
+ return symbol;
+ }
+
+ /**
+ * Converts a symbol to a EOD symbol
+ *
+ * Currency: USDCHF -> CHF.FOREX
+ */
+ private convertToEodSymbol(aSymbol: string) {
+ if (
+ aSymbol.startsWith(this.baseCurrency) &&
+ aSymbol.length > this.baseCurrency.length
+ ) {
+ if (
+ isCurrency(
+ aSymbol.substring(0, aSymbol.length - this.baseCurrency.length)
+ )
+ ) {
+ return `${aSymbol
+ .replace('GBp', 'GBX')
+ .replace(this.baseCurrency, '')}.FOREX`;
+ }
+ }
+
+ return aSymbol;
+ }
+
+ private getConvertedValue({
+ symbol,
+ value
+ }: {
+ symbol: string;
+ value: number;
+ }) {
+ if (symbol === `${this.baseCurrency}GBp`) {
+ // Convert GPB to GBp (pence)
+ return new Big(value).mul(100).toNumber();
+ } else if (symbol === `${this.baseCurrency}ILA`) {
+ // Convert ILS to ILA
+ return new Big(value).mul(100).toNumber();
+ }
+
+ return value;
+ }
+
private async getSearchResult(aQuery: string): Promise<
(LookupItem & {
assetClass: AssetClass;
diff --git a/apps/api/src/services/data-provider/yahoo-finance/yahoo-finance.service.ts b/apps/api/src/services/data-provider/yahoo-finance/yahoo-finance.service.ts
index 9d20d94936..0671cf8c84 100644
--- a/apps/api/src/services/data-provider/yahoo-finance/yahoo-finance.service.ts
+++ b/apps/api/src/services/data-provider/yahoo-finance/yahoo-finance.service.ts
@@ -203,9 +203,10 @@ export class YahooFinanceService implements DataProviderInterface {
response[`${this.baseCurrency}GBp`] = {
...response[symbol],
currency: 'GBp',
- marketPrice: new Big(response[symbol].marketPrice)
- .mul(100)
- .toNumber()
+ marketPrice: this.getConvertedValue({
+ symbol: `${this.baseCurrency}GBp`,
+ value: response[symbol].marketPrice
+ })
};
} else if (
symbol === `${this.baseCurrency}ILS` &&
@@ -215,9 +216,10 @@ export class YahooFinanceService implements DataProviderInterface {
response[`${this.baseCurrency}ILA`] = {
...response[symbol],
currency: 'ILA',
- marketPrice: new Big(response[symbol].marketPrice)
- .mul(100)
- .toNumber()
+ marketPrice: this.getConvertedValue({
+ symbol: `${this.baseCurrency}ILA`,
+ value: response[symbol].marketPrice
+ })
};
} else if (
symbol === `${this.baseCurrency}ZAR` &&
@@ -227,9 +229,10 @@ export class YahooFinanceService implements DataProviderInterface {
response[`${this.baseCurrency}ZAc`] = {
...response[symbol],
currency: 'ZAc',
- marketPrice: new Big(response[symbol].marketPrice)
- .mul(100)
- .toNumber()
+ marketPrice: this.getConvertedValue({
+ symbol: `${this.baseCurrency}ZAc`,
+ value: response[symbol].marketPrice
+ })
};
}
}
diff --git a/apps/api/src/services/exchange-rate-data/exchange-rate-data.service.ts b/apps/api/src/services/exchange-rate-data/exchange-rate-data.service.ts
index 10aef9411e..e39e926a91 100644
--- a/apps/api/src/services/exchange-rate-data/exchange-rate-data.service.ts
+++ b/apps/api/src/services/exchange-rate-data/exchange-rate-data.service.ts
@@ -61,42 +61,41 @@ export class ExchangeRateDataService {
getYesterday()
);
- // TODO: add fallback
- /*if (Object.keys(result).length !== this.currencyPairs.length) {
+ if (Object.keys(result).length !== this.currencyPairs.length) {
// Load currencies directly from data provider as a fallback
// if historical data is not fully available
- const historicalData = await this.dataProviderService.getQuotes(
+ const quotes = await this.dataProviderService.getQuotes(
this.currencyPairs.map(({ dataSource, symbol }) => {
return { dataSource, symbol };
})
);
- Object.keys(historicalData).forEach((key) => {
- if (isNumber(historicalData[key].marketPrice)) {
- result[key] = {
+ for (const symbol of Object.keys(quotes)) {
+ if (isNumber(quotes[symbol].marketPrice)) {
+ result[symbol] = {
[format(getYesterday(), DATE_FORMAT)]: {
- marketPrice: historicalData[key].marketPrice
+ marketPrice: quotes[symbol].marketPrice
}
};
}
- });
- }*/
+ }
+ }
const resultExtended = result;
- Object.keys(result).forEach((pair) => {
- const [currency1, currency2] = pair.match(/.{1,3}/g);
- const [date] = Object.keys(result[pair]);
+ for (const symbol of Object.keys(result)) {
+ const [currency1, currency2] = symbol.match(/.{1,3}/g);
+ const [date] = Object.keys(result[symbol]);
// Calculate the opposite direction
resultExtended[`${currency2}${currency1}`] = {
[date]: {
- marketPrice: 1 / result[pair][date].marketPrice
+ marketPrice: 1 / result[symbol][date].marketPrice
}
};
- });
+ }
- Object.keys(resultExtended).forEach((symbol) => {
+ for (const symbol of Object.keys(resultExtended)) {
const [currency1, currency2] = symbol.match(/.{1,3}/g);
const date = format(getYesterday(), DATE_FORMAT);
@@ -114,7 +113,7 @@ export class ExchangeRateDataService {
this.exchangeRates[`${currency2}${currency1}`] =
1 / this.exchangeRates[symbol];
}
- });
+ }
}
public toCurrency(
@@ -173,7 +172,8 @@ export class ExchangeRateDataService {
let factor: number;
if (aFromCurrency !== aToCurrency) {
- const dataSource = this.dataProviderService.getPrimaryDataSource();
+ const dataSource =
+ this.dataProviderService.getDataSourceForExchangeRates();
const symbol = `${aFromCurrency}${aToCurrency}`;
const marketData = await this.marketDataService.get({
@@ -274,7 +274,7 @@ export class ExchangeRateDataService {
return {
currency1: this.baseCurrency,
currency2: currency,
- dataSource: this.dataProviderService.getPrimaryDataSource(),
+ dataSource: this.dataProviderService.getDataSourceForExchangeRates(),
symbol: `${this.baseCurrency}${currency}`
};
});
diff --git a/apps/api/src/services/interfaces/environment.interface.ts b/apps/api/src/services/interfaces/environment.interface.ts
index b8f05e98c5..586f199977 100644
--- a/apps/api/src/services/interfaces/environment.interface.ts
+++ b/apps/api/src/services/interfaces/environment.interface.ts
@@ -5,7 +5,8 @@ export interface Environment extends CleanedEnvAccessors {
ALPHA_VANTAGE_API_KEY: string;
BASE_CURRENCY: string;
CACHE_TTL: number;
- DATA_SOURCE_PRIMARY: string;
+ DATA_SOURCE_EXCHANGE_RATES: string;
+ DATA_SOURCE_IMPORT: string;
DATA_SOURCES: string[];
ENABLE_FEATURE_BLOG: boolean;
ENABLE_FEATURE_FEAR_AND_GREED_INDEX: boolean;