Open Source Wealth Management Software. Angular + NestJS + Prisma + Nx + TypeScript 🤍
TypeScript angularetffinancefintechghostfoliohacktoberfestinvestingnestjsosspersonal-financeportfolioprismasoftwarestocktrackertradingtypescriptwealth-managementweb
Feature/introduce max number of symbols per data provider request (#1111)
Thomas Kaul9f2a49a1c700
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 9f90870cd4..28c77e173f 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -5,6 +5,13 @@ All notable changes to this project will be documented in this file.The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).+## Unreleased++### Changed++- Improved the performance of data provider requests by introducing a maximum number of symbols per request (chunk size)+- Changed the log level settings+## 1.175.0 - 29.07.2022### Addeddiff --git a/apps/api/src/main.ts b/apps/api/src/main.tsindex 7b4ea21e95..fd8237cd29 100644--- a/apps/api/src/main.ts+++ b/apps/api/src/main.ts@@ -5,7 +5,12 @@ import { AppModule } from './app/app.module';import { environment } from './environments/environment';async function bootstrap() {- const app = await NestFactory.create(AppModule);+ const app = await NestFactory.create(AppModule, {+ logger:+ process.env.NODE_ENV === 'production'+ ? ['error', 'log', 'warn']+ : ['debug', 'error', 'log', 'verbose', 'warn']+ });app.enableCors();app.enableVersioning({defaultVersion: '1',diff --git a/apps/api/src/services/data-provider/data-provider.service.ts b/apps/api/src/services/data-provider/data-provider.service.tsindex fd44f24269..7f10dc3a0b 100644--- a/apps/api/src/services/data-provider/data-provider.service.ts+++ b/apps/api/src/services/data-provider/data-provider.service.ts@@ -168,6 +168,7 @@ export class DataProviderService {const response: {[symbol: string]: IDataProviderResponse;} = {};+ const startTimeTotal = performance.now();const itemsGroupedByDataSource = groupBy(items, (item) => item.dataSource);@@ -176,25 +177,59 @@ export class DataProviderService {for (const [dataSource, dataGatheringItems] of Object.entries(itemsGroupedByDataSource)) {+ const dataProvider = this.getDataProvider(DataSource[dataSource]);+const symbols = dataGatheringItems.map((dataGatheringItem) => {return dataGatheringItem.symbol;});- const promise = Promise.resolve(- this.getDataProvider(DataSource[dataSource]).getQuotes(symbols)- );+ const maximumNumberOfSymbolsPerRequest =+ dataProvider.getMaxNumberOfSymbolsPerRequest?.() ??+ Number.MAX_SAFE_INTEGER;+ for (+ let i = 0;+ i < symbols.length;+ i += maximumNumberOfSymbolsPerRequest+ ) {+ const startTimeDataSource = performance.now();++ const symbolsChunk = symbols.slice(+ i,+ i + maximumNumberOfSymbolsPerRequest+ );- promises.push(- promise.then((result) => {- for (const [symbol, dataProviderResponse] of Object.entries(result)) {- response[symbol] = dataProviderResponse;- }- })- );+ const promise = Promise.resolve(dataProvider.getQuotes(symbolsChunk));++ promises.push(+ promise.then((result) => {+ for (const [symbol, dataProviderResponse] of Object.entries(+ result+ )) {+ response[symbol] = dataProviderResponse;+ }++ Logger.debug(+ `Fetched ${symbolsChunk.length} quotes from ${dataSource} in ${(+ (performance.now() - startTimeDataSource) /+ 1000+ ).toFixed(3)} seconds`+ );+ })+ );+ }}await Promise.all(promises);+ Logger.debug('------------------------------------------------');+ Logger.debug(+ `Fetched ${items.length} quotes in ${(+ (performance.now() - startTimeTotal) /+ 1000+ ).toFixed(3)} seconds`+ );+ Logger.debug('================================================');+return response;}diff --git a/apps/api/src/services/data-provider/eod-historical-data/eod-historical-data.service.ts b/apps/api/src/services/data-provider/eod-historical-data/eod-historical-data.service.tsindex 1dd3c7aff2..c87c6ec3ed 100644--- a/apps/api/src/services/data-provider/eod-historical-data/eod-historical-data.service.ts+++ b/apps/api/src/services/data-provider/eod-historical-data/eod-historical-data.service.ts@@ -81,6 +81,12 @@ export class EodHistoricalDataService implements DataProviderInterface {}}+ public getMaxNumberOfSymbolsPerRequest() {+ // It is not recommended using more than 15-20 tickers per request+ // https://eodhistoricaldata.com/financial-apis/live-realtime-stocks-api+ return 20;+ }+public getName(): DataSource {return DataSource.EOD_HISTORICAL_DATA;}diff --git a/apps/api/src/services/data-provider/interfaces/data-provider.interface.ts b/apps/api/src/services/data-provider/interfaces/data-provider.interface.tsindex 16cf44603a..6719f30991 100644--- a/apps/api/src/services/data-provider/interfaces/data-provider.interface.ts+++ b/apps/api/src/services/data-provider/interfaces/data-provider.interface.ts@@ -20,6 +20,8 @@ export interface DataProviderInterface {[symbol: string]: { [date: string]: IDataProviderHistoricalResponse };}>; // TODO: Return only one symbol+ getMaxNumberOfSymbolsPerRequest?(): number;+getName(): DataSource;getQuotes(diff --git a/apps/api/src/services/data-provider/yahoo-finance/yahoo-finance.service.ts b/apps/api/src/services/data-provider/yahoo-finance/yahoo-finance.service.tsindex bddd529c2b..d6a6a84123 100644--- a/apps/api/src/services/data-provider/yahoo-finance/yahoo-finance.service.ts+++ b/apps/api/src/services/data-provider/yahoo-finance/yahoo-finance.service.ts@@ -208,6 +208,10 @@ export class YahooFinanceService implements DataProviderInterface {}}+ public getMaxNumberOfSymbolsPerRequest() {+ return 50;+ }+public getName(): DataSource {return DataSource.YAHOO;}