Skip to content

Open Source Wealth Management Software. Angular + NestJS + Prisma + Nx + TypeScript 🤍

Feature/introduce max number of symbols per data provider request (#1111)

Thomas Kaul9f2a49a1c700
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 9f90870cd4..28c77e173f 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -5,6 +5,13 @@ All notable changes to this project will be documented in this file.
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+## Unreleased
+
+### Changed
+
+- Improved the performance of data provider requests by introducing a maximum number of symbols per request (chunk size)
+- Changed the log level settings
+
## 1.175.0 - 29.07.2022
### Added
diff --git a/apps/api/src/main.ts b/apps/api/src/main.ts
index 7b4ea21e95..fd8237cd29 100644
--- a/apps/api/src/main.ts
+++ b/apps/api/src/main.ts
@@ -5,7 +5,12 @@ import { AppModule } from './app/app.module';
import { environment } from './environments/environment';
async function bootstrap() {
- const app = await NestFactory.create(AppModule);
+ const app = await NestFactory.create(AppModule, {
+ logger:
+ process.env.NODE_ENV === 'production'
+ ? ['error', 'log', 'warn']
+ : ['debug', 'error', 'log', 'verbose', 'warn']
+ });
app.enableCors();
app.enableVersioning({
defaultVersion: '1',
diff --git a/apps/api/src/services/data-provider/data-provider.service.ts b/apps/api/src/services/data-provider/data-provider.service.ts
index fd44f24269..7f10dc3a0b 100644
--- a/apps/api/src/services/data-provider/data-provider.service.ts
+++ b/apps/api/src/services/data-provider/data-provider.service.ts
@@ -168,6 +168,7 @@ export class DataProviderService {
const response: {
[symbol: string]: IDataProviderResponse;
} = {};
+ const startTimeTotal = performance.now();
const itemsGroupedByDataSource = groupBy(items, (item) => item.dataSource);
@@ -176,25 +177,59 @@ export class DataProviderService {
for (const [dataSource, dataGatheringItems] of Object.entries(
itemsGroupedByDataSource
)) {
+ const dataProvider = this.getDataProvider(DataSource[dataSource]);
+
const symbols = dataGatheringItems.map((dataGatheringItem) => {
return dataGatheringItem.symbol;
});
- const promise = Promise.resolve(
- this.getDataProvider(DataSource[dataSource]).getQuotes(symbols)
- );
+ const maximumNumberOfSymbolsPerRequest =
+ dataProvider.getMaxNumberOfSymbolsPerRequest?.() ??
+ Number.MAX_SAFE_INTEGER;
+ for (
+ let i = 0;
+ i < symbols.length;
+ i += maximumNumberOfSymbolsPerRequest
+ ) {
+ const startTimeDataSource = performance.now();
+
+ const symbolsChunk = symbols.slice(
+ i,
+ i + maximumNumberOfSymbolsPerRequest
+ );
- promises.push(
- promise.then((result) => {
- for (const [symbol, dataProviderResponse] of Object.entries(result)) {
- response[symbol] = dataProviderResponse;
- }
- })
- );
+ const promise = Promise.resolve(dataProvider.getQuotes(symbolsChunk));
+
+ promises.push(
+ promise.then((result) => {
+ for (const [symbol, dataProviderResponse] of Object.entries(
+ result
+ )) {
+ response[symbol] = dataProviderResponse;
+ }
+
+ Logger.debug(
+ `Fetched ${symbolsChunk.length} quotes from ${dataSource} in ${(
+ (performance.now() - startTimeDataSource) /
+ 1000
+ ).toFixed(3)} seconds`
+ );
+ })
+ );
+ }
}
await Promise.all(promises);
+ Logger.debug('------------------------------------------------');
+ Logger.debug(
+ `Fetched ${items.length} quotes in ${(
+ (performance.now() - startTimeTotal) /
+ 1000
+ ).toFixed(3)} seconds`
+ );
+ Logger.debug('================================================');
+
return response;
}
diff --git a/apps/api/src/services/data-provider/eod-historical-data/eod-historical-data.service.ts b/apps/api/src/services/data-provider/eod-historical-data/eod-historical-data.service.ts
index 1dd3c7aff2..c87c6ec3ed 100644
--- a/apps/api/src/services/data-provider/eod-historical-data/eod-historical-data.service.ts
+++ b/apps/api/src/services/data-provider/eod-historical-data/eod-historical-data.service.ts
@@ -81,6 +81,12 @@ export class EodHistoricalDataService implements DataProviderInterface {
}
}
+ public getMaxNumberOfSymbolsPerRequest() {
+ // It is not recommended using more than 15-20 tickers per request
+ // https://eodhistoricaldata.com/financial-apis/live-realtime-stocks-api
+ return 20;
+ }
+
public getName(): DataSource {
return DataSource.EOD_HISTORICAL_DATA;
}
diff --git a/apps/api/src/services/data-provider/interfaces/data-provider.interface.ts b/apps/api/src/services/data-provider/interfaces/data-provider.interface.ts
index 16cf44603a..6719f30991 100644
--- a/apps/api/src/services/data-provider/interfaces/data-provider.interface.ts
+++ b/apps/api/src/services/data-provider/interfaces/data-provider.interface.ts
@@ -20,6 +20,8 @@ export interface DataProviderInterface {
[symbol: string]: { [date: string]: IDataProviderHistoricalResponse };
}>; // TODO: Return only one symbol
+ getMaxNumberOfSymbolsPerRequest?(): number;
+
getName(): DataSource;
getQuotes(
diff --git a/apps/api/src/services/data-provider/yahoo-finance/yahoo-finance.service.ts b/apps/api/src/services/data-provider/yahoo-finance/yahoo-finance.service.ts
index bddd529c2b..d6a6a84123 100644
--- a/apps/api/src/services/data-provider/yahoo-finance/yahoo-finance.service.ts
+++ b/apps/api/src/services/data-provider/yahoo-finance/yahoo-finance.service.ts
@@ -208,6 +208,10 @@ export class YahooFinanceService implements DataProviderInterface {
}
}
+ public getMaxNumberOfSymbolsPerRequest() {
+ return 50;
+ }
+
public getName(): DataSource {
return DataSource.YAHOO;
}