Skip to content

Open Source Wealth Management Software. Angular + NestJS + Prisma + Nx + TypeScript 🤍

Feature/increase timeout in data provider and enhancer health check endpoint (#2621)

Thomas Kauldec1d89c5c5a
diff --git a/CHANGELOG.md b/CHANGELOG.md
index d42cccdc35..e6e55c6c46 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -10,6 +10,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
### Changed
- Improved the import of historical market data in the admin control panel
+- Increased the timeout in the health check endpoint for data enhancers
+- Increased the timeout in the health check endpoint for data providers
- Removed the account type from the `Account` database schema
## 2.19.0 - 2023-11-06
@@ -368,7 +370,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
### Added
-- Added health check endpoints for data enhancers
+- Added a health check endpoint for data enhancers
### Changed
@@ -544,7 +546,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
### Changed
- Improved the usability of the login dialog
-- Disabled the caching in the health check endpoints for data providers
+- Disabled the caching in the health check endpoint for data providers
- Improved the content of the Frequently Asked Questions (FAQ) page
- Upgraded `prisma` from version `4.15.0` to `4.16.2`
@@ -932,7 +934,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
- Added a fallback to historical market data if a data provider does not provide live data
- Added a general health check endpoint
-- Added health check endpoints for data providers
+- Added a health check endpoint for data providers
### Changed
diff --git a/apps/api/src/services/data-provider/alpha-vantage/alpha-vantage.service.ts b/apps/api/src/services/data-provider/alpha-vantage/alpha-vantage.service.ts
index fbbfffc67c..2b03f519da 100644
--- a/apps/api/src/services/data-provider/alpha-vantage/alpha-vantage.service.ts
+++ b/apps/api/src/services/data-provider/alpha-vantage/alpha-vantage.service.ts
@@ -5,6 +5,7 @@ import {
IDataProviderHistoricalResponse,
IDataProviderResponse
} from '@ghostfolio/api/services/interfaces/interfaces';
+import { DEFAULT_REQUEST_TIMEOUT } from '@ghostfolio/common/config';
import { DATE_FORMAT } from '@ghostfolio/common/helper';
import { Granularity } from '@ghostfolio/common/types';
import { Injectable } from '@nestjs/common';
@@ -106,8 +107,10 @@ export class AlphaVantageService implements DataProviderInterface {
}
public async getQuotes({
+ requestTimeout = DEFAULT_REQUEST_TIMEOUT,
symbols
}: {
+ requestTimeout?: number;
symbols: string[];
}): Promise<{ [symbol: string]: IDataProviderResponse }> {
return {};
diff --git a/apps/api/src/services/data-provider/coingecko/coingecko.service.ts b/apps/api/src/services/data-provider/coingecko/coingecko.service.ts
index 3e93d42bf5..6ed67bf8b5 100644
--- a/apps/api/src/services/data-provider/coingecko/coingecko.service.ts
+++ b/apps/api/src/services/data-provider/coingecko/coingecko.service.ts
@@ -135,8 +135,10 @@ export class CoinGeckoService implements DataProviderInterface {
}
public async getQuotes({
+ requestTimeout = DEFAULT_REQUEST_TIMEOUT,
symbols
}: {
+ requestTimeout?: number;
symbols: string[];
}): Promise<{ [symbol: string]: IDataProviderResponse }> {
const response: { [symbol: string]: IDataProviderResponse } = {};
@@ -150,7 +152,7 @@ export class CoinGeckoService implements DataProviderInterface {
setTimeout(() => {
abortController.abort();
- }, DEFAULT_REQUEST_TIMEOUT);
+ }, requestTimeout);
const quotes = await got(
`${this.URL}/simple/price?ids=${symbols.join(
diff --git a/apps/api/src/services/data-provider/data-enhancer/data-enhancer.service.ts b/apps/api/src/services/data-provider/data-enhancer/data-enhancer.service.ts
index e5038c7c66..d9b509f42a 100644
--- a/apps/api/src/services/data-provider/data-enhancer/data-enhancer.service.ts
+++ b/apps/api/src/services/data-provider/data-enhancer/data-enhancer.service.ts
@@ -2,6 +2,7 @@ import { DataEnhancerInterface } from '@ghostfolio/api/services/data-provider/in
import { HttpException, Inject, Injectable } from '@nestjs/common';
import { Prisma } from '@prisma/client';
import { StatusCodes, getReasonPhrase } from 'http-status-codes';
+import ms from 'ms';
@Injectable()
export class DataEnhancerService {
@@ -24,6 +25,7 @@ export class DataEnhancerService {
try {
const assetProfile = await dataEnhancer.enhance({
+ requestTimeout: ms('30 seconds'),
response: {
assetClass: 'EQUITY',
assetSubClass: 'ETF'
diff --git a/apps/api/src/services/data-provider/data-enhancer/openfigi/openfigi.service.ts b/apps/api/src/services/data-provider/data-enhancer/openfigi/openfigi.service.ts
index 363cbb167d..dda30ba1af 100644
--- a/apps/api/src/services/data-provider/data-enhancer/openfigi/openfigi.service.ts
+++ b/apps/api/src/services/data-provider/data-enhancer/openfigi/openfigi.service.ts
@@ -15,9 +15,11 @@ export class OpenFigiDataEnhancerService implements DataEnhancerInterface {
) {}
public async enhance({
+ requestTimeout = DEFAULT_REQUEST_TIMEOUT,
response,
symbol
}: {
+ requestTimeout?: number;
response: Partial<SymbolProfile>;
symbol: string;
}): Promise<Partial<SymbolProfile>> {
@@ -45,7 +47,7 @@ export class OpenFigiDataEnhancerService implements DataEnhancerInterface {
setTimeout(() => {
abortController.abort();
- }, DEFAULT_REQUEST_TIMEOUT);
+ }, requestTimeout);
const mappings = await got
.post(`${OpenFigiDataEnhancerService.baseUrl}/v3/mapping`, {
diff --git a/apps/api/src/services/data-provider/data-enhancer/trackinsight/trackinsight.service.ts b/apps/api/src/services/data-provider/data-enhancer/trackinsight/trackinsight.service.ts
index 36eb22dad5..4de08fcef9 100644
--- a/apps/api/src/services/data-provider/data-enhancer/trackinsight/trackinsight.service.ts
+++ b/apps/api/src/services/data-provider/data-enhancer/trackinsight/trackinsight.service.ts
@@ -21,9 +21,11 @@ export class TrackinsightDataEnhancerService implements DataEnhancerInterface {
};
public async enhance({
+ requestTimeout = DEFAULT_REQUEST_TIMEOUT,
response,
symbol
}: {
+ requestTimeout?: number;
response: Partial<SymbolProfile>;
symbol: string;
}): Promise<Partial<SymbolProfile>> {
@@ -37,7 +39,7 @@ export class TrackinsightDataEnhancerService implements DataEnhancerInterface {
setTimeout(() => {
abortController.abort();
- }, DEFAULT_REQUEST_TIMEOUT);
+ }, requestTimeout);
const profile = await got(
`${TrackinsightDataEnhancerService.baseUrl}/funds/${symbol}.json`,
diff --git a/apps/api/src/services/data-provider/data-enhancer/yahoo-finance/yahoo-finance.service.ts b/apps/api/src/services/data-provider/data-enhancer/yahoo-finance/yahoo-finance.service.ts
index d46af66a7e..67fc0a9c16 100644
--- a/apps/api/src/services/data-provider/data-enhancer/yahoo-finance/yahoo-finance.service.ts
+++ b/apps/api/src/services/data-provider/data-enhancer/yahoo-finance/yahoo-finance.service.ts
@@ -1,6 +1,10 @@
import { CryptocurrencyService } from '@ghostfolio/api/services/cryptocurrency/cryptocurrency.service';
import { DataEnhancerInterface } from '@ghostfolio/api/services/data-provider/interfaces/data-enhancer.interface';
-import { DEFAULT_CURRENCY, UNKNOWN_KEY } from '@ghostfolio/common/config';
+import {
+ DEFAULT_CURRENCY,
+ DEFAULT_REQUEST_TIMEOUT,
+ UNKNOWN_KEY
+} from '@ghostfolio/common/config';
import { isCurrency } from '@ghostfolio/common/helper';
import { Injectable, Logger } from '@nestjs/common';
import {
@@ -72,9 +76,11 @@ export class YahooFinanceDataEnhancerService implements DataEnhancerInterface {
}
public async enhance({
+ requestTimeout = DEFAULT_REQUEST_TIMEOUT,
response,
symbol
}: {
+ requestTimeout?: number;
response: Partial<SymbolProfile>;
symbol: string;
}): Promise<Partial<SymbolProfile>> {
diff --git a/apps/api/src/services/data-provider/data-provider.service.ts b/apps/api/src/services/data-provider/data-provider.service.ts
index 7a998eeb3e..ef5143475f 100644
--- a/apps/api/src/services/data-provider/data-provider.service.ts
+++ b/apps/api/src/services/data-provider/data-provider.service.ts
@@ -17,6 +17,7 @@ import { Inject, Injectable, Logger } from '@nestjs/common';
import { DataSource, MarketData, SymbolProfile } from '@prisma/client';
import { format, isValid } from 'date-fns';
import { groupBy, isEmpty, isNumber } from 'lodash';
+import ms from 'ms';
@Injectable()
export class DataProviderService {
@@ -52,6 +53,7 @@ export class DataProviderService {
symbol
}
],
+ requestTimeout: ms('30 seconds'),
useCache: false
});
@@ -236,9 +238,11 @@ export class DataProviderService {
public async getQuotes({
items,
+ requestTimeout,
useCache = true
}: {
items: UniqueAsset[];
+ requestTimeout?: number;
useCache?: boolean;
}): Promise<{
[symbol: string]: IDataProviderResponse;
@@ -312,7 +316,7 @@ export class DataProviderService {
);
const promise = Promise.resolve(
- dataProvider.getQuotes({ symbols: symbolsChunk })
+ dataProvider.getQuotes({ requestTimeout, symbols: symbolsChunk })
);
promises.push(
diff --git a/apps/api/src/services/data-provider/eod-historical-data/eod-historical-data.service.ts b/apps/api/src/services/data-provider/eod-historical-data/eod-historical-data.service.ts
index 37c37e3898..a582a953da 100644
--- a/apps/api/src/services/data-provider/eod-historical-data/eod-historical-data.service.ts
+++ b/apps/api/src/services/data-provider/eod-historical-data/eod-historical-data.service.ts
@@ -132,8 +132,10 @@ export class EodHistoricalDataService implements DataProviderInterface {
}
public async getQuotes({
+ requestTimeout = DEFAULT_REQUEST_TIMEOUT,
symbols
}: {
+ requestTimeout?: number;
symbols: string[];
}): Promise<{ [symbol: string]: IDataProviderResponse }> {
let response: { [symbol: string]: IDataProviderResponse } = {};
@@ -151,7 +153,7 @@ export class EodHistoricalDataService implements DataProviderInterface {
setTimeout(() => {
abortController.abort();
- }, DEFAULT_REQUEST_TIMEOUT);
+ }, requestTimeout);
const realTimeResponse = await got(
`${this.URL}/real-time/${eodHistoricalDataSymbols[0]}?api_token=${
diff --git a/apps/api/src/services/data-provider/financial-modeling-prep/financial-modeling-prep.service.ts b/apps/api/src/services/data-provider/financial-modeling-prep/financial-modeling-prep.service.ts
index b08bc099e7..c74cfdc003 100644
--- a/apps/api/src/services/data-provider/financial-modeling-prep/financial-modeling-prep.service.ts
+++ b/apps/api/src/services/data-provider/financial-modeling-prep/financial-modeling-prep.service.ts
@@ -114,8 +114,10 @@ export class FinancialModelingPrepService implements DataProviderInterface {
}
public async getQuotes({
+ requestTimeout = DEFAULT_REQUEST_TIMEOUT,
symbols
}: {
+ requestTimeout?: number;
symbols: string[];
}): Promise<{ [symbol: string]: IDataProviderResponse }> {
const response: { [symbol: string]: IDataProviderResponse } = {};
@@ -129,7 +131,7 @@ export class FinancialModelingPrepService implements DataProviderInterface {
setTimeout(() => {
abortController.abort();
- }, DEFAULT_REQUEST_TIMEOUT);
+ }, requestTimeout);
const response = await got(
`${this.URL}/quote/${symbols.join(',')}?apikey=${this.apiKey}`,
diff --git a/apps/api/src/services/data-provider/google-sheets/google-sheets.service.ts b/apps/api/src/services/data-provider/google-sheets/google-sheets.service.ts
index a541fcd121..6dc8dc80ba 100644
--- a/apps/api/src/services/data-provider/google-sheets/google-sheets.service.ts
+++ b/apps/api/src/services/data-provider/google-sheets/google-sheets.service.ts
@@ -7,6 +7,7 @@ import {
} from '@ghostfolio/api/services/interfaces/interfaces';
import { PrismaService } from '@ghostfolio/api/services/prisma/prisma.service';
import { SymbolProfileService } from '@ghostfolio/api/services/symbol-profile/symbol-profile.service';
+import { DEFAULT_REQUEST_TIMEOUT } from '@ghostfolio/common/config';
import { DATE_FORMAT, parseDate } from '@ghostfolio/common/helper';
import { Granularity } from '@ghostfolio/common/types';
import { Injectable, Logger } from '@nestjs/common';
@@ -100,8 +101,10 @@ export class GoogleSheetsService implements DataProviderInterface {
}
public async getQuotes({
+ requestTimeout = DEFAULT_REQUEST_TIMEOUT,
symbols
}: {
+ requestTimeout?: number;
symbols: string[];
}): Promise<{ [symbol: string]: IDataProviderResponse }> {
const response: { [symbol: string]: IDataProviderResponse } = {};
diff --git a/apps/api/src/services/data-provider/interfaces/data-enhancer.interface.ts b/apps/api/src/services/data-provider/interfaces/data-enhancer.interface.ts
index 9c6db91964..73e0cc69d6 100644
--- a/apps/api/src/services/data-provider/interfaces/data-enhancer.interface.ts
+++ b/apps/api/src/services/data-provider/interfaces/data-enhancer.interface.ts
@@ -2,9 +2,11 @@ import { SymbolProfile } from '@prisma/client';
export interface DataEnhancerInterface {
enhance({
+ requestTimeout,
response,
symbol
}: {
+ requestTimeout?: number;
response: Partial<SymbolProfile>;
symbol: string;
}): Promise<Partial<SymbolProfile>>;
diff --git a/apps/api/src/services/data-provider/interfaces/data-provider.interface.ts b/apps/api/src/services/data-provider/interfaces/data-provider.interface.ts
index f4daeb108a..a27c8544cb 100644
--- a/apps/api/src/services/data-provider/interfaces/data-provider.interface.ts
+++ b/apps/api/src/services/data-provider/interfaces/data-provider.interface.ts
@@ -37,8 +37,10 @@ export interface DataProviderInterface {
getName(): DataSource;
getQuotes({
+ requestTimeout,
symbols
}: {
+ requestTimeout?: number;
symbols: string[];
}): Promise<{ [symbol: string]: IDataProviderResponse }>;
diff --git a/apps/api/src/services/data-provider/manual/manual.service.ts b/apps/api/src/services/data-provider/manual/manual.service.ts
index bea4b60ce7..1464a526d7 100644
--- a/apps/api/src/services/data-provider/manual/manual.service.ts
+++ b/apps/api/src/services/data-provider/manual/manual.service.ts
@@ -134,8 +134,10 @@ export class ManualService implements DataProviderInterface {
}
public async getQuotes({
+ requestTimeout = DEFAULT_REQUEST_TIMEOUT,
symbols
}: {
+ requestTimeout?: number;
symbols: string[];
}): Promise<{ [symbol: string]: IDataProviderResponse }> {
const response: { [symbol: string]: IDataProviderResponse } = {};
diff --git a/apps/api/src/services/data-provider/rapid-api/rapid-api.service.ts b/apps/api/src/services/data-provider/rapid-api/rapid-api.service.ts
index ce188ffe05..5e833cbf1f 100644
--- a/apps/api/src/services/data-provider/rapid-api/rapid-api.service.ts
+++ b/apps/api/src/services/data-provider/rapid-api/rapid-api.service.ts
@@ -88,8 +88,10 @@ export class RapidApiService implements DataProviderInterface {
}
public async getQuotes({
+ requestTimeout = DEFAULT_REQUEST_TIMEOUT,
symbols
}: {
+ requestTimeout?: number;
symbols: string[];
}): Promise<{ [symbol: string]: IDataProviderResponse }> {
if (symbols.length <= 0) {
diff --git a/apps/api/src/services/data-provider/yahoo-finance/yahoo-finance.service.ts b/apps/api/src/services/data-provider/yahoo-finance/yahoo-finance.service.ts
index 2f69448f91..96bffa7ea1 100644
--- a/apps/api/src/services/data-provider/yahoo-finance/yahoo-finance.service.ts
+++ b/apps/api/src/services/data-provider/yahoo-finance/yahoo-finance.service.ts
@@ -6,7 +6,10 @@ import {
IDataProviderHistoricalResponse,
IDataProviderResponse
} from '@ghostfolio/api/services/interfaces/interfaces';
-import { DEFAULT_CURRENCY } from '@ghostfolio/common/config';
+import {
+ DEFAULT_CURRENCY,
+ DEFAULT_REQUEST_TIMEOUT
+} from '@ghostfolio/common/config';
import { DATE_FORMAT } from '@ghostfolio/common/helper';
import { Granularity } from '@ghostfolio/common/types';
import { Injectable, Logger } from '@nestjs/common';
@@ -157,8 +160,10 @@ export class YahooFinanceService implements DataProviderInterface {
}
public async getQuotes({
+ requestTimeout = DEFAULT_REQUEST_TIMEOUT,
symbols
}: {
+ requestTimeout?: number;
symbols: string[];
}): Promise<{ [symbol: string]: IDataProviderResponse }> {
const response: { [symbol: string]: IDataProviderResponse } = {};