Open Source Wealth Management Software. Angular + NestJS + Prisma + Nx + TypeScript 🤍
TypeScript angularetffinancefintechghostfoliohacktoberfestinvestingnestjsosspersonal-financeportfolioprismasoftwarestocktrackertradingtypescriptwealth-managementweb
Feature/improve timeout of data source requests (#2330)
Thomas Kaul8fc567644348
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 1e6ad9bba1..a6e4e5b8f6 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -10,6 +10,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0### Added- Added support to drop a file in the import activities dialog+- Added a timeout to all data source requests### Changed@@ -22,6 +23,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0### Fixed+- Fixed the timeout in _EOD Historical Data_ requests- Fixed an issue with the portfolio summary caused by the language localization for Dutch (`nl`)## 2.0.0 - 2023-09-09diff --git a/apps/api/src/app/info/info.service.ts b/apps/api/src/app/info/info.service.tsindex 3ac1cfbc15..f2c45a72ba 100644--- a/apps/api/src/app/info/info.service.ts+++ b/apps/api/src/app/info/info.service.ts@@ -8,6 +8,7 @@ import { PropertyService } from '@ghostfolio/api/services/property/property.servimport { TagService } from '@ghostfolio/api/services/tag/tag.service';import {DEFAULT_CURRENCY,+ DEFAULT_REQUEST_TIMEOUT,PROPERTY_BETTER_UPTIME_MONITOR_ID,PROPERTY_COUNTRIES_OF_SUBSCRIBERS,PROPERTY_DEMO_USER_ID,@@ -168,10 +169,18 @@ export class InfoService {private async countDockerHubPulls(): Promise<number> {try {+ const abortController = new AbortController();++ setTimeout(() => {+ abortController.abort();+ }, DEFAULT_REQUEST_TIMEOUT);+const { pull_count } = await got(`https://hub.docker.com/v2/repositories/ghostfolio/ghostfolio`,{- headers: { 'User-Agent': 'request' }+ headers: { 'User-Agent': 'request' },+ // @ts-ignore+ signal: abortController.signal}).json<any>();@@ -185,7 +194,16 @@ export class InfoService {private async countGitHubContributors(): Promise<number> {try {- const { body } = await got('https://github.com/ghostfolio/ghostfolio');+ const abortController = new AbortController();++ setTimeout(() => {+ abortController.abort();+ }, DEFAULT_REQUEST_TIMEOUT);++ const { body } = await got('https://github.com/ghostfolio/ghostfolio', {+ // @ts-ignore+ signal: abortController.signal+ });const $ = cheerio.load(body);@@ -203,10 +221,18 @@ export class InfoService {private async countGitHubStargazers(): Promise<number> {try {+ const abortController = new AbortController();++ setTimeout(() => {+ abortController.abort();+ }, DEFAULT_REQUEST_TIMEOUT);+const { stargazers_count } = await got(`https://api.github.com/repos/ghostfolio/ghostfolio`,{- headers: { 'User-Agent': 'request' }+ headers: { 'User-Agent': 'request' },+ // @ts-ignore+ signal: abortController.signal}).json<any>();@@ -323,18 +349,25 @@ export class InfoService {PROPERTY_BETTER_UPTIME_MONITOR_ID)) as string;+ const abortController = new AbortController();++ setTimeout(() => {+ abortController.abort();+ }, DEFAULT_REQUEST_TIMEOUT);+const { data } = await got(`https://uptime.betterstack.com/api/v2/monitors/${monitorId}/sla?from=${format(subDays(new Date(), 90),DATE_FORMAT)}&to${format(new Date(), DATE_FORMAT)}`,-{headers: {Authorization: `Bearer ${this.configurationService.get('BETTER_UPTIME_API_KEY')}`- }+ },+ // @ts-ignore+ signal: abortController.signal}).json<any>();diff --git a/apps/api/src/app/logo/logo.service.ts b/apps/api/src/app/logo/logo.service.tsindex 166143a759..80ae1d6a9a 100644--- a/apps/api/src/app/logo/logo.service.ts+++ b/apps/api/src/app/logo/logo.service.ts@@ -1,4 +1,5 @@import { SymbolProfileService } from '@ghostfolio/api/services/symbol-profile/symbol-profile.service';+import { DEFAULT_REQUEST_TIMEOUT } from '@ghostfolio/common/config';import { UniqueAsset } from '@ghostfolio/common/interfaces';import { HttpException, Injectable } from '@nestjs/common';import { DataSource } from '@prisma/client';@@ -41,10 +42,18 @@ export class LogoService {}private getBuffer(aUrl: string) {+ const abortController = new AbortController();++ setTimeout(() => {+ abortController.abort();+ }, DEFAULT_REQUEST_TIMEOUT);+return got(`https://t0.gstatic.com/faviconV2?client=SOCIAL&type=FAVICON&fallback_opts=TYPE,SIZE,URL&url=${aUrl}&size=64`,{- headers: { 'User-Agent': 'request' }+ headers: { 'User-Agent': 'request' },+ // @ts-ignore+ signal: abortController.signal}).buffer();}diff --git a/apps/api/src/services/data-provider/coingecko/coingecko.service.ts b/apps/api/src/services/data-provider/coingecko/coingecko.service.tsindex 56594ca40d..4360822f04 100644--- a/apps/api/src/services/data-provider/coingecko/coingecko.service.ts+++ b/apps/api/src/services/data-provider/coingecko/coingecko.service.ts@@ -4,7 +4,10 @@ import {IDataProviderHistoricalResponse,IDataProviderResponse} from '@ghostfolio/api/services/interfaces/interfaces';-import { DEFAULT_CURRENCY } from '@ghostfolio/common/config';+import {+ DEFAULT_CURRENCY,+ DEFAULT_REQUEST_TIMEOUT+} from '@ghostfolio/common/config';import { DATE_FORMAT } from '@ghostfolio/common/helper';import { DataProviderInfo } from '@ghostfolio/common/interfaces';import { Granularity } from '@ghostfolio/common/types';@@ -40,7 +43,16 @@ export class CoinGeckoService implements DataProviderInterface {};try {- const { name } = await got(`${this.URL}/coins/${aSymbol}`).json<any>();+ const abortController = new AbortController();++ setTimeout(() => {+ abortController.abort();+ }, DEFAULT_REQUEST_TIMEOUT);++ const { name } = await got(`${this.URL}/coins/${aSymbol}`, {+ // @ts-ignore+ signal: abortController.signal+ }).json<any>();response.name = name;} catch (error) {@@ -73,12 +85,22 @@ export class CoinGeckoService implements DataProviderInterface {[symbol: string]: { [date: string]: IDataProviderHistoricalResponse };}> {try {+ const abortController = new AbortController();++ setTimeout(() => {+ abortController.abort();+ }, DEFAULT_REQUEST_TIMEOUT);+const { prices } = await got(`${this.URL}/coins/${aSymbol}/market_chart/range?vs_currency=${DEFAULT_CURRENCY.toLowerCase()}&from=${getUnixTime(from- )}&to=${getUnixTime(to)}`+ )}&to=${getUnixTime(to)}`,+ {+ // @ts-ignore+ signal: abortController.signal+ }).json<any>();const result: {@@ -122,10 +144,20 @@ export class CoinGeckoService implements DataProviderInterface {}try {+ const abortController = new AbortController();++ setTimeout(() => {+ abortController.abort();+ }, DEFAULT_REQUEST_TIMEOUT);+const response = await got(`${this.URL}/simple/price?ids=${aSymbols.join(','- )}&vs_currencies=${DEFAULT_CURRENCY.toLowerCase()}`+ )}&vs_currencies=${DEFAULT_CURRENCY.toLowerCase()}`,+ {+ // @ts-ignore+ signal: abortController.signal+ }).json<any>();for (const symbol in response) {@@ -160,9 +192,16 @@ export class CoinGeckoService implements DataProviderInterface {let items: LookupItem[] = [];try {- const { coins } = await got(- `${this.URL}/search?query=${query}`- ).json<any>();+ const abortController = new AbortController();++ setTimeout(() => {+ abortController.abort();+ }, DEFAULT_REQUEST_TIMEOUT);++ const { coins } = await got(`${this.URL}/search?query=${query}`, {+ // @ts-ignore+ signal: abortController.signal+ }).json<any>();items = coins.map(({ id: symbol, name }) => {return {diff --git a/apps/api/src/services/data-provider/data-enhancer/trackinsight/trackinsight.service.ts b/apps/api/src/services/data-provider/data-enhancer/trackinsight/trackinsight.service.tsindex 07c0234b6d..36eb22dad5 100644--- a/apps/api/src/services/data-provider/data-enhancer/trackinsight/trackinsight.service.ts+++ b/apps/api/src/services/data-provider/data-enhancer/trackinsight/trackinsight.service.ts@@ -1,4 +1,5 @@import { DataEnhancerInterface } from '@ghostfolio/api/services/data-provider/interfaces/data-enhancer.interface';+import { DEFAULT_REQUEST_TIMEOUT } from '@ghostfolio/common/config';import { Country } from '@ghostfolio/common/interfaces/country.interface';import { Sector } from '@ghostfolio/common/interfaces/sector.interface';import { Injectable } from '@nestjs/common';@@ -32,15 +33,35 @@ export class TrackinsightDataEnhancerService implements DataEnhancerInterface {return response;}+ let abortController = new AbortController();++ setTimeout(() => {+ abortController.abort();+ }, DEFAULT_REQUEST_TIMEOUT);+const profile = await got(- `${TrackinsightDataEnhancerService.baseUrl}/funds/${symbol}.json`+ `${TrackinsightDataEnhancerService.baseUrl}/funds/${symbol}.json`,+ {+ // @ts-ignore+ signal: abortController.signal+ }).json<any>().catch(() => {+ const abortController = new AbortController();++ setTimeout(() => {+ abortController.abort();+ }, DEFAULT_REQUEST_TIMEOUT);+return got(`${TrackinsightDataEnhancerService.baseUrl}/funds/${symbol.split('.'- )?.[0]}.json`+ )?.[0]}.json`,+ {+ // @ts-ignore+ signal: abortController.signal+ }).json<any>().catch(() => {@@ -54,15 +75,35 @@ export class TrackinsightDataEnhancerService implements DataEnhancerInterface {response.isin = isin;}+ abortController = new AbortController();++ setTimeout(() => {+ abortController.abort();+ }, DEFAULT_REQUEST_TIMEOUT);+const holdings = await got(- `${TrackinsightDataEnhancerService.baseUrl}/holdings/${symbol}.json`+ `${TrackinsightDataEnhancerService.baseUrl}/holdings/${symbol}.json`,+ {+ // @ts-ignore+ signal: abortController.signal+ }).json<any>().catch(() => {+ const abortController = new AbortController();++ setTimeout(() => {+ abortController.abort();+ }, DEFAULT_REQUEST_TIMEOUT);+return got(`${TrackinsightDataEnhancerService.baseUrl}/holdings/${symbol.split('.'- )?.[0]}.json`+ )?.[0]}.json`,+ {+ // @ts-ignore+ signal: abortController.signal+ }).json<any>().catch(() => {diff --git a/apps/api/src/services/data-provider/eod-historical-data/eod-historical-data.service.ts b/apps/api/src/services/data-provider/eod-historical-data/eod-historical-data.service.tsindex fd8114ad65..307f6127a8 100644--- a/apps/api/src/services/data-provider/eod-historical-data/eod-historical-data.service.ts+++ b/apps/api/src/services/data-provider/eod-historical-data/eod-historical-data.service.ts@@ -78,6 +78,12 @@ export class EodHistoricalDataService implements DataProviderInterface {const symbol = this.convertToEodSymbol(aSymbol);try {+ const abortController = new AbortController();++ setTimeout(() => {+ abortController.abort();+ }, DEFAULT_REQUEST_TIMEOUT);+const response = await got(`${this.URL}/eod/${symbol}?api_token=${this.apiKey@@ -86,9 +92,8 @@ export class EodHistoricalDataService implements DataProviderInterface {DATE_FORMAT)}&period={aGranularity}`,{- timeout: {- request: DEFAULT_REQUEST_TIMEOUT- }+ // @ts-ignore+ signal: abortController.signal}).json<any>();@@ -138,14 +143,19 @@ export class EodHistoricalDataService implements DataProviderInterface {}try {+ const abortController = new AbortController();++ setTimeout(() => {+ abortController.abort();+ }, DEFAULT_REQUEST_TIMEOUT);+const realTimeResponse = await got(`${this.URL}/real-time/${symbols[0]}?api_token=${this.apiKey}&fmt=json&s=${symbols.join(',')}`,{- timeout: {- request: DEFAULT_REQUEST_TIMEOUT- }+ // @ts-ignore+ signal: abortController.signal}).json<any>();@@ -331,12 +341,17 @@ export class EodHistoricalDataService implements DataProviderInterface {let searchResult = [];try {+ const abortController = new AbortController();++ setTimeout(() => {+ abortController.abort();+ }, DEFAULT_REQUEST_TIMEOUT);+const response = await got(`${this.URL}/search/${aQuery}?api_token=${this.apiKey}`,{- timeout: {- request: DEFAULT_REQUEST_TIMEOUT- }+ // @ts-ignore+ signal: abortController.signal}).json<any>();diff --git a/apps/api/src/services/data-provider/financial-modeling-prep/financial-modeling-prep.service.ts b/apps/api/src/services/data-provider/financial-modeling-prep/financial-modeling-prep.service.tsindex c5d163456d..4fd1d4ebd3 100644--- a/apps/api/src/services/data-provider/financial-modeling-prep/financial-modeling-prep.service.ts+++ b/apps/api/src/services/data-provider/financial-modeling-prep/financial-modeling-prep.service.ts@@ -5,7 +5,10 @@ import {IDataProviderHistoricalResponse,IDataProviderResponse} from '@ghostfolio/api/services/interfaces/interfaces';-import { DEFAULT_CURRENCY } from '@ghostfolio/common/config';+import {+ DEFAULT_CURRENCY,+ DEFAULT_REQUEST_TIMEOUT+} from '@ghostfolio/common/config';import { DATE_FORMAT, parseDate } from '@ghostfolio/common/helper';import { DataProviderInfo } from '@ghostfolio/common/interfaces';import { Granularity } from '@ghostfolio/common/types';@@ -63,8 +66,18 @@ export class FinancialModelingPrepService implements DataProviderInterface {[symbol: string]: { [date: string]: IDataProviderHistoricalResponse };}> {try {+ const abortController = new AbortController();++ setTimeout(() => {+ abortController.abort();+ }, DEFAULT_REQUEST_TIMEOUT);+const { historical } = await got(- `${this.URL}/historical-price-full/${aSymbol}?apikey=${this.apiKey}`+ `${this.URL}/historical-price-full/${aSymbol}?apikey=${this.apiKey}`,+ {+ // @ts-ignore+ signal: abortController.signal+ }).json<any>();const result: {@@ -110,8 +123,18 @@ export class FinancialModelingPrepService implements DataProviderInterface {}try {+ const abortController = new AbortController();++ setTimeout(() => {+ abortController.abort();+ }, DEFAULT_REQUEST_TIMEOUT);+const response = await got(- `${this.URL}/quote/${aSymbols.join(',')}?apikey=${this.apiKey}`+ `${this.URL}/quote/${aSymbols.join(',')}?apikey=${this.apiKey}`,+ {+ // @ts-ignore+ signal: abortController.signal+ }).json<any>();for (const { price, symbol } of response) {@@ -144,8 +167,18 @@ export class FinancialModelingPrepService implements DataProviderInterface {let items: LookupItem[] = [];try {+ const abortController = new AbortController();++ setTimeout(() => {+ abortController.abort();+ }, DEFAULT_REQUEST_TIMEOUT);+const result = await got(- `${this.URL}/search?query=${query}&apikey=${this.apiKey}`+ `${this.URL}/search?query=${query}&apikey=${this.apiKey}`,+ {+ // @ts-ignore+ signal: abortController.signal+ }).json<any>();items = result.map(({ currency, name, symbol }) => {diff --git a/apps/api/src/services/data-provider/manual/manual.service.ts b/apps/api/src/services/data-provider/manual/manual.service.tsindex adf14e43f1..5c84a9c922 100644--- a/apps/api/src/services/data-provider/manual/manual.service.ts+++ b/apps/api/src/services/data-provider/manual/manual.service.ts@@ -6,6 +6,7 @@ import {} from '@ghostfolio/api/services/interfaces/interfaces';import { PrismaService } from '@ghostfolio/api/services/prisma/prisma.service';import { SymbolProfileService } from '@ghostfolio/api/services/symbol-profile/symbol-profile.service';+import { DEFAULT_REQUEST_TIMEOUT } from '@ghostfolio/common/config';import {DATE_FORMAT,extractNumberFromString,@@ -95,7 +96,17 @@ export class ManualService implements DataProviderInterface {return {};}- const { body } = await got(url, { headers });+ const abortController = new AbortController();++ setTimeout(() => {+ abortController.abort();+ }, DEFAULT_REQUEST_TIMEOUT);++ const { body } = await got(url, {+ headers,+ // @ts-ignore+ signal: abortController.signal+ });const $ = cheerio.load(body);diff --git a/apps/api/src/services/data-provider/rapid-api/rapid-api.service.ts b/apps/api/src/services/data-provider/rapid-api/rapid-api.service.tsindex 307855aafb..7743d78053 100644--- a/apps/api/src/services/data-provider/rapid-api/rapid-api.service.ts+++ b/apps/api/src/services/data-provider/rapid-api/rapid-api.service.ts@@ -5,7 +5,10 @@ import {IDataProviderHistoricalResponse,IDataProviderResponse} from '@ghostfolio/api/services/interfaces/interfaces';-import { ghostfolioFearAndGreedIndexSymbol } from '@ghostfolio/common/config';+import {+ DEFAULT_REQUEST_TIMEOUT,+ ghostfolioFearAndGreedIndexSymbol+} from '@ghostfolio/common/config';import { DATE_FORMAT, getYesterday } from '@ghostfolio/common/helper';import { Granularity } from '@ghostfolio/common/types';import { Injectable, Logger } from '@nestjs/common';@@ -135,6 +138,12 @@ export class RapidApiService implements DataProviderInterface {oneYearAgo: { value: number; valueText: string };}> {try {+ const abortController = new AbortController();++ setTimeout(() => {+ abortController.abort();+ }, DEFAULT_REQUEST_TIMEOUT);+const { fgi } = await got(`https://fear-and-greed-index.p.rapidapi.com/v1/fgi`,{@@ -142,7 +151,9 @@ export class RapidApiService implements DataProviderInterface {useQueryString: 'true','x-rapidapi-host': 'fear-and-greed-index.p.rapidapi.com','x-rapidapi-key': this.configurationService.get('RAPID_API_API_KEY')- }+ },+ // @ts-ignore+ signal: abortController.signal}).json<any>();diff --git a/libs/common/src/lib/config.ts b/libs/common/src/lib/config.tsindex 345641f976..467986fa10 100644--- a/libs/common/src/lib/config.ts+++ b/libs/common/src/lib/config.ts@@ -39,7 +39,7 @@ export const DEFAULT_CURRENCY = 'USD';export const DEFAULT_DATE_FORMAT_MONTH_YEAR = 'MMM yyyy';export const DEFAULT_LANGUAGE_CODE = 'en';export const DEFAULT_PAGE_SIZE = 50;-export const DEFAULT_REQUEST_TIMEOUT = ms('3 seconds');+export const DEFAULT_REQUEST_TIMEOUT = ms('2 seconds');export const DEFAULT_ROOT_URL = 'http://localhost:4200';export const EMERGENCY_FUND_TAG_ID = '4452656d-9fa4-4bd0-ba38-70492e31d180';