Open Source Wealth Management Software. Angular + NestJS + Prisma + Nx + TypeScript 🤍
TypeScript angularetffinancefintechghostfoliohacktoberfestinvestingnestjsosspersonal-financeportfolioprismasoftwarestocktrackertradingtypescriptwealth-managementweb
Feature/disable caching in health check endpoints for data providers (#2147)
Thomas Kauled735e0b2921
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 22650dc376..344d9f77db 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -5,6 +5,12 @@ All notable changes to this project will be documented in this file.The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).+## Unreleased++### Changed++- Disabled the caching in the health check endpoints for data providers+## 1.289.0 - 2023-07-14### Changeddiff --git a/apps/api/src/app/benchmark/benchmark.service.ts b/apps/api/src/app/benchmark/benchmark.service.tsindex 73b48068b3..785c2801a4 100644--- a/apps/api/src/app/benchmark/benchmark.service.ts+++ b/apps/api/src/app/benchmark/benchmark.service.ts@@ -66,11 +66,11 @@ export class BenchmarkService {const promises: Promise<number>[] = [];- const quotes = await this.dataProviderService.getQuotes(- benchmarkAssetProfiles.map(({ dataSource, symbol }) => {+ const quotes = await this.dataProviderService.getQuotes({+ items: benchmarkAssetProfiles.map(({ dataSource, symbol }) => {return { dataSource, symbol };})- );+ });for (const { dataSource, symbol } of benchmarkAssetProfiles) {promises.push(this.marketDataService.getMax({ dataSource, symbol }));diff --git a/apps/api/src/app/portfolio/current-rate.service.ts b/apps/api/src/app/portfolio/current-rate.service.tsindex 6d48573f7b..a4ee317bb0 100644--- a/apps/api/src/app/portfolio/current-rate.service.ts+++ b/apps/api/src/app/portfolio/current-rate.service.ts@@ -38,7 +38,7 @@ export class CurrentRateService {if (includeToday) {promises.push(this.dataProviderService- .getQuotes(dataGatheringItems)+ .getQuotes({ items: dataGatheringItems }).then((dataResultProvider) => {const result: GetValueObject[] = [];for (const dataGatheringItem of dataGatheringItems) {diff --git a/apps/api/src/app/portfolio/portfolio.service.ts b/apps/api/src/app/portfolio/portfolio.service.tsindex 66f3841a47..7d13728d2d 100644--- a/apps/api/src/app/portfolio/portfolio.service.ts+++ b/apps/api/src/app/portfolio/portfolio.service.ts@@ -504,15 +504,17 @@ export class PortfolioService {);}- const dataGatheringItems = currentPositions.positions.map((position) => {- return {- dataSource: position.dataSource,- symbol: position.symbol- };- });+ const dataGatheringItems = currentPositions.positions.map(+ ({ dataSource, symbol }) => {+ return {+ dataSource,+ symbol+ };+ }+ );const [dataProviderResponses, symbolProfiles] = await Promise.all([- this.dataProviderService.getQuotes(dataGatheringItems),+ this.dataProviderService.getQuotes({ items: dataGatheringItems }),this.symbolProfileService.getSymbolProfiles(dataGatheringItems)]);@@ -897,9 +899,9 @@ export class PortfolioService {)};} else {- const currentData = await this.dataProviderService.getQuotes([- { dataSource: DataSource.YAHOO, symbol: aSymbol }- ]);+ const currentData = await this.dataProviderService.getQuotes({+ items: [{ dataSource: DataSource.YAHOO, symbol: aSymbol }]+ });const marketPrice = currentData[aSymbol]?.marketPrice;let historicalData = await this.dataProviderService.getHistorical(@@ -1000,15 +1002,15 @@ export class PortfolioService {(item) => !item.quantity.eq(0));- const dataGatheringItem = positions.map((position) => {+ const dataGatheringItems = positions.map(({ dataSource, symbol }) => {return {- dataSource: position.dataSource,- symbol: position.symbol+ dataSource,+ symbol};});const [dataProviderResponses, symbolProfiles] = await Promise.all([- this.dataProviderService.getQuotes(dataGatheringItem),+ this.dataProviderService.getQuotes({ items: dataGatheringItems }),this.symbolProfileService.getSymbolProfiles(positions.map(({ dataSource, symbol }) => {return { dataSource, symbol };diff --git a/apps/api/src/app/symbol/symbol.service.ts b/apps/api/src/app/symbol/symbol.service.tsindex bc626a97f2..5eacbb1b02 100644--- a/apps/api/src/app/symbol/symbol.service.ts+++ b/apps/api/src/app/symbol/symbol.service.ts@@ -27,9 +27,9 @@ export class SymbolService {dataGatheringItem: IDataGatheringItem;includeHistoricalData?: number;}): Promise<SymbolItem> {- const quotes = await this.dataProviderService.getQuotes([- dataGatheringItem- ]);+ const quotes = await this.dataProviderService.getQuotes({+ items: [dataGatheringItem]+ });const { currency, marketPrice } = quotes[dataGatheringItem.symbol] ?? {};if (dataGatheringItem.dataSource && marketPrice >= 0) {diff --git a/apps/api/src/services/data-provider/data-provider.service.ts b/apps/api/src/services/data-provider/data-provider.service.tsindex b4f4e10b58..557699495d 100644--- a/apps/api/src/services/data-provider/data-provider.service.ts+++ b/apps/api/src/services/data-provider/data-provider.service.ts@@ -3,7 +3,6 @@ import { LookupItem } from '@ghostfolio/api/app/symbol/interfaces/lookup-item.inimport { ConfigurationService } from '@ghostfolio/api/services/configuration/configuration.service';import { DataProviderInterface } from '@ghostfolio/api/services/data-provider/interfaces/data-provider.interface';import {- IDataGatheringItem,IDataProviderHistoricalResponse,IDataProviderResponse} from '@ghostfolio/api/services/interfaces/interfaces';@@ -12,6 +11,7 @@ import { PrismaService } from '@ghostfolio/api/services/prisma/prisma.service';import { PropertyService } from '@ghostfolio/api/services/property/property.service';import { PROPERTY_DATA_SOURCE_MAPPING } from '@ghostfolio/common/config';import { DATE_FORMAT, getStartOfUtcDate } from '@ghostfolio/common/helper';+import { UniqueAsset } from '@ghostfolio/common/interfaces';import type { Granularity, UserWithSettings } from '@ghostfolio/common/types';import { Inject, Injectable, Logger } from '@nestjs/common';import { DataSource, MarketData, SymbolProfile } from '@prisma/client';@@ -45,12 +45,15 @@ export class DataProviderService {const dataProvider = this.getDataProvider(dataSource);const symbol = dataProvider.getTestSymbol();- const quotes = await this.getQuotes([- {- dataSource,- symbol- }- ]);+ const quotes = await this.getQuotes({+ items: [+ {+ dataSource,+ symbol+ }+ ],+ useCache: false+ });if (quotes[symbol]?.marketPrice > 0) {return true;@@ -59,14 +62,16 @@ export class DataProviderService {return false;}- public async getAssetProfiles(items: IDataGatheringItem[]): Promise<{+ public async getAssetProfiles(items: UniqueAsset[]): Promise<{[symbol: string]: Partial<SymbolProfile>;}> {const response: {[symbol: string]: Partial<SymbolProfile>;} = {};- const itemsGroupedByDataSource = groupBy(items, (item) => item.dataSource);+ const itemsGroupedByDataSource = groupBy(items, ({ dataSource }) => {+ return dataSource;+ });const promises = [];@@ -127,7 +132,7 @@ export class DataProviderService {}public async getHistorical(- aItems: IDataGatheringItem[],+ aItems: UniqueAsset[],aGranularity: Granularity = 'month',from: Date,to: Date@@ -155,11 +160,11 @@ export class DataProviderService {)}'`: '';- const dataSources = aItems.map((item) => {- return item.dataSource;+ const dataSources = aItems.map(({ dataSource }) => {+ return dataSource;});- const symbols = aItems.map((item) => {- return item.symbol;+ const symbols = aItems.map(({ symbol }) => {+ return symbol;});try {@@ -192,7 +197,7 @@ export class DataProviderService {}public async getHistoricalRaw(- aDataGatheringItems: IDataGatheringItem[],+ aDataGatheringItems: UniqueAsset[],from: Date,to: Date): Promise<{@@ -229,7 +234,13 @@ export class DataProviderService {return result;}- public async getQuotes(items: IDataGatheringItem[]): Promise<{+ public async getQuotes({+ items,+ useCache = true+ }: {+ items: UniqueAsset[];+ useCache?: boolean;+ }): Promise<{[symbol: string]: IDataProviderResponse;}> {const response: {@@ -238,23 +249,24 @@ export class DataProviderService {const startTimeTotal = performance.now();// Get items from cache- const itemsToFetch: IDataGatheringItem[] = [];+ const itemsToFetch: UniqueAsset[] = [];for (const { dataSource, symbol } of items) {- const quoteString = await this.redisCacheService.get(- this.redisCacheService.getQuoteKey({ dataSource, symbol })- );+ if (useCache) {+ const quoteString = await this.redisCacheService.get(+ this.redisCacheService.getQuoteKey({ dataSource, symbol })+ );- if (quoteString) {- try {- const cachedDataProviderResponse = JSON.parse(quoteString);- response[symbol] = cachedDataProviderResponse;- } catch {}+ if (quoteString) {+ try {+ const cachedDataProviderResponse = JSON.parse(quoteString);+ response[symbol] = cachedDataProviderResponse;+ continue;+ } catch {}+ }}- if (!quoteString) {- itemsToFetch.push({ dataSource, symbol });- }+ itemsToFetch.push({ dataSource, symbol });}const numberOfItemsInCache = Object.keys(response)?.length;diff --git a/apps/api/src/services/exchange-rate-data/exchange-rate-data.service.ts b/apps/api/src/services/exchange-rate-data/exchange-rate-data.service.tsindex 448933b421..9e1daa2241 100644--- a/apps/api/src/services/exchange-rate-data/exchange-rate-data.service.ts+++ b/apps/api/src/services/exchange-rate-data/exchange-rate-data.service.ts@@ -64,11 +64,11 @@ export class ExchangeRateDataService {if (Object.keys(result).length !== this.currencyPairs.length) {// Load currencies directly from data provider as a fallback// if historical data is not fully available- const quotes = await this.dataProviderService.getQuotes(- this.currencyPairs.map(({ dataSource, symbol }) => {+ const quotes = await this.dataProviderService.getQuotes({+ items: this.currencyPairs.map(({ dataSource, symbol }) => {return { dataSource, symbol };})- );+ });for (const symbol of Object.keys(quotes)) {if (isNumber(quotes[symbol].marketPrice)) {