Skip to content

Open Source Wealth Management Software. Angular + NestJS + Prisma + Nx + TypeScript 🤍

Feature/disable caching in health check endpoints for data providers (#2147)

Thomas Kauled735e0b2921
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 22650dc376..344d9f77db 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -5,6 +5,12 @@ All notable changes to this project will be documented in this file.
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+## Unreleased
+
+### Changed
+
+- Disabled the caching in the health check endpoints for data providers
+
## 1.289.0 - 2023-07-14
### Changed
diff --git a/apps/api/src/app/benchmark/benchmark.service.ts b/apps/api/src/app/benchmark/benchmark.service.ts
index 73b48068b3..785c2801a4 100644
--- a/apps/api/src/app/benchmark/benchmark.service.ts
+++ b/apps/api/src/app/benchmark/benchmark.service.ts
@@ -66,11 +66,11 @@ export class BenchmarkService {
const promises: Promise<number>[] = [];
- const quotes = await this.dataProviderService.getQuotes(
- benchmarkAssetProfiles.map(({ dataSource, symbol }) => {
+ const quotes = await this.dataProviderService.getQuotes({
+ items: benchmarkAssetProfiles.map(({ dataSource, symbol }) => {
return { dataSource, symbol };
})
- );
+ });
for (const { dataSource, symbol } of benchmarkAssetProfiles) {
promises.push(this.marketDataService.getMax({ dataSource, symbol }));
diff --git a/apps/api/src/app/portfolio/current-rate.service.ts b/apps/api/src/app/portfolio/current-rate.service.ts
index 6d48573f7b..a4ee317bb0 100644
--- a/apps/api/src/app/portfolio/current-rate.service.ts
+++ b/apps/api/src/app/portfolio/current-rate.service.ts
@@ -38,7 +38,7 @@ export class CurrentRateService {
if (includeToday) {
promises.push(
this.dataProviderService
- .getQuotes(dataGatheringItems)
+ .getQuotes({ items: dataGatheringItems })
.then((dataResultProvider) => {
const result: GetValueObject[] = [];
for (const dataGatheringItem of dataGatheringItems) {
diff --git a/apps/api/src/app/portfolio/portfolio.service.ts b/apps/api/src/app/portfolio/portfolio.service.ts
index 66f3841a47..7d13728d2d 100644
--- a/apps/api/src/app/portfolio/portfolio.service.ts
+++ b/apps/api/src/app/portfolio/portfolio.service.ts
@@ -504,15 +504,17 @@ export class PortfolioService {
);
}
- const dataGatheringItems = currentPositions.positions.map((position) => {
- return {
- dataSource: position.dataSource,
- symbol: position.symbol
- };
- });
+ const dataGatheringItems = currentPositions.positions.map(
+ ({ dataSource, symbol }) => {
+ return {
+ dataSource,
+ symbol
+ };
+ }
+ );
const [dataProviderResponses, symbolProfiles] = await Promise.all([
- this.dataProviderService.getQuotes(dataGatheringItems),
+ this.dataProviderService.getQuotes({ items: dataGatheringItems }),
this.symbolProfileService.getSymbolProfiles(dataGatheringItems)
]);
@@ -897,9 +899,9 @@ export class PortfolioService {
)
};
} else {
- const currentData = await this.dataProviderService.getQuotes([
- { dataSource: DataSource.YAHOO, symbol: aSymbol }
- ]);
+ const currentData = await this.dataProviderService.getQuotes({
+ items: [{ dataSource: DataSource.YAHOO, symbol: aSymbol }]
+ });
const marketPrice = currentData[aSymbol]?.marketPrice;
let historicalData = await this.dataProviderService.getHistorical(
@@ -1000,15 +1002,15 @@ export class PortfolioService {
(item) => !item.quantity.eq(0)
);
- const dataGatheringItem = positions.map((position) => {
+ const dataGatheringItems = positions.map(({ dataSource, symbol }) => {
return {
- dataSource: position.dataSource,
- symbol: position.symbol
+ dataSource,
+ symbol
};
});
const [dataProviderResponses, symbolProfiles] = await Promise.all([
- this.dataProviderService.getQuotes(dataGatheringItem),
+ this.dataProviderService.getQuotes({ items: dataGatheringItems }),
this.symbolProfileService.getSymbolProfiles(
positions.map(({ dataSource, symbol }) => {
return { dataSource, symbol };
diff --git a/apps/api/src/app/symbol/symbol.service.ts b/apps/api/src/app/symbol/symbol.service.ts
index bc626a97f2..5eacbb1b02 100644
--- a/apps/api/src/app/symbol/symbol.service.ts
+++ b/apps/api/src/app/symbol/symbol.service.ts
@@ -27,9 +27,9 @@ export class SymbolService {
dataGatheringItem: IDataGatheringItem;
includeHistoricalData?: number;
}): Promise<SymbolItem> {
- const quotes = await this.dataProviderService.getQuotes([
- dataGatheringItem
- ]);
+ const quotes = await this.dataProviderService.getQuotes({
+ items: [dataGatheringItem]
+ });
const { currency, marketPrice } = quotes[dataGatheringItem.symbol] ?? {};
if (dataGatheringItem.dataSource && marketPrice >= 0) {
diff --git a/apps/api/src/services/data-provider/data-provider.service.ts b/apps/api/src/services/data-provider/data-provider.service.ts
index b4f4e10b58..557699495d 100644
--- a/apps/api/src/services/data-provider/data-provider.service.ts
+++ b/apps/api/src/services/data-provider/data-provider.service.ts
@@ -3,7 +3,6 @@ import { LookupItem } from '@ghostfolio/api/app/symbol/interfaces/lookup-item.in
import { ConfigurationService } from '@ghostfolio/api/services/configuration/configuration.service';
import { DataProviderInterface } from '@ghostfolio/api/services/data-provider/interfaces/data-provider.interface';
import {
- IDataGatheringItem,
IDataProviderHistoricalResponse,
IDataProviderResponse
} from '@ghostfolio/api/services/interfaces/interfaces';
@@ -12,6 +11,7 @@ import { PrismaService } from '@ghostfolio/api/services/prisma/prisma.service';
import { PropertyService } from '@ghostfolio/api/services/property/property.service';
import { PROPERTY_DATA_SOURCE_MAPPING } from '@ghostfolio/common/config';
import { DATE_FORMAT, getStartOfUtcDate } from '@ghostfolio/common/helper';
+import { UniqueAsset } from '@ghostfolio/common/interfaces';
import type { Granularity, UserWithSettings } from '@ghostfolio/common/types';
import { Inject, Injectable, Logger } from '@nestjs/common';
import { DataSource, MarketData, SymbolProfile } from '@prisma/client';
@@ -45,12 +45,15 @@ export class DataProviderService {
const dataProvider = this.getDataProvider(dataSource);
const symbol = dataProvider.getTestSymbol();
- const quotes = await this.getQuotes([
- {
- dataSource,
- symbol
- }
- ]);
+ const quotes = await this.getQuotes({
+ items: [
+ {
+ dataSource,
+ symbol
+ }
+ ],
+ useCache: false
+ });
if (quotes[symbol]?.marketPrice > 0) {
return true;
@@ -59,14 +62,16 @@ export class DataProviderService {
return false;
}
- public async getAssetProfiles(items: IDataGatheringItem[]): Promise<{
+ public async getAssetProfiles(items: UniqueAsset[]): Promise<{
[symbol: string]: Partial<SymbolProfile>;
}> {
const response: {
[symbol: string]: Partial<SymbolProfile>;
} = {};
- const itemsGroupedByDataSource = groupBy(items, (item) => item.dataSource);
+ const itemsGroupedByDataSource = groupBy(items, ({ dataSource }) => {
+ return dataSource;
+ });
const promises = [];
@@ -127,7 +132,7 @@ export class DataProviderService {
}
public async getHistorical(
- aItems: IDataGatheringItem[],
+ aItems: UniqueAsset[],
aGranularity: Granularity = 'month',
from: Date,
to: Date
@@ -155,11 +160,11 @@ export class DataProviderService {
)}'`
: '';
- const dataSources = aItems.map((item) => {
- return item.dataSource;
+ const dataSources = aItems.map(({ dataSource }) => {
+ return dataSource;
});
- const symbols = aItems.map((item) => {
- return item.symbol;
+ const symbols = aItems.map(({ symbol }) => {
+ return symbol;
});
try {
@@ -192,7 +197,7 @@ export class DataProviderService {
}
public async getHistoricalRaw(
- aDataGatheringItems: IDataGatheringItem[],
+ aDataGatheringItems: UniqueAsset[],
from: Date,
to: Date
): Promise<{
@@ -229,7 +234,13 @@ export class DataProviderService {
return result;
}
- public async getQuotes(items: IDataGatheringItem[]): Promise<{
+ public async getQuotes({
+ items,
+ useCache = true
+ }: {
+ items: UniqueAsset[];
+ useCache?: boolean;
+ }): Promise<{
[symbol: string]: IDataProviderResponse;
}> {
const response: {
@@ -238,23 +249,24 @@ export class DataProviderService {
const startTimeTotal = performance.now();
// Get items from cache
- const itemsToFetch: IDataGatheringItem[] = [];
+ const itemsToFetch: UniqueAsset[] = [];
for (const { dataSource, symbol } of items) {
- const quoteString = await this.redisCacheService.get(
- this.redisCacheService.getQuoteKey({ dataSource, symbol })
- );
+ if (useCache) {
+ const quoteString = await this.redisCacheService.get(
+ this.redisCacheService.getQuoteKey({ dataSource, symbol })
+ );
- if (quoteString) {
- try {
- const cachedDataProviderResponse = JSON.parse(quoteString);
- response[symbol] = cachedDataProviderResponse;
- } catch {}
+ if (quoteString) {
+ try {
+ const cachedDataProviderResponse = JSON.parse(quoteString);
+ response[symbol] = cachedDataProviderResponse;
+ continue;
+ } catch {}
+ }
}
- if (!quoteString) {
- itemsToFetch.push({ dataSource, symbol });
- }
+ itemsToFetch.push({ dataSource, symbol });
}
const numberOfItemsInCache = Object.keys(response)?.length;
diff --git a/apps/api/src/services/exchange-rate-data/exchange-rate-data.service.ts b/apps/api/src/services/exchange-rate-data/exchange-rate-data.service.ts
index 448933b421..9e1daa2241 100644
--- a/apps/api/src/services/exchange-rate-data/exchange-rate-data.service.ts
+++ b/apps/api/src/services/exchange-rate-data/exchange-rate-data.service.ts
@@ -64,11 +64,11 @@ export class ExchangeRateDataService {
if (Object.keys(result).length !== this.currencyPairs.length) {
// Load currencies directly from data provider as a fallback
// if historical data is not fully available
- const quotes = await this.dataProviderService.getQuotes(
- this.currencyPairs.map(({ dataSource, symbol }) => {
+ const quotes = await this.dataProviderService.getQuotes({
+ items: this.currencyPairs.map(({ dataSource, symbol }) => {
return { dataSource, symbol };
})
- );
+ });
for (const symbol of Object.keys(quotes)) {
if (isNumber(quotes[symbol].marketPrice)) {