Open Source Wealth Management Software. Angular + NestJS + Prisma + Nx + TypeScript 🤍
TypeScript angularetffinancefintechghostfoliohacktoberfestinvestingnestjsosspersonal-financeportfolioprismasoftwarestocktrackertradingtypescriptwealth-managementweb
Feature/add caching for quotes (#2095)
Thomas Kaul54ea6c84b4ff
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex cc54f98b46..22c03989b0 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0### Added+- Added the caching for current market prices- Added a loading indicator to the import dividends dialog### Changeddiff --git a/apps/api/src/app/portfolio/current-rate.service.spec.ts b/apps/api/src/app/portfolio/current-rate.service.spec.tsindex 0dd2b3a687..c9711aa7b5 100644--- a/apps/api/src/app/portfolio/current-rate.service.spec.ts+++ b/apps/api/src/app/portfolio/current-rate.service.spec.ts@@ -98,7 +98,8 @@ describe('CurrentRateService', () => {[],null,null,- propertyService+ propertyService,+ null);exchangeRateDataService = new ExchangeRateDataService(null,diff --git a/apps/api/src/app/redis-cache/redis-cache.service.ts b/apps/api/src/app/redis-cache/redis-cache.service.tsindex 49f736bc5a..fb75460ed9 100644--- a/apps/api/src/app/redis-cache/redis-cache.service.ts+++ b/apps/api/src/app/redis-cache/redis-cache.service.ts@@ -1,4 +1,5 @@import { ConfigurationService } from '@ghostfolio/api/services/configuration/configuration.service';+import { UniqueAsset } from '@ghostfolio/common/interfaces';import { CACHE_MANAGER, Inject, Injectable } from '@nestjs/common';import { Cache } from 'cache-manager';@@ -13,6 +14,10 @@ export class RedisCacheService {return await this.cache.get(key);}+ public getQuoteKey({ dataSource, symbol }: UniqueAsset) {+ return `quote-${dataSource}-${symbol}`;+ }+public async remove(key: string) {await this.cache.del(key);}diff --git a/apps/api/src/services/configuration/configuration.service.ts b/apps/api/src/services/configuration/configuration.service.tsindex c5a4689056..e522aeccde 100644--- a/apps/api/src/services/configuration/configuration.service.ts+++ b/apps/api/src/services/configuration/configuration.service.ts@@ -16,6 +16,7 @@ export class ConfigurationService {default: 'USD'}),BETTER_UPTIME_API_KEY: str({ default: '' }),+ CACHE_QUOTES_TTL: num({ default: 1 }),CACHE_TTL: num({ default: 1 }),DATA_SOURCE_EXCHANGE_RATES: str({ default: DataSource.YAHOO }),DATA_SOURCE_IMPORT: str({ default: DataSource.YAHOO }),diff --git a/apps/api/src/services/data-provider/data-provider.module.ts b/apps/api/src/services/data-provider/data-provider.module.tsindex 3ba678631f..b3a219a50a 100644--- a/apps/api/src/services/data-provider/data-provider.module.ts+++ b/apps/api/src/services/data-provider/data-provider.module.ts@@ -1,3 +1,4 @@+import { RedisCacheModule } from '@ghostfolio/api/app/redis-cache/redis-cache.module';import { ConfigurationModule } from '@ghostfolio/api/services/configuration/configuration.module';import { CryptocurrencyModule } from '@ghostfolio/api/services/cryptocurrency/cryptocurrency.module';import { AlphaVantageService } from '@ghostfolio/api/services/data-provider/alpha-vantage/alpha-vantage.service';@@ -26,6 +27,7 @@ import { DataProviderService } from './data-provider.service';MarketDataModule,PrismaModule,PropertyModule,+ RedisCacheModule,SymbolProfileModule],providers: [diff --git a/apps/api/src/services/data-provider/data-provider.service.ts b/apps/api/src/services/data-provider/data-provider.service.tsindex c05fc3e4fc..6209b89aae 100644--- a/apps/api/src/services/data-provider/data-provider.service.ts+++ b/apps/api/src/services/data-provider/data-provider.service.ts@@ -1,3 +1,4 @@+import { RedisCacheService } from '@ghostfolio/api/app/redis-cache/redis-cache.service';import { LookupItem } from '@ghostfolio/api/app/symbol/interfaces/lookup-item.interface';import { ConfigurationService } from '@ghostfolio/api/services/configuration/configuration.service';import { DataProviderInterface } from '@ghostfolio/api/services/data-provider/interfaces/data-provider.interface';@@ -27,7 +28,8 @@ export class DataProviderService {private readonly dataProviderInterfaces: DataProviderInterface[],private readonly marketDataService: MarketDataService,private readonly prismaService: PrismaService,- private readonly propertyService: PropertyService+ private readonly propertyService: PropertyService,+ private readonly redisCacheService: RedisCacheService) {this.initialize();}@@ -235,9 +237,43 @@ export class DataProviderService {} = {};const startTimeTotal = performance.now();- const itemsGroupedByDataSource = groupBy(items, (item) => item.dataSource);+ // Get items from cache+ const itemsToFetch: IDataGatheringItem[] = [];- const promises = [];+ for (const { dataSource, symbol } of items) {+ const quoteString = await this.redisCacheService.get(+ this.redisCacheService.getQuoteKey({ dataSource, symbol })+ );++ if (quoteString) {+ try {+ const cachedDataProviderResponse = JSON.parse(quoteString);+ response[symbol] = cachedDataProviderResponse;+ } catch {}+ }++ if (!quoteString) {+ itemsToFetch.push({ dataSource, symbol });+ }+ }++ const numberOfItemsInCache = Object.keys(response)?.length;++ if (numberOfItemsInCache) {+ Logger.debug(+ `Fetched ${numberOfItemsInCache} quote${+ numberOfItemsInCache > 1 ? 's' : ''+ } from cache in ${((performance.now() - startTimeTotal) / 1000).toFixed(+ 3+ )} seconds`+ );+ }++ const itemsGroupedByDataSource = groupBy(itemsToFetch, ({ dataSource }) => {+ return dataSource;+ });++ const promises: Promise<any>[] = [];for (const [dataSource, dataGatheringItems] of Object.entries(itemsGroupedByDataSource@@ -271,6 +307,15 @@ export class DataProviderService {result)) {response[symbol] = dataProviderResponse;++ this.redisCacheService.set(+ this.redisCacheService.getQuoteKey({+ dataSource: DataSource[dataSource],+ symbol+ }),+ JSON.stringify(dataProviderResponse),+ this.configurationService.get('CACHE_QUOTES_TTL')+ );}Logger.debug(@@ -283,7 +328,7 @@ export class DataProviderService {);try {- await this.marketDataService.updateMany({+ this.marketDataService.updateMany({data: Object.keys(response).filter((symbol) => {return (diff --git a/apps/api/src/services/interfaces/environment.interface.ts b/apps/api/src/services/interfaces/environment.interface.tsindex 1256f0f2f5..88fa4c3f55 100644--- a/apps/api/src/services/interfaces/environment.interface.ts+++ b/apps/api/src/services/interfaces/environment.interface.ts@@ -5,6 +5,7 @@ export interface Environment extends CleanedEnvAccessors {ALPHA_VANTAGE_API_KEY: string;BASE_CURRENCY: string;BETTER_UPTIME_API_KEY: string;+ CACHE_QUOTES_TTL: number;CACHE_TTL: number;DATA_SOURCE_EXCHANGE_RATES: string;DATA_SOURCE_IMPORT: string;