Skip to content

Open Source Wealth Management Software. Angular + NestJS + Prisma + Nx + TypeScript 🤍

Feature/add caching for quotes (#2095)

Thomas Kaul54ea6c84b4ff
diff --git a/CHANGELOG.md b/CHANGELOG.md
index cc54f98b46..22c03989b0 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
### Added
+- Added the caching for current market prices
- Added a loading indicator to the import dividends dialog
### Changed
diff --git a/apps/api/src/app/portfolio/current-rate.service.spec.ts b/apps/api/src/app/portfolio/current-rate.service.spec.ts
index 0dd2b3a687..c9711aa7b5 100644
--- a/apps/api/src/app/portfolio/current-rate.service.spec.ts
+++ b/apps/api/src/app/portfolio/current-rate.service.spec.ts
@@ -98,7 +98,8 @@ describe('CurrentRateService', () => {
[],
null,
null,
- propertyService
+ propertyService,
+ null
);
exchangeRateDataService = new ExchangeRateDataService(
null,
diff --git a/apps/api/src/app/redis-cache/redis-cache.service.ts b/apps/api/src/app/redis-cache/redis-cache.service.ts
index 49f736bc5a..fb75460ed9 100644
--- a/apps/api/src/app/redis-cache/redis-cache.service.ts
+++ b/apps/api/src/app/redis-cache/redis-cache.service.ts
@@ -1,4 +1,5 @@
import { ConfigurationService } from '@ghostfolio/api/services/configuration/configuration.service';
+import { UniqueAsset } from '@ghostfolio/common/interfaces';
import { CACHE_MANAGER, Inject, Injectable } from '@nestjs/common';
import { Cache } from 'cache-manager';
@@ -13,6 +14,10 @@ export class RedisCacheService {
return await this.cache.get(key);
}
+ public getQuoteKey({ dataSource, symbol }: UniqueAsset) {
+ return `quote-${dataSource}-${symbol}`;
+ }
+
public async remove(key: string) {
await this.cache.del(key);
}
diff --git a/apps/api/src/services/configuration/configuration.service.ts b/apps/api/src/services/configuration/configuration.service.ts
index c5a4689056..e522aeccde 100644
--- a/apps/api/src/services/configuration/configuration.service.ts
+++ b/apps/api/src/services/configuration/configuration.service.ts
@@ -16,6 +16,7 @@ export class ConfigurationService {
default: 'USD'
}),
BETTER_UPTIME_API_KEY: str({ default: '' }),
+ CACHE_QUOTES_TTL: num({ default: 1 }),
CACHE_TTL: num({ default: 1 }),
DATA_SOURCE_EXCHANGE_RATES: str({ default: DataSource.YAHOO }),
DATA_SOURCE_IMPORT: str({ default: DataSource.YAHOO }),
diff --git a/apps/api/src/services/data-provider/data-provider.module.ts b/apps/api/src/services/data-provider/data-provider.module.ts
index 3ba678631f..b3a219a50a 100644
--- a/apps/api/src/services/data-provider/data-provider.module.ts
+++ b/apps/api/src/services/data-provider/data-provider.module.ts
@@ -1,3 +1,4 @@
+import { RedisCacheModule } from '@ghostfolio/api/app/redis-cache/redis-cache.module';
import { ConfigurationModule } from '@ghostfolio/api/services/configuration/configuration.module';
import { CryptocurrencyModule } from '@ghostfolio/api/services/cryptocurrency/cryptocurrency.module';
import { AlphaVantageService } from '@ghostfolio/api/services/data-provider/alpha-vantage/alpha-vantage.service';
@@ -26,6 +27,7 @@ import { DataProviderService } from './data-provider.service';
MarketDataModule,
PrismaModule,
PropertyModule,
+ RedisCacheModule,
SymbolProfileModule
],
providers: [
diff --git a/apps/api/src/services/data-provider/data-provider.service.ts b/apps/api/src/services/data-provider/data-provider.service.ts
index c05fc3e4fc..6209b89aae 100644
--- a/apps/api/src/services/data-provider/data-provider.service.ts
+++ b/apps/api/src/services/data-provider/data-provider.service.ts
@@ -1,3 +1,4 @@
+import { RedisCacheService } from '@ghostfolio/api/app/redis-cache/redis-cache.service';
import { LookupItem } from '@ghostfolio/api/app/symbol/interfaces/lookup-item.interface';
import { ConfigurationService } from '@ghostfolio/api/services/configuration/configuration.service';
import { DataProviderInterface } from '@ghostfolio/api/services/data-provider/interfaces/data-provider.interface';
@@ -27,7 +28,8 @@ export class DataProviderService {
private readonly dataProviderInterfaces: DataProviderInterface[],
private readonly marketDataService: MarketDataService,
private readonly prismaService: PrismaService,
- private readonly propertyService: PropertyService
+ private readonly propertyService: PropertyService,
+ private readonly redisCacheService: RedisCacheService
) {
this.initialize();
}
@@ -235,9 +237,43 @@ export class DataProviderService {
} = {};
const startTimeTotal = performance.now();
- const itemsGroupedByDataSource = groupBy(items, (item) => item.dataSource);
+ // Get items from cache
+ const itemsToFetch: IDataGatheringItem[] = [];
- const promises = [];
+ for (const { dataSource, symbol } of items) {
+ const quoteString = await this.redisCacheService.get(
+ this.redisCacheService.getQuoteKey({ dataSource, symbol })
+ );
+
+ if (quoteString) {
+ try {
+ const cachedDataProviderResponse = JSON.parse(quoteString);
+ response[symbol] = cachedDataProviderResponse;
+ } catch {}
+ }
+
+ if (!quoteString) {
+ itemsToFetch.push({ dataSource, symbol });
+ }
+ }
+
+ const numberOfItemsInCache = Object.keys(response)?.length;
+
+ if (numberOfItemsInCache) {
+ Logger.debug(
+ `Fetched ${numberOfItemsInCache} quote${
+ numberOfItemsInCache > 1 ? 's' : ''
+ } from cache in ${((performance.now() - startTimeTotal) / 1000).toFixed(
+ 3
+ )} seconds`
+ );
+ }
+
+ const itemsGroupedByDataSource = groupBy(itemsToFetch, ({ dataSource }) => {
+ return dataSource;
+ });
+
+ const promises: Promise<any>[] = [];
for (const [dataSource, dataGatheringItems] of Object.entries(
itemsGroupedByDataSource
@@ -271,6 +307,15 @@ export class DataProviderService {
result
)) {
response[symbol] = dataProviderResponse;
+
+ this.redisCacheService.set(
+ this.redisCacheService.getQuoteKey({
+ dataSource: DataSource[dataSource],
+ symbol
+ }),
+ JSON.stringify(dataProviderResponse),
+ this.configurationService.get('CACHE_QUOTES_TTL')
+ );
}
Logger.debug(
@@ -283,7 +328,7 @@ export class DataProviderService {
);
try {
- await this.marketDataService.updateMany({
+ this.marketDataService.updateMany({
data: Object.keys(response)
.filter((symbol) => {
return (
diff --git a/apps/api/src/services/interfaces/environment.interface.ts b/apps/api/src/services/interfaces/environment.interface.ts
index 1256f0f2f5..88fa4c3f55 100644
--- a/apps/api/src/services/interfaces/environment.interface.ts
+++ b/apps/api/src/services/interfaces/environment.interface.ts
@@ -5,6 +5,7 @@ export interface Environment extends CleanedEnvAccessors {
ALPHA_VANTAGE_API_KEY: string;
BASE_CURRENCY: string;
BETTER_UPTIME_API_KEY: string;
+ CACHE_QUOTES_TTL: number;
CACHE_TTL: number;
DATA_SOURCE_EXCHANGE_RATES: string;
DATA_SOURCE_IMPORT: string;