Open Source Wealth Management Software. Angular + NestJS + Prisma + Nx + TypeScript 🤍
TypeScript angularetffinancefintechghostfoliohacktoberfestinvestingnestjsosspersonal-financeportfolioprismasoftwarestocktrackertradingtypescriptwealth-managementweb
Feature/optimize get range query in market data service (#4527)
Thomas Kaul6122da3f1454
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 100e43b5f4..673233c5f1 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -10,6 +10,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0### Changed- Deactivated asset profiles automatically on delisting in the _Yahoo Finance_ service+- Optimized the query of the data range functionality (`getRange()`) in the market data service- Upgraded `Nx` from version `20.7.1` to `20.8.0`- Upgraded `prisma` from version `6.5.0` to `6.6.0`- Upgraded `storybook` from version `8.4.7` to `8.6.12`diff --git a/apps/api/src/app/portfolio/current-rate.service.spec.ts b/apps/api/src/app/portfolio/current-rate.service.spec.tsindex d0e61c8ceb..d8b7482e72 100644--- a/apps/api/src/app/portfolio/current-rate.service.spec.ts+++ b/apps/api/src/app/portfolio/current-rate.service.spec.ts@@ -6,6 +6,7 @@ import { AssetProfileIdentifier } from '@ghostfolio/common/interfaces';import { DataSource, MarketData } from '@prisma/client';import { CurrentRateService } from './current-rate.service';+import { DateQuery } from './interfaces/date-query.interface';import { GetValuesObject } from './interfaces/get-values-object.interface';jest.mock('@ghostfolio/api/services/market-data/market-data.service', () => {@@ -25,33 +26,40 @@ jest.mock('@ghostfolio/api/services/market-data/market-data.service', () => {},getRange: ({assetProfileIdentifiers,- dateRangeEnd,- dateRangeStart+ dateQuery}: {assetProfileIdentifiers: AssetProfileIdentifier[];- dateRangeEnd: Date;- dateRangeStart: Date;+ dateQuery: DateQuery;+ skip?: number;+ take?: number;}) => {return Promise.resolve<MarketData[]>([{- createdAt: dateRangeStart,+ createdAt: dateQuery.gte,dataSource: assetProfileIdentifiers[0].dataSource,- date: dateRangeStart,+ date: dateQuery.gte,id: '8fa48fde-f397-4b0d-adbc-fb940e830e6d',marketPrice: 1841.823902,state: 'CLOSE',symbol: assetProfileIdentifiers[0].symbol},{- createdAt: dateRangeEnd,+ createdAt: dateQuery.lt,dataSource: assetProfileIdentifiers[0].dataSource,- date: dateRangeEnd,+ date: dateQuery.lt,id: '082d6893-df27-4c91-8a5d-092e84315b56',marketPrice: 1847.839966,state: 'CLOSE',symbol: assetProfileIdentifiers[0].symbol}]);+ },+ getRangeCount: ({}: {+ assetProfileIdentifiers: AssetProfileIdentifier[];+ dateRangeEnd: Date;+ dateRangeStart: Date;+ }) => {+ return Promise.resolve<number>(2);}};})@@ -128,9 +136,15 @@ describe('CurrentRateService', () => {values: [{dataSource: 'YAHOO',- date: undefined,+ date: new Date('2020-01-01T00:00:00.000Z'),marketPrice: 1841.823902,symbol: 'AMZN'+ },+ {+ dataSource: 'YAHOO',+ date: new Date('2020-01-02T00:00:00.000Z'),+ marketPrice: 1847.839966,+ symbol: 'AMZN'}]});diff --git a/apps/api/src/app/portfolio/current-rate.service.ts b/apps/api/src/app/portfolio/current-rate.service.tsindex 058bf1dd53..5d39a54bb3 100644--- a/apps/api/src/app/portfolio/current-rate.service.ts+++ b/apps/api/src/app/portfolio/current-rate.service.ts@@ -21,6 +21,8 @@ import { GetValuesParams } from './interfaces/get-values-params.interface';@Injectable()export class CurrentRateService {+ private static readonly MARKET_DATA_PAGE_SIZE = 50000;+public constructor(private readonly dataProviderService: DataProviderService,private readonly marketDataService: MarketDataService,@@ -41,42 +43,37 @@ export class CurrentRateService {(!dateQuery.gte || isBefore(dateQuery.gte, new Date())) &&(!dateQuery.in || this.containsToday(dateQuery.in));- const promises: Promise<GetValueObject[]>[] = [];const quoteErrors: ResponseError['errors'] = [];const today = resetHours(new Date());+ const values: GetValueObject[] = [];if (includesToday) {- promises.push(- this.dataProviderService- .getQuotes({ items: dataGatheringItems, user: this.request?.user })- .then((dataResultProvider) => {- const result: GetValueObject[] = [];-- for (const { dataSource, symbol } of dataGatheringItems) {- if (dataResultProvider?.[symbol]?.dataProviderInfo) {- dataProviderInfos.push(- dataResultProvider[symbol].dataProviderInfo- );- }+ const quotesBySymbol = await this.dataProviderService.getQuotes({+ items: dataGatheringItems,+ user: this.request?.user+ });- if (dataResultProvider?.[symbol]?.marketPrice) {- result.push({- dataSource,- symbol,- date: today,- marketPrice: dataResultProvider?.[symbol]?.marketPrice- });- } else {- quoteErrors.push({- dataSource,- symbol- });- }- }+ for (const { dataSource, symbol } of dataGatheringItems) {+ const quote = quotesBySymbol[symbol];- return result;- })- );+ if (quote?.dataProviderInfo) {+ dataProviderInfos.push(quote.dataProviderInfo);+ }++ if (quote?.marketPrice) {+ values.push({+ dataSource,+ symbol,+ date: today,+ marketPrice: quote.marketPrice+ });+ } else {+ quoteErrors.push({+ dataSource,+ symbol+ });+ }+ }}const assetProfileIdentifiers: AssetProfileIdentifier[] =@@ -84,34 +81,42 @@ export class CurrentRateService {return { dataSource, symbol };});- promises.push(- this.marketDataService- .getRange({- assetProfileIdentifiers,- dateQuery- })- .then((data) => {- return data.map(({ dataSource, date, marketPrice, symbol }) => {- return {- dataSource,- date,- marketPrice,- symbol- };- });- })- );-- const values = await Promise.all(promises).then((array) => {- return array.flat();+ const marketDataCount = await this.marketDataService.getRangeCount({+ assetProfileIdentifiers,+ dateQuery});+ for (+ let i = 0;+ i < marketDataCount;+ i += CurrentRateService.MARKET_DATA_PAGE_SIZE+ ) {+ // Use page size to limit the number of records fetched at once+ const data = await this.marketDataService.getRange({+ assetProfileIdentifiers,+ dateQuery,+ skip: i,+ take: CurrentRateService.MARKET_DATA_PAGE_SIZE+ });++ values.push(+ ...data.map(({ dataSource, date, marketPrice, symbol }) => ({+ dataSource,+ date,+ marketPrice,+ symbol+ }))+ );+ }+const response: GetValuesObject = {dataProviderInfos,errors: quoteErrors.map(({ dataSource, symbol }) => {return { dataSource, symbol };}),- values: uniqBy(values, ({ date, symbol }) => `${date}-${symbol}`)+ values: uniqBy(values, ({ date, symbol }) => {+ return `${date}-${symbol}`;+ })};if (!isEmpty(quoteErrors)) {diff --git a/apps/api/src/services/market-data/market-data.service.ts b/apps/api/src/services/market-data/market-data.service.tsindex 390586b370..58b9b09ec0 100644--- a/apps/api/src/services/market-data/market-data.service.ts+++ b/apps/api/src/services/market-data/market-data.service.ts@@ -60,12 +60,18 @@ export class MarketDataService {public async getRange({assetProfileIdentifiers,- dateQuery+ dateQuery,+ skip,+ take}: {assetProfileIdentifiers: AssetProfileIdentifier[];dateQuery: DateQuery;+ skip?: number;+ take?: number;}): Promise<MarketData[]> {return this.prismaService.marketData.findMany({+ skip,+ take,orderBy: [{date: 'asc'@@ -75,17 +81,33 @@ export class MarketDataService {}],where: {- dataSource: {- in: assetProfileIdentifiers.map(({ dataSource }) => {- return dataSource;- })- },date: dateQuery,- symbol: {- in: assetProfileIdentifiers.map(({ symbol }) => {- return symbol;- })- }+ OR: assetProfileIdentifiers.map(({ dataSource, symbol }) => {+ return {+ dataSource,+ symbol+ };+ })+ }+ });+ }++ public async getRangeCount({+ assetProfileIdentifiers,+ dateQuery+ }: {+ assetProfileIdentifiers: AssetProfileIdentifier[];+ dateQuery: DateQuery;+ }): Promise<number> {+ return this.prismaService.marketData.count({+ where: {+ date: dateQuery,+ OR: assetProfileIdentifiers.map(({ dataSource, symbol }) => {+ return {+ dataSource,+ symbol+ };+ })}});}