Skip to content

Open Source Wealth Management Software. Angular + NestJS + Prisma + Nx + TypeScript 🤍

Feature/optimize get range query in market data service (#4527)

Thomas Kaul6122da3f1454
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 100e43b5f4..673233c5f1 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -10,6 +10,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
### Changed
- Deactivated asset profiles automatically on delisting in the _Yahoo Finance_ service
+- Optimized the query of the data range functionality (`getRange()`) in the market data service
- Upgraded `Nx` from version `20.7.1` to `20.8.0`
- Upgraded `prisma` from version `6.5.0` to `6.6.0`
- Upgraded `storybook` from version `8.4.7` to `8.6.12`
diff --git a/apps/api/src/app/portfolio/current-rate.service.spec.ts b/apps/api/src/app/portfolio/current-rate.service.spec.ts
index d0e61c8ceb..d8b7482e72 100644
--- a/apps/api/src/app/portfolio/current-rate.service.spec.ts
+++ b/apps/api/src/app/portfolio/current-rate.service.spec.ts
@@ -6,6 +6,7 @@ import { AssetProfileIdentifier } from '@ghostfolio/common/interfaces';
import { DataSource, MarketData } from '@prisma/client';
import { CurrentRateService } from './current-rate.service';
+import { DateQuery } from './interfaces/date-query.interface';
import { GetValuesObject } from './interfaces/get-values-object.interface';
jest.mock('@ghostfolio/api/services/market-data/market-data.service', () => {
@@ -25,33 +26,40 @@ jest.mock('@ghostfolio/api/services/market-data/market-data.service', () => {
},
getRange: ({
assetProfileIdentifiers,
- dateRangeEnd,
- dateRangeStart
+ dateQuery
}: {
assetProfileIdentifiers: AssetProfileIdentifier[];
- dateRangeEnd: Date;
- dateRangeStart: Date;
+ dateQuery: DateQuery;
+ skip?: number;
+ take?: number;
}) => {
return Promise.resolve<MarketData[]>([
{
- createdAt: dateRangeStart,
+ createdAt: dateQuery.gte,
dataSource: assetProfileIdentifiers[0].dataSource,
- date: dateRangeStart,
+ date: dateQuery.gte,
id: '8fa48fde-f397-4b0d-adbc-fb940e830e6d',
marketPrice: 1841.823902,
state: 'CLOSE',
symbol: assetProfileIdentifiers[0].symbol
},
{
- createdAt: dateRangeEnd,
+ createdAt: dateQuery.lt,
dataSource: assetProfileIdentifiers[0].dataSource,
- date: dateRangeEnd,
+ date: dateQuery.lt,
id: '082d6893-df27-4c91-8a5d-092e84315b56',
marketPrice: 1847.839966,
state: 'CLOSE',
symbol: assetProfileIdentifiers[0].symbol
}
]);
+ },
+ getRangeCount: ({}: {
+ assetProfileIdentifiers: AssetProfileIdentifier[];
+ dateRangeEnd: Date;
+ dateRangeStart: Date;
+ }) => {
+ return Promise.resolve<number>(2);
}
};
})
@@ -128,9 +136,15 @@ describe('CurrentRateService', () => {
values: [
{
dataSource: 'YAHOO',
- date: undefined,
+ date: new Date('2020-01-01T00:00:00.000Z'),
marketPrice: 1841.823902,
symbol: 'AMZN'
+ },
+ {
+ dataSource: 'YAHOO',
+ date: new Date('2020-01-02T00:00:00.000Z'),
+ marketPrice: 1847.839966,
+ symbol: 'AMZN'
}
]
});
diff --git a/apps/api/src/app/portfolio/current-rate.service.ts b/apps/api/src/app/portfolio/current-rate.service.ts
index 058bf1dd53..5d39a54bb3 100644
--- a/apps/api/src/app/portfolio/current-rate.service.ts
+++ b/apps/api/src/app/portfolio/current-rate.service.ts
@@ -21,6 +21,8 @@ import { GetValuesParams } from './interfaces/get-values-params.interface';
@Injectable()
export class CurrentRateService {
+ private static readonly MARKET_DATA_PAGE_SIZE = 50000;
+
public constructor(
private readonly dataProviderService: DataProviderService,
private readonly marketDataService: MarketDataService,
@@ -41,42 +43,37 @@ export class CurrentRateService {
(!dateQuery.gte || isBefore(dateQuery.gte, new Date())) &&
(!dateQuery.in || this.containsToday(dateQuery.in));
- const promises: Promise<GetValueObject[]>[] = [];
const quoteErrors: ResponseError['errors'] = [];
const today = resetHours(new Date());
+ const values: GetValueObject[] = [];
if (includesToday) {
- promises.push(
- this.dataProviderService
- .getQuotes({ items: dataGatheringItems, user: this.request?.user })
- .then((dataResultProvider) => {
- const result: GetValueObject[] = [];
-
- for (const { dataSource, symbol } of dataGatheringItems) {
- if (dataResultProvider?.[symbol]?.dataProviderInfo) {
- dataProviderInfos.push(
- dataResultProvider[symbol].dataProviderInfo
- );
- }
+ const quotesBySymbol = await this.dataProviderService.getQuotes({
+ items: dataGatheringItems,
+ user: this.request?.user
+ });
- if (dataResultProvider?.[symbol]?.marketPrice) {
- result.push({
- dataSource,
- symbol,
- date: today,
- marketPrice: dataResultProvider?.[symbol]?.marketPrice
- });
- } else {
- quoteErrors.push({
- dataSource,
- symbol
- });
- }
- }
+ for (const { dataSource, symbol } of dataGatheringItems) {
+ const quote = quotesBySymbol[symbol];
- return result;
- })
- );
+ if (quote?.dataProviderInfo) {
+ dataProviderInfos.push(quote.dataProviderInfo);
+ }
+
+ if (quote?.marketPrice) {
+ values.push({
+ dataSource,
+ symbol,
+ date: today,
+ marketPrice: quote.marketPrice
+ });
+ } else {
+ quoteErrors.push({
+ dataSource,
+ symbol
+ });
+ }
+ }
}
const assetProfileIdentifiers: AssetProfileIdentifier[] =
@@ -84,34 +81,42 @@ export class CurrentRateService {
return { dataSource, symbol };
});
- promises.push(
- this.marketDataService
- .getRange({
- assetProfileIdentifiers,
- dateQuery
- })
- .then((data) => {
- return data.map(({ dataSource, date, marketPrice, symbol }) => {
- return {
- dataSource,
- date,
- marketPrice,
- symbol
- };
- });
- })
- );
-
- const values = await Promise.all(promises).then((array) => {
- return array.flat();
+ const marketDataCount = await this.marketDataService.getRangeCount({
+ assetProfileIdentifiers,
+ dateQuery
});
+ for (
+ let i = 0;
+ i < marketDataCount;
+ i += CurrentRateService.MARKET_DATA_PAGE_SIZE
+ ) {
+ // Use page size to limit the number of records fetched at once
+ const data = await this.marketDataService.getRange({
+ assetProfileIdentifiers,
+ dateQuery,
+ skip: i,
+ take: CurrentRateService.MARKET_DATA_PAGE_SIZE
+ });
+
+ values.push(
+ ...data.map(({ dataSource, date, marketPrice, symbol }) => ({
+ dataSource,
+ date,
+ marketPrice,
+ symbol
+ }))
+ );
+ }
+
const response: GetValuesObject = {
dataProviderInfos,
errors: quoteErrors.map(({ dataSource, symbol }) => {
return { dataSource, symbol };
}),
- values: uniqBy(values, ({ date, symbol }) => `${date}-${symbol}`)
+ values: uniqBy(values, ({ date, symbol }) => {
+ return `${date}-${symbol}`;
+ })
};
if (!isEmpty(quoteErrors)) {
diff --git a/apps/api/src/services/market-data/market-data.service.ts b/apps/api/src/services/market-data/market-data.service.ts
index 390586b370..58b9b09ec0 100644
--- a/apps/api/src/services/market-data/market-data.service.ts
+++ b/apps/api/src/services/market-data/market-data.service.ts
@@ -60,12 +60,18 @@ export class MarketDataService {
public async getRange({
assetProfileIdentifiers,
- dateQuery
+ dateQuery,
+ skip,
+ take
}: {
assetProfileIdentifiers: AssetProfileIdentifier[];
dateQuery: DateQuery;
+ skip?: number;
+ take?: number;
}): Promise<MarketData[]> {
return this.prismaService.marketData.findMany({
+ skip,
+ take,
orderBy: [
{
date: 'asc'
@@ -75,17 +81,33 @@ export class MarketDataService {
}
],
where: {
- dataSource: {
- in: assetProfileIdentifiers.map(({ dataSource }) => {
- return dataSource;
- })
- },
date: dateQuery,
- symbol: {
- in: assetProfileIdentifiers.map(({ symbol }) => {
- return symbol;
- })
- }
+ OR: assetProfileIdentifiers.map(({ dataSource, symbol }) => {
+ return {
+ dataSource,
+ symbol
+ };
+ })
+ }
+ });
+ }
+
+ public async getRangeCount({
+ assetProfileIdentifiers,
+ dateQuery
+ }: {
+ assetProfileIdentifiers: AssetProfileIdentifier[];
+ dateQuery: DateQuery;
+ }): Promise<number> {
+ return this.prismaService.marketData.count({
+ where: {
+ date: dateQuery,
+ OR: assetProfileIdentifiers.map(({ dataSource, symbol }) => {
+ return {
+ dataSource,
+ symbol
+ };
+ })
}
});
}