Open Source Wealth Management Software. Angular + NestJS + Prisma + Nx + TypeScript 🤍
TypeScript angularetffinancefintechghostfoliohacktoberfestinvestingnestjsosspersonal-financeportfolioprismasoftwarestocktrackertradingtypescriptwealth-managementweb
Feature/optimize 7d data gathering by prioritization (#3575)
Thomas Kauld5c56fb16c93
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex b26a39a5c9..5f466e0935 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0## Unreleased+### Changed++- Optimized the 7d data gathering by prioritizing the currencies+### Fixed- Fixed the table sorting of the holdings tab on the home pagediff --git a/apps/api/src/app/admin/admin.controller.ts b/apps/api/src/app/admin/admin.controller.tsindex 4494fef7ab..69e6955c1c 100644--- a/apps/api/src/app/admin/admin.controller.ts+++ b/apps/api/src/app/admin/admin.controller.ts@@ -81,10 +81,11 @@ export class AdminController {@Post('gather/max')@UseGuards(AuthGuard('jwt'), HasPermissionGuard)public async gatherMax(): Promise<void> {- const uniqueAssets = await this.dataGatheringService.getUniqueAssets();+ const assetProfileIdentifiers =+ await this.dataGatheringService.getAllAssetProfileIdentifiers();await this.dataGatheringService.addJobsToQueue(- uniqueAssets.map(({ dataSource, symbol }) => {+ assetProfileIdentifiers.map(({ dataSource, symbol }) => {return {data: {dataSource,@@ -107,10 +108,11 @@ export class AdminController {@Post('gather/profile-data')@UseGuards(AuthGuard('jwt'), HasPermissionGuard)public async gatherProfileData(): Promise<void> {- const uniqueAssets = await this.dataGatheringService.getUniqueAssets();+ const assetProfileIdentifiers =+ await this.dataGatheringService.getAllAssetProfileIdentifiers();await this.dataGatheringService.addJobsToQueue(- uniqueAssets.map(({ dataSource, symbol }) => {+ assetProfileIdentifiers.map(({ dataSource, symbol }) => {return {data: {dataSource,diff --git a/apps/api/src/services/cron.service.ts b/apps/api/src/services/cron.service.tsindex fc5d613a20..864891c6a6 100644--- a/apps/api/src/services/cron.service.ts+++ b/apps/api/src/services/cron.service.ts@@ -45,10 +45,11 @@ export class CronService {@Cron(CronService.EVERY_SUNDAY_AT_LUNCH_TIME)public async runEverySundayAtTwelvePm() {if (await this.isDataGatheringEnabled()) {- const uniqueAssets = await this.dataGatheringService.getUniqueAssets();+ const assetProfileIdentifiers =+ await this.dataGatheringService.getAllAssetProfileIdentifiers();await this.dataGatheringService.addJobsToQueue(- uniqueAssets.map(({ dataSource, symbol }) => {+ assetProfileIdentifiers.map(({ dataSource, symbol }) => {return {data: {dataSource,diff --git a/apps/api/src/services/data-gathering/data-gathering.service.ts b/apps/api/src/services/data-gathering/data-gathering.service.tsindex a80d68d6b0..2bf6cc1b20 100644--- a/apps/api/src/services/data-gathering/data-gathering.service.ts+++ b/apps/api/src/services/data-gathering/data-gathering.service.ts@@ -10,6 +10,7 @@ import {DATA_GATHERING_QUEUE,DATA_GATHERING_QUEUE_PRIORITY_HIGH,DATA_GATHERING_QUEUE_PRIORITY_LOW,+ DATA_GATHERING_QUEUE_PRIORITY_MEDIUM,GATHER_HISTORICAL_MARKET_DATA_PROCESS,GATHER_HISTORICAL_MARKET_DATA_PROCESS_OPTIONS,PROPERTY_BENCHMARKS@@ -62,9 +63,22 @@ export class DataGatheringService {}public async gather7Days() {- const dataGatheringItems = await this.getSymbols7D();await this.gatherSymbols({- dataGatheringItems,+ dataGatheringItems: await this.getCurrencies7D(),+ priority: DATA_GATHERING_QUEUE_PRIORITY_HIGH+ });++ await this.gatherSymbols({+ dataGatheringItems: await this.getSymbols7D({+ withUserSubscription: true+ }),+ priority: DATA_GATHERING_QUEUE_PRIORITY_MEDIUM+ });++ await this.gatherSymbols({+ dataGatheringItems: await this.getSymbols7D({+ withUserSubscription: false+ }),priority: DATA_GATHERING_QUEUE_PRIORITY_LOW});}@@ -138,7 +152,7 @@ export class DataGatheringService {});if (!uniqueAssets) {- uniqueAssets = await this.getUniqueAssets();+ uniqueAssets = await this.getAllAssetProfileIdentifiers();}if (uniqueAssets.length <= 0) {@@ -270,7 +284,7 @@ export class DataGatheringService {);}- public async getUniqueAssets(): Promise<UniqueAsset[]> {+ public async getAllAssetProfileIdentifiers(): Promise<UniqueAsset[]> {const symbolProfiles = await this.prismaService.symbolProfile.findMany({orderBy: [{ symbol: 'asc' }]});@@ -290,73 +304,83 @@ export class DataGatheringService {});}- private getEarliestDate(aStartDate: Date) {- return min([aStartDate, subYears(new Date(), 10)]);- }-- private async getSymbols7D(): Promise<IDataGatheringItem[]> {- const startDate = subDays(resetHours(new Date()), 7);-- const symbolProfiles = await this.prismaService.symbolProfile.findMany({- orderBy: [{ symbol: 'asc' }],- select: {- dataSource: true,- scraperConfiguration: true,- symbol: true- }- });-- // Only consider symbols with incomplete market data for the last- // 7 days- const symbolsWithCompleteMarketData = (+ private async getAssetProfileIdentifiersWithCompleteMarketData(): Promise<+ UniqueAsset[]+ > {+ return (await this.prismaService.marketData.groupBy({_count: true,- by: ['symbol'],+ by: ['dataSource', 'symbol'],orderBy: [{ symbol: 'asc' }],where: {- date: { gt: startDate },+ date: { gt: subDays(resetHours(new Date()), 7) },state: 'CLOSE'}}))- .filter((group) => {- return group._count >= 6;+ .filter(({ _count }) => {+ return _count >= 6;})- .map((group) => {- return group.symbol;+ .map(({ dataSource, symbol }) => {+ return { dataSource, symbol };});+ }++ private async getCurrencies7D(): Promise<IDataGatheringItem[]> {+ const assetProfileIdentifiersWithCompleteMarketData =+ await this.getAssetProfileIdentifiersWithCompleteMarketData();- const symbolProfilesToGather = symbolProfiles+ return this.exchangeRateDataService+ .getCurrencyPairs()+ .filter(({ dataSource, symbol }) => {+ return !assetProfileIdentifiersWithCompleteMarketData.some((item) => {+ return item.dataSource === dataSource && item.symbol === symbol;+ });+ })+ .map(({ dataSource, symbol }) => {+ return {+ dataSource,+ symbol,+ date: subDays(resetHours(new Date()), 7)+ };+ });+ }++ private getEarliestDate(aStartDate: Date) {+ return min([aStartDate, subYears(new Date(), 10)]);+ }++ private async getSymbols7D({+ withUserSubscription = false+ }: {+ withUserSubscription?: boolean;+ }): Promise<IDataGatheringItem[]> {+ const symbolProfiles =+ await this.symbolProfileService.getSymbolProfilesByUserSubscription({+ withUserSubscription+ });++ const assetProfileIdentifiersWithCompleteMarketData =+ await this.getAssetProfileIdentifiersWithCompleteMarketData();++ return symbolProfiles.filter(({ dataSource, scraperConfiguration, symbol }) => {const manualDataSourceWithScraperConfiguration =dataSource === 'MANUAL' && !isEmpty(scraperConfiguration);return (- !symbolsWithCompleteMarketData.includes(symbol) &&+ !assetProfileIdentifiersWithCompleteMarketData.some((item) => {+ return item.dataSource === dataSource && item.symbol === symbol;+ }) &&(dataSource !== 'MANUAL' || manualDataSourceWithScraperConfiguration));}).map((symbolProfile) => {return {...symbolProfile,- date: startDate+ date: subDays(resetHours(new Date()), 7)};});-- const currencyPairsToGather = this.exchangeRateDataService- .getCurrencyPairs()- .filter(({ symbol }) => {- return !symbolsWithCompleteMarketData.includes(symbol);- })- .map(({ dataSource, symbol }) => {- return {- dataSource,- symbol,- date: startDate- };- });-- return [...currencyPairsToGather, ...symbolProfilesToGather];}private async getSymbolsMax(): Promise<IDataGatheringItem[]> {diff --git a/apps/api/src/services/symbol-profile/symbol-profile.service.ts b/apps/api/src/services/symbol-profile/symbol-profile.service.tsindex 1d7ea556b9..e0cfed2925 100644--- a/apps/api/src/services/symbol-profile/symbol-profile.service.ts+++ b/apps/api/src/services/symbol-profile/symbol-profile.service.ts@@ -91,6 +91,40 @@ export class SymbolProfileService {});}+ public async getSymbolProfilesByUserSubscription({+ withUserSubscription = false+ }: {+ withUserSubscription?: boolean;+ }) {+ return this.prismaService.symbolProfile.findMany({+ include: {+ Order: {+ include: {+ User: true+ }+ }+ },+ orderBy: [{ symbol: 'asc' }],+ where: {+ Order: withUserSubscription+ ? {+ some: {+ User: {+ Subscription: { some: { expiresAt: { gt: new Date() } } }+ }+ }+ }+ : {+ every: {+ User: {+ Subscription: { none: { expiresAt: { gt: new Date() } } }+ }+ }+ }+ }+ });+ }+public updateSymbolProfile({assetClass,assetSubClass,