Open Source Wealth Management Software. Angular + NestJS + Prisma + Nx + TypeScript 🤍
TypeScript angularetffinancefintechghostfoliohacktoberfestinvestingnestjsosspersonal-financeportfolioprismasoftwarestocktrackertradingtypescriptwealth-managementweb
Feature/improve asset profile data gathering (#5997)
Thomas Kauld341c4804aba
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex aa78ea5c00..5acae5a13e 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0### Changed+- Restricted the asset profile data gathering on Sundays to only process outdated asset profiles- Eliminated `uuid` in favor of using `randomUUID` from `node:crypto`- Upgraded `color` from version `5.0.0` to `5.0.3`diff --git a/apps/api/src/app/admin/admin.controller.ts b/apps/api/src/app/admin/admin.controller.tsindex 8b5da49650..24467c7324 100644--- a/apps/api/src/app/admin/admin.controller.ts+++ b/apps/api/src/app/admin/admin.controller.ts@@ -93,7 +93,7 @@ export class AdminController {@UseGuards(AuthGuard('jwt'), HasPermissionGuard)public async gatherMax(): Promise<void> {const assetProfileIdentifiers =- await this.dataGatheringService.getAllActiveAssetProfileIdentifiers();+ await this.dataGatheringService.getActiveAssetProfileIdentifiers();await this.dataGatheringService.addJobsToQueue(assetProfileIdentifiers.map(({ dataSource, symbol }) => {@@ -120,7 +120,7 @@ export class AdminController {@UseGuards(AuthGuard('jwt'), HasPermissionGuard)public async gatherProfileData(): Promise<void> {const assetProfileIdentifiers =- await this.dataGatheringService.getAllActiveAssetProfileIdentifiers();+ await this.dataGatheringService.getActiveAssetProfileIdentifiers();await this.dataGatheringService.addJobsToQueue(assetProfileIdentifiers.map(({ dataSource, symbol }) => {diff --git a/apps/api/src/services/cron/cron.service.ts b/apps/api/src/services/cron/cron.service.tsindex 88fcabce2e..ee91a811e6 100644--- a/apps/api/src/services/cron/cron.service.ts+++ b/apps/api/src/services/cron/cron.service.ts@@ -59,7 +59,9 @@ export class CronService {public async runEverySundayAtTwelvePm() {if (await this.isDataGatheringEnabled()) {const assetProfileIdentifiers =- await this.dataGatheringService.getAllActiveAssetProfileIdentifiers();+ await this.dataGatheringService.getActiveAssetProfileIdentifiers({+ maxAge: '60 days'+ });await this.dataGatheringService.addJobsToQueue(assetProfileIdentifiers.map(({ dataSource, symbol }) => {diff --git a/apps/api/src/services/queues/data-gathering/data-gathering.service.ts b/apps/api/src/services/queues/data-gathering/data-gathering.service.tsindex c433f692f0..cec63c3ebc 100644--- a/apps/api/src/services/queues/data-gathering/data-gathering.service.ts+++ b/apps/api/src/services/queues/data-gathering/data-gathering.service.ts@@ -28,8 +28,9 @@ import { InjectQueue } from '@nestjs/bull';import { Inject, Injectable, Logger } from '@nestjs/common';import { DataSource } from '@prisma/client';import { JobOptions, Queue } from 'bull';-import { format, min, subDays, subYears } from 'date-fns';+import { format, min, subDays, subMilliseconds, subYears } from 'date-fns';import { isEmpty } from 'lodash';+import ms, { StringValue } from 'ms';@Injectable()export class DataGatheringService {@@ -160,8 +161,7 @@ export class DataGatheringService {);if (!assetProfileIdentifiers) {- assetProfileIdentifiers =- await this.getAllActiveAssetProfileIdentifiers();+ assetProfileIdentifiers = await this.getActiveAssetProfileIdentifiers();}if (assetProfileIdentifiers.length <= 0) {@@ -301,29 +301,36 @@ export class DataGatheringService {);}- public async getAllActiveAssetProfileIdentifiers(): Promise<- AssetProfileIdentifier[]- > {- const symbolProfiles = await this.prismaService.symbolProfile.findMany({- orderBy: [{ symbol: 'asc' }],+ /**+ * Returns active asset profile identifiers+ *+ * @param {StringValue} maxAge - Optional. Specifies the maximum allowed age+ * of a profile’s last update timestamp. Only asset profiles considered stale+ * are returned.+ */+ public async getActiveAssetProfileIdentifiers({+ maxAge+ }: {+ maxAge?: StringValue;+ } = {}): Promise<AssetProfileIdentifier[]> {+ return this.prismaService.symbolProfile.findMany({+ orderBy: [{ symbol: 'asc' }, { dataSource: 'asc' }],+ select: {+ dataSource: true,+ symbol: true+ },where: {- isActive: true+ dataSource: {+ notIn: ['MANUAL', 'RAPID_API']+ },+ isActive: true,+ ...(maxAge && {+ updatedAt: {+ lt: subMilliseconds(new Date(), ms(maxAge))+ }+ })}});-- return symbolProfiles- .filter(({ dataSource }) => {- return (- dataSource !== DataSource.MANUAL &&- dataSource !== DataSource.RAPID_API- );- })- .map(({ dataSource, symbol }) => {- return {- dataSource,- symbol- };- });}private async getAssetProfileIdentifiersWithCompleteMarketData(): Promise<