Skip to content

Open Source Wealth Management Software. Angular + NestJS + Prisma + Nx + TypeScript 🤍

Feature/improve asset profile data gathering (#5997)

Thomas Kauld341c4804aba
diff --git a/CHANGELOG.md b/CHANGELOG.md
index aa78ea5c00..5acae5a13e 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
### Changed
+- Restricted the asset profile data gathering on Sundays to only process outdated asset profiles
- Eliminated `uuid` in favor of using `randomUUID` from `node:crypto`
- Upgraded `color` from version `5.0.0` to `5.0.3`
diff --git a/apps/api/src/app/admin/admin.controller.ts b/apps/api/src/app/admin/admin.controller.ts
index 8b5da49650..24467c7324 100644
--- a/apps/api/src/app/admin/admin.controller.ts
+++ b/apps/api/src/app/admin/admin.controller.ts
@@ -93,7 +93,7 @@ export class AdminController {
@UseGuards(AuthGuard('jwt'), HasPermissionGuard)
public async gatherMax(): Promise<void> {
const assetProfileIdentifiers =
- await this.dataGatheringService.getAllActiveAssetProfileIdentifiers();
+ await this.dataGatheringService.getActiveAssetProfileIdentifiers();
await this.dataGatheringService.addJobsToQueue(
assetProfileIdentifiers.map(({ dataSource, symbol }) => {
@@ -120,7 +120,7 @@ export class AdminController {
@UseGuards(AuthGuard('jwt'), HasPermissionGuard)
public async gatherProfileData(): Promise<void> {
const assetProfileIdentifiers =
- await this.dataGatheringService.getAllActiveAssetProfileIdentifiers();
+ await this.dataGatheringService.getActiveAssetProfileIdentifiers();
await this.dataGatheringService.addJobsToQueue(
assetProfileIdentifiers.map(({ dataSource, symbol }) => {
diff --git a/apps/api/src/services/cron/cron.service.ts b/apps/api/src/services/cron/cron.service.ts
index 88fcabce2e..ee91a811e6 100644
--- a/apps/api/src/services/cron/cron.service.ts
+++ b/apps/api/src/services/cron/cron.service.ts
@@ -59,7 +59,9 @@ export class CronService {
public async runEverySundayAtTwelvePm() {
if (await this.isDataGatheringEnabled()) {
const assetProfileIdentifiers =
- await this.dataGatheringService.getAllActiveAssetProfileIdentifiers();
+ await this.dataGatheringService.getActiveAssetProfileIdentifiers({
+ maxAge: '60 days'
+ });
await this.dataGatheringService.addJobsToQueue(
assetProfileIdentifiers.map(({ dataSource, symbol }) => {
diff --git a/apps/api/src/services/queues/data-gathering/data-gathering.service.ts b/apps/api/src/services/queues/data-gathering/data-gathering.service.ts
index c433f692f0..cec63c3ebc 100644
--- a/apps/api/src/services/queues/data-gathering/data-gathering.service.ts
+++ b/apps/api/src/services/queues/data-gathering/data-gathering.service.ts
@@ -28,8 +28,9 @@ import { InjectQueue } from '@nestjs/bull';
import { Inject, Injectable, Logger } from '@nestjs/common';
import { DataSource } from '@prisma/client';
import { JobOptions, Queue } from 'bull';
-import { format, min, subDays, subYears } from 'date-fns';
+import { format, min, subDays, subMilliseconds, subYears } from 'date-fns';
import { isEmpty } from 'lodash';
+import ms, { StringValue } from 'ms';
@Injectable()
export class DataGatheringService {
@@ -160,8 +161,7 @@ export class DataGatheringService {
);
if (!assetProfileIdentifiers) {
- assetProfileIdentifiers =
- await this.getAllActiveAssetProfileIdentifiers();
+ assetProfileIdentifiers = await this.getActiveAssetProfileIdentifiers();
}
if (assetProfileIdentifiers.length <= 0) {
@@ -301,29 +301,36 @@ export class DataGatheringService {
);
}
- public async getAllActiveAssetProfileIdentifiers(): Promise<
- AssetProfileIdentifier[]
- > {
- const symbolProfiles = await this.prismaService.symbolProfile.findMany({
- orderBy: [{ symbol: 'asc' }],
+ /**
+ * Returns active asset profile identifiers
+ *
+ * @param {StringValue} maxAge - Optional. Specifies the maximum allowed age
+ * of a profile’s last update timestamp. Only asset profiles considered stale
+ * are returned.
+ */
+ public async getActiveAssetProfileIdentifiers({
+ maxAge
+ }: {
+ maxAge?: StringValue;
+ } = {}): Promise<AssetProfileIdentifier[]> {
+ return this.prismaService.symbolProfile.findMany({
+ orderBy: [{ symbol: 'asc' }, { dataSource: 'asc' }],
+ select: {
+ dataSource: true,
+ symbol: true
+ },
where: {
- isActive: true
+ dataSource: {
+ notIn: ['MANUAL', 'RAPID_API']
+ },
+ isActive: true,
+ ...(maxAge && {
+ updatedAt: {
+ lt: subMilliseconds(new Date(), ms(maxAge))
+ }
+ })
}
});
-
- return symbolProfiles
- .filter(({ dataSource }) => {
- return (
- dataSource !== DataSource.MANUAL &&
- dataSource !== DataSource.RAPID_API
- );
- })
- .map(({ dataSource, symbol }) => {
- return {
- dataSource,
- symbol
- };
- });
}
private async getAssetProfileIdentifiersWithCompleteMarketData(): Promise<