Open Source Wealth Management Software. Angular + NestJS + Prisma + Nx + TypeScript 🤍
TypeScript angularetffinancefintechghostfoliohacktoberfestinvestingnestjsosspersonal-financeportfolioprismasoftwarestocktrackertradingtypescriptwealth-managementweb
Bugfix/data gathering blocked by failed jobs (#7720)
Thomas Kauldef48c6adff6
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex bd29210eaf..c1bf235532 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -12,11 +12,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0- Harmonized the icons and labels in the access table to share the portfolio- Migrated the create and edit access dialogs to dedicated routes- Improved the response of the historical market data gathering endpoint for a specific date+- Introduced a timeout for the asset profile and the historical market data gathering jobs+- Reduced the number of attempts of the asset profile and the historical market data gathering jobs- Improved the historical market data gathering by loading the asset profiles with recent market data in a single database query per run### Fixed- Fixed the country mapping of Macau in the _Financial Modeling Prep_ service+- Fixed the asset profile and historical market data gathering of a symbol getting blocked permanently by a failed job by discarding the failed jobs+- Fixed the asset profile data gathering of a symbol in the admin control panel by removing an existing job before enqueueing a new one- Fixed the date of the gathered historical market data for instances running in a time zone other than UTC- Fixed the repeated historical market data gathering for instances running in a time zone other than UTCdiff --git a/apps/api/src/app/admin/admin.controller.ts b/apps/api/src/app/admin/admin.controller.tsindex 6d3f7cfeb8..8c3faa25d6 100644--- a/apps/api/src/app/admin/admin.controller.ts+++ b/apps/api/src/app/admin/admin.controller.ts@@ -159,6 +159,11 @@ export class AdminController {@Param('dataSource') dataSource: DataSource,@Param('symbol') symbol: string): Promise<void> {+ await this.dataGatheringService.removeAssetProfileJobFromQueue({+ dataSource,+ symbol+ });+await this.dataGatheringService.addJobToQueue({data: {dataSource,diff --git a/apps/api/src/services/queues/data-gathering/data-gathering.service.ts b/apps/api/src/services/queues/data-gathering/data-gathering.service.tsindex 6f280f5c98..4cbe6969ae 100644--- a/apps/api/src/services/queues/data-gathering/data-gathering.service.ts+++ b/apps/api/src/services/queues/data-gathering/data-gathering.service.ts@@ -430,6 +430,20 @@ export class DataGatheringService {});}+ /**+ * Removes the asset profile job of a symbol from the queue+ */+ public async removeAssetProfileJobFromQueue({+ dataSource,+ symbol+ }: AssetProfileIdentifier) {+ const job = await this.dataGatheringQueue.getJob(+ `${getAssetProfileIdentifier({ dataSource, symbol })}:${dataSource}`+ );++ return job?.remove();+ }+private async getAssetProfileIdentifiersWithRecentMarketData(): Promise<AssetProfileIdentifier[]> {diff --git a/libs/common/src/lib/config.ts b/libs/common/src/lib/config.tsindex 04186cfa97..3adddf39a8 100644--- a/libs/common/src/lib/config.ts+++ b/libs/common/src/lib/config.ts@@ -195,28 +195,32 @@ export const E_MAIL_LINE_BREAK = '%0D%0A';export const GATHER_ASSET_PROFILE_PROCESS_JOB_NAME = 'GATHER_ASSET_PROFILE';export const GATHER_ASSET_PROFILE_PROCESS_JOB_OPTIONS: JobOptions = {- attempts: 12,+ attempts: 6, // Retries after 1, 3, 7, 15 and 31 minutes (57 minutes in total)backoff: {delay: ms('1 minute'),type: 'exponential'},- removeOnComplete: true+ removeOnComplete: true,+ removeOnFail: true,+ timeout: ms('5 minutes')};export const GATHER_HISTORICAL_MARKET_DATA_COOLDOWN_IN_MS = ms('12 hours');export const GATHER_HISTORICAL_MARKET_DATA_PROCESS_JOB_NAME ='GATHER_HISTORICAL_MARKET_DATA';export const GATHER_HISTORICAL_MARKET_DATA_PROCESS_JOB_OPTIONS: JobOptions = {- attempts: 12,+ attempts: 6, // Retries after 1, 3, 7, 15 and 31 minutes (57 minutes in total)backoff: {delay: ms('1 minute'),type: 'exponential'},- removeOnComplete: true+ removeOnComplete: true,+ removeOnFail: true,+ timeout: ms('5 minutes')};export const GATHER_STATISTICS_PROCESS_JOB_OPTIONS: JobOptions = {- attempts: 5,+ attempts: 6, // Retries after 1, 3, 7, 15 and 31 minutes (57 minutes in total)backoff: {delay: ms('1 minute'),type: 'exponential'