Open Source Wealth Management Software. Angular + NestJS + Prisma + Nx + TypeScript 🤍
TypeScript angularetffinancefintechghostfoliohacktoberfestinvestingnestjsosspersonal-financeportfolioprismasoftwarestocktrackertradingtypescriptwealth-managementweb
Feature/adapt priorities of data gathering jobs (#3262)
Thomas Kaul4e7d93db13c6
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 68a85fd95b..73999efb7d 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -10,9 +10,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0### Added- Added support to immediately execute a queue job from the admin control panel+- Added a priority column to the queue jobs view in the admin control panel### Changed+- Adapted the priorities of queue jobs- Upgraded `angular` from version `17.2.4` to `17.3.3`- Upgraded `Nx` from version `18.1.2` to `18.2.3`- Upgraded `prisma` from version `5.11.0` to `5.12.1`diff --git a/apps/api/src/app/admin/admin.controller.ts b/apps/api/src/app/admin/admin.controller.tsindex dab8fb8b2c..298a471c3e 100644--- a/apps/api/src/app/admin/admin.controller.ts+++ b/apps/api/src/app/admin/admin.controller.ts@@ -7,13 +7,12 @@ import { ManualService } from '@ghostfolio/api/services/data-provider/manual/manimport { MarketDataService } from '@ghostfolio/api/services/market-data/market-data.service';import { PropertyDto } from '@ghostfolio/api/services/property/property.dto';import {+ DATA_GATHERING_QUEUE_PRIORITY_HIGH,+ DATA_GATHERING_QUEUE_PRIORITY_MEDIUM,GATHER_ASSET_PROFILE_PROCESS,GATHER_ASSET_PROFILE_PROCESS_OPTIONS} from '@ghostfolio/common/config';-import {- getAssetProfileIdentifier,- resetHours-} from '@ghostfolio/common/helper';+import { getAssetProfileIdentifier } from '@ghostfolio/common/helper';import {AdminData,AdminMarketData,@@ -94,7 +93,8 @@ export class AdminController {name: GATHER_ASSET_PROFILE_PROCESS,opts: {...GATHER_ASSET_PROFILE_PROCESS_OPTIONS,- jobId: getAssetProfileIdentifier({ dataSource, symbol })+ jobId: getAssetProfileIdentifier({ dataSource, symbol }),+ priority: DATA_GATHERING_QUEUE_PRIORITY_MEDIUM}};})@@ -119,7 +119,8 @@ export class AdminController {name: GATHER_ASSET_PROFILE_PROCESS,opts: {...GATHER_ASSET_PROFILE_PROCESS_OPTIONS,- jobId: getAssetProfileIdentifier({ dataSource, symbol })+ jobId: getAssetProfileIdentifier({ dataSource, symbol }),+ priority: DATA_GATHERING_QUEUE_PRIORITY_MEDIUM}};})@@ -141,7 +142,8 @@ export class AdminController {name: GATHER_ASSET_PROFILE_PROCESS,opts: {...GATHER_ASSET_PROFILE_PROCESS_OPTIONS,- jobId: getAssetProfileIdentifier({ dataSource, symbol })+ jobId: getAssetProfileIdentifier({ dataSource, symbol }),+ priority: DATA_GATHERING_QUEUE_PRIORITY_HIGH}});}diff --git a/apps/api/src/app/admin/queue/queue.service.ts b/apps/api/src/app/admin/queue/queue.service.tsindex e4f29bc601..abae3cad14 100644--- a/apps/api/src/app/admin/queue/queue.service.ts+++ b/apps/api/src/app/admin/queue/queue.service.ts@@ -58,6 +58,7 @@ export class QueueService {finishedOn: job.finishedOn,id: job.id,name: job.name,+ opts: job.opts,stacktrace: job.stacktrace,state: await job.getState(),timestamp: job.timestampdiff --git a/apps/api/src/app/import/import.service.ts b/apps/api/src/app/import/import.service.tsindex cbdff87c0b..26df9d0699 100644--- a/apps/api/src/app/import/import.service.ts+++ b/apps/api/src/app/import/import.service.ts@@ -13,6 +13,10 @@ import { DataGatheringService } from '@ghostfolio/api/services/data-gathering/daimport { DataProviderService } from '@ghostfolio/api/services/data-provider/data-provider.service';import { ExchangeRateDataService } from '@ghostfolio/api/services/exchange-rate-data/exchange-rate-data.service';import { SymbolProfileService } from '@ghostfolio/api/services/symbol-profile/symbol-profile.service';+import {+ DATA_GATHERING_QUEUE_PRIORITY_HIGH,+ DATA_GATHERING_QUEUE_PRIORITY_MEDIUM+} from '@ghostfolio/common/config';import {DATE_FORMAT,getAssetProfileIdentifier,@@ -448,15 +452,16 @@ export class ImportService {});});- this.dataGatheringService.gatherSymbols(- uniqueActivities.map(({ date, SymbolProfile }) => {+ this.dataGatheringService.gatherSymbols({+ dataGatheringItems: uniqueActivities.map(({ date, SymbolProfile }) => {return {date,dataSource: SymbolProfile.dataSource,symbol: SymbolProfile.symbol};- })- );+ }),+ priority: DATA_GATHERING_QUEUE_PRIORITY_HIGH+ });}return activities;diff --git a/apps/api/src/app/order/order.controller.ts b/apps/api/src/app/order/order.controller.tsindex c7fec0dac9..3dadedcaf3 100644--- a/apps/api/src/app/order/order.controller.ts+++ b/apps/api/src/app/order/order.controller.ts@@ -7,7 +7,10 @@ import { TransformDataSourceInResponseInterceptor } from '@ghostfolio/api/intercimport { ApiService } from '@ghostfolio/api/services/api/api.service';import { DataGatheringService } from '@ghostfolio/api/services/data-gathering/data-gathering.service';import { ImpersonationService } from '@ghostfolio/api/services/impersonation/impersonation.service';-import { HEADER_KEY_IMPERSONATION } from '@ghostfolio/common/config';+import {+ DATA_GATHERING_QUEUE_PRIORITY_HIGH,+ HEADER_KEY_IMPERSONATION+} from '@ghostfolio/common/config';import { hasPermission, permissions } from '@ghostfolio/common/permissions';import type { DateRange, RequestWithUser } from '@ghostfolio/common/types';@@ -160,13 +163,16 @@ export class OrderController {if (data.dataSource && !order.isDraft) {// Gather symbol data in the background, if data source is set// (not MANUAL) and not draft- this.dataGatheringService.gatherSymbols([- {- dataSource: data.dataSource,- date: order.date,- symbol: data.symbol- }- ]);+ this.dataGatheringService.gatherSymbols({+ dataGatheringItems: [+ {+ dataSource: data.dataSource,+ date: order.date,+ symbol: data.symbol+ }+ ],+ priority: DATA_GATHERING_QUEUE_PRIORITY_HIGH+ });}return order;diff --git a/apps/api/src/app/order/order.service.ts b/apps/api/src/app/order/order.service.tsindex 20b2d5f157..35bfa1bcfe 100644--- a/apps/api/src/app/order/order.service.ts+++ b/apps/api/src/app/order/order.service.ts@@ -4,6 +4,7 @@ import { ExchangeRateDataService } from '@ghostfolio/api/services/exchange-rate-import { PrismaService } from '@ghostfolio/api/services/prisma/prisma.service';import { SymbolProfileService } from '@ghostfolio/api/services/symbol-profile/symbol-profile.service';import {+ DATA_GATHERING_QUEUE_PRIORITY_HIGH,GATHER_ASSET_PROFILE_PROCESS,GATHER_ASSET_PROFILE_PROCESS_OPTIONS} from '@ghostfolio/common/config';@@ -101,7 +102,8 @@ export class OrderService {jobId: getAssetProfileIdentifier({dataSource: data.SymbolProfile.connectOrCreate.create.dataSource,symbol: data.SymbolProfile.connectOrCreate.create.symbol- })+ }),+ priority: DATA_GATHERING_QUEUE_PRIORITY_HIGH}});}@@ -427,13 +429,17 @@ export class OrderService {if (!isDraft) {// Gather symbol data of order in the background, if not draft- this.dataGatheringService.gatherSymbols([- {- dataSource: data.SymbolProfile.connect.dataSource_symbol.dataSource,- date: <Date>data.date,- symbol: data.SymbolProfile.connect.dataSource_symbol.symbol- }- ]);+ this.dataGatheringService.gatherSymbols({+ dataGatheringItems: [+ {+ dataSource:+ data.SymbolProfile.connect.dataSource_symbol.dataSource,+ date: <Date>data.date,+ symbol: data.SymbolProfile.connect.dataSource_symbol.symbol+ }+ ],+ priority: DATA_GATHERING_QUEUE_PRIORITY_HIGH+ });}}diff --git a/apps/api/src/services/cron.service.ts b/apps/api/src/services/cron.service.tsindex d74ad6a94a..fc5d613a20 100644--- a/apps/api/src/services/cron.service.ts+++ b/apps/api/src/services/cron.service.ts@@ -1,4 +1,5 @@import {+ DATA_GATHERING_QUEUE_PRIORITY_LOW,GATHER_ASSET_PROFILE_PROCESS,GATHER_ASSET_PROFILE_PROCESS_OPTIONS,PROPERTY_IS_DATA_GATHERING_ENABLED@@ -56,7 +57,8 @@ export class CronService {name: GATHER_ASSET_PROFILE_PROCESS,opts: {...GATHER_ASSET_PROFILE_PROCESS_OPTIONS,- jobId: getAssetProfileIdentifier({ dataSource, symbol })+ jobId: getAssetProfileIdentifier({ dataSource, symbol }),+ priority: DATA_GATHERING_QUEUE_PRIORITY_LOW}};})diff --git a/apps/api/src/services/data-gathering/data-gathering.service.ts b/apps/api/src/services/data-gathering/data-gathering.service.tsindex 6dccd645e1..b2b0c371cb 100644--- a/apps/api/src/services/data-gathering/data-gathering.service.ts+++ b/apps/api/src/services/data-gathering/data-gathering.service.ts@@ -8,6 +8,8 @@ import { PropertyService } from '@ghostfolio/api/services/property/property.servimport { SymbolProfileService } from '@ghostfolio/api/services/symbol-profile/symbol-profile.service';import {DATA_GATHERING_QUEUE,+ DATA_GATHERING_QUEUE_PRIORITY_HIGH,+ DATA_GATHERING_QUEUE_PRIORITY_LOW,GATHER_HISTORICAL_MARKET_DATA_PROCESS,GATHER_HISTORICAL_MARKET_DATA_PROCESS_OPTIONS,PROPERTY_BENCHMARKS@@ -61,24 +63,35 @@ export class DataGatheringService {public async gather7Days() {const dataGatheringItems = await this.getSymbols7D();- await this.gatherSymbols(dataGatheringItems);+ await this.gatherSymbols({+ dataGatheringItems,+ priority: DATA_GATHERING_QUEUE_PRIORITY_LOW+ });}public async gatherMax() {const dataGatheringItems = await this.getSymbolsMax();- await this.gatherSymbols(dataGatheringItems);+ await this.gatherSymbols({+ dataGatheringItems,+ priority: DATA_GATHERING_QUEUE_PRIORITY_LOW+ });}public async gatherSymbol({ dataSource, symbol }: UniqueAsset) {await this.marketDataService.deleteMany({ dataSource, symbol });- const symbols = (await this.getSymbolsMax()).filter((dataGatheringItem) => {- return (- dataGatheringItem.dataSource === dataSource &&- dataGatheringItem.symbol === symbol- );+ const dataGatheringItems = (await this.getSymbolsMax()).filter(+ (dataGatheringItem) => {+ return (+ dataGatheringItem.dataSource === dataSource &&+ dataGatheringItem.symbol === symbol+ );+ }+ );+ await this.gatherSymbols({+ dataGatheringItems,+ priority: DATA_GATHERING_QUEUE_PRIORITY_HIGH});- await this.gatherSymbols(symbols);}public async gatherSymbolForDate({@@ -230,9 +243,15 @@ export class DataGatheringService {);}- public async gatherSymbols(aSymbolsWithStartDate: IDataGatheringItem[]) {+ public async gatherSymbols({+ dataGatheringItems,+ priority+ }: {+ dataGatheringItems: IDataGatheringItem[];+ priority: number;+ }) {await this.addJobsToQueue(- aSymbolsWithStartDate.map(({ dataSource, date, symbol }) => {+ dataGatheringItems.map(({ dataSource, date, symbol }) => {return {data: {dataSource,@@ -242,6 +261,7 @@ export class DataGatheringService {name: GATHER_HISTORICAL_MARKET_DATA_PROCESS,opts: {...GATHER_HISTORICAL_MARKET_DATA_PROCESS_OPTIONS,+ priority,jobId: `${getAssetProfileIdentifier({dataSource,symboldiff --git a/apps/client/src/app/components/admin-jobs/admin-jobs.component.ts b/apps/client/src/app/components/admin-jobs/admin-jobs.component.tsindex 99d67dea2f..23730f3aa3 100644--- a/apps/client/src/app/components/admin-jobs/admin-jobs.component.ts+++ b/apps/client/src/app/components/admin-jobs/admin-jobs.component.ts@@ -1,6 +1,11 @@import { AdminService } from '@ghostfolio/client/services/admin.service';import { UserService } from '@ghostfolio/client/services/user/user.service';-import { QUEUE_JOB_STATUS_LIST } from '@ghostfolio/common/config';+import {+ DATA_GATHERING_QUEUE_PRIORITY_HIGH,+ DATA_GATHERING_QUEUE_PRIORITY_LOW,+ DATA_GATHERING_QUEUE_PRIORITY_MEDIUM,+ QUEUE_JOB_STATUS_LIST+} from '@ghostfolio/common/config';import { getDateWithTimeFormatString } from '@ghostfolio/common/helper';import { AdminJobs, User } from '@ghostfolio/common/interfaces';@@ -24,6 +29,11 @@ import { takeUntil } from 'rxjs/operators';templateUrl: './admin-jobs.html'})export class AdminJobsComponent implements OnDestroy, OnInit {+ public DATA_GATHERING_QUEUE_PRIORITY_LOW = DATA_GATHERING_QUEUE_PRIORITY_LOW;+ public DATA_GATHERING_QUEUE_PRIORITY_HIGH =+ DATA_GATHERING_QUEUE_PRIORITY_HIGH;+ public DATA_GATHERING_QUEUE_PRIORITY_MEDIUM =+ DATA_GATHERING_QUEUE_PRIORITY_MEDIUM;public defaultDateTimeFormat: string;public filterForm: FormGroup;public dataSource: MatTableDataSource<AdminJobs['jobs'][0]> =@@ -33,6 +43,7 @@ export class AdminJobsComponent implements OnDestroy, OnInit {'type','symbol','dataSource',+ 'priority','attempts','created','finished',diff --git a/apps/client/src/app/components/admin-jobs/admin-jobs.html b/apps/client/src/app/components/admin-jobs/admin-jobs.htmlindex 31dad0ca65..5da929fe19 100644--- a/apps/client/src/app/components/admin-jobs/admin-jobs.html+++ b/apps/client/src/app/components/admin-jobs/admin-jobs.html@@ -58,6 +58,25 @@</td></ng-container>+ <ng-container matColumnDef="priority">+ <th *matHeaderCellDef class="px-1 py-2" mat-header-cell>+ <ng-container i18n>Priority</ng-container>+ </th>+ <td *matCellDef="let element" class="px-1 py-2" mat-cell>+ @if (element.opts.priority === DATA_GATHERING_QUEUE_PRIORITY_LOW) {+ <ion-icon class="h6 mb-0" name="chevron-down-circle-outline" />+ } @else if (+ element.opts.priority === DATA_GATHERING_QUEUE_PRIORITY_MEDIUM+ ) {+ <ion-icon class="h6 mb-0" name="remove-circle-outline" />+ } @else if (+ element.opts.priority === DATA_GATHERING_QUEUE_PRIORITY_HIGH+ ) {+ <ion-icon class="h6 mb-0" name="chevron-up-circle-outline" />+ }+ </td>+ </ng-container>+<ng-container matColumnDef="attempts"><th *matHeaderCellDef class="px-1 py-2 text-right" mat-header-cell><ng-container i18n>Attempts</ng-container>@@ -90,24 +109,37 @@<ng-container i18n>Status</ng-container></th><td *matCellDef="let element" class="px-1 py-2" mat-cell>- <ion-icon *ngIf="element.state === 'active'" name="play-outline" />+ <ion-icon+ *ngIf="element.state === 'active'"+ class="h6 mb-0"+ name="play-outline"+ /><ion-icon*ngIf="element.state === 'completed'"- class="text-success"+ class="h6 mb-0 text-success"name="checkmark-circle-outline"/><ion-icon*ngIf="element.state === 'delayed'"+ class="h6 mb-0"name="time-outline"[ngClass]="{ 'text-danger': element.stacktrace?.length > 0 }"/><ion-icon*ngIf="element.state === 'failed'"- class="text-danger"+ class="h6 mb-0 text-danger"name="alert-circle-outline"/>- <ion-icon *ngIf="element.state === 'paused'" name="pause-outline" />- <ion-icon *ngIf="element.state === 'waiting'" name="cafe-outline" />+ <ion-icon+ *ngIf="element.state === 'paused'"+ class="h6 mb-0"+ name="pause-outline"+ />+ <ion-icon+ *ngIf="element.state === 'waiting'"+ class="h6 mb-0"+ name="cafe-outline"+ /></td></ng-container>diff --git a/libs/common/src/lib/config.ts b/libs/common/src/lib/config.tsindex 293f77488f..5e1366ce21 100644--- a/libs/common/src/lib/config.ts+++ b/libs/common/src/lib/config.ts@@ -32,8 +32,11 @@ export const warnColorRgb = {};export const DATA_GATHERING_QUEUE = 'DATA_GATHERING_QUEUE';-export const DATA_GATHERING_QUEUE_PRIORITY_LOW = Number.MAX_SAFE_INTEGER;export const DATA_GATHERING_QUEUE_PRIORITY_HIGH = 1;+export const DATA_GATHERING_QUEUE_PRIORITY_LOW = Number.MAX_SAFE_INTEGER;+export const DATA_GATHERING_QUEUE_PRIORITY_MEDIUM = Math.round(+ DATA_GATHERING_QUEUE_PRIORITY_LOW / 2+);export const DEFAULT_CURRENCY = 'USD';export const DEFAULT_DATE_FORMAT_MONTH_YEAR = 'MMM yyyy';@@ -69,7 +72,6 @@ export const GATHER_ASSET_PROFILE_PROCESS_OPTIONS: JobOptions = {delay: ms('1 minute'),type: 'exponential'},- priority: DATA_GATHERING_QUEUE_PRIORITY_HIGH,removeOnComplete: true};export const GATHER_HISTORICAL_MARKET_DATA_PROCESS =@@ -80,7 +82,6 @@ export const GATHER_HISTORICAL_MARKET_DATA_PROCESS_OPTIONS: JobOptions = {delay: ms('1 minute'),type: 'exponential'},- priority: DATA_GATHERING_QUEUE_PRIORITY_LOW,removeOnComplete: true};diff --git a/libs/common/src/lib/interfaces/admin-jobs.interface.ts b/libs/common/src/lib/interfaces/admin-jobs.interface.tsindex 25e9376261..b4c91ebc0a 100644--- a/libs/common/src/lib/interfaces/admin-jobs.interface.ts+++ b/libs/common/src/lib/interfaces/admin-jobs.interface.ts@@ -8,6 +8,7 @@ export interface AdminJobs {| 'finishedOn'| 'id'| 'name'+ | 'opts'| 'stacktrace'| 'timestamp'> & {