Skip to content

Open Source Wealth Management Software. Angular + NestJS + Prisma + Nx + TypeScript 🤍

Feature/adapt priorities of data gathering jobs (#3262)

Thomas Kaul4e7d93db13c6
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 68a85fd95b..73999efb7d 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -10,9 +10,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
### Added
- Added support to immediately execute a queue job from the admin control panel
+- Added a priority column to the queue jobs view in the admin control panel
### Changed
+- Adapted the priorities of queue jobs
- Upgraded `angular` from version `17.2.4` to `17.3.3`
- Upgraded `Nx` from version `18.1.2` to `18.2.3`
- Upgraded `prisma` from version `5.11.0` to `5.12.1`
diff --git a/apps/api/src/app/admin/admin.controller.ts b/apps/api/src/app/admin/admin.controller.ts
index dab8fb8b2c..298a471c3e 100644
--- a/apps/api/src/app/admin/admin.controller.ts
+++ b/apps/api/src/app/admin/admin.controller.ts
@@ -7,13 +7,12 @@ import { ManualService } from '@ghostfolio/api/services/data-provider/manual/man
import { MarketDataService } from '@ghostfolio/api/services/market-data/market-data.service';
import { PropertyDto } from '@ghostfolio/api/services/property/property.dto';
import {
+ DATA_GATHERING_QUEUE_PRIORITY_HIGH,
+ DATA_GATHERING_QUEUE_PRIORITY_MEDIUM,
GATHER_ASSET_PROFILE_PROCESS,
GATHER_ASSET_PROFILE_PROCESS_OPTIONS
} from '@ghostfolio/common/config';
-import {
- getAssetProfileIdentifier,
- resetHours
-} from '@ghostfolio/common/helper';
+import { getAssetProfileIdentifier } from '@ghostfolio/common/helper';
import {
AdminData,
AdminMarketData,
@@ -94,7 +93,8 @@ export class AdminController {
name: GATHER_ASSET_PROFILE_PROCESS,
opts: {
...GATHER_ASSET_PROFILE_PROCESS_OPTIONS,
- jobId: getAssetProfileIdentifier({ dataSource, symbol })
+ jobId: getAssetProfileIdentifier({ dataSource, symbol }),
+ priority: DATA_GATHERING_QUEUE_PRIORITY_MEDIUM
}
};
})
@@ -119,7 +119,8 @@ export class AdminController {
name: GATHER_ASSET_PROFILE_PROCESS,
opts: {
...GATHER_ASSET_PROFILE_PROCESS_OPTIONS,
- jobId: getAssetProfileIdentifier({ dataSource, symbol })
+ jobId: getAssetProfileIdentifier({ dataSource, symbol }),
+ priority: DATA_GATHERING_QUEUE_PRIORITY_MEDIUM
}
};
})
@@ -141,7 +142,8 @@ export class AdminController {
name: GATHER_ASSET_PROFILE_PROCESS,
opts: {
...GATHER_ASSET_PROFILE_PROCESS_OPTIONS,
- jobId: getAssetProfileIdentifier({ dataSource, symbol })
+ jobId: getAssetProfileIdentifier({ dataSource, symbol }),
+ priority: DATA_GATHERING_QUEUE_PRIORITY_HIGH
}
});
}
diff --git a/apps/api/src/app/admin/queue/queue.service.ts b/apps/api/src/app/admin/queue/queue.service.ts
index e4f29bc601..abae3cad14 100644
--- a/apps/api/src/app/admin/queue/queue.service.ts
+++ b/apps/api/src/app/admin/queue/queue.service.ts
@@ -58,6 +58,7 @@ export class QueueService {
finishedOn: job.finishedOn,
id: job.id,
name: job.name,
+ opts: job.opts,
stacktrace: job.stacktrace,
state: await job.getState(),
timestamp: job.timestamp
diff --git a/apps/api/src/app/import/import.service.ts b/apps/api/src/app/import/import.service.ts
index cbdff87c0b..26df9d0699 100644
--- a/apps/api/src/app/import/import.service.ts
+++ b/apps/api/src/app/import/import.service.ts
@@ -13,6 +13,10 @@ import { DataGatheringService } from '@ghostfolio/api/services/data-gathering/da
import { DataProviderService } from '@ghostfolio/api/services/data-provider/data-provider.service';
import { ExchangeRateDataService } from '@ghostfolio/api/services/exchange-rate-data/exchange-rate-data.service';
import { SymbolProfileService } from '@ghostfolio/api/services/symbol-profile/symbol-profile.service';
+import {
+ DATA_GATHERING_QUEUE_PRIORITY_HIGH,
+ DATA_GATHERING_QUEUE_PRIORITY_MEDIUM
+} from '@ghostfolio/common/config';
import {
DATE_FORMAT,
getAssetProfileIdentifier,
@@ -448,15 +452,16 @@ export class ImportService {
});
});
- this.dataGatheringService.gatherSymbols(
- uniqueActivities.map(({ date, SymbolProfile }) => {
+ this.dataGatheringService.gatherSymbols({
+ dataGatheringItems: uniqueActivities.map(({ date, SymbolProfile }) => {
return {
date,
dataSource: SymbolProfile.dataSource,
symbol: SymbolProfile.symbol
};
- })
- );
+ }),
+ priority: DATA_GATHERING_QUEUE_PRIORITY_HIGH
+ });
}
return activities;
diff --git a/apps/api/src/app/order/order.controller.ts b/apps/api/src/app/order/order.controller.ts
index c7fec0dac9..3dadedcaf3 100644
--- a/apps/api/src/app/order/order.controller.ts
+++ b/apps/api/src/app/order/order.controller.ts
@@ -7,7 +7,10 @@ import { TransformDataSourceInResponseInterceptor } from '@ghostfolio/api/interc
import { ApiService } from '@ghostfolio/api/services/api/api.service';
import { DataGatheringService } from '@ghostfolio/api/services/data-gathering/data-gathering.service';
import { ImpersonationService } from '@ghostfolio/api/services/impersonation/impersonation.service';
-import { HEADER_KEY_IMPERSONATION } from '@ghostfolio/common/config';
+import {
+ DATA_GATHERING_QUEUE_PRIORITY_HIGH,
+ HEADER_KEY_IMPERSONATION
+} from '@ghostfolio/common/config';
import { hasPermission, permissions } from '@ghostfolio/common/permissions';
import type { DateRange, RequestWithUser } from '@ghostfolio/common/types';
@@ -160,13 +163,16 @@ export class OrderController {
if (data.dataSource && !order.isDraft) {
// Gather symbol data in the background, if data source is set
// (not MANUAL) and not draft
- this.dataGatheringService.gatherSymbols([
- {
- dataSource: data.dataSource,
- date: order.date,
- symbol: data.symbol
- }
- ]);
+ this.dataGatheringService.gatherSymbols({
+ dataGatheringItems: [
+ {
+ dataSource: data.dataSource,
+ date: order.date,
+ symbol: data.symbol
+ }
+ ],
+ priority: DATA_GATHERING_QUEUE_PRIORITY_HIGH
+ });
}
return order;
diff --git a/apps/api/src/app/order/order.service.ts b/apps/api/src/app/order/order.service.ts
index 20b2d5f157..35bfa1bcfe 100644
--- a/apps/api/src/app/order/order.service.ts
+++ b/apps/api/src/app/order/order.service.ts
@@ -4,6 +4,7 @@ import { ExchangeRateDataService } from '@ghostfolio/api/services/exchange-rate-
import { PrismaService } from '@ghostfolio/api/services/prisma/prisma.service';
import { SymbolProfileService } from '@ghostfolio/api/services/symbol-profile/symbol-profile.service';
import {
+ DATA_GATHERING_QUEUE_PRIORITY_HIGH,
GATHER_ASSET_PROFILE_PROCESS,
GATHER_ASSET_PROFILE_PROCESS_OPTIONS
} from '@ghostfolio/common/config';
@@ -101,7 +102,8 @@ export class OrderService {
jobId: getAssetProfileIdentifier({
dataSource: data.SymbolProfile.connectOrCreate.create.dataSource,
symbol: data.SymbolProfile.connectOrCreate.create.symbol
- })
+ }),
+ priority: DATA_GATHERING_QUEUE_PRIORITY_HIGH
}
});
}
@@ -427,13 +429,17 @@ export class OrderService {
if (!isDraft) {
// Gather symbol data of order in the background, if not draft
- this.dataGatheringService.gatherSymbols([
- {
- dataSource: data.SymbolProfile.connect.dataSource_symbol.dataSource,
- date: <Date>data.date,
- symbol: data.SymbolProfile.connect.dataSource_symbol.symbol
- }
- ]);
+ this.dataGatheringService.gatherSymbols({
+ dataGatheringItems: [
+ {
+ dataSource:
+ data.SymbolProfile.connect.dataSource_symbol.dataSource,
+ date: <Date>data.date,
+ symbol: data.SymbolProfile.connect.dataSource_symbol.symbol
+ }
+ ],
+ priority: DATA_GATHERING_QUEUE_PRIORITY_HIGH
+ });
}
}
diff --git a/apps/api/src/services/cron.service.ts b/apps/api/src/services/cron.service.ts
index d74ad6a94a..fc5d613a20 100644
--- a/apps/api/src/services/cron.service.ts
+++ b/apps/api/src/services/cron.service.ts
@@ -1,4 +1,5 @@
import {
+ DATA_GATHERING_QUEUE_PRIORITY_LOW,
GATHER_ASSET_PROFILE_PROCESS,
GATHER_ASSET_PROFILE_PROCESS_OPTIONS,
PROPERTY_IS_DATA_GATHERING_ENABLED
@@ -56,7 +57,8 @@ export class CronService {
name: GATHER_ASSET_PROFILE_PROCESS,
opts: {
...GATHER_ASSET_PROFILE_PROCESS_OPTIONS,
- jobId: getAssetProfileIdentifier({ dataSource, symbol })
+ jobId: getAssetProfileIdentifier({ dataSource, symbol }),
+ priority: DATA_GATHERING_QUEUE_PRIORITY_LOW
}
};
})
diff --git a/apps/api/src/services/data-gathering/data-gathering.service.ts b/apps/api/src/services/data-gathering/data-gathering.service.ts
index 6dccd645e1..b2b0c371cb 100644
--- a/apps/api/src/services/data-gathering/data-gathering.service.ts
+++ b/apps/api/src/services/data-gathering/data-gathering.service.ts
@@ -8,6 +8,8 @@ import { PropertyService } from '@ghostfolio/api/services/property/property.serv
import { SymbolProfileService } from '@ghostfolio/api/services/symbol-profile/symbol-profile.service';
import {
DATA_GATHERING_QUEUE,
+ DATA_GATHERING_QUEUE_PRIORITY_HIGH,
+ DATA_GATHERING_QUEUE_PRIORITY_LOW,
GATHER_HISTORICAL_MARKET_DATA_PROCESS,
GATHER_HISTORICAL_MARKET_DATA_PROCESS_OPTIONS,
PROPERTY_BENCHMARKS
@@ -61,24 +63,35 @@ export class DataGatheringService {
public async gather7Days() {
const dataGatheringItems = await this.getSymbols7D();
- await this.gatherSymbols(dataGatheringItems);
+ await this.gatherSymbols({
+ dataGatheringItems,
+ priority: DATA_GATHERING_QUEUE_PRIORITY_LOW
+ });
}
public async gatherMax() {
const dataGatheringItems = await this.getSymbolsMax();
- await this.gatherSymbols(dataGatheringItems);
+ await this.gatherSymbols({
+ dataGatheringItems,
+ priority: DATA_GATHERING_QUEUE_PRIORITY_LOW
+ });
}
public async gatherSymbol({ dataSource, symbol }: UniqueAsset) {
await this.marketDataService.deleteMany({ dataSource, symbol });
- const symbols = (await this.getSymbolsMax()).filter((dataGatheringItem) => {
- return (
- dataGatheringItem.dataSource === dataSource &&
- dataGatheringItem.symbol === symbol
- );
+ const dataGatheringItems = (await this.getSymbolsMax()).filter(
+ (dataGatheringItem) => {
+ return (
+ dataGatheringItem.dataSource === dataSource &&
+ dataGatheringItem.symbol === symbol
+ );
+ }
+ );
+ await this.gatherSymbols({
+ dataGatheringItems,
+ priority: DATA_GATHERING_QUEUE_PRIORITY_HIGH
});
- await this.gatherSymbols(symbols);
}
public async gatherSymbolForDate({
@@ -230,9 +243,15 @@ export class DataGatheringService {
);
}
- public async gatherSymbols(aSymbolsWithStartDate: IDataGatheringItem[]) {
+ public async gatherSymbols({
+ dataGatheringItems,
+ priority
+ }: {
+ dataGatheringItems: IDataGatheringItem[];
+ priority: number;
+ }) {
await this.addJobsToQueue(
- aSymbolsWithStartDate.map(({ dataSource, date, symbol }) => {
+ dataGatheringItems.map(({ dataSource, date, symbol }) => {
return {
data: {
dataSource,
@@ -242,6 +261,7 @@ export class DataGatheringService {
name: GATHER_HISTORICAL_MARKET_DATA_PROCESS,
opts: {
...GATHER_HISTORICAL_MARKET_DATA_PROCESS_OPTIONS,
+ priority,
jobId: `${getAssetProfileIdentifier({
dataSource,
symbol
diff --git a/apps/client/src/app/components/admin-jobs/admin-jobs.component.ts b/apps/client/src/app/components/admin-jobs/admin-jobs.component.ts
index 99d67dea2f..23730f3aa3 100644
--- a/apps/client/src/app/components/admin-jobs/admin-jobs.component.ts
+++ b/apps/client/src/app/components/admin-jobs/admin-jobs.component.ts
@@ -1,6 +1,11 @@
import { AdminService } from '@ghostfolio/client/services/admin.service';
import { UserService } from '@ghostfolio/client/services/user/user.service';
-import { QUEUE_JOB_STATUS_LIST } from '@ghostfolio/common/config';
+import {
+ DATA_GATHERING_QUEUE_PRIORITY_HIGH,
+ DATA_GATHERING_QUEUE_PRIORITY_LOW,
+ DATA_GATHERING_QUEUE_PRIORITY_MEDIUM,
+ QUEUE_JOB_STATUS_LIST
+} from '@ghostfolio/common/config';
import { getDateWithTimeFormatString } from '@ghostfolio/common/helper';
import { AdminJobs, User } from '@ghostfolio/common/interfaces';
@@ -24,6 +29,11 @@ import { takeUntil } from 'rxjs/operators';
templateUrl: './admin-jobs.html'
})
export class AdminJobsComponent implements OnDestroy, OnInit {
+ public DATA_GATHERING_QUEUE_PRIORITY_LOW = DATA_GATHERING_QUEUE_PRIORITY_LOW;
+ public DATA_GATHERING_QUEUE_PRIORITY_HIGH =
+ DATA_GATHERING_QUEUE_PRIORITY_HIGH;
+ public DATA_GATHERING_QUEUE_PRIORITY_MEDIUM =
+ DATA_GATHERING_QUEUE_PRIORITY_MEDIUM;
public defaultDateTimeFormat: string;
public filterForm: FormGroup;
public dataSource: MatTableDataSource<AdminJobs['jobs'][0]> =
@@ -33,6 +43,7 @@ export class AdminJobsComponent implements OnDestroy, OnInit {
'type',
'symbol',
'dataSource',
+ 'priority',
'attempts',
'created',
'finished',
diff --git a/apps/client/src/app/components/admin-jobs/admin-jobs.html b/apps/client/src/app/components/admin-jobs/admin-jobs.html
index 31dad0ca65..5da929fe19 100644
--- a/apps/client/src/app/components/admin-jobs/admin-jobs.html
+++ b/apps/client/src/app/components/admin-jobs/admin-jobs.html
@@ -58,6 +58,25 @@
</td>
</ng-container>
+ <ng-container matColumnDef="priority">
+ <th *matHeaderCellDef class="px-1 py-2" mat-header-cell>
+ <ng-container i18n>Priority</ng-container>
+ </th>
+ <td *matCellDef="let element" class="px-1 py-2" mat-cell>
+ @if (element.opts.priority === DATA_GATHERING_QUEUE_PRIORITY_LOW) {
+ <ion-icon class="h6 mb-0" name="chevron-down-circle-outline" />
+ } @else if (
+ element.opts.priority === DATA_GATHERING_QUEUE_PRIORITY_MEDIUM
+ ) {
+ <ion-icon class="h6 mb-0" name="remove-circle-outline" />
+ } @else if (
+ element.opts.priority === DATA_GATHERING_QUEUE_PRIORITY_HIGH
+ ) {
+ <ion-icon class="h6 mb-0" name="chevron-up-circle-outline" />
+ }
+ </td>
+ </ng-container>
+
<ng-container matColumnDef="attempts">
<th *matHeaderCellDef class="px-1 py-2 text-right" mat-header-cell>
<ng-container i18n>Attempts</ng-container>
@@ -90,24 +109,37 @@
<ng-container i18n>Status</ng-container>
</th>
<td *matCellDef="let element" class="px-1 py-2" mat-cell>
- <ion-icon *ngIf="element.state === 'active'" name="play-outline" />
+ <ion-icon
+ *ngIf="element.state === 'active'"
+ class="h6 mb-0"
+ name="play-outline"
+ />
<ion-icon
*ngIf="element.state === 'completed'"
- class="text-success"
+ class="h6 mb-0 text-success"
name="checkmark-circle-outline"
/>
<ion-icon
*ngIf="element.state === 'delayed'"
+ class="h6 mb-0"
name="time-outline"
[ngClass]="{ 'text-danger': element.stacktrace?.length > 0 }"
/>
<ion-icon
*ngIf="element.state === 'failed'"
- class="text-danger"
+ class="h6 mb-0 text-danger"
name="alert-circle-outline"
/>
- <ion-icon *ngIf="element.state === 'paused'" name="pause-outline" />
- <ion-icon *ngIf="element.state === 'waiting'" name="cafe-outline" />
+ <ion-icon
+ *ngIf="element.state === 'paused'"
+ class="h6 mb-0"
+ name="pause-outline"
+ />
+ <ion-icon
+ *ngIf="element.state === 'waiting'"
+ class="h6 mb-0"
+ name="cafe-outline"
+ />
</td>
</ng-container>
diff --git a/libs/common/src/lib/config.ts b/libs/common/src/lib/config.ts
index 293f77488f..5e1366ce21 100644
--- a/libs/common/src/lib/config.ts
+++ b/libs/common/src/lib/config.ts
@@ -32,8 +32,11 @@ export const warnColorRgb = {
};
export const DATA_GATHERING_QUEUE = 'DATA_GATHERING_QUEUE';
-export const DATA_GATHERING_QUEUE_PRIORITY_LOW = Number.MAX_SAFE_INTEGER;
export const DATA_GATHERING_QUEUE_PRIORITY_HIGH = 1;
+export const DATA_GATHERING_QUEUE_PRIORITY_LOW = Number.MAX_SAFE_INTEGER;
+export const DATA_GATHERING_QUEUE_PRIORITY_MEDIUM = Math.round(
+ DATA_GATHERING_QUEUE_PRIORITY_LOW / 2
+);
export const DEFAULT_CURRENCY = 'USD';
export const DEFAULT_DATE_FORMAT_MONTH_YEAR = 'MMM yyyy';
@@ -69,7 +72,6 @@ export const GATHER_ASSET_PROFILE_PROCESS_OPTIONS: JobOptions = {
delay: ms('1 minute'),
type: 'exponential'
},
- priority: DATA_GATHERING_QUEUE_PRIORITY_HIGH,
removeOnComplete: true
};
export const GATHER_HISTORICAL_MARKET_DATA_PROCESS =
@@ -80,7 +82,6 @@ export const GATHER_HISTORICAL_MARKET_DATA_PROCESS_OPTIONS: JobOptions = {
delay: ms('1 minute'),
type: 'exponential'
},
- priority: DATA_GATHERING_QUEUE_PRIORITY_LOW,
removeOnComplete: true
};
diff --git a/libs/common/src/lib/interfaces/admin-jobs.interface.ts b/libs/common/src/lib/interfaces/admin-jobs.interface.ts
index 25e9376261..b4c91ebc0a 100644
--- a/libs/common/src/lib/interfaces/admin-jobs.interface.ts
+++ b/libs/common/src/lib/interfaces/admin-jobs.interface.ts
@@ -8,6 +8,7 @@ export interface AdminJobs {
| 'finishedOn'
| 'id'
| 'name'
+ | 'opts'
| 'stacktrace'
| 'timestamp'
> & {