Open Source Wealth Management Software. Angular + NestJS + Prisma + Nx + TypeScript 🤍
TypeScript angularetffinancefintechghostfoliohacktoberfestinvestingnestjsosspersonal-financeportfolioprismasoftwarestocktrackertradingtypescriptwealth-managementweb
Feature/expose PROCESSOR_GATHER_STATISTICS_CONCURRENCY environment variable (#7100)
Thomas Kaulfcad7917be07
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 3dd4209e13..671c9f935e 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -10,6 +10,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0### Added- Exposed the `ENABLE_FEATURE_CRON` environment variable to control scheduled cron job execution+- Exposed the `PROCESSOR_GATHER_STATISTICS_CONCURRENCY` environment variable to control the concurrency of the statistics gathering queue processor### Changeddiff --git a/apps/api/src/services/configuration/configuration.service.ts b/apps/api/src/services/configuration/configuration.service.tsindex 328620164c..703a90c3a5 100644--- a/apps/api/src/services/configuration/configuration.service.ts+++ b/apps/api/src/services/configuration/configuration.service.ts@@ -6,6 +6,7 @@ import {DEFAULT_PORT,DEFAULT_PROCESSOR_GATHER_ASSET_PROFILE_CONCURRENCY,DEFAULT_PROCESSOR_GATHER_HISTORICAL_MARKET_DATA_CONCURRENCY,+ DEFAULT_PROCESSOR_GATHER_STATISTICS_CONCURRENCY,DEFAULT_PROCESSOR_PORTFOLIO_SNAPSHOT_COMPUTATION_CONCURRENCY,DEFAULT_PROCESSOR_PORTFOLIO_SNAPSHOT_COMPUTATION_TIMEOUT} from '@ghostfolio/common/config';@@ -89,6 +90,9 @@ export class ConfigurationService {PROCESSOR_GATHER_HISTORICAL_MARKET_DATA_CONCURRENCY: num({default: DEFAULT_PROCESSOR_GATHER_HISTORICAL_MARKET_DATA_CONCURRENCY}),+ PROCESSOR_GATHER_STATISTICS_CONCURRENCY: num({+ default: DEFAULT_PROCESSOR_GATHER_STATISTICS_CONCURRENCY+ }),PROCESSOR_PORTFOLIO_SNAPSHOT_COMPUTATION_CONCURRENCY: num({default: DEFAULT_PROCESSOR_PORTFOLIO_SNAPSHOT_COMPUTATION_CONCURRENCY}),diff --git a/apps/api/src/services/interfaces/environment.interface.ts b/apps/api/src/services/interfaces/environment.interface.tsindex 45654a2e3a..11bdeabf60 100644--- a/apps/api/src/services/interfaces/environment.interface.ts+++ b/apps/api/src/services/interfaces/environment.interface.ts@@ -45,6 +45,7 @@ export interface Environment extends CleanedEnvAccessors {PORT: number;PROCESSOR_GATHER_ASSET_PROFILE_CONCURRENCY: number;PROCESSOR_GATHER_HISTORICAL_MARKET_DATA_CONCURRENCY: number;+ PROCESSOR_GATHER_STATISTICS_CONCURRENCY: number;PROCESSOR_PORTFOLIO_SNAPSHOT_COMPUTATION_CONCURRENCY: number;PROCESSOR_PORTFOLIO_SNAPSHOT_COMPUTATION_TIMEOUT: number;REDIS_DB: number;diff --git a/apps/api/src/services/queues/statistics-gathering/statistics-gathering.processor.ts b/apps/api/src/services/queues/statistics-gathering/statistics-gathering.processor.tsindex 7eefc101f9..82f362d258 100644--- a/apps/api/src/services/queues/statistics-gathering/statistics-gathering.processor.ts+++ b/apps/api/src/services/queues/statistics-gathering/statistics-gathering.processor.ts@@ -2,6 +2,7 @@ import { ConfigurationService } from '@ghostfolio/api/services/configuration/conimport { FetchService } from '@ghostfolio/api/services/fetch/fetch.service';import { PropertyService } from '@ghostfolio/api/services/property/property.service';import {+ DEFAULT_PROCESSOR_GATHER_STATISTICS_CONCURRENCY,GATHER_STATISTICS_DOCKER_HUB_PULLS_PROCESS_JOB_NAME,GATHER_STATISTICS_GITHUB_CONTRIBUTORS_PROCESS_JOB_NAME,GATHER_STATISTICS_GITHUB_STARGAZERS_PROCESS_JOB_NAME,@@ -35,7 +36,14 @@ export class StatisticsGatheringProcessor {private readonly propertyService: PropertyService) {}- @Process(GATHER_STATISTICS_DOCKER_HUB_PULLS_PROCESS_JOB_NAME)+ @Process({+ concurrency: parseInt(+ process.env.PROCESSOR_GATHER_STATISTICS_CONCURRENCY ??+ DEFAULT_PROCESSOR_GATHER_STATISTICS_CONCURRENCY.toString(),+ 10+ ),+ name: GATHER_STATISTICS_DOCKER_HUB_PULLS_PROCESS_JOB_NAME+ })public async gatherDockerHubPullsStatistics() {this.logger.log('Docker Hub pulls statistics gathering has been started');@@ -49,7 +57,14 @@ export class StatisticsGatheringProcessor {this.logger.log('Docker Hub pulls statistics gathering has been completed');}- @Process(GATHER_STATISTICS_GITHUB_CONTRIBUTORS_PROCESS_JOB_NAME)+ @Process({+ concurrency: parseInt(+ process.env.PROCESSOR_GATHER_STATISTICS_CONCURRENCY ??+ DEFAULT_PROCESSOR_GATHER_STATISTICS_CONCURRENCY.toString(),+ 10+ ),+ name: GATHER_STATISTICS_GITHUB_CONTRIBUTORS_PROCESS_JOB_NAME+ })public async gatherGitHubContributorsStatistics() {this.logger.log('GitHub contributors statistics gathering has been started'@@ -67,7 +82,14 @@ export class StatisticsGatheringProcessor {);}- @Process(GATHER_STATISTICS_GITHUB_STARGAZERS_PROCESS_JOB_NAME)+ @Process({+ concurrency: parseInt(+ process.env.PROCESSOR_GATHER_STATISTICS_CONCURRENCY ??+ DEFAULT_PROCESSOR_GATHER_STATISTICS_CONCURRENCY.toString(),+ 10+ ),+ name: GATHER_STATISTICS_GITHUB_STARGAZERS_PROCESS_JOB_NAME+ })public async gatherGitHubStargazersStatistics() {this.logger.log('GitHub stargazers statistics gathering has been started');@@ -83,7 +105,14 @@ export class StatisticsGatheringProcessor {);}- @Process(GATHER_STATISTICS_UPTIME_PROCESS_JOB_NAME)+ @Process({+ concurrency: parseInt(+ process.env.PROCESSOR_GATHER_STATISTICS_CONCURRENCY ??+ DEFAULT_PROCESSOR_GATHER_STATISTICS_CONCURRENCY.toString(),+ 10+ ),+ name: GATHER_STATISTICS_UPTIME_PROCESS_JOB_NAME+ })public async gatherUptimeStatistics() {const monitorId = await this.propertyService.getByKey<string>(PROPERTY_BETTER_UPTIME_MONITOR_IDdiff --git a/libs/common/src/lib/config.ts b/libs/common/src/lib/config.tsindex 4924aaeea6..e21271ca1e 100644--- a/libs/common/src/lib/config.ts+++ b/libs/common/src/lib/config.ts@@ -88,6 +88,7 @@ export const DEFAULT_PAGE_SIZE = 50;export const DEFAULT_PORT = 3333;export const DEFAULT_PROCESSOR_GATHER_ASSET_PROFILE_CONCURRENCY = 1;export const DEFAULT_PROCESSOR_GATHER_HISTORICAL_MARKET_DATA_CONCURRENCY = 1;+export const DEFAULT_PROCESSOR_GATHER_STATISTICS_CONCURRENCY = 1;export const DEFAULT_PROCESSOR_PORTFOLIO_SNAPSHOT_COMPUTATION_CONCURRENCY = 1;export const DEFAULT_PROCESSOR_PORTFOLIO_SNAPSHOT_COMPUTATION_TIMEOUT = 30000;