Open Source Wealth Management Software. Angular + NestJS + Prisma + Nx + TypeScript 🤍
TypeScript angularetffinancefintechghostfoliohacktoberfestinvestingnestjsosspersonal-financeportfolioprismasoftwarestocktrackertradingtypescriptwealth-managementweb
Feature/setup statistics gathering queue (#6696)
Thomas Kaulcdef2afa48af
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 29d71c9fe9..1cc28c4eb4 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -15,6 +15,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0- Extended the asset profile details dialog in the admin control panel to support editing countries for all asset types- Extended the asset profile details dialog in the admin control panel to support editing sectors for all asset types+- Migrated the data collection for the _Open Startup_ (`/open`) page to the queue design pattern- Improved the language localization for German (`de`)- Upgraded `lodash` from version `4.17.23` to `4.18.1`diff --git a/apps/api/src/app/activities/activities.module.ts b/apps/api/src/app/activities/activities.module.tsindex 7476ad66ad..f4e592c3fc 100644--- a/apps/api/src/app/activities/activities.module.ts+++ b/apps/api/src/app/activities/activities.module.ts@@ -10,7 +10,7 @@ import { DataProviderModule } from '@ghostfolio/api/services/data-provider/data-import { ExchangeRateDataModule } from '@ghostfolio/api/services/exchange-rate-data/exchange-rate-data.module';import { ImpersonationModule } from '@ghostfolio/api/services/impersonation/impersonation.module';import { PrismaModule } from '@ghostfolio/api/services/prisma/prisma.module';-import { DataGatheringModule } from '@ghostfolio/api/services/queues/data-gathering/data-gathering.module';+import { DataGatheringQueueModule } from '@ghostfolio/api/services/queues/data-gathering/data-gathering.module';import { SymbolProfileModule } from '@ghostfolio/api/services/symbol-profile/symbol-profile.module';import { Module } from '@nestjs/common';@@ -24,7 +24,7 @@ import { ActivitiesService } from './activities.service';imports: [ApiModule,CacheModule,- DataGatheringModule,+ DataGatheringQueueModule,DataProviderModule,ExchangeRateDataModule,ImpersonationModule,diff --git a/apps/api/src/app/admin/admin.module.ts b/apps/api/src/app/admin/admin.module.tsindex 960a366292..e87df9e749 100644--- a/apps/api/src/app/admin/admin.module.ts+++ b/apps/api/src/app/admin/admin.module.ts@@ -9,7 +9,7 @@ import { ExchangeRateDataModule } from '@ghostfolio/api/services/exchange-rate-dimport { MarketDataModule } from '@ghostfolio/api/services/market-data/market-data.module';import { PrismaModule } from '@ghostfolio/api/services/prisma/prisma.module';import { PropertyModule } from '@ghostfolio/api/services/property/property.module';-import { DataGatheringModule } from '@ghostfolio/api/services/queues/data-gathering/data-gathering.module';+import { DataGatheringQueueModule } from '@ghostfolio/api/services/queues/data-gathering/data-gathering.module';import { SymbolProfileModule } from '@ghostfolio/api/services/symbol-profile/symbol-profile.module';import { Module } from '@nestjs/common';@@ -24,7 +24,7 @@ import { QueueModule } from './queue/queue.module';ApiModule,BenchmarkModule,ConfigurationModule,- DataGatheringModule,+ DataGatheringQueueModule,DataProviderModule,DemoModule,ExchangeRateDataModule,diff --git a/apps/api/src/app/admin/queue/queue.module.ts b/apps/api/src/app/admin/queue/queue.module.tsindex 22d1cefc6a..4cfb794921 100644--- a/apps/api/src/app/admin/queue/queue.module.ts+++ b/apps/api/src/app/admin/queue/queue.module.ts@@ -1,5 +1,6 @@-import { DataGatheringModule } from '@ghostfolio/api/services/queues/data-gathering/data-gathering.module';+import { DataGatheringQueueModule } from '@ghostfolio/api/services/queues/data-gathering/data-gathering.module';import { PortfolioSnapshotQueueModule } from '@ghostfolio/api/services/queues/portfolio-snapshot/portfolio-snapshot.module';+import { StatisticsGatheringQueueModule } from '@ghostfolio/api/services/queues/statistics-gathering/statistics-gathering.module';import { Module } from '@nestjs/common';@@ -8,7 +9,11 @@ import { QueueService } from './queue.service';@Module({controllers: [QueueController],- imports: [DataGatheringModule, PortfolioSnapshotQueueModule],+ imports: [+ DataGatheringQueueModule,+ PortfolioSnapshotQueueModule,+ StatisticsGatheringQueueModule+ ],providers: [QueueService]})export class QueueModule {}diff --git a/apps/api/src/app/admin/queue/queue.service.ts b/apps/api/src/app/admin/queue/queue.service.tsindex 747c4d6fbe..f47b3d3a1e 100644--- a/apps/api/src/app/admin/queue/queue.service.ts+++ b/apps/api/src/app/admin/queue/queue.service.ts@@ -1,7 +1,8 @@import {DATA_GATHERING_QUEUE,PORTFOLIO_SNAPSHOT_COMPUTATION_QUEUE,- QUEUE_JOB_STATUS_LIST+ QUEUE_JOB_STATUS_LIST,+ STATISTICS_GATHERING_QUEUE} from '@ghostfolio/common/config';import { AdminJobs } from '@ghostfolio/common/interfaces';@@ -15,7 +16,9 @@ export class QueueService {@InjectQueue(DATA_GATHERING_QUEUE)private readonly dataGatheringQueue: Queue,@InjectQueue(PORTFOLIO_SNAPSHOT_COMPUTATION_QUEUE)- private readonly portfolioSnapshotQueue: Queue+ private readonly portfolioSnapshotQueue: Queue,+ @InjectQueue(STATISTICS_GATHERING_QUEUE)+ private readonly statisticsGatheringQueue: Queue) {}public async deleteJob(aId: string) {@@ -38,6 +41,7 @@ export class QueueService {await this.dataGatheringQueue.clean(300, queueStatus);await this.portfolioSnapshotQueue.clean(300, queueStatus);+ await this.statisticsGatheringQueue.clean(300, queueStatus);}}@@ -58,13 +62,19 @@ export class QueueService {limit?: number;status?: JobStatus[];}): Promise<AdminJobs> {- const [dataGatheringJobs, portfolioSnapshotJobs] = await Promise.all([- this.dataGatheringQueue.getJobs(status),- this.portfolioSnapshotQueue.getJobs(status)- ]);+ const [dataGatheringJobs, portfolioSnapshotJobs, statisticsGatheringJobs] =+ await Promise.all([+ this.dataGatheringQueue.getJobs(status),+ this.portfolioSnapshotQueue.getJobs(status),+ this.statisticsGatheringQueue.getJobs(status)+ ]);const jobsWithState = await Promise.all(- [...dataGatheringJobs, ...portfolioSnapshotJobs]+ [+ ...dataGatheringJobs,+ ...portfolioSnapshotJobs,+ ...statisticsGatheringJobs+ ].filter((job) => {return job;})diff --git a/apps/api/src/app/app.module.ts b/apps/api/src/app/app.module.tsindex 8ebe05928f..3316f9ce4d 100644--- a/apps/api/src/app/app.module.ts+++ b/apps/api/src/app/app.module.ts@@ -8,7 +8,7 @@ import { ExchangeRateDataModule } from '@ghostfolio/api/services/exchange-rate-dimport { I18nService } from '@ghostfolio/api/services/i18n/i18n.service';import { PrismaModule } from '@ghostfolio/api/services/prisma/prisma.module';import { PropertyModule } from '@ghostfolio/api/services/property/property.module';-import { DataGatheringModule } from '@ghostfolio/api/services/queues/data-gathering/data-gathering.module';+import { DataGatheringQueueModule } from '@ghostfolio/api/services/queues/data-gathering/data-gathering.module';import { PortfolioSnapshotQueueModule } from '@ghostfolio/api/services/queues/portfolio-snapshot/portfolio-snapshot.module';import {BULL_BOARD_ROUTE,@@ -109,7 +109,7 @@ import { UserModule } from './user/user.module';ConfigModule.forRoot(),ConfigurationModule,CronModule,- DataGatheringModule,+ DataGatheringQueueModule,DataProviderModule,EventEmitterModule.forRoot(),EventsModule,diff --git a/apps/api/src/app/endpoints/watchlist/watchlist.module.ts b/apps/api/src/app/endpoints/watchlist/watchlist.module.tsindex ce9ae12bbf..9b4b960a08 100644--- a/apps/api/src/app/endpoints/watchlist/watchlist.module.ts+++ b/apps/api/src/app/endpoints/watchlist/watchlist.module.ts@@ -5,7 +5,7 @@ import { DataProviderModule } from '@ghostfolio/api/services/data-provider/data-import { ImpersonationModule } from '@ghostfolio/api/services/impersonation/impersonation.module';import { MarketDataModule } from '@ghostfolio/api/services/market-data/market-data.module';import { PrismaModule } from '@ghostfolio/api/services/prisma/prisma.module';-import { DataGatheringModule } from '@ghostfolio/api/services/queues/data-gathering/data-gathering.module';+import { DataGatheringQueueModule } from '@ghostfolio/api/services/queues/data-gathering/data-gathering.module';import { SymbolProfileModule } from '@ghostfolio/api/services/symbol-profile/symbol-profile.module';import { Module } from '@nestjs/common';@@ -17,7 +17,7 @@ import { WatchlistService } from './watchlist.service';controllers: [WatchlistController],imports: [BenchmarkModule,- DataGatheringModule,+ DataGatheringQueueModule,DataProviderModule,ImpersonationModule,MarketDataModule,diff --git a/apps/api/src/app/import/import.module.ts b/apps/api/src/app/import/import.module.tsindex ca9b5667b5..8aebcfa082 100644--- a/apps/api/src/app/import/import.module.ts+++ b/apps/api/src/app/import/import.module.ts@@ -12,7 +12,7 @@ import { DataProviderModule } from '@ghostfolio/api/services/data-provider/data-import { ExchangeRateDataModule } from '@ghostfolio/api/services/exchange-rate-data/exchange-rate-data.module';import { MarketDataModule } from '@ghostfolio/api/services/market-data/market-data.module';import { PrismaModule } from '@ghostfolio/api/services/prisma/prisma.module';-import { DataGatheringModule } from '@ghostfolio/api/services/queues/data-gathering/data-gathering.module';+import { DataGatheringQueueModule } from '@ghostfolio/api/services/queues/data-gathering/data-gathering.module';import { SymbolProfileModule } from '@ghostfolio/api/services/symbol-profile/symbol-profile.module';import { TagModule } from '@ghostfolio/api/services/tag/tag.module';@@ -29,7 +29,7 @@ import { ImportService } from './import.service';ApiModule,CacheModule,ConfigurationModule,- DataGatheringModule,+ DataGatheringQueueModule,DataProviderModule,ExchangeRateDataModule,MarketDataModule,diff --git a/apps/api/src/app/info/info.module.ts b/apps/api/src/app/info/info.module.tsindex 9ded446005..e33c5e0c2e 100644--- a/apps/api/src/app/info/info.module.ts+++ b/apps/api/src/app/info/info.module.ts@@ -8,7 +8,7 @@ import { ConfigurationModule } from '@ghostfolio/api/services/configuration/confimport { DataProviderModule } from '@ghostfolio/api/services/data-provider/data-provider.module';import { ExchangeRateDataModule } from '@ghostfolio/api/services/exchange-rate-data/exchange-rate-data.module';import { PropertyModule } from '@ghostfolio/api/services/property/property.module';-import { DataGatheringModule } from '@ghostfolio/api/services/queues/data-gathering/data-gathering.module';+import { DataGatheringQueueModule } from '@ghostfolio/api/services/queues/data-gathering/data-gathering.module';import { SymbolProfileModule } from '@ghostfolio/api/services/symbol-profile/symbol-profile.module';import { Module } from '@nestjs/common';@@ -22,7 +22,7 @@ import { InfoService } from './info.service';imports: [BenchmarkModule,ConfigurationModule,- DataGatheringModule,+ DataGatheringQueueModule,DataProviderModule,ExchangeRateDataModule,JwtModule.register({diff --git a/apps/api/src/app/info/info.service.ts b/apps/api/src/app/info/info.service.tsindex 9b4a4d5977..86630db53f 100644--- a/apps/api/src/app/info/info.service.ts+++ b/apps/api/src/app/info/info.service.ts@@ -7,26 +7,24 @@ import { ExchangeRateDataService } from '@ghostfolio/api/services/exchange-rate-import { PropertyService } from '@ghostfolio/api/services/property/property.service';import {DEFAULT_CURRENCY,- HEADER_KEY_TOKEN,- PROPERTY_BETTER_UPTIME_MONITOR_ID,PROPERTY_COUNTRIES_OF_SUBSCRIBERS,PROPERTY_DEMO_USER_ID,+ PROPERTY_DOCKER_HUB_PULLS,+ PROPERTY_GITHUB_CONTRIBUTORS,+ PROPERTY_GITHUB_STARGAZERS,PROPERTY_IS_READ_ONLY_MODE,PROPERTY_SLACK_COMMUNITY_USERS,+ PROPERTY_UPTIME,ghostfolioFearAndGreedIndexDataSourceStocks} from '@ghostfolio/common/config';-import {- DATE_FORMAT,- encodeDataSource,- extractNumberFromString-} from '@ghostfolio/common/helper';+import { encodeDataSource } from '@ghostfolio/common/helper';import { InfoItem, Statistics } from '@ghostfolio/common/interfaces';import { permissions } from '@ghostfolio/common/permissions';-import { Injectable, Logger } from '@nestjs/common';+import { Injectable } from '@nestjs/common';import { JwtService } from '@nestjs/jwt';-import * as cheerio from 'cheerio';-import { format, subDays } from 'date-fns';+import { subDays } from 'date-fns';+import { isNil } from 'lodash';@Injectable()export class InfoService {@@ -149,68 +147,6 @@ export class InfoService {});}- private async countDockerHubPulls(): Promise<number> {- try {- const { pull_count } = (await fetch(- 'https://hub.docker.com/v2/repositories/ghostfolio/ghostfolio',- {- headers: { 'User-Agent': 'request' },- signal: AbortSignal.timeout(- this.configurationService.get('REQUEST_TIMEOUT')- )- }- ).then((res) => res.json())) as { pull_count: number };-- return pull_count;- } catch (error) {- Logger.error(error, 'InfoService - DockerHub');-- return undefined;- }- }-- private async countGitHubContributors(): Promise<number> {- try {- const body = await fetch('https://github.com/ghostfolio/ghostfolio', {- signal: AbortSignal.timeout(- this.configurationService.get('REQUEST_TIMEOUT')- )- }).then((res) => res.text());-- const $ = cheerio.load(body);-- return extractNumberFromString({- value: $(- 'a[href="/ghostfolio/ghostfolio/graphs/contributors"] .Counter'- ).text()- });- } catch (error) {- Logger.error(error, 'InfoService - GitHub');-- return undefined;- }- }-- private async countGitHubStargazers(): Promise<number> {- try {- const { stargazers_count } = (await fetch(- 'https://api.github.com/repos/ghostfolio/ghostfolio',- {- headers: { 'User-Agent': 'request' },- signal: AbortSignal.timeout(- this.configurationService.get('REQUEST_TIMEOUT')- )- }- ).then((res) => res.json())) as { stargazers_count: number };-- return stargazers_count;- } catch (error) {- Logger.error(error, 'InfoService - GitHub');-- return undefined;- }- }-private async countNewUsers(aDays: number) {return this.userService.count({where: {@@ -230,12 +166,6 @@ export class InfoService {});}- private async countSlackCommunityUsers() {- return await this.propertyService.getByKey<string>(- PROPERTY_SLACK_COMMUNITY_USERS- );- }-private async getDemoAuthToken() {const demoUserId = await this.propertyService.getByKey<string>(PROPERTY_DEMO_USER_ID@@ -267,65 +197,56 @@ export class InfoService {}} catch {}- const activeUsers1d = await this.countActiveUsers(1);- const activeUsers30d = await this.countActiveUsers(30);- const newUsers30d = await this.countNewUsers(30);-- const dockerHubPulls = await this.countDockerHubPulls();- const gitHubContributors = await this.countGitHubContributors();- const gitHubStargazers = await this.countGitHubStargazers();- const slackCommunityUsers = await this.countSlackCommunityUsers();- const uptime = await this.getUptime();-- statistics = {+ const [activeUsers1d,activeUsers30d,+ newUsers30d,dockerHubPulls,gitHubContributors,gitHubStargazers,- newUsers30d,slackCommunityUsers,uptime+ ] = await Promise.all([+ this.countActiveUsers(1),+ this.countActiveUsers(30),+ this.countNewUsers(30),+ this.propertyService.getByKey<string>(PROPERTY_DOCKER_HUB_PULLS),+ this.propertyService.getByKey<string>(PROPERTY_GITHUB_CONTRIBUTORS),+ this.propertyService.getByKey<string>(PROPERTY_GITHUB_STARGAZERS),+ this.propertyService.getByKey<string>(PROPERTY_SLACK_COMMUNITY_USERS),+ this.propertyService.getByKey<string>(PROPERTY_UPTIME)+ ]);++ statistics = {+ activeUsers1d,+ activeUsers30d,+ newUsers30d,+ dockerHubPulls: dockerHubPulls+ ? Number.parseInt(dockerHubPulls, 10)+ : undefined,+ gitHubContributors: gitHubContributors+ ? Number.parseInt(gitHubContributors, 10)+ : undefined,+ gitHubStargazers: gitHubStargazers+ ? Number.parseInt(gitHubStargazers, 10)+ : undefined,+ slackCommunityUsers: slackCommunityUsers+ ? Number.parseInt(slackCommunityUsers, 10)+ : undefined,+ uptime: uptime ? Number.parseFloat(uptime) : undefined};- await this.redisCacheService.set(- InfoService.CACHE_KEY_STATISTICS,- JSON.stringify(statistics)- );+ if (+ Object.values(statistics).every((value) => {+ return !isNil(value);+ })+ ) {+ await this.redisCacheService.set(+ InfoService.CACHE_KEY_STATISTICS,+ JSON.stringify(statistics)+ );+ }return statistics;}-- private async getUptime(): Promise<number> {- {- try {- const monitorId = await this.propertyService.getByKey<string>(- PROPERTY_BETTER_UPTIME_MONITOR_ID- );-- const { data } = await fetch(- `https://uptime.betterstack.com/api/v2/monitors/${monitorId}/sla?from=${format(- subDays(new Date(), 90),- DATE_FORMAT- )}&to${format(new Date(), DATE_FORMAT)}`,- {- headers: {- [HEADER_KEY_TOKEN]: `Bearer ${this.configurationService.get(- 'API_KEY_BETTER_UPTIME'- )}`- },- signal: AbortSignal.timeout(- this.configurationService.get('REQUEST_TIMEOUT')- )- }- ).then((res) => res.json());-- return data.attributes.availability / 100;- } catch (error) {- Logger.error(error, 'InfoService - Better Stack');-- return undefined;- }- }- }}diff --git a/apps/api/src/app/portfolio/portfolio.module.ts b/apps/api/src/app/portfolio/portfolio.module.tsindex 65a9b71aa6..d818195ca8 100644--- a/apps/api/src/app/portfolio/portfolio.module.ts+++ b/apps/api/src/app/portfolio/portfolio.module.ts@@ -17,7 +17,7 @@ import { I18nModule } from '@ghostfolio/api/services/i18n/i18n.module';import { ImpersonationModule } from '@ghostfolio/api/services/impersonation/impersonation.module';import { MarketDataModule } from '@ghostfolio/api/services/market-data/market-data.module';import { PrismaModule } from '@ghostfolio/api/services/prisma/prisma.module';-import { DataGatheringModule } from '@ghostfolio/api/services/queues/data-gathering/data-gathering.module';+import { DataGatheringQueueModule } from '@ghostfolio/api/services/queues/data-gathering/data-gathering.module';import { PortfolioSnapshotQueueModule } from '@ghostfolio/api/services/queues/portfolio-snapshot/portfolio-snapshot.module';import { SymbolProfileModule } from '@ghostfolio/api/services/symbol-profile/symbol-profile.module';@@ -38,7 +38,7 @@ import { RulesService } from './rules.service';ApiModule,BenchmarkModule,ConfigurationModule,- DataGatheringModule,+ DataGatheringQueueModule,DataProviderModule,ExchangeRateDataModule,I18nModule,diff --git a/apps/api/src/events/events.module.ts b/apps/api/src/events/events.module.tsindex 7727669451..df943a3c95 100644--- a/apps/api/src/events/events.module.ts+++ b/apps/api/src/events/events.module.ts@@ -3,7 +3,7 @@ import { RedisCacheModule } from '@ghostfolio/api/app/redis-cache/redis-cache.moimport { ConfigurationModule } from '@ghostfolio/api/services/configuration/configuration.module';import { DataProviderModule } from '@ghostfolio/api/services/data-provider/data-provider.module';import { ExchangeRateDataModule } from '@ghostfolio/api/services/exchange-rate-data/exchange-rate-data.module';-import { DataGatheringModule } from '@ghostfolio/api/services/queues/data-gathering/data-gathering.module';+import { DataGatheringQueueModule } from '@ghostfolio/api/services/queues/data-gathering/data-gathering.module';import { Module } from '@nestjs/common';@@ -14,7 +14,7 @@ import { PortfolioChangedListener } from './portfolio-changed.listener';imports: [ActivitiesModule,ConfigurationModule,- DataGatheringModule,+ DataGatheringQueueModule,DataProviderModule,ExchangeRateDataModule,RedisCacheModulediff --git a/apps/api/src/services/cron/cron.module.ts b/apps/api/src/services/cron/cron.module.tsindex 06f9d2caae..bcc9d3360b 100644--- a/apps/api/src/services/cron/cron.module.ts+++ b/apps/api/src/services/cron/cron.module.ts@@ -2,7 +2,8 @@ import { UserModule } from '@ghostfolio/api/app/user/user.module';import { ConfigurationModule } from '@ghostfolio/api/services/configuration/configuration.module';import { ExchangeRateDataModule } from '@ghostfolio/api/services/exchange-rate-data/exchange-rate-data.module';import { PropertyModule } from '@ghostfolio/api/services/property/property.module';-import { DataGatheringModule } from '@ghostfolio/api/services/queues/data-gathering/data-gathering.module';+import { DataGatheringQueueModule } from '@ghostfolio/api/services/queues/data-gathering/data-gathering.module';+import { StatisticsGatheringQueueModule } from '@ghostfolio/api/services/queues/statistics-gathering/statistics-gathering.module';import { TwitterBotModule } from '@ghostfolio/api/services/twitter-bot/twitter-bot.module';import { Module } from '@nestjs/common';@@ -12,9 +13,10 @@ import { CronService } from './cron.service';@Module({imports: [ConfigurationModule,- DataGatheringModule,+ DataGatheringQueueModule,ExchangeRateDataModule,PropertyModule,+ StatisticsGatheringQueueModule,TwitterBotModule,UserModule],diff --git a/apps/api/src/services/cron/cron.service.ts b/apps/api/src/services/cron/cron.service.tsindex ee91a811e6..e680f00639 100644--- a/apps/api/src/services/cron/cron.service.ts+++ b/apps/api/src/services/cron/cron.service.ts@@ -3,6 +3,7 @@ import { ConfigurationService } from '@ghostfolio/api/services/configuration/conimport { ExchangeRateDataService } from '@ghostfolio/api/services/exchange-rate-data/exchange-rate-data.service';import { PropertyService } from '@ghostfolio/api/services/property/property.service';import { DataGatheringService } from '@ghostfolio/api/services/queues/data-gathering/data-gathering.service';+import { StatisticsGatheringService } from '@ghostfolio/api/services/queues/statistics-gathering/statistics-gathering.service';import { TwitterBotService } from '@ghostfolio/api/services/twitter-bot/twitter-bot.service';import {DATA_GATHERING_QUEUE_PRIORITY_LOW,@@ -25,10 +26,18 @@ export class CronService {private readonly dataGatheringService: DataGatheringService,private readonly exchangeRateDataService: ExchangeRateDataService,private readonly propertyService: PropertyService,+ private readonly statisticsGatheringService: StatisticsGatheringService,private readonly twitterBotService: TwitterBotService,private readonly userService: UserService) {}+ @Cron(CronExpression.EVERY_HOUR)+ public async runEveryHour() {+ if (this.configurationService.get('ENABLE_FEATURE_STATISTICS')) {+ await this.statisticsGatheringService.addJobsToQueue();+ }+ }+@Cron(CronService.EVERY_HOUR_AT_RANDOM_MINUTE)public async runEveryHourAtRandomMinute() {if (await this.isDataGatheringEnabled()) {diff --git a/apps/api/src/services/queues/data-gathering/data-gathering.module.ts b/apps/api/src/services/queues/data-gathering/data-gathering.module.tsindex f251c8d0c9..d163f0d29d 100644--- a/apps/api/src/services/queues/data-gathering/data-gathering.module.ts+++ b/apps/api/src/services/queues/data-gathering/data-gathering.module.ts@@ -50,4 +50,4 @@ import { DataGatheringProcessor } from './data-gathering.processor';providers: [DataGatheringProcessor, DataGatheringService],exports: [BullModule, DataEnhancerModule, DataGatheringService]})-export class DataGatheringModule {}+export class DataGatheringQueueModule {}diff --git a/apps/api/src/services/queues/statistics-gathering/statistics-gathering.module.ts b/apps/api/src/services/queues/statistics-gathering/statistics-gathering.module.tsnew file mode 100644index 0000000000..1818dd4ec5--- /dev/null+++ b/apps/api/src/services/queues/statistics-gathering/statistics-gathering.module.ts@@ -0,0 +1,36 @@+import { ConfigurationModule } from '@ghostfolio/api/services/configuration/configuration.module';+import { PropertyModule } from '@ghostfolio/api/services/property/property.module';+import { STATISTICS_GATHERING_QUEUE } from '@ghostfolio/common/config';++import { BullAdapter } from '@bull-board/api/bullAdapter';+import { BullBoardModule } from '@bull-board/nestjs';+import { BullModule } from '@nestjs/bull';+import { Module } from '@nestjs/common';++import { StatisticsGatheringProcessor } from './statistics-gathering.processor';+import { StatisticsGatheringService } from './statistics-gathering.service';++@Module({+ exports: [BullModule, StatisticsGatheringService],+ imports: [+ ...(process.env.ENABLE_FEATURE_BULL_BOARD === 'true'+ ? [+ BullBoardModule.forFeature({+ adapter: BullAdapter,+ name: STATISTICS_GATHERING_QUEUE,+ options: {+ displayName: 'Statistics Gathering',+ readOnlyMode: process.env.BULL_BOARD_IS_READ_ONLY !== 'false'+ }+ })+ ]+ : []),+ BullModule.registerQueue({+ name: STATISTICS_GATHERING_QUEUE+ }),+ ConfigurationModule,+ PropertyModule+ ],+ providers: [StatisticsGatheringProcessor, StatisticsGatheringService]+})+export class StatisticsGatheringQueueModule {}diff --git a/apps/api/src/services/queues/statistics-gathering/statistics-gathering.processor.ts b/apps/api/src/services/queues/statistics-gathering/statistics-gathering.processor.tsnew file mode 100644index 0000000000..07fc32585a--- /dev/null+++ b/apps/api/src/services/queues/statistics-gathering/statistics-gathering.processor.ts@@ -0,0 +1,212 @@+import { ConfigurationService } from '@ghostfolio/api/services/configuration/configuration.service';+import { PropertyService } from '@ghostfolio/api/services/property/property.service';+import {+ GATHER_STATISTICS_DOCKER_HUB_PULLS_PROCESS_JOB_NAME,+ GATHER_STATISTICS_GITHUB_CONTRIBUTORS_PROCESS_JOB_NAME,+ GATHER_STATISTICS_GITHUB_STARGAZERS_PROCESS_JOB_NAME,+ GATHER_STATISTICS_UPTIME_PROCESS_JOB_NAME,+ HEADER_KEY_TOKEN,+ PROPERTY_BETTER_UPTIME_MONITOR_ID,+ PROPERTY_DOCKER_HUB_PULLS,+ PROPERTY_GITHUB_CONTRIBUTORS,+ PROPERTY_GITHUB_STARGAZERS,+ PROPERTY_UPTIME,+ STATISTICS_GATHERING_QUEUE+} from '@ghostfolio/common/config';+import {+ DATE_FORMAT,+ extractNumberFromString+} from '@ghostfolio/common/helper';++import { Process, Processor } from '@nestjs/bull';+import { Injectable, Logger } from '@nestjs/common';+import * as cheerio from 'cheerio';+import { format, subDays } from 'date-fns';++@Injectable()+@Processor(STATISTICS_GATHERING_QUEUE)+export class StatisticsGatheringProcessor {+ public constructor(+ private readonly configurationService: ConfigurationService,+ private readonly propertyService: PropertyService+ ) {}++ @Process(GATHER_STATISTICS_DOCKER_HUB_PULLS_PROCESS_JOB_NAME)+ public async gatherDockerHubPullsStatistics() {+ Logger.log(+ 'Docker Hub pulls statistics gathering has been started',+ 'StatisticsGatheringProcessor'+ );++ const dockerHubPulls = await this.countDockerHubPulls();++ await this.propertyService.put({+ key: PROPERTY_DOCKER_HUB_PULLS,+ value: String(dockerHubPulls)+ });++ Logger.log(+ 'Docker Hub pulls statistics gathering has been completed',+ 'StatisticsGatheringProcessor'+ );+ }++ @Process(GATHER_STATISTICS_GITHUB_CONTRIBUTORS_PROCESS_JOB_NAME)+ public async gatherGitHubContributorsStatistics() {+ Logger.log(+ 'GitHub contributors statistics gathering has been started',+ 'StatisticsGatheringProcessor'+ );++ const gitHubContributors = await this.countGitHubContributors();++ await this.propertyService.put({+ key: PROPERTY_GITHUB_CONTRIBUTORS,+ value: String(gitHubContributors)+ });++ Logger.log(+ 'GitHub contributors statistics gathering has been completed',+ 'StatisticsGatheringProcessor'+ );+ }++ @Process(GATHER_STATISTICS_GITHUB_STARGAZERS_PROCESS_JOB_NAME)+ public async gatherGitHubStargazersStatistics() {+ Logger.log(+ 'GitHub stargazers statistics gathering has been started',+ 'StatisticsGatheringProcessor'+ );++ const gitHubStargazers = await this.countGitHubStargazers();++ await this.propertyService.put({+ key: PROPERTY_GITHUB_STARGAZERS,+ value: String(gitHubStargazers)+ });++ Logger.log(+ 'GitHub stargazers statistics gathering has been completed',+ 'StatisticsGatheringProcessor'+ );+ }++ @Process(GATHER_STATISTICS_UPTIME_PROCESS_JOB_NAME)+ public async gatherUptimeStatistics() {+ Logger.log(+ 'Uptime statistics gathering has been started',+ 'StatisticsGatheringProcessor'+ );++ const uptime = await this.getUptime();++ await this.propertyService.put({+ key: PROPERTY_UPTIME,+ value: String(uptime)+ });++ Logger.log(+ 'Uptime statistics gathering has been completed',+ 'StatisticsGatheringProcessor'+ );+ }++ private async countDockerHubPulls(): Promise<number> {+ try {+ const { pull_count } = (await fetch(+ 'https://hub.docker.com/v2/repositories/ghostfolio/ghostfolio',+ {+ headers: { 'User-Agent': 'request' },+ signal: AbortSignal.timeout(+ this.configurationService.get('REQUEST_TIMEOUT')+ )+ }+ ).then((res) => res.json())) as { pull_count: number };++ return pull_count;+ } catch (error) {+ Logger.error(error, 'StatisticsGatheringProcessor - DockerHub');++ throw error;+ }+ }++ private async countGitHubContributors(): Promise<number> {+ try {+ const body = await fetch('https://github.com/ghostfolio/ghostfolio', {+ signal: AbortSignal.timeout(+ this.configurationService.get('REQUEST_TIMEOUT')+ )+ }).then((res) => res.text());++ const $ = cheerio.load(body);++ const value = $(+ 'a[href="/ghostfolio/ghostfolio/graphs/contributors"] .Counter'+ ).text();++ if (!value) {+ throw new Error('Could not find the contributors count in the page');+ }++ return extractNumberFromString({+ value+ });+ } catch (error) {+ Logger.error(error, 'StatisticsGatheringProcessor - GitHub');++ throw error;+ }+ }++ private async countGitHubStargazers(): Promise<number> {+ try {+ const { stargazers_count } = (await fetch(+ 'https://api.github.com/repos/ghostfolio/ghostfolio',+ {+ headers: { 'User-Agent': 'request' },+ signal: AbortSignal.timeout(+ this.configurationService.get('REQUEST_TIMEOUT')+ )+ }+ ).then((res) => res.json())) as { stargazers_count: number };++ return stargazers_count;+ } catch (error) {+ Logger.error(error, 'StatisticsGatheringProcessor - GitHub');++ throw error;+ }+ }++ private async getUptime(): Promise<number> {+ try {+ const monitorId = await this.propertyService.getByKey<string>(+ PROPERTY_BETTER_UPTIME_MONITOR_ID+ );++ const { data } = await fetch(+ `https://uptime.betterstack.com/api/v2/monitors/${monitorId}/sla?from=${format(+ subDays(new Date(), 90),+ DATE_FORMAT+ )}&to${format(new Date(), DATE_FORMAT)}`,+ {+ headers: {+ [HEADER_KEY_TOKEN]: `Bearer ${this.configurationService.get(+ 'API_KEY_BETTER_UPTIME'+ )}`+ },+ signal: AbortSignal.timeout(+ this.configurationService.get('REQUEST_TIMEOUT')+ )+ }+ ).then((res) => res.json());++ return data.attributes.availability / 100;+ } catch (error) {+ Logger.error(error, 'StatisticsGatheringProcessor - Better Stack');++ throw error;+ }+ }+}diff --git a/apps/api/src/services/queues/statistics-gathering/statistics-gathering.service.ts b/apps/api/src/services/queues/statistics-gathering/statistics-gathering.service.tsnew file mode 100644index 0000000000..68c6746880--- /dev/null+++ b/apps/api/src/services/queues/statistics-gathering/statistics-gathering.service.ts@@ -0,0 +1,40 @@+import {+ GATHER_STATISTICS_DOCKER_HUB_PULLS_PROCESS_JOB_NAME,+ GATHER_STATISTICS_GITHUB_CONTRIBUTORS_PROCESS_JOB_NAME,+ GATHER_STATISTICS_GITHUB_STARGAZERS_PROCESS_JOB_NAME,+ GATHER_STATISTICS_PROCESS_JOB_OPTIONS,+ GATHER_STATISTICS_UPTIME_PROCESS_JOB_NAME,+ STATISTICS_GATHERING_QUEUE+} from '@ghostfolio/common/config';++import { InjectQueue } from '@nestjs/bull';+import { Injectable } from '@nestjs/common';+import { Queue } from 'bull';++@Injectable()+export class StatisticsGatheringService {+ public constructor(+ @InjectQueue(STATISTICS_GATHERING_QUEUE)+ private readonly statisticsGatheringQueue: Queue+ ) {}++ public async addJobsToQueue() {+ return Promise.all(+ [+ GATHER_STATISTICS_DOCKER_HUB_PULLS_PROCESS_JOB_NAME,+ GATHER_STATISTICS_GITHUB_CONTRIBUTORS_PROCESS_JOB_NAME,+ GATHER_STATISTICS_GITHUB_STARGAZERS_PROCESS_JOB_NAME,+ GATHER_STATISTICS_UPTIME_PROCESS_JOB_NAME+ ].map((jobName) => {+ return this.statisticsGatheringQueue.add(+ jobName,+ {},+ {+ ...GATHER_STATISTICS_PROCESS_JOB_OPTIONS,+ jobId: jobName+ }+ );+ })+ );+ }+}diff --git a/libs/common/src/lib/config.ts b/libs/common/src/lib/config.tsindex b442f2b7a3..42e1f6b630 100644--- a/libs/common/src/lib/config.ts+++ b/libs/common/src/lib/config.ts@@ -75,6 +75,8 @@ export const PORTFOLIO_SNAPSHOT_COMPUTATION_QUEUE_PRIORITY_HIGH = 1;export const PORTFOLIO_SNAPSHOT_COMPUTATION_QUEUE_PRIORITY_LOW =Number.MAX_SAFE_INTEGER;+export const STATISTICS_GATHERING_QUEUE = 'STATISTICS_GATHERING_QUEUE';+export const DEFAULT_CURRENCY = 'USD';export const DEFAULT_DATE_FORMAT_MONTH_YEAR = 'MMM yyyy';export const DEFAULT_HOST = '0.0.0.0';@@ -183,6 +185,27 @@ export const GATHER_HISTORICAL_MARKET_DATA_PROCESS_JOB_OPTIONS: JobOptions = {removeOnComplete: true};+export const GATHER_STATISTICS_PROCESS_JOB_OPTIONS: JobOptions = {+ attempts: 5,+ backoff: {+ delay: ms('1 minute'),+ type: 'exponential'+ },+ removeOnComplete: true+};++export const GATHER_STATISTICS_DOCKER_HUB_PULLS_PROCESS_JOB_NAME =+ 'GATHER_STATISTICS_DOCKER_HUB_PULLS';++export const GATHER_STATISTICS_GITHUB_CONTRIBUTORS_PROCESS_JOB_NAME =+ 'GATHER_STATISTICS_GITHUB_CONTRIBUTORS';++export const GATHER_STATISTICS_GITHUB_STARGAZERS_PROCESS_JOB_NAME =+ 'GATHER_STATISTICS_GITHUB_STARGAZERS';++export const GATHER_STATISTICS_UPTIME_PROCESS_JOB_NAME =+ 'GATHER_STATISTICS_UPTIME';+export const INVESTMENT_ACTIVITY_TYPES = [Type.BUY,Type.DIVIDEND,@@ -209,6 +232,9 @@ export const PROPERTY_API_KEY_GHOSTFOLIO = 'API_KEY_GHOSTFOLIO';export const PROPERTY_API_KEY_OPENROUTER = 'API_KEY_OPENROUTER';export const PROPERTY_BENCHMARKS = 'BENCHMARKS';export const PROPERTY_BETTER_UPTIME_MONITOR_ID = 'BETTER_UPTIME_MONITOR_ID';+export const PROPERTY_DOCKER_HUB_PULLS = 'DOCKER_HUB_PULLS';+export const PROPERTY_GITHUB_CONTRIBUTORS = 'GITHUB_CONTRIBUTORS';+export const PROPERTY_GITHUB_STARGAZERS = 'GITHUB_STARGAZERS';export const PROPERTY_COUNTRIES_OF_SUBSCRIBERS = 'COUNTRIES_OF_SUBSCRIBERS';export const PROPERTY_COUPONS = 'COUPONS';export const PROPERTY_CURRENCIES = 'CURRENCIES';@@ -225,6 +251,7 @@ export const PROPERTY_OPENROUTER_MODEL = 'OPENROUTER_MODEL';export const PROPERTY_SLACK_COMMUNITY_USERS = 'SLACK_COMMUNITY_USERS';export const PROPERTY_STRIPE_CONFIG = 'STRIPE_CONFIG';export const PROPERTY_SYSTEM_MESSAGE = 'SYSTEM_MESSAGE';+export const PROPERTY_UPTIME = 'UPTIME';export const QUEUE_JOB_STATUS_LIST = ['active',diff --git a/libs/common/src/lib/interfaces/statistics.interface.ts b/libs/common/src/lib/interfaces/statistics.interface.tsindex 2852d34ab4..670cdaf4c5 100644--- a/libs/common/src/lib/interfaces/statistics.interface.ts+++ b/libs/common/src/lib/interfaces/statistics.interface.ts@@ -5,6 +5,6 @@ export interface Statistics {gitHubContributors: number;gitHubStargazers: number;newUsers30d: number;- slackCommunityUsers: string;+ slackCommunityUsers: number;uptime: number;}