Skip to content

Open Source Wealth Management Software. Angular + NestJS + Prisma + Nx + TypeScript 🤍

Feature/setup statistics gathering queue (#6696)

Thomas Kaulcdef2afa48af
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 29d71c9fe9..1cc28c4eb4 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -15,6 +15,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
- Extended the asset profile details dialog in the admin control panel to support editing countries for all asset types
- Extended the asset profile details dialog in the admin control panel to support editing sectors for all asset types
+- Migrated the data collection for the _Open Startup_ (`/open`) page to the queue design pattern
- Improved the language localization for German (`de`)
- Upgraded `lodash` from version `4.17.23` to `4.18.1`
diff --git a/apps/api/src/app/activities/activities.module.ts b/apps/api/src/app/activities/activities.module.ts
index 7476ad66ad..f4e592c3fc 100644
--- a/apps/api/src/app/activities/activities.module.ts
+++ b/apps/api/src/app/activities/activities.module.ts
@@ -10,7 +10,7 @@ import { DataProviderModule } from '@ghostfolio/api/services/data-provider/data-
import { ExchangeRateDataModule } from '@ghostfolio/api/services/exchange-rate-data/exchange-rate-data.module';
import { ImpersonationModule } from '@ghostfolio/api/services/impersonation/impersonation.module';
import { PrismaModule } from '@ghostfolio/api/services/prisma/prisma.module';
-import { DataGatheringModule } from '@ghostfolio/api/services/queues/data-gathering/data-gathering.module';
+import { DataGatheringQueueModule } from '@ghostfolio/api/services/queues/data-gathering/data-gathering.module';
import { SymbolProfileModule } from '@ghostfolio/api/services/symbol-profile/symbol-profile.module';
import { Module } from '@nestjs/common';
@@ -24,7 +24,7 @@ import { ActivitiesService } from './activities.service';
imports: [
ApiModule,
CacheModule,
- DataGatheringModule,
+ DataGatheringQueueModule,
DataProviderModule,
ExchangeRateDataModule,
ImpersonationModule,
diff --git a/apps/api/src/app/admin/admin.module.ts b/apps/api/src/app/admin/admin.module.ts
index 960a366292..e87df9e749 100644
--- a/apps/api/src/app/admin/admin.module.ts
+++ b/apps/api/src/app/admin/admin.module.ts
@@ -9,7 +9,7 @@ import { ExchangeRateDataModule } from '@ghostfolio/api/services/exchange-rate-d
import { MarketDataModule } from '@ghostfolio/api/services/market-data/market-data.module';
import { PrismaModule } from '@ghostfolio/api/services/prisma/prisma.module';
import { PropertyModule } from '@ghostfolio/api/services/property/property.module';
-import { DataGatheringModule } from '@ghostfolio/api/services/queues/data-gathering/data-gathering.module';
+import { DataGatheringQueueModule } from '@ghostfolio/api/services/queues/data-gathering/data-gathering.module';
import { SymbolProfileModule } from '@ghostfolio/api/services/symbol-profile/symbol-profile.module';
import { Module } from '@nestjs/common';
@@ -24,7 +24,7 @@ import { QueueModule } from './queue/queue.module';
ApiModule,
BenchmarkModule,
ConfigurationModule,
- DataGatheringModule,
+ DataGatheringQueueModule,
DataProviderModule,
DemoModule,
ExchangeRateDataModule,
diff --git a/apps/api/src/app/admin/queue/queue.module.ts b/apps/api/src/app/admin/queue/queue.module.ts
index 22d1cefc6a..4cfb794921 100644
--- a/apps/api/src/app/admin/queue/queue.module.ts
+++ b/apps/api/src/app/admin/queue/queue.module.ts
@@ -1,5 +1,6 @@
-import { DataGatheringModule } from '@ghostfolio/api/services/queues/data-gathering/data-gathering.module';
+import { DataGatheringQueueModule } from '@ghostfolio/api/services/queues/data-gathering/data-gathering.module';
import { PortfolioSnapshotQueueModule } from '@ghostfolio/api/services/queues/portfolio-snapshot/portfolio-snapshot.module';
+import { StatisticsGatheringQueueModule } from '@ghostfolio/api/services/queues/statistics-gathering/statistics-gathering.module';
import { Module } from '@nestjs/common';
@@ -8,7 +9,11 @@ import { QueueService } from './queue.service';
@Module({
controllers: [QueueController],
- imports: [DataGatheringModule, PortfolioSnapshotQueueModule],
+ imports: [
+ DataGatheringQueueModule,
+ PortfolioSnapshotQueueModule,
+ StatisticsGatheringQueueModule
+ ],
providers: [QueueService]
})
export class QueueModule {}
diff --git a/apps/api/src/app/admin/queue/queue.service.ts b/apps/api/src/app/admin/queue/queue.service.ts
index 747c4d6fbe..f47b3d3a1e 100644
--- a/apps/api/src/app/admin/queue/queue.service.ts
+++ b/apps/api/src/app/admin/queue/queue.service.ts
@@ -1,7 +1,8 @@
import {
DATA_GATHERING_QUEUE,
PORTFOLIO_SNAPSHOT_COMPUTATION_QUEUE,
- QUEUE_JOB_STATUS_LIST
+ QUEUE_JOB_STATUS_LIST,
+ STATISTICS_GATHERING_QUEUE
} from '@ghostfolio/common/config';
import { AdminJobs } from '@ghostfolio/common/interfaces';
@@ -15,7 +16,9 @@ export class QueueService {
@InjectQueue(DATA_GATHERING_QUEUE)
private readonly dataGatheringQueue: Queue,
@InjectQueue(PORTFOLIO_SNAPSHOT_COMPUTATION_QUEUE)
- private readonly portfolioSnapshotQueue: Queue
+ private readonly portfolioSnapshotQueue: Queue,
+ @InjectQueue(STATISTICS_GATHERING_QUEUE)
+ private readonly statisticsGatheringQueue: Queue
) {}
public async deleteJob(aId: string) {
@@ -38,6 +41,7 @@ export class QueueService {
await this.dataGatheringQueue.clean(300, queueStatus);
await this.portfolioSnapshotQueue.clean(300, queueStatus);
+ await this.statisticsGatheringQueue.clean(300, queueStatus);
}
}
@@ -58,13 +62,19 @@ export class QueueService {
limit?: number;
status?: JobStatus[];
}): Promise<AdminJobs> {
- const [dataGatheringJobs, portfolioSnapshotJobs] = await Promise.all([
- this.dataGatheringQueue.getJobs(status),
- this.portfolioSnapshotQueue.getJobs(status)
- ]);
+ const [dataGatheringJobs, portfolioSnapshotJobs, statisticsGatheringJobs] =
+ await Promise.all([
+ this.dataGatheringQueue.getJobs(status),
+ this.portfolioSnapshotQueue.getJobs(status),
+ this.statisticsGatheringQueue.getJobs(status)
+ ]);
const jobsWithState = await Promise.all(
- [...dataGatheringJobs, ...portfolioSnapshotJobs]
+ [
+ ...dataGatheringJobs,
+ ...portfolioSnapshotJobs,
+ ...statisticsGatheringJobs
+ ]
.filter((job) => {
return job;
})
diff --git a/apps/api/src/app/app.module.ts b/apps/api/src/app/app.module.ts
index 8ebe05928f..3316f9ce4d 100644
--- a/apps/api/src/app/app.module.ts
+++ b/apps/api/src/app/app.module.ts
@@ -8,7 +8,7 @@ import { ExchangeRateDataModule } from '@ghostfolio/api/services/exchange-rate-d
import { I18nService } from '@ghostfolio/api/services/i18n/i18n.service';
import { PrismaModule } from '@ghostfolio/api/services/prisma/prisma.module';
import { PropertyModule } from '@ghostfolio/api/services/property/property.module';
-import { DataGatheringModule } from '@ghostfolio/api/services/queues/data-gathering/data-gathering.module';
+import { DataGatheringQueueModule } from '@ghostfolio/api/services/queues/data-gathering/data-gathering.module';
import { PortfolioSnapshotQueueModule } from '@ghostfolio/api/services/queues/portfolio-snapshot/portfolio-snapshot.module';
import {
BULL_BOARD_ROUTE,
@@ -109,7 +109,7 @@ import { UserModule } from './user/user.module';
ConfigModule.forRoot(),
ConfigurationModule,
CronModule,
- DataGatheringModule,
+ DataGatheringQueueModule,
DataProviderModule,
EventEmitterModule.forRoot(),
EventsModule,
diff --git a/apps/api/src/app/endpoints/watchlist/watchlist.module.ts b/apps/api/src/app/endpoints/watchlist/watchlist.module.ts
index ce9ae12bbf..9b4b960a08 100644
--- a/apps/api/src/app/endpoints/watchlist/watchlist.module.ts
+++ b/apps/api/src/app/endpoints/watchlist/watchlist.module.ts
@@ -5,7 +5,7 @@ import { DataProviderModule } from '@ghostfolio/api/services/data-provider/data-
import { ImpersonationModule } from '@ghostfolio/api/services/impersonation/impersonation.module';
import { MarketDataModule } from '@ghostfolio/api/services/market-data/market-data.module';
import { PrismaModule } from '@ghostfolio/api/services/prisma/prisma.module';
-import { DataGatheringModule } from '@ghostfolio/api/services/queues/data-gathering/data-gathering.module';
+import { DataGatheringQueueModule } from '@ghostfolio/api/services/queues/data-gathering/data-gathering.module';
import { SymbolProfileModule } from '@ghostfolio/api/services/symbol-profile/symbol-profile.module';
import { Module } from '@nestjs/common';
@@ -17,7 +17,7 @@ import { WatchlistService } from './watchlist.service';
controllers: [WatchlistController],
imports: [
BenchmarkModule,
- DataGatheringModule,
+ DataGatheringQueueModule,
DataProviderModule,
ImpersonationModule,
MarketDataModule,
diff --git a/apps/api/src/app/import/import.module.ts b/apps/api/src/app/import/import.module.ts
index ca9b5667b5..8aebcfa082 100644
--- a/apps/api/src/app/import/import.module.ts
+++ b/apps/api/src/app/import/import.module.ts
@@ -12,7 +12,7 @@ import { DataProviderModule } from '@ghostfolio/api/services/data-provider/data-
import { ExchangeRateDataModule } from '@ghostfolio/api/services/exchange-rate-data/exchange-rate-data.module';
import { MarketDataModule } from '@ghostfolio/api/services/market-data/market-data.module';
import { PrismaModule } from '@ghostfolio/api/services/prisma/prisma.module';
-import { DataGatheringModule } from '@ghostfolio/api/services/queues/data-gathering/data-gathering.module';
+import { DataGatheringQueueModule } from '@ghostfolio/api/services/queues/data-gathering/data-gathering.module';
import { SymbolProfileModule } from '@ghostfolio/api/services/symbol-profile/symbol-profile.module';
import { TagModule } from '@ghostfolio/api/services/tag/tag.module';
@@ -29,7 +29,7 @@ import { ImportService } from './import.service';
ApiModule,
CacheModule,
ConfigurationModule,
- DataGatheringModule,
+ DataGatheringQueueModule,
DataProviderModule,
ExchangeRateDataModule,
MarketDataModule,
diff --git a/apps/api/src/app/info/info.module.ts b/apps/api/src/app/info/info.module.ts
index 9ded446005..e33c5e0c2e 100644
--- a/apps/api/src/app/info/info.module.ts
+++ b/apps/api/src/app/info/info.module.ts
@@ -8,7 +8,7 @@ import { ConfigurationModule } from '@ghostfolio/api/services/configuration/conf
import { DataProviderModule } from '@ghostfolio/api/services/data-provider/data-provider.module';
import { ExchangeRateDataModule } from '@ghostfolio/api/services/exchange-rate-data/exchange-rate-data.module';
import { PropertyModule } from '@ghostfolio/api/services/property/property.module';
-import { DataGatheringModule } from '@ghostfolio/api/services/queues/data-gathering/data-gathering.module';
+import { DataGatheringQueueModule } from '@ghostfolio/api/services/queues/data-gathering/data-gathering.module';
import { SymbolProfileModule } from '@ghostfolio/api/services/symbol-profile/symbol-profile.module';
import { Module } from '@nestjs/common';
@@ -22,7 +22,7 @@ import { InfoService } from './info.service';
imports: [
BenchmarkModule,
ConfigurationModule,
- DataGatheringModule,
+ DataGatheringQueueModule,
DataProviderModule,
ExchangeRateDataModule,
JwtModule.register({
diff --git a/apps/api/src/app/info/info.service.ts b/apps/api/src/app/info/info.service.ts
index 9b4a4d5977..86630db53f 100644
--- a/apps/api/src/app/info/info.service.ts
+++ b/apps/api/src/app/info/info.service.ts
@@ -7,26 +7,24 @@ import { ExchangeRateDataService } from '@ghostfolio/api/services/exchange-rate-
import { PropertyService } from '@ghostfolio/api/services/property/property.service';
import {
DEFAULT_CURRENCY,
- HEADER_KEY_TOKEN,
- PROPERTY_BETTER_UPTIME_MONITOR_ID,
PROPERTY_COUNTRIES_OF_SUBSCRIBERS,
PROPERTY_DEMO_USER_ID,
+ PROPERTY_DOCKER_HUB_PULLS,
+ PROPERTY_GITHUB_CONTRIBUTORS,
+ PROPERTY_GITHUB_STARGAZERS,
PROPERTY_IS_READ_ONLY_MODE,
PROPERTY_SLACK_COMMUNITY_USERS,
+ PROPERTY_UPTIME,
ghostfolioFearAndGreedIndexDataSourceStocks
} from '@ghostfolio/common/config';
-import {
- DATE_FORMAT,
- encodeDataSource,
- extractNumberFromString
-} from '@ghostfolio/common/helper';
+import { encodeDataSource } from '@ghostfolio/common/helper';
import { InfoItem, Statistics } from '@ghostfolio/common/interfaces';
import { permissions } from '@ghostfolio/common/permissions';
-import { Injectable, Logger } from '@nestjs/common';
+import { Injectable } from '@nestjs/common';
import { JwtService } from '@nestjs/jwt';
-import * as cheerio from 'cheerio';
-import { format, subDays } from 'date-fns';
+import { subDays } from 'date-fns';
+import { isNil } from 'lodash';
@Injectable()
export class InfoService {
@@ -149,68 +147,6 @@ export class InfoService {
});
}
- private async countDockerHubPulls(): Promise<number> {
- try {
- const { pull_count } = (await fetch(
- 'https://hub.docker.com/v2/repositories/ghostfolio/ghostfolio',
- {
- headers: { 'User-Agent': 'request' },
- signal: AbortSignal.timeout(
- this.configurationService.get('REQUEST_TIMEOUT')
- )
- }
- ).then((res) => res.json())) as { pull_count: number };
-
- return pull_count;
- } catch (error) {
- Logger.error(error, 'InfoService - DockerHub');
-
- return undefined;
- }
- }
-
- private async countGitHubContributors(): Promise<number> {
- try {
- const body = await fetch('https://github.com/ghostfolio/ghostfolio', {
- signal: AbortSignal.timeout(
- this.configurationService.get('REQUEST_TIMEOUT')
- )
- }).then((res) => res.text());
-
- const $ = cheerio.load(body);
-
- return extractNumberFromString({
- value: $(
- 'a[href="/ghostfolio/ghostfolio/graphs/contributors"] .Counter'
- ).text()
- });
- } catch (error) {
- Logger.error(error, 'InfoService - GitHub');
-
- return undefined;
- }
- }
-
- private async countGitHubStargazers(): Promise<number> {
- try {
- const { stargazers_count } = (await fetch(
- 'https://api.github.com/repos/ghostfolio/ghostfolio',
- {
- headers: { 'User-Agent': 'request' },
- signal: AbortSignal.timeout(
- this.configurationService.get('REQUEST_TIMEOUT')
- )
- }
- ).then((res) => res.json())) as { stargazers_count: number };
-
- return stargazers_count;
- } catch (error) {
- Logger.error(error, 'InfoService - GitHub');
-
- return undefined;
- }
- }
-
private async countNewUsers(aDays: number) {
return this.userService.count({
where: {
@@ -230,12 +166,6 @@ export class InfoService {
});
}
- private async countSlackCommunityUsers() {
- return await this.propertyService.getByKey<string>(
- PROPERTY_SLACK_COMMUNITY_USERS
- );
- }
-
private async getDemoAuthToken() {
const demoUserId = await this.propertyService.getByKey<string>(
PROPERTY_DEMO_USER_ID
@@ -267,65 +197,56 @@ export class InfoService {
}
} catch {}
- const activeUsers1d = await this.countActiveUsers(1);
- const activeUsers30d = await this.countActiveUsers(30);
- const newUsers30d = await this.countNewUsers(30);
-
- const dockerHubPulls = await this.countDockerHubPulls();
- const gitHubContributors = await this.countGitHubContributors();
- const gitHubStargazers = await this.countGitHubStargazers();
- const slackCommunityUsers = await this.countSlackCommunityUsers();
- const uptime = await this.getUptime();
-
- statistics = {
+ const [
activeUsers1d,
activeUsers30d,
+ newUsers30d,
dockerHubPulls,
gitHubContributors,
gitHubStargazers,
- newUsers30d,
slackCommunityUsers,
uptime
+ ] = await Promise.all([
+ this.countActiveUsers(1),
+ this.countActiveUsers(30),
+ this.countNewUsers(30),
+ this.propertyService.getByKey<string>(PROPERTY_DOCKER_HUB_PULLS),
+ this.propertyService.getByKey<string>(PROPERTY_GITHUB_CONTRIBUTORS),
+ this.propertyService.getByKey<string>(PROPERTY_GITHUB_STARGAZERS),
+ this.propertyService.getByKey<string>(PROPERTY_SLACK_COMMUNITY_USERS),
+ this.propertyService.getByKey<string>(PROPERTY_UPTIME)
+ ]);
+
+ statistics = {
+ activeUsers1d,
+ activeUsers30d,
+ newUsers30d,
+ dockerHubPulls: dockerHubPulls
+ ? Number.parseInt(dockerHubPulls, 10)
+ : undefined,
+ gitHubContributors: gitHubContributors
+ ? Number.parseInt(gitHubContributors, 10)
+ : undefined,
+ gitHubStargazers: gitHubStargazers
+ ? Number.parseInt(gitHubStargazers, 10)
+ : undefined,
+ slackCommunityUsers: slackCommunityUsers
+ ? Number.parseInt(slackCommunityUsers, 10)
+ : undefined,
+ uptime: uptime ? Number.parseFloat(uptime) : undefined
};
- await this.redisCacheService.set(
- InfoService.CACHE_KEY_STATISTICS,
- JSON.stringify(statistics)
- );
+ if (
+ Object.values(statistics).every((value) => {
+ return !isNil(value);
+ })
+ ) {
+ await this.redisCacheService.set(
+ InfoService.CACHE_KEY_STATISTICS,
+ JSON.stringify(statistics)
+ );
+ }
return statistics;
}
-
- private async getUptime(): Promise<number> {
- {
- try {
- const monitorId = await this.propertyService.getByKey<string>(
- PROPERTY_BETTER_UPTIME_MONITOR_ID
- );
-
- const { data } = await fetch(
- `https://uptime.betterstack.com/api/v2/monitors/${monitorId}/sla?from=${format(
- subDays(new Date(), 90),
- DATE_FORMAT
- )}&to${format(new Date(), DATE_FORMAT)}`,
- {
- headers: {
- [HEADER_KEY_TOKEN]: `Bearer ${this.configurationService.get(
- 'API_KEY_BETTER_UPTIME'
- )}`
- },
- signal: AbortSignal.timeout(
- this.configurationService.get('REQUEST_TIMEOUT')
- )
- }
- ).then((res) => res.json());
-
- return data.attributes.availability / 100;
- } catch (error) {
- Logger.error(error, 'InfoService - Better Stack');
-
- return undefined;
- }
- }
- }
}
diff --git a/apps/api/src/app/portfolio/portfolio.module.ts b/apps/api/src/app/portfolio/portfolio.module.ts
index 65a9b71aa6..d818195ca8 100644
--- a/apps/api/src/app/portfolio/portfolio.module.ts
+++ b/apps/api/src/app/portfolio/portfolio.module.ts
@@ -17,7 +17,7 @@ import { I18nModule } from '@ghostfolio/api/services/i18n/i18n.module';
import { ImpersonationModule } from '@ghostfolio/api/services/impersonation/impersonation.module';
import { MarketDataModule } from '@ghostfolio/api/services/market-data/market-data.module';
import { PrismaModule } from '@ghostfolio/api/services/prisma/prisma.module';
-import { DataGatheringModule } from '@ghostfolio/api/services/queues/data-gathering/data-gathering.module';
+import { DataGatheringQueueModule } from '@ghostfolio/api/services/queues/data-gathering/data-gathering.module';
import { PortfolioSnapshotQueueModule } from '@ghostfolio/api/services/queues/portfolio-snapshot/portfolio-snapshot.module';
import { SymbolProfileModule } from '@ghostfolio/api/services/symbol-profile/symbol-profile.module';
@@ -38,7 +38,7 @@ import { RulesService } from './rules.service';
ApiModule,
BenchmarkModule,
ConfigurationModule,
- DataGatheringModule,
+ DataGatheringQueueModule,
DataProviderModule,
ExchangeRateDataModule,
I18nModule,
diff --git a/apps/api/src/events/events.module.ts b/apps/api/src/events/events.module.ts
index 7727669451..df943a3c95 100644
--- a/apps/api/src/events/events.module.ts
+++ b/apps/api/src/events/events.module.ts
@@ -3,7 +3,7 @@ import { RedisCacheModule } from '@ghostfolio/api/app/redis-cache/redis-cache.mo
import { ConfigurationModule } from '@ghostfolio/api/services/configuration/configuration.module';
import { DataProviderModule } from '@ghostfolio/api/services/data-provider/data-provider.module';
import { ExchangeRateDataModule } from '@ghostfolio/api/services/exchange-rate-data/exchange-rate-data.module';
-import { DataGatheringModule } from '@ghostfolio/api/services/queues/data-gathering/data-gathering.module';
+import { DataGatheringQueueModule } from '@ghostfolio/api/services/queues/data-gathering/data-gathering.module';
import { Module } from '@nestjs/common';
@@ -14,7 +14,7 @@ import { PortfolioChangedListener } from './portfolio-changed.listener';
imports: [
ActivitiesModule,
ConfigurationModule,
- DataGatheringModule,
+ DataGatheringQueueModule,
DataProviderModule,
ExchangeRateDataModule,
RedisCacheModule
diff --git a/apps/api/src/services/cron/cron.module.ts b/apps/api/src/services/cron/cron.module.ts
index 06f9d2caae..bcc9d3360b 100644
--- a/apps/api/src/services/cron/cron.module.ts
+++ b/apps/api/src/services/cron/cron.module.ts
@@ -2,7 +2,8 @@ import { UserModule } from '@ghostfolio/api/app/user/user.module';
import { ConfigurationModule } from '@ghostfolio/api/services/configuration/configuration.module';
import { ExchangeRateDataModule } from '@ghostfolio/api/services/exchange-rate-data/exchange-rate-data.module';
import { PropertyModule } from '@ghostfolio/api/services/property/property.module';
-import { DataGatheringModule } from '@ghostfolio/api/services/queues/data-gathering/data-gathering.module';
+import { DataGatheringQueueModule } from '@ghostfolio/api/services/queues/data-gathering/data-gathering.module';
+import { StatisticsGatheringQueueModule } from '@ghostfolio/api/services/queues/statistics-gathering/statistics-gathering.module';
import { TwitterBotModule } from '@ghostfolio/api/services/twitter-bot/twitter-bot.module';
import { Module } from '@nestjs/common';
@@ -12,9 +13,10 @@ import { CronService } from './cron.service';
@Module({
imports: [
ConfigurationModule,
- DataGatheringModule,
+ DataGatheringQueueModule,
ExchangeRateDataModule,
PropertyModule,
+ StatisticsGatheringQueueModule,
TwitterBotModule,
UserModule
],
diff --git a/apps/api/src/services/cron/cron.service.ts b/apps/api/src/services/cron/cron.service.ts
index ee91a811e6..e680f00639 100644
--- a/apps/api/src/services/cron/cron.service.ts
+++ b/apps/api/src/services/cron/cron.service.ts
@@ -3,6 +3,7 @@ import { ConfigurationService } from '@ghostfolio/api/services/configuration/con
import { ExchangeRateDataService } from '@ghostfolio/api/services/exchange-rate-data/exchange-rate-data.service';
import { PropertyService } from '@ghostfolio/api/services/property/property.service';
import { DataGatheringService } from '@ghostfolio/api/services/queues/data-gathering/data-gathering.service';
+import { StatisticsGatheringService } from '@ghostfolio/api/services/queues/statistics-gathering/statistics-gathering.service';
import { TwitterBotService } from '@ghostfolio/api/services/twitter-bot/twitter-bot.service';
import {
DATA_GATHERING_QUEUE_PRIORITY_LOW,
@@ -25,10 +26,18 @@ export class CronService {
private readonly dataGatheringService: DataGatheringService,
private readonly exchangeRateDataService: ExchangeRateDataService,
private readonly propertyService: PropertyService,
+ private readonly statisticsGatheringService: StatisticsGatheringService,
private readonly twitterBotService: TwitterBotService,
private readonly userService: UserService
) {}
+ @Cron(CronExpression.EVERY_HOUR)
+ public async runEveryHour() {
+ if (this.configurationService.get('ENABLE_FEATURE_STATISTICS')) {
+ await this.statisticsGatheringService.addJobsToQueue();
+ }
+ }
+
@Cron(CronService.EVERY_HOUR_AT_RANDOM_MINUTE)
public async runEveryHourAtRandomMinute() {
if (await this.isDataGatheringEnabled()) {
diff --git a/apps/api/src/services/queues/data-gathering/data-gathering.module.ts b/apps/api/src/services/queues/data-gathering/data-gathering.module.ts
index f251c8d0c9..d163f0d29d 100644
--- a/apps/api/src/services/queues/data-gathering/data-gathering.module.ts
+++ b/apps/api/src/services/queues/data-gathering/data-gathering.module.ts
@@ -50,4 +50,4 @@ import { DataGatheringProcessor } from './data-gathering.processor';
providers: [DataGatheringProcessor, DataGatheringService],
exports: [BullModule, DataEnhancerModule, DataGatheringService]
})
-export class DataGatheringModule {}
+export class DataGatheringQueueModule {}
diff --git a/apps/api/src/services/queues/statistics-gathering/statistics-gathering.module.ts b/apps/api/src/services/queues/statistics-gathering/statistics-gathering.module.ts
new file mode 100644
index 0000000000..1818dd4ec5
--- /dev/null
+++ b/apps/api/src/services/queues/statistics-gathering/statistics-gathering.module.ts
@@ -0,0 +1,36 @@
+import { ConfigurationModule } from '@ghostfolio/api/services/configuration/configuration.module';
+import { PropertyModule } from '@ghostfolio/api/services/property/property.module';
+import { STATISTICS_GATHERING_QUEUE } from '@ghostfolio/common/config';
+
+import { BullAdapter } from '@bull-board/api/bullAdapter';
+import { BullBoardModule } from '@bull-board/nestjs';
+import { BullModule } from '@nestjs/bull';
+import { Module } from '@nestjs/common';
+
+import { StatisticsGatheringProcessor } from './statistics-gathering.processor';
+import { StatisticsGatheringService } from './statistics-gathering.service';
+
+@Module({
+ exports: [BullModule, StatisticsGatheringService],
+ imports: [
+ ...(process.env.ENABLE_FEATURE_BULL_BOARD === 'true'
+ ? [
+ BullBoardModule.forFeature({
+ adapter: BullAdapter,
+ name: STATISTICS_GATHERING_QUEUE,
+ options: {
+ displayName: 'Statistics Gathering',
+ readOnlyMode: process.env.BULL_BOARD_IS_READ_ONLY !== 'false'
+ }
+ })
+ ]
+ : []),
+ BullModule.registerQueue({
+ name: STATISTICS_GATHERING_QUEUE
+ }),
+ ConfigurationModule,
+ PropertyModule
+ ],
+ providers: [StatisticsGatheringProcessor, StatisticsGatheringService]
+})
+export class StatisticsGatheringQueueModule {}
diff --git a/apps/api/src/services/queues/statistics-gathering/statistics-gathering.processor.ts b/apps/api/src/services/queues/statistics-gathering/statistics-gathering.processor.ts
new file mode 100644
index 0000000000..07fc32585a
--- /dev/null
+++ b/apps/api/src/services/queues/statistics-gathering/statistics-gathering.processor.ts
@@ -0,0 +1,212 @@
+import { ConfigurationService } from '@ghostfolio/api/services/configuration/configuration.service';
+import { PropertyService } from '@ghostfolio/api/services/property/property.service';
+import {
+ GATHER_STATISTICS_DOCKER_HUB_PULLS_PROCESS_JOB_NAME,
+ GATHER_STATISTICS_GITHUB_CONTRIBUTORS_PROCESS_JOB_NAME,
+ GATHER_STATISTICS_GITHUB_STARGAZERS_PROCESS_JOB_NAME,
+ GATHER_STATISTICS_UPTIME_PROCESS_JOB_NAME,
+ HEADER_KEY_TOKEN,
+ PROPERTY_BETTER_UPTIME_MONITOR_ID,
+ PROPERTY_DOCKER_HUB_PULLS,
+ PROPERTY_GITHUB_CONTRIBUTORS,
+ PROPERTY_GITHUB_STARGAZERS,
+ PROPERTY_UPTIME,
+ STATISTICS_GATHERING_QUEUE
+} from '@ghostfolio/common/config';
+import {
+ DATE_FORMAT,
+ extractNumberFromString
+} from '@ghostfolio/common/helper';
+
+import { Process, Processor } from '@nestjs/bull';
+import { Injectable, Logger } from '@nestjs/common';
+import * as cheerio from 'cheerio';
+import { format, subDays } from 'date-fns';
+
+@Injectable()
+@Processor(STATISTICS_GATHERING_QUEUE)
+export class StatisticsGatheringProcessor {
+ public constructor(
+ private readonly configurationService: ConfigurationService,
+ private readonly propertyService: PropertyService
+ ) {}
+
+ @Process(GATHER_STATISTICS_DOCKER_HUB_PULLS_PROCESS_JOB_NAME)
+ public async gatherDockerHubPullsStatistics() {
+ Logger.log(
+ 'Docker Hub pulls statistics gathering has been started',
+ 'StatisticsGatheringProcessor'
+ );
+
+ const dockerHubPulls = await this.countDockerHubPulls();
+
+ await this.propertyService.put({
+ key: PROPERTY_DOCKER_HUB_PULLS,
+ value: String(dockerHubPulls)
+ });
+
+ Logger.log(
+ 'Docker Hub pulls statistics gathering has been completed',
+ 'StatisticsGatheringProcessor'
+ );
+ }
+
+ @Process(GATHER_STATISTICS_GITHUB_CONTRIBUTORS_PROCESS_JOB_NAME)
+ public async gatherGitHubContributorsStatistics() {
+ Logger.log(
+ 'GitHub contributors statistics gathering has been started',
+ 'StatisticsGatheringProcessor'
+ );
+
+ const gitHubContributors = await this.countGitHubContributors();
+
+ await this.propertyService.put({
+ key: PROPERTY_GITHUB_CONTRIBUTORS,
+ value: String(gitHubContributors)
+ });
+
+ Logger.log(
+ 'GitHub contributors statistics gathering has been completed',
+ 'StatisticsGatheringProcessor'
+ );
+ }
+
+ @Process(GATHER_STATISTICS_GITHUB_STARGAZERS_PROCESS_JOB_NAME)
+ public async gatherGitHubStargazersStatistics() {
+ Logger.log(
+ 'GitHub stargazers statistics gathering has been started',
+ 'StatisticsGatheringProcessor'
+ );
+
+ const gitHubStargazers = await this.countGitHubStargazers();
+
+ await this.propertyService.put({
+ key: PROPERTY_GITHUB_STARGAZERS,
+ value: String(gitHubStargazers)
+ });
+
+ Logger.log(
+ 'GitHub stargazers statistics gathering has been completed',
+ 'StatisticsGatheringProcessor'
+ );
+ }
+
+ @Process(GATHER_STATISTICS_UPTIME_PROCESS_JOB_NAME)
+ public async gatherUptimeStatistics() {
+ Logger.log(
+ 'Uptime statistics gathering has been started',
+ 'StatisticsGatheringProcessor'
+ );
+
+ const uptime = await this.getUptime();
+
+ await this.propertyService.put({
+ key: PROPERTY_UPTIME,
+ value: String(uptime)
+ });
+
+ Logger.log(
+ 'Uptime statistics gathering has been completed',
+ 'StatisticsGatheringProcessor'
+ );
+ }
+
+ private async countDockerHubPulls(): Promise<number> {
+ try {
+ const { pull_count } = (await fetch(
+ 'https://hub.docker.com/v2/repositories/ghostfolio/ghostfolio',
+ {
+ headers: { 'User-Agent': 'request' },
+ signal: AbortSignal.timeout(
+ this.configurationService.get('REQUEST_TIMEOUT')
+ )
+ }
+ ).then((res) => res.json())) as { pull_count: number };
+
+ return pull_count;
+ } catch (error) {
+ Logger.error(error, 'StatisticsGatheringProcessor - DockerHub');
+
+ throw error;
+ }
+ }
+
+ private async countGitHubContributors(): Promise<number> {
+ try {
+ const body = await fetch('https://github.com/ghostfolio/ghostfolio', {
+ signal: AbortSignal.timeout(
+ this.configurationService.get('REQUEST_TIMEOUT')
+ )
+ }).then((res) => res.text());
+
+ const $ = cheerio.load(body);
+
+ const value = $(
+ 'a[href="/ghostfolio/ghostfolio/graphs/contributors"] .Counter'
+ ).text();
+
+ if (!value) {
+ throw new Error('Could not find the contributors count in the page');
+ }
+
+ return extractNumberFromString({
+ value
+ });
+ } catch (error) {
+ Logger.error(error, 'StatisticsGatheringProcessor - GitHub');
+
+ throw error;
+ }
+ }
+
+ private async countGitHubStargazers(): Promise<number> {
+ try {
+ const { stargazers_count } = (await fetch(
+ 'https://api.github.com/repos/ghostfolio/ghostfolio',
+ {
+ headers: { 'User-Agent': 'request' },
+ signal: AbortSignal.timeout(
+ this.configurationService.get('REQUEST_TIMEOUT')
+ )
+ }
+ ).then((res) => res.json())) as { stargazers_count: number };
+
+ return stargazers_count;
+ } catch (error) {
+ Logger.error(error, 'StatisticsGatheringProcessor - GitHub');
+
+ throw error;
+ }
+ }
+
+ private async getUptime(): Promise<number> {
+ try {
+ const monitorId = await this.propertyService.getByKey<string>(
+ PROPERTY_BETTER_UPTIME_MONITOR_ID
+ );
+
+ const { data } = await fetch(
+ `https://uptime.betterstack.com/api/v2/monitors/${monitorId}/sla?from=${format(
+ subDays(new Date(), 90),
+ DATE_FORMAT
+ )}&to${format(new Date(), DATE_FORMAT)}`,
+ {
+ headers: {
+ [HEADER_KEY_TOKEN]: `Bearer ${this.configurationService.get(
+ 'API_KEY_BETTER_UPTIME'
+ )}`
+ },
+ signal: AbortSignal.timeout(
+ this.configurationService.get('REQUEST_TIMEOUT')
+ )
+ }
+ ).then((res) => res.json());
+
+ return data.attributes.availability / 100;
+ } catch (error) {
+ Logger.error(error, 'StatisticsGatheringProcessor - Better Stack');
+
+ throw error;
+ }
+ }
+}
diff --git a/apps/api/src/services/queues/statistics-gathering/statistics-gathering.service.ts b/apps/api/src/services/queues/statistics-gathering/statistics-gathering.service.ts
new file mode 100644
index 0000000000..68c6746880
--- /dev/null
+++ b/apps/api/src/services/queues/statistics-gathering/statistics-gathering.service.ts
@@ -0,0 +1,40 @@
+import {
+ GATHER_STATISTICS_DOCKER_HUB_PULLS_PROCESS_JOB_NAME,
+ GATHER_STATISTICS_GITHUB_CONTRIBUTORS_PROCESS_JOB_NAME,
+ GATHER_STATISTICS_GITHUB_STARGAZERS_PROCESS_JOB_NAME,
+ GATHER_STATISTICS_PROCESS_JOB_OPTIONS,
+ GATHER_STATISTICS_UPTIME_PROCESS_JOB_NAME,
+ STATISTICS_GATHERING_QUEUE
+} from '@ghostfolio/common/config';
+
+import { InjectQueue } from '@nestjs/bull';
+import { Injectable } from '@nestjs/common';
+import { Queue } from 'bull';
+
+@Injectable()
+export class StatisticsGatheringService {
+ public constructor(
+ @InjectQueue(STATISTICS_GATHERING_QUEUE)
+ private readonly statisticsGatheringQueue: Queue
+ ) {}
+
+ public async addJobsToQueue() {
+ return Promise.all(
+ [
+ GATHER_STATISTICS_DOCKER_HUB_PULLS_PROCESS_JOB_NAME,
+ GATHER_STATISTICS_GITHUB_CONTRIBUTORS_PROCESS_JOB_NAME,
+ GATHER_STATISTICS_GITHUB_STARGAZERS_PROCESS_JOB_NAME,
+ GATHER_STATISTICS_UPTIME_PROCESS_JOB_NAME
+ ].map((jobName) => {
+ return this.statisticsGatheringQueue.add(
+ jobName,
+ {},
+ {
+ ...GATHER_STATISTICS_PROCESS_JOB_OPTIONS,
+ jobId: jobName
+ }
+ );
+ })
+ );
+ }
+}
diff --git a/libs/common/src/lib/config.ts b/libs/common/src/lib/config.ts
index b442f2b7a3..42e1f6b630 100644
--- a/libs/common/src/lib/config.ts
+++ b/libs/common/src/lib/config.ts
@@ -75,6 +75,8 @@ export const PORTFOLIO_SNAPSHOT_COMPUTATION_QUEUE_PRIORITY_HIGH = 1;
export const PORTFOLIO_SNAPSHOT_COMPUTATION_QUEUE_PRIORITY_LOW =
Number.MAX_SAFE_INTEGER;
+export const STATISTICS_GATHERING_QUEUE = 'STATISTICS_GATHERING_QUEUE';
+
export const DEFAULT_CURRENCY = 'USD';
export const DEFAULT_DATE_FORMAT_MONTH_YEAR = 'MMM yyyy';
export const DEFAULT_HOST = '0.0.0.0';
@@ -183,6 +185,27 @@ export const GATHER_HISTORICAL_MARKET_DATA_PROCESS_JOB_OPTIONS: JobOptions = {
removeOnComplete: true
};
+export const GATHER_STATISTICS_PROCESS_JOB_OPTIONS: JobOptions = {
+ attempts: 5,
+ backoff: {
+ delay: ms('1 minute'),
+ type: 'exponential'
+ },
+ removeOnComplete: true
+};
+
+export const GATHER_STATISTICS_DOCKER_HUB_PULLS_PROCESS_JOB_NAME =
+ 'GATHER_STATISTICS_DOCKER_HUB_PULLS';
+
+export const GATHER_STATISTICS_GITHUB_CONTRIBUTORS_PROCESS_JOB_NAME =
+ 'GATHER_STATISTICS_GITHUB_CONTRIBUTORS';
+
+export const GATHER_STATISTICS_GITHUB_STARGAZERS_PROCESS_JOB_NAME =
+ 'GATHER_STATISTICS_GITHUB_STARGAZERS';
+
+export const GATHER_STATISTICS_UPTIME_PROCESS_JOB_NAME =
+ 'GATHER_STATISTICS_UPTIME';
+
export const INVESTMENT_ACTIVITY_TYPES = [
Type.BUY,
Type.DIVIDEND,
@@ -209,6 +232,9 @@ export const PROPERTY_API_KEY_GHOSTFOLIO = 'API_KEY_GHOSTFOLIO';
export const PROPERTY_API_KEY_OPENROUTER = 'API_KEY_OPENROUTER';
export const PROPERTY_BENCHMARKS = 'BENCHMARKS';
export const PROPERTY_BETTER_UPTIME_MONITOR_ID = 'BETTER_UPTIME_MONITOR_ID';
+export const PROPERTY_DOCKER_HUB_PULLS = 'DOCKER_HUB_PULLS';
+export const PROPERTY_GITHUB_CONTRIBUTORS = 'GITHUB_CONTRIBUTORS';
+export const PROPERTY_GITHUB_STARGAZERS = 'GITHUB_STARGAZERS';
export const PROPERTY_COUNTRIES_OF_SUBSCRIBERS = 'COUNTRIES_OF_SUBSCRIBERS';
export const PROPERTY_COUPONS = 'COUPONS';
export const PROPERTY_CURRENCIES = 'CURRENCIES';
@@ -225,6 +251,7 @@ export const PROPERTY_OPENROUTER_MODEL = 'OPENROUTER_MODEL';
export const PROPERTY_SLACK_COMMUNITY_USERS = 'SLACK_COMMUNITY_USERS';
export const PROPERTY_STRIPE_CONFIG = 'STRIPE_CONFIG';
export const PROPERTY_SYSTEM_MESSAGE = 'SYSTEM_MESSAGE';
+export const PROPERTY_UPTIME = 'UPTIME';
export const QUEUE_JOB_STATUS_LIST = [
'active',
diff --git a/libs/common/src/lib/interfaces/statistics.interface.ts b/libs/common/src/lib/interfaces/statistics.interface.ts
index 2852d34ab4..670cdaf4c5 100644
--- a/libs/common/src/lib/interfaces/statistics.interface.ts
+++ b/libs/common/src/lib/interfaces/statistics.interface.ts
@@ -5,6 +5,6 @@ export interface Statistics {
gitHubContributors: number;
gitHubStargazers: number;
newUsers30d: number;
- slackCommunityUsers: string;
+ slackCommunityUsers: number;
uptime: number;
}