Skip to content

Open Source Wealth Management Software. Angular + NestJS + Prisma + Nx + TypeScript 🤍

Task/consolidate exchange rates gathering together with hourly market data (#7095)

Thomas Kaul46d740e381c6
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 7bd5629787..8568c74f90 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -14,6 +14,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
### Changed
+- Consolidated the exchange rates to be gathered with hourly market data
- Improved the language localization for German (`de`)
- Upgraded `@openrouter/ai-sdk-provider` from version `2.9.0` to `2.9.1`
- Upgraded `undici` from version `7.24.4` to `8.5.0`
diff --git a/apps/api/src/app/admin/admin.controller.ts b/apps/api/src/app/admin/admin.controller.ts
index 9eb045fba7..a6f6e8b6f3 100644
--- a/apps/api/src/app/admin/admin.controller.ts
+++ b/apps/api/src/app/admin/admin.controller.ts
@@ -86,8 +86,8 @@ export class AdminController {
@HasPermission(permissions.accessAdminControl)
@Post('gather')
@UseGuards(AuthGuard('jwt'), HasPermissionGuard)
- public async gather7Days(): Promise<void> {
- this.dataGatheringService.gather7Days();
+ public async gatherRecentMarketData(): Promise<void> {
+ this.dataGatheringService.gatherRecentMarketData();
}
@HasPermission(permissions.accessAdminControl)
diff --git a/apps/api/src/services/cron/cron.module.ts b/apps/api/src/services/cron/cron.module.ts
index 3d999e3977..bc14990c15 100644
--- a/apps/api/src/services/cron/cron.module.ts
+++ b/apps/api/src/services/cron/cron.module.ts
@@ -2,8 +2,6 @@ import { UserModule } from '@ghostfolio/api/app/user/user.module';
import { UserService } from '@ghostfolio/api/app/user/user.service';
import { ConfigurationModule } from '@ghostfolio/api/services/configuration/configuration.module';
import { ConfigurationService } from '@ghostfolio/api/services/configuration/configuration.service';
-import { ExchangeRateDataModule } from '@ghostfolio/api/services/exchange-rate-data/exchange-rate-data.module';
-import { ExchangeRateDataService } from '@ghostfolio/api/services/exchange-rate-data/exchange-rate-data.service';
import { PropertyModule } from '@ghostfolio/api/services/property/property.module';
import { PropertyService } from '@ghostfolio/api/services/property/property.service';
import { DataGatheringQueueModule } from '@ghostfolio/api/services/queues/data-gathering/data-gathering.module';
@@ -21,7 +19,6 @@ import { CronService } from './cron.service';
imports: [
ConfigurationModule,
DataGatheringQueueModule,
- ExchangeRateDataModule,
PropertyModule,
StatisticsGatheringQueueModule,
TwitterBotModule,
@@ -32,7 +29,6 @@ import { CronService } from './cron.service';
inject: [
ConfigurationService,
DataGatheringService,
- ExchangeRateDataService,
PropertyService,
StatisticsGatheringService,
TwitterBotService,
@@ -42,7 +38,6 @@ import { CronService } from './cron.service';
useFactory: (
configurationService: ConfigurationService,
dataGatheringService: DataGatheringService,
- exchangeRateDataService: ExchangeRateDataService,
propertyService: PropertyService,
statisticsGatheringService: StatisticsGatheringService,
twitterBotService: TwitterBotService,
@@ -57,7 +52,6 @@ import { CronService } from './cron.service';
return new CronService(
configurationService,
dataGatheringService,
- exchangeRateDataService,
propertyService,
statisticsGatheringService,
twitterBotService,
diff --git a/apps/api/src/services/cron/cron.service.ts b/apps/api/src/services/cron/cron.service.ts
index 7299cbbf4a..b3e60ae59b 100644
--- a/apps/api/src/services/cron/cron.service.ts
+++ b/apps/api/src/services/cron/cron.service.ts
@@ -1,6 +1,5 @@
import { UserService } from '@ghostfolio/api/app/user/user.service';
import { ConfigurationService } from '@ghostfolio/api/services/configuration/configuration.service';
-import { ExchangeRateDataService } from '@ghostfolio/api/services/exchange-rate-data/exchange-rate-data.service';
import { PropertyService } from '@ghostfolio/api/services/property/property.service';
import { DataGatheringService } from '@ghostfolio/api/services/queues/data-gathering/data-gathering.service';
import { StatisticsGatheringService } from '@ghostfolio/api/services/queues/statistics-gathering/statistics-gathering.service';
@@ -24,7 +23,6 @@ export class CronService {
public constructor(
private readonly configurationService: ConfigurationService,
private readonly dataGatheringService: DataGatheringService,
- private readonly exchangeRateDataService: ExchangeRateDataService,
private readonly propertyService: PropertyService,
private readonly statisticsGatheringService: StatisticsGatheringService,
private readonly twitterBotService: TwitterBotService,
@@ -41,16 +39,11 @@ export class CronService {
@Cron(CronService.EVERY_HOUR_AT_RANDOM_MINUTE)
public async runEveryHourAtRandomMinute() {
if (await this.isDataGatheringEnabled()) {
- await this.dataGatheringService.gather7Days();
- await this.dataGatheringService.gatherHourlySymbols();
+ await this.dataGatheringService.gatherHourlyMarketData();
+ await this.dataGatheringService.gatherRecentMarketData();
}
}
- @Cron(CronExpression.EVERY_12_HOURS)
- public async runEveryTwelveHours() {
- await this.exchangeRateDataService.loadCurrencies();
- }
-
@Cron(CronExpression.EVERY_DAY_AT_5PM)
public async runEveryDayAtFivePm() {
if (this.configurationService.get('ENABLE_FEATURE_SUBSCRIPTION')) {
diff --git a/apps/api/src/services/queues/data-gathering/data-gathering.service.ts b/apps/api/src/services/queues/data-gathering/data-gathering.service.ts
index ff65ebc837..1a2fa720eb 100644
--- a/apps/api/src/services/queues/data-gathering/data-gathering.service.ts
+++ b/apps/api/src/services/queues/data-gathering/data-gathering.service.ts
@@ -69,91 +69,6 @@ export class DataGatheringService {
return this.dataGatheringQueue.addBulk(jobs);
}
- public async gather7Days() {
- await this.gatherSymbols({
- dataGatheringItems: await this.getCurrencies7D(),
- priority: DATA_GATHERING_QUEUE_PRIORITY_HIGH
- });
-
- await this.gatherSymbols({
- dataGatheringItems: await this.getSymbols7D({
- withUserSubscription: true
- }),
- priority: DATA_GATHERING_QUEUE_PRIORITY_MEDIUM
- });
-
- await this.gatherSymbols({
- dataGatheringItems: await this.getSymbols7D({
- withUserSubscription: false
- }),
- priority: DATA_GATHERING_QUEUE_PRIORITY_LOW
- });
- }
-
- public async gatherMax() {
- const dataGatheringItems = await this.getSymbolsMax();
- await this.gatherSymbols({
- dataGatheringItems,
- priority: DATA_GATHERING_QUEUE_PRIORITY_LOW
- });
- }
-
- public async gatherSymbol({ dataSource, date, symbol }: DataGatheringItem) {
- const dataGatheringItems = (await this.getSymbolsMax())
- .filter((dataGatheringItem) => {
- return (
- dataGatheringItem.dataSource === dataSource &&
- dataGatheringItem.symbol === symbol
- );
- })
- .map((item) => ({
- ...item,
- date: date ?? item.date
- }));
-
- await this.gatherSymbols({
- dataGatheringItems,
- force: true,
- priority: DATA_GATHERING_QUEUE_PRIORITY_HIGH
- });
- }
-
- public async gatherSymbolForDate({
- dataSource,
- date,
- symbol
- }: { date: Date } & AssetProfileIdentifier) {
- try {
- const historicalData = await this.dataProviderService.getHistoricalRaw({
- assetProfileIdentifiers: [{ dataSource, symbol }],
- from: date,
- to: date
- });
-
- const marketPrice =
- historicalData[getAssetProfileIdentifier({ dataSource, symbol })][
- format(date, DATE_FORMAT)
- ].marketPrice;
-
- if (marketPrice) {
- return await this.prismaService.marketData.upsert({
- create: {
- dataSource,
- date,
- marketPrice,
- symbol
- },
- update: { marketPrice },
- where: { dataSource_date_symbol: { dataSource, date, symbol } }
- });
- }
- } catch (error) {
- this.logger.error(error);
- } finally {
- return undefined;
- }
- }
-
public async gatherAssetProfiles(
aAssetProfileIdentifiers?: AssetProfileIdentifier[]
) {
@@ -282,7 +197,13 @@ export class DataGatheringService {
}
}
- public async gatherHourlySymbols() {
+ public async gatherHourlyMarketData() {
+ try {
+ await this.exchangeRateDataService.loadCurrencies();
+ } catch (error) {
+ this.logger.error('Could not gather exchange rates', error);
+ }
+
const assetProfileIdentifiers =
await this.getHourlyAssetProfileIdentifiers();
@@ -322,6 +243,91 @@ export class DataGatheringService {
}
}
+ public async gatherMax() {
+ const dataGatheringItems = await this.getSymbolsMax();
+ await this.gatherSymbols({
+ dataGatheringItems,
+ priority: DATA_GATHERING_QUEUE_PRIORITY_LOW
+ });
+ }
+
+ public async gatherRecentMarketData() {
+ await this.gatherSymbols({
+ dataGatheringItems: await this.getCurrencies7D(),
+ priority: DATA_GATHERING_QUEUE_PRIORITY_HIGH
+ });
+
+ await this.gatherSymbols({
+ dataGatheringItems: await this.getSymbols7D({
+ withUserSubscription: true
+ }),
+ priority: DATA_GATHERING_QUEUE_PRIORITY_MEDIUM
+ });
+
+ await this.gatherSymbols({
+ dataGatheringItems: await this.getSymbols7D({
+ withUserSubscription: false
+ }),
+ priority: DATA_GATHERING_QUEUE_PRIORITY_LOW
+ });
+ }
+
+ public async gatherSymbol({ dataSource, date, symbol }: DataGatheringItem) {
+ const dataGatheringItems = (await this.getSymbolsMax())
+ .filter((dataGatheringItem) => {
+ return (
+ dataGatheringItem.dataSource === dataSource &&
+ dataGatheringItem.symbol === symbol
+ );
+ })
+ .map((item) => ({
+ ...item,
+ date: date ?? item.date
+ }));
+
+ await this.gatherSymbols({
+ dataGatheringItems,
+ force: true,
+ priority: DATA_GATHERING_QUEUE_PRIORITY_HIGH
+ });
+ }
+
+ public async gatherSymbolForDate({
+ dataSource,
+ date,
+ symbol
+ }: { date: Date } & AssetProfileIdentifier) {
+ try {
+ const historicalData = await this.dataProviderService.getHistoricalRaw({
+ assetProfileIdentifiers: [{ dataSource, symbol }],
+ from: date,
+ to: date
+ });
+
+ const marketPrice =
+ historicalData[getAssetProfileIdentifier({ dataSource, symbol })][
+ format(date, DATE_FORMAT)
+ ].marketPrice;
+
+ if (marketPrice) {
+ return await this.prismaService.marketData.upsert({
+ create: {
+ dataSource,
+ date,
+ marketPrice,
+ symbol
+ },
+ update: { marketPrice },
+ where: { dataSource_date_symbol: { dataSource, date, symbol } }
+ });
+ }
+ } catch (error) {
+ this.logger.error(error);
+ } finally {
+ return undefined;
+ }
+ }
+
public async gatherSymbols({
dataGatheringItems,
force = false,
diff --git a/apps/client/src/app/components/admin-market-data/admin-market-data.component.ts b/apps/client/src/app/components/admin-market-data/admin-market-data.component.ts
index 511d6df988..ae0f8fda2f 100644
--- a/apps/client/src/app/components/admin-market-data/admin-market-data.component.ts
+++ b/apps/client/src/app/components/admin-market-data/admin-market-data.component.ts
@@ -305,17 +305,6 @@ export class GfAdminMarketDataComponent implements AfterViewInit, OnInit {
);
}
- protected onGather7Days() {
- this.adminService
- .gather7Days()
- .pipe(takeUntilDestroyed(this.destroyRef))
- .subscribe(() => {
- setTimeout(() => {
- window.location.reload();
- }, 300);
- });
- }
-
protected onGatherMax() {
this.adminService
.gatherMax()
@@ -334,6 +323,17 @@ export class GfAdminMarketDataComponent implements AfterViewInit, OnInit {
.subscribe();
}
+ protected onGatherRecentMarketData() {
+ this.adminService
+ .gatherRecentMarketData()
+ .pipe(takeUntilDestroyed(this.destroyRef))
+ .subscribe(() => {
+ setTimeout(() => {
+ window.location.reload();
+ }, 300);
+ });
+ }
+
protected onOpenAssetProfileDialog({
dataSource,
symbol
diff --git a/apps/client/src/app/components/admin-market-data/admin-market-data.html b/apps/client/src/app/components/admin-market-data/admin-market-data.html
index e86e3e6319..270722a4bb 100644
--- a/apps/client/src/app/components/admin-market-data/admin-market-data.html
+++ b/apps/client/src/app/components/admin-market-data/admin-market-data.html
@@ -220,7 +220,7 @@
class="no-max-width"
xPosition="before"
>
- <button mat-menu-item (click)="onGather7Days()">
+ <button mat-menu-item (click)="onGatherRecentMarketData()">
<ng-container i18n
>Gather Recent Historical Market Data</ng-container
>
diff --git a/libs/ui/src/lib/services/admin.service.ts b/libs/ui/src/lib/services/admin.service.ts
index acfd15ef55..c767c4f48a 100644
--- a/libs/ui/src/lib/services/admin.service.ts
+++ b/libs/ui/src/lib/services/admin.service.ts
@@ -127,10 +127,6 @@ export class AdminService {
return this.http.get<AdminUsersResponse>('/api/v1/admin/user', { params });
}
- public gather7Days() {
- return this.http.post<void>('/api/v1/admin/gather', {});
- }
-
public gatherMax() {
return this.http.post<void>('/api/v1/admin/gather/max', {});
}
@@ -149,6 +145,10 @@ export class AdminService {
);
}
+ public gatherRecentMarketData() {
+ return this.http.post<void>('/api/v1/admin/gather', {});
+ }
+
public gatherSymbol({
dataSource,
range,