Open Source Wealth Management Software. Angular + NestJS + Prisma + Nx + TypeScript 🤍
TypeScript angularetffinancefintechghostfoliohacktoberfestinvestingnestjsosspersonal-financeportfolioprismasoftwarestocktrackertradingtypescriptwealth-managementweb
Task/consolidate exchange rates gathering together with hourly market data (#7095)
Thomas Kaul46d740e381c6
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 7bd5629787..8568c74f90 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -14,6 +14,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0### Changed+- Consolidated the exchange rates to be gathered with hourly market data- Improved the language localization for German (`de`)- Upgraded `@openrouter/ai-sdk-provider` from version `2.9.0` to `2.9.1`- Upgraded `undici` from version `7.24.4` to `8.5.0`diff --git a/apps/api/src/app/admin/admin.controller.ts b/apps/api/src/app/admin/admin.controller.tsindex 9eb045fba7..a6f6e8b6f3 100644--- a/apps/api/src/app/admin/admin.controller.ts+++ b/apps/api/src/app/admin/admin.controller.ts@@ -86,8 +86,8 @@ export class AdminController {@HasPermission(permissions.accessAdminControl)@Post('gather')@UseGuards(AuthGuard('jwt'), HasPermissionGuard)- public async gather7Days(): Promise<void> {- this.dataGatheringService.gather7Days();+ public async gatherRecentMarketData(): Promise<void> {+ this.dataGatheringService.gatherRecentMarketData();}@HasPermission(permissions.accessAdminControl)diff --git a/apps/api/src/services/cron/cron.module.ts b/apps/api/src/services/cron/cron.module.tsindex 3d999e3977..bc14990c15 100644--- a/apps/api/src/services/cron/cron.module.ts+++ b/apps/api/src/services/cron/cron.module.ts@@ -2,8 +2,6 @@ import { UserModule } from '@ghostfolio/api/app/user/user.module';import { UserService } from '@ghostfolio/api/app/user/user.service';import { ConfigurationModule } from '@ghostfolio/api/services/configuration/configuration.module';import { ConfigurationService } from '@ghostfolio/api/services/configuration/configuration.service';-import { ExchangeRateDataModule } from '@ghostfolio/api/services/exchange-rate-data/exchange-rate-data.module';-import { ExchangeRateDataService } from '@ghostfolio/api/services/exchange-rate-data/exchange-rate-data.service';import { PropertyModule } from '@ghostfolio/api/services/property/property.module';import { PropertyService } from '@ghostfolio/api/services/property/property.service';import { DataGatheringQueueModule } from '@ghostfolio/api/services/queues/data-gathering/data-gathering.module';@@ -21,7 +19,6 @@ import { CronService } from './cron.service';imports: [ConfigurationModule,DataGatheringQueueModule,- ExchangeRateDataModule,PropertyModule,StatisticsGatheringQueueModule,TwitterBotModule,@@ -32,7 +29,6 @@ import { CronService } from './cron.service';inject: [ConfigurationService,DataGatheringService,- ExchangeRateDataService,PropertyService,StatisticsGatheringService,TwitterBotService,@@ -42,7 +38,6 @@ import { CronService } from './cron.service';useFactory: (configurationService: ConfigurationService,dataGatheringService: DataGatheringService,- exchangeRateDataService: ExchangeRateDataService,propertyService: PropertyService,statisticsGatheringService: StatisticsGatheringService,twitterBotService: TwitterBotService,@@ -57,7 +52,6 @@ import { CronService } from './cron.service';return new CronService(configurationService,dataGatheringService,- exchangeRateDataService,propertyService,statisticsGatheringService,twitterBotService,diff --git a/apps/api/src/services/cron/cron.service.ts b/apps/api/src/services/cron/cron.service.tsindex 7299cbbf4a..b3e60ae59b 100644--- a/apps/api/src/services/cron/cron.service.ts+++ b/apps/api/src/services/cron/cron.service.ts@@ -1,6 +1,5 @@import { UserService } from '@ghostfolio/api/app/user/user.service';import { ConfigurationService } from '@ghostfolio/api/services/configuration/configuration.service';-import { ExchangeRateDataService } from '@ghostfolio/api/services/exchange-rate-data/exchange-rate-data.service';import { PropertyService } from '@ghostfolio/api/services/property/property.service';import { DataGatheringService } from '@ghostfolio/api/services/queues/data-gathering/data-gathering.service';import { StatisticsGatheringService } from '@ghostfolio/api/services/queues/statistics-gathering/statistics-gathering.service';@@ -24,7 +23,6 @@ export class CronService {public constructor(private readonly configurationService: ConfigurationService,private readonly dataGatheringService: DataGatheringService,- private readonly exchangeRateDataService: ExchangeRateDataService,private readonly propertyService: PropertyService,private readonly statisticsGatheringService: StatisticsGatheringService,private readonly twitterBotService: TwitterBotService,@@ -41,16 +39,11 @@ export class CronService {@Cron(CronService.EVERY_HOUR_AT_RANDOM_MINUTE)public async runEveryHourAtRandomMinute() {if (await this.isDataGatheringEnabled()) {- await this.dataGatheringService.gather7Days();- await this.dataGatheringService.gatherHourlySymbols();+ await this.dataGatheringService.gatherHourlyMarketData();+ await this.dataGatheringService.gatherRecentMarketData();}}- @Cron(CronExpression.EVERY_12_HOURS)- public async runEveryTwelveHours() {- await this.exchangeRateDataService.loadCurrencies();- }-@Cron(CronExpression.EVERY_DAY_AT_5PM)public async runEveryDayAtFivePm() {if (this.configurationService.get('ENABLE_FEATURE_SUBSCRIPTION')) {diff --git a/apps/api/src/services/queues/data-gathering/data-gathering.service.ts b/apps/api/src/services/queues/data-gathering/data-gathering.service.tsindex ff65ebc837..1a2fa720eb 100644--- a/apps/api/src/services/queues/data-gathering/data-gathering.service.ts+++ b/apps/api/src/services/queues/data-gathering/data-gathering.service.ts@@ -69,91 +69,6 @@ export class DataGatheringService {return this.dataGatheringQueue.addBulk(jobs);}- public async gather7Days() {- await this.gatherSymbols({- dataGatheringItems: await this.getCurrencies7D(),- priority: DATA_GATHERING_QUEUE_PRIORITY_HIGH- });-- await this.gatherSymbols({- dataGatheringItems: await this.getSymbols7D({- withUserSubscription: true- }),- priority: DATA_GATHERING_QUEUE_PRIORITY_MEDIUM- });-- await this.gatherSymbols({- dataGatheringItems: await this.getSymbols7D({- withUserSubscription: false- }),- priority: DATA_GATHERING_QUEUE_PRIORITY_LOW- });- }-- public async gatherMax() {- const dataGatheringItems = await this.getSymbolsMax();- await this.gatherSymbols({- dataGatheringItems,- priority: DATA_GATHERING_QUEUE_PRIORITY_LOW- });- }-- public async gatherSymbol({ dataSource, date, symbol }: DataGatheringItem) {- const dataGatheringItems = (await this.getSymbolsMax())- .filter((dataGatheringItem) => {- return (- dataGatheringItem.dataSource === dataSource &&- dataGatheringItem.symbol === symbol- );- })- .map((item) => ({- ...item,- date: date ?? item.date- }));-- await this.gatherSymbols({- dataGatheringItems,- force: true,- priority: DATA_GATHERING_QUEUE_PRIORITY_HIGH- });- }-- public async gatherSymbolForDate({- dataSource,- date,- symbol- }: { date: Date } & AssetProfileIdentifier) {- try {- const historicalData = await this.dataProviderService.getHistoricalRaw({- assetProfileIdentifiers: [{ dataSource, symbol }],- from: date,- to: date- });-- const marketPrice =- historicalData[getAssetProfileIdentifier({ dataSource, symbol })][- format(date, DATE_FORMAT)- ].marketPrice;-- if (marketPrice) {- return await this.prismaService.marketData.upsert({- create: {- dataSource,- date,- marketPrice,- symbol- },- update: { marketPrice },- where: { dataSource_date_symbol: { dataSource, date, symbol } }- });- }- } catch (error) {- this.logger.error(error);- } finally {- return undefined;- }- }-public async gatherAssetProfiles(aAssetProfileIdentifiers?: AssetProfileIdentifier[]) {@@ -282,7 +197,13 @@ export class DataGatheringService {}}- public async gatherHourlySymbols() {+ public async gatherHourlyMarketData() {+ try {+ await this.exchangeRateDataService.loadCurrencies();+ } catch (error) {+ this.logger.error('Could not gather exchange rates', error);+ }+const assetProfileIdentifiers =await this.getHourlyAssetProfileIdentifiers();@@ -322,6 +243,91 @@ export class DataGatheringService {}}+ public async gatherMax() {+ const dataGatheringItems = await this.getSymbolsMax();+ await this.gatherSymbols({+ dataGatheringItems,+ priority: DATA_GATHERING_QUEUE_PRIORITY_LOW+ });+ }++ public async gatherRecentMarketData() {+ await this.gatherSymbols({+ dataGatheringItems: await this.getCurrencies7D(),+ priority: DATA_GATHERING_QUEUE_PRIORITY_HIGH+ });++ await this.gatherSymbols({+ dataGatheringItems: await this.getSymbols7D({+ withUserSubscription: true+ }),+ priority: DATA_GATHERING_QUEUE_PRIORITY_MEDIUM+ });++ await this.gatherSymbols({+ dataGatheringItems: await this.getSymbols7D({+ withUserSubscription: false+ }),+ priority: DATA_GATHERING_QUEUE_PRIORITY_LOW+ });+ }++ public async gatherSymbol({ dataSource, date, symbol }: DataGatheringItem) {+ const dataGatheringItems = (await this.getSymbolsMax())+ .filter((dataGatheringItem) => {+ return (+ dataGatheringItem.dataSource === dataSource &&+ dataGatheringItem.symbol === symbol+ );+ })+ .map((item) => ({+ ...item,+ date: date ?? item.date+ }));++ await this.gatherSymbols({+ dataGatheringItems,+ force: true,+ priority: DATA_GATHERING_QUEUE_PRIORITY_HIGH+ });+ }++ public async gatherSymbolForDate({+ dataSource,+ date,+ symbol+ }: { date: Date } & AssetProfileIdentifier) {+ try {+ const historicalData = await this.dataProviderService.getHistoricalRaw({+ assetProfileIdentifiers: [{ dataSource, symbol }],+ from: date,+ to: date+ });++ const marketPrice =+ historicalData[getAssetProfileIdentifier({ dataSource, symbol })][+ format(date, DATE_FORMAT)+ ].marketPrice;++ if (marketPrice) {+ return await this.prismaService.marketData.upsert({+ create: {+ dataSource,+ date,+ marketPrice,+ symbol+ },+ update: { marketPrice },+ where: { dataSource_date_symbol: { dataSource, date, symbol } }+ });+ }+ } catch (error) {+ this.logger.error(error);+ } finally {+ return undefined;+ }+ }+public async gatherSymbols({dataGatheringItems,force = false,diff --git a/apps/client/src/app/components/admin-market-data/admin-market-data.component.ts b/apps/client/src/app/components/admin-market-data/admin-market-data.component.tsindex 511d6df988..ae0f8fda2f 100644--- a/apps/client/src/app/components/admin-market-data/admin-market-data.component.ts+++ b/apps/client/src/app/components/admin-market-data/admin-market-data.component.ts@@ -305,17 +305,6 @@ export class GfAdminMarketDataComponent implements AfterViewInit, OnInit {);}- protected onGather7Days() {- this.adminService- .gather7Days()- .pipe(takeUntilDestroyed(this.destroyRef))- .subscribe(() => {- setTimeout(() => {- window.location.reload();- }, 300);- });- }-protected onGatherMax() {this.adminService.gatherMax()@@ -334,6 +323,17 @@ export class GfAdminMarketDataComponent implements AfterViewInit, OnInit {.subscribe();}+ protected onGatherRecentMarketData() {+ this.adminService+ .gatherRecentMarketData()+ .pipe(takeUntilDestroyed(this.destroyRef))+ .subscribe(() => {+ setTimeout(() => {+ window.location.reload();+ }, 300);+ });+ }+protected onOpenAssetProfileDialog({dataSource,symboldiff --git a/apps/client/src/app/components/admin-market-data/admin-market-data.html b/apps/client/src/app/components/admin-market-data/admin-market-data.htmlindex e86e3e6319..270722a4bb 100644--- a/apps/client/src/app/components/admin-market-data/admin-market-data.html+++ b/apps/client/src/app/components/admin-market-data/admin-market-data.html@@ -220,7 +220,7 @@class="no-max-width"xPosition="before">- <button mat-menu-item (click)="onGather7Days()">+ <button mat-menu-item (click)="onGatherRecentMarketData()"><ng-container i18n>Gather Recent Historical Market Data</ng-container>diff --git a/libs/ui/src/lib/services/admin.service.ts b/libs/ui/src/lib/services/admin.service.tsindex acfd15ef55..c767c4f48a 100644--- a/libs/ui/src/lib/services/admin.service.ts+++ b/libs/ui/src/lib/services/admin.service.ts@@ -127,10 +127,6 @@ export class AdminService {return this.http.get<AdminUsersResponse>('/api/v1/admin/user', { params });}- public gather7Days() {- return this.http.post<void>('/api/v1/admin/gather', {});- }-public gatherMax() {return this.http.post<void>('/api/v1/admin/gather/max', {});}@@ -149,6 +145,10 @@ export class AdminService {);}+ public gatherRecentMarketData() {+ return this.http.post<void>('/api/v1/admin/gather', {});+ }+public gatherSymbol({dataSource,range,