Skip to content

Open Source Wealth Management Software. Angular + NestJS + Prisma + Nx + TypeScript 🤍

Task/simplify recent market data gathering (#7714)

Thomas Kaulb4e8a4090d6d
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 653cae509d..bd29210eaf 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -12,10 +12,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
- Harmonized the icons and labels in the access table to share the portfolio
- Migrated the create and edit access dialogs to dedicated routes
- Improved the response of the historical market data gathering endpoint for a specific date
+- Improved the historical market data gathering by loading the asset profiles with recent market data in a single database query per run
### Fixed
- Fixed the country mapping of Macau in the _Financial Modeling Prep_ service
+- Fixed the date of the gathered historical market data for instances running in a time zone other than UTC
+- Fixed the repeated historical market data gathering for instances running in a time zone other than UTC
## 3.60.0 - 2026-08-24
diff --git a/apps/api/jest-environment-tz.js b/apps/api/jest-environment-tz.js
new file mode 100644
index 0000000000..ef0d72576f
--- /dev/null
+++ b/apps/api/jest-environment-tz.js
@@ -0,0 +1,35 @@
+const NodeEnvironment = require('jest-environment-node').TestEnvironment;
+
+// Jest gives each test file a copy of `process`, so a test cannot change the
+// time zone at run time. This environment sets `TZ` on the real process of the
+// worker, which resets the internal date cache of Node.
+class TimeZoneEnvironment extends NodeEnvironment {
+ constructor(config, context) {
+ super(config, context);
+
+ this.previousTimeZone = process.env.TZ;
+ this.timeZone = config.projectConfig.testEnvironmentOptions?.timeZone;
+ }
+
+ async setup() {
+ if (this.timeZone) {
+ process.env.TZ = this.timeZone;
+ }
+
+ await super.setup();
+ }
+
+ async teardown() {
+ try {
+ await super.teardown();
+ } finally {
+ if (this.previousTimeZone === undefined) {
+ delete process.env.TZ;
+ } else {
+ process.env.TZ = this.previousTimeZone;
+ }
+ }
+ }
+}
+
+module.exports = TimeZoneEnvironment;
diff --git a/apps/api/src/app/admin/admin.controller.ts b/apps/api/src/app/admin/admin.controller.ts
index f47d216f4d..6d3f7cfeb8 100644
--- a/apps/api/src/app/admin/admin.controller.ts
+++ b/apps/api/src/app/admin/admin.controller.ts
@@ -37,6 +37,7 @@ import type {
RequestWithUser
} from '@ghostfolio/common/types';
+import { utc } from '@date-fns/utc';
import {
Body,
Controller,
@@ -204,7 +205,7 @@ export class AdminController {
@Param('dateString') dateString: string,
@Param('symbol') symbol: string
): Promise<MarketData> {
- const date = parseISO(dateString);
+ const date = parseISO(dateString, { in: utc });
if (!isDate(date)) {
throw new HttpException(
diff --git a/apps/api/src/services/data-provider/coingecko/coingecko.service.ts b/apps/api/src/services/data-provider/coingecko/coingecko.service.ts
index 96bc00561a..208e23269c 100644
--- a/apps/api/src/services/data-provider/coingecko/coingecko.service.ts
+++ b/apps/api/src/services/data-provider/coingecko/coingecko.service.ts
@@ -18,6 +18,7 @@ import {
LookupResponse
} from '@ghostfolio/common/interfaces';
+import { utc } from '@date-fns/utc';
import { Injectable, Logger, OnModuleInit } from '@nestjs/common';
import {
AssetClass,
@@ -147,7 +148,9 @@ export class CoinGeckoService implements DataProviderInterface, OnModuleInit {
} = {};
for (const [timestamp, marketPrice] of prices) {
- result[format(fromUnixTime(timestamp / 1000), DATE_FORMAT)] = {
+ result[
+ format(fromUnixTime(timestamp / 1000), DATE_FORMAT, { in: utc })
+ ] = {
marketPrice
};
}
diff --git a/apps/api/src/services/data-provider/data-provider.service.ts b/apps/api/src/services/data-provider/data-provider.service.ts
index 85b4a068f5..e9449be9e0 100644
--- a/apps/api/src/services/data-provider/data-provider.service.ts
+++ b/apps/api/src/services/data-provider/data-provider.service.ts
@@ -35,6 +35,7 @@ import {
} from '@ghostfolio/common/interfaces';
import type { Granularity, UserWithSettings } from '@ghostfolio/common/types';
+import { utc } from '@date-fns/utc';
import { Inject, Injectable, Logger, OnModuleInit } from '@nestjs/common';
import { DataSource, MarketData, Prisma, SymbolProfile } from '@prisma/client';
import { Big } from 'big.js';
@@ -494,8 +495,11 @@ export class DataProviderService implements OnModuleInit {
[date: string]: DataProviderHistoricalResponse;
} = {};
- for (const date of eachDayOfInterval({ end: to, start: from })) {
- data[format(date, DATE_FORMAT)] = { marketPrice: 100 };
+ for (const date of eachDayOfInterval(
+ { end: to, start: from },
+ { in: utc }
+ )) {
+ data[format(date, DATE_FORMAT, { in: utc })] = { marketPrice: 100 };
}
promises.push(
diff --git a/apps/api/src/services/data-provider/manual/manual.service.ts b/apps/api/src/services/data-provider/manual/manual.service.ts
index 1d09c91fef..a187335a0e 100644
--- a/apps/api/src/services/data-provider/manual/manual.service.ts
+++ b/apps/api/src/services/data-provider/manual/manual.service.ts
@@ -14,7 +14,7 @@ import { SymbolProfileService } from '@ghostfolio/api/services/symbol-profile/sy
import {
DATE_FORMAT,
extractNumberFromString,
- getYesterday
+ getStartOfUtcDateOfYesterday
} from '@ghostfolio/common/helper';
import {
DataProviderHistoricalResponse,
@@ -24,6 +24,7 @@ import {
ScraperConfiguration
} from '@ghostfolio/common/interfaces';
+import { utc } from '@date-fns/utc';
import { Injectable, Logger } from '@nestjs/common';
import { DataSource, SymbolProfile } from '@prisma/client';
import * as cheerio from 'cheerio';
@@ -96,11 +97,11 @@ export class ManualService implements DataProviderInterface {
let date = from;
while (isBefore(date, to)) {
- historical[format(date, DATE_FORMAT)] = {
+ historical[format(date, DATE_FORMAT, { in: utc })] = {
marketPrice: defaultMarketPrice
};
- date = addDays(date, 1);
+ date = addDays(date, 1, { in: utc });
}
return historical;
@@ -114,7 +115,7 @@ export class ManualService implements DataProviderInterface {
});
return {
- [format(getYesterday(), DATE_FORMAT)]: {
+ [format(getStartOfUtcDateOfYesterday(), DATE_FORMAT, { in: utc })]: {
marketPrice: value
}
};
diff --git a/apps/api/src/services/data-provider/rapid-api/rapid-api.service.ts b/apps/api/src/services/data-provider/rapid-api/rapid-api.service.ts
index f14bfec14f..07dc62393a 100644
--- a/apps/api/src/services/data-provider/rapid-api/rapid-api.service.ts
+++ b/apps/api/src/services/data-provider/rapid-api/rapid-api.service.ts
@@ -9,7 +9,10 @@ import {
} from '@ghostfolio/api/services/data-provider/interfaces/data-provider.interface';
import { FetchService } from '@ghostfolio/api/services/fetch/fetch.service';
import { ghostfolioFearAndGreedIndexSymbolStocks } from '@ghostfolio/common/config';
-import { DATE_FORMAT, getYesterday } from '@ghostfolio/common/helper';
+import {
+ DATE_FORMAT,
+ getStartOfUtcDateOfYesterday
+} from '@ghostfolio/common/helper';
import {
DataProviderHistoricalResponse,
DataProviderInfo,
@@ -17,6 +20,7 @@ import {
LookupResponse
} from '@ghostfolio/common/interfaces';
+import { utc } from '@date-fns/utc';
import { Injectable, Logger } from '@nestjs/common';
import { DataSource, SymbolProfile } from '@prisma/client';
import { format } from 'date-fns';
@@ -66,7 +70,9 @@ export class RapidApiService implements DataProviderInterface {
if (fgi) {
return {
- [format(getYesterday(), DATE_FORMAT)]: {
+ [format(getStartOfUtcDateOfYesterday(), DATE_FORMAT, {
+ in: utc
+ })]: {
marketPrice: fgi.previousClose.value
}
};
diff --git a/apps/api/src/services/data-provider/yahoo-finance/yahoo-finance.service.ts b/apps/api/src/services/data-provider/yahoo-finance/yahoo-finance.service.ts
index d0c7a204a8..ca6357e257 100644
--- a/apps/api/src/services/data-provider/yahoo-finance/yahoo-finance.service.ts
+++ b/apps/api/src/services/data-provider/yahoo-finance/yahoo-finance.service.ts
@@ -19,6 +19,7 @@ import {
LookupResponse
} from '@ghostfolio/common/interfaces';
+import { utc } from '@date-fns/utc';
import { Injectable, Logger } from '@nestjs/common';
import { DataSource, SymbolProfile } from '@prisma/client';
import { addDays, format, isSameDay } from 'date-fns';
@@ -103,7 +104,7 @@ export class YahooFinanceService implements DataProviderInterface {
} = {};
for (const historicalItem of historicalResult) {
- response[format(historicalItem.date, DATE_FORMAT)] = {
+ response[format(historicalItem.date, DATE_FORMAT, { in: utc })] = {
marketPrice: historicalItem.dividends
};
}
@@ -174,7 +175,7 @@ export class YahooFinanceService implements DataProviderInterface {
: undefined);
if (marketPrice) {
- response[format(date, DATE_FORMAT)] = { marketPrice };
+ response[format(date, DATE_FORMAT, { in: utc })] = { marketPrice };
}
}
diff --git a/apps/api/src/services/queues/data-gathering/data-gathering.processor.time-zone.spec.ts b/apps/api/src/services/queues/data-gathering/data-gathering.processor.time-zone.spec.ts
new file mode 100644
index 0000000000..10b9098b10
--- /dev/null
+++ b/apps/api/src/services/queues/data-gathering/data-gathering.processor.time-zone.spec.ts
@@ -0,0 +1,70 @@
+/**
+ * @jest-environment <rootDir>/jest-environment-tz.js
+ * @jest-environment-options {"timeZone": "America/New_York"}
+ */
+import { DataGatheringItem } from '@ghostfolio/api/services/interfaces/interfaces';
+
+import { Job } from 'bull';
+
+import { DataGatheringProcessor } from './data-gathering.processor';
+
+describe('DataGatheringProcessor in a time zone behind UTC', () => {
+ let dataGatheringProcessor: DataGatheringProcessor;
+ let dataProviderService: { getHistoricalRaw: jest.Mock };
+ let marketDataService: { replaceForSymbol: jest.Mock; updateMany: jest.Mock };
+
+ beforeAll(() => {
+ // 2026-08-23 21:30 in New York, but already 2026-08-24 in UTC
+ jest.useFakeTimers().setSystemTime(new Date('2026-08-24T01:30:00.000Z'));
+ });
+
+ beforeEach(() => {
+ dataProviderService = {
+ getHistoricalRaw: jest.fn().mockResolvedValue({
+ 'COINGECKO-bitcoin': {
+ '2026-08-21': { marketPrice: 5 },
+ '2026-08-22': { marketPrice: 6 },
+ '2026-08-23': { marketPrice: 7 }
+ }
+ })
+ };
+ marketDataService = {
+ replaceForSymbol: jest.fn(),
+ updateMany: jest.fn()
+ };
+
+ dataGatheringProcessor = new DataGatheringProcessor(
+ null,
+ dataProviderService as any,
+ marketDataService as any,
+ null
+ );
+ });
+
+ afterAll(() => {
+ jest.useRealTimers();
+ });
+
+ it('gathers up to the last complete UTC day and does not shift the dates', async () => {
+ await dataGatheringProcessor.gatherHistoricalMarketData({
+ data: {
+ dataSource: 'COINGECKO',
+ // The queue enqueues dates at midnight (UTC), not at local midnight
+ date: new Date('2026-08-21T00:00:00.000Z').toISOString(),
+ symbol: 'bitcoin'
+ }
+ } as unknown as Job<DataGatheringItem>);
+
+ const { data } = marketDataService.updateMany.mock.calls[0][0];
+
+ expect(
+ data.map(({ date, marketPrice }) => {
+ return { date, marketPrice };
+ })
+ ).toEqual([
+ { date: new Date('2026-08-21T00:00:00.000Z'), marketPrice: 5 },
+ { date: new Date('2026-08-22T00:00:00.000Z'), marketPrice: 6 },
+ { date: new Date('2026-08-23T00:00:00.000Z'), marketPrice: 7 }
+ ]);
+ });
+});
diff --git a/apps/api/src/services/queues/data-gathering/data-gathering.processor.ts b/apps/api/src/services/queues/data-gathering/data-gathering.processor.ts
index f1e8d01cdb..ce9c6fe4b4 100644
--- a/apps/api/src/services/queues/data-gathering/data-gathering.processor.ts
+++ b/apps/api/src/services/queues/data-gathering/data-gathering.processor.ts
@@ -17,19 +17,12 @@ import {
} from '@ghostfolio/common/helper';
import { AssetProfileIdentifier } from '@ghostfolio/common/interfaces';
+import { utc } from '@date-fns/utc';
import { Process, Processor } from '@nestjs/bull';
import { Injectable, Logger } from '@nestjs/common';
import { Prisma } from '@prisma/client';
import { Job } from 'bull';
-import {
- addDays,
- format,
- getDate,
- getMonth,
- getYear,
- isBefore,
- parseISO
-} from 'date-fns';
+import { addDays, format, isBefore, parseISO } from 'date-fns';
import { DataGatheringService } from './data-gathering.service';
@@ -108,7 +101,8 @@ export class DataGatheringProcessor {
this.logger.log(
`Historical market data gathering has been started for ${symbol} (${dataSource}) at ${format(
currentDate,
- DATE_FORMAT
+ DATE_FORMAT,
+ { in: utc }
)}${force ? ' (forced update)' : ''}`
);
@@ -124,24 +118,13 @@ export class DataGatheringProcessor {
});
const data: Prisma.MarketDataUpdateInput[] = [];
+ const startOfUtcDateOfToday = getStartOfUtcDate(new Date());
let lastMarketPrice: number;
- while (
- isBefore(
- currentDate,
- new Date(
- Date.UTC(
- getYear(new Date()),
- getMonth(new Date()),
- getDate(new Date()),
- 0
- )
- )
- )
- ) {
+ while (isBefore(currentDate, startOfUtcDateOfToday)) {
const marketPriceOfDataProvider =
historicalData[assetProfileIdentifier]?.[
- format(currentDate, DATE_FORMAT)
+ format(currentDate, DATE_FORMAT, { in: utc })
]?.marketPrice;
if (marketPriceOfDataProvider) {
@@ -159,7 +142,7 @@ export class DataGatheringProcessor {
});
}
- currentDate = addDays(currentDate, 1);
+ currentDate = addDays(currentDate, 1, { in: utc });
}
if (force) {
@@ -175,7 +158,8 @@ export class DataGatheringProcessor {
this.logger.log(
`Historical market data gathering has been completed for ${symbol} (${dataSource}) at ${format(
currentDate,
- DATE_FORMAT
+ DATE_FORMAT,
+ { in: utc }
)}`
);
} catch (error) {
diff --git a/apps/api/src/services/queues/data-gathering/data-gathering.service.spec.ts b/apps/api/src/services/queues/data-gathering/data-gathering.service.spec.ts
index 7830d428e2..4fc80d3d64 100644
--- a/apps/api/src/services/queues/data-gathering/data-gathering.service.spec.ts
+++ b/apps/api/src/services/queues/data-gathering/data-gathering.service.spec.ts
@@ -42,8 +42,8 @@ describe('DataGatheringService', () => {
});
describe('getAssetProfileIdentifiersWithRecentMarketData', () => {
- it('excludes carried forward market prices from the query', async () => {
- jest.useFakeTimers().setSystemTime(parseDate('2026-08-23').getTime());
+ it('queries real market prices since the start of yesterday (UTC)', async () => {
+ jest.useFakeTimers().setSystemTime(new Date('2026-08-24T14:00:00.000Z'));
await dataGatheringService[
'getAssetProfileIdentifiersWithRecentMarketData'
@@ -51,28 +51,51 @@ describe('DataGatheringService', () => {
expect(prismaService.marketData.groupBy).toHaveBeenCalledWith(
expect.objectContaining({
- where: expect.objectContaining({
+ where: {
+ date: { gte: new Date('2026-08-23T00:00:00.000Z') },
isCarriedForward: false,
state: 'CLOSE'
+ }
+ })
+ );
+ });
+
+ it('includes the Friday close when it runs on Saturday', async () => {
+ jest.useFakeTimers().setSystemTime(new Date('2026-08-22T14:00:00.000Z'));
+
+ await dataGatheringService[
+ 'getAssetProfileIdentifiersWithRecentMarketData'
+ ]();
+
+ expect(prismaService.marketData.groupBy).toHaveBeenCalledWith(
+ expect.objectContaining({
+ where: expect.objectContaining({
+ date: { gte: new Date('2026-08-21T00:00:00.000Z') }
})
})
);
});
- it('keeps a cryptocurrency with a real market price of yesterday on Sunday', async () => {
- jest.useFakeTimers().setSystemTime(parseDate('2026-08-23').getTime());
+ it('excludes the Friday close when it runs on Sunday', async () => {
+ jest.useFakeTimers().setSystemTime(new Date('2026-08-23T14:00:00.000Z'));
+ await dataGatheringService[
+ 'getAssetProfileIdentifiersWithRecentMarketData'
+ ]();
+
+ expect(prismaService.marketData.groupBy).toHaveBeenCalledWith(
+ expect.objectContaining({
+ where: expect.objectContaining({
+ date: { gte: new Date('2026-08-22T00:00:00.000Z') }
+ })
+ })
+ );
+ });
+
+ it('maps the query result to asset profile identifiers', async () => {
prismaService.marketData.groupBy.mockResolvedValue([
- {
- _max: { date: parseDate('2026-08-22') },
- dataSource: 'COINGECKO',
- symbol: 'bitcoin'
- },
- {
- _max: { date: parseDate('2026-08-21') },
- dataSource: 'YAHOO',
- symbol: 'AAPL'
- }
+ { dataSource: 'COINGECKO', symbol: 'bitcoin' },
+ { dataSource: 'YAHOO', symbol: 'AAPL' }
]);
const assetProfileIdentifiers =
@@ -81,68 +104,52 @@ describe('DataGatheringService', () => {
]();
expect(assetProfileIdentifiers).toEqual([
- { dataSource: 'COINGECKO', symbol: 'bitcoin' }
+ { dataSource: 'COINGECKO', symbol: 'bitcoin' },
+ { dataSource: 'YAHOO', symbol: 'AAPL' }
]);
});
+ });
- it('drops a stock with a real market price of Friday on Monday', async () => {
- jest.useFakeTimers().setSystemTime(parseDate('2026-08-24').getTime());
+ describe('gatherRecentMarketData', () => {
+ it('queries the asset profiles with recent market data once and reuses them', async () => {
+ const assetProfileIdentifiersWithRecentMarketData = [
+ { dataSource: 'COINGECKO', symbol: 'bitcoin' }
+ ];
- prismaService.marketData.groupBy.mockResolvedValue([
- {
- _max: { date: parseDate('2026-08-23') },
- dataSource: 'COINGECKO',
- symbol: 'bitcoin'
- },
- {
- _max: { date: parseDate('2026-08-21') },
- dataSource: 'YAHOO',
- symbol: 'AAPL'
- }
- ]);
+ prismaService.marketData.groupBy.mockResolvedValue(
+ assetProfileIdentifiersWithRecentMarketData
+ );
- const assetProfileIdentifiers =
- await dataGatheringService[
- 'getAssetProfileIdentifiersWithRecentMarketData'
- ]();
+ const getCurrencies7D = jest
+ .spyOn(dataGatheringService as any, 'getCurrencies7D')
+ .mockReturnValue([]);
+ const getSymbols7D = jest
+ .spyOn(dataGatheringService as any, 'getSymbols7D')
+ .mockResolvedValue([]);
- expect(assetProfileIdentifiers).toEqual([
- { dataSource: 'COINGECKO', symbol: 'bitcoin' }
- ]);
- });
+ await dataGatheringService.gatherRecentMarketData();
- it('drops a stock with a late Friday close on Saturday', async () => {
- jest.useFakeTimers().setSystemTime(parseDate('2026-08-22').getTime());
+ expect(prismaService.marketData.groupBy).toHaveBeenCalledTimes(1);
- prismaService.marketData.groupBy.mockResolvedValue([
- {
- _max: { date: parseDate('2026-08-20') },
- dataSource: 'YAHOO',
- symbol: 'AAPL'
- },
- {
- _max: { date: parseDate('2026-08-21') },
- dataSource: 'YAHOO',
- symbol: 'MSFT'
- }
- ]);
+ expect(getCurrencies7D).toHaveBeenCalledWith({
+ assetProfileIdentifiersWithRecentMarketData
+ });
- const assetProfileIdentifiers =
- await dataGatheringService[
- 'getAssetProfileIdentifiersWithRecentMarketData'
- ]();
+ expect(getSymbols7D).toHaveBeenCalledWith({
+ assetProfileIdentifiersWithRecentMarketData,
+ withUserSubscription: true
+ });
- expect(assetProfileIdentifiers).toEqual([
- { dataSource: 'YAHOO', symbol: 'MSFT' }
- ]);
+ expect(getSymbols7D).toHaveBeenCalledWith({
+ assetProfileIdentifiersWithRecentMarketData,
+ withUserSubscription: false
+ });
});
- });
- describe('gatherRecentMarketData', () => {
it('expires completed jobs which are older than the cooldown', async () => {
jest
.spyOn(dataGatheringService as any, 'getCurrencies7D')
- .mockResolvedValue([]);
+ .mockReturnValue([]);
jest
.spyOn(dataGatheringService as any, 'getSymbols7D')
.mockResolvedValue([]);
@@ -158,7 +165,7 @@ describe('DataGatheringService', () => {
it('retains its completed jobs for the duration of the cooldown', async () => {
jest
.spyOn(dataGatheringService as any, 'getCurrencies7D')
- .mockResolvedValue([
+ .mockReturnValue([
{
dataSource: 'YAHOO',
date: parseDate('2026-08-01'),
diff --git a/apps/api/src/services/queues/data-gathering/data-gathering.service.ts b/apps/api/src/services/queues/data-gathering/data-gathering.service.ts
index 010fe5f90b..6f280f5c98 100644
--- a/apps/api/src/services/queues/data-gathering/data-gathering.service.ts
+++ b/apps/api/src/services/queues/data-gathering/data-gathering.service.ts
@@ -20,25 +20,19 @@ import {
DATE_FORMAT,
getAssetProfileIdentifier,
getStartOfUtcDate,
- resetHours
+ getStartOfUtcDateOfYesterday
} from '@ghostfolio/common/helper';
import {
AssetProfileIdentifier,
BenchmarkProperty
} from '@ghostfolio/common/interfaces';
+import { utc } from '@date-fns/utc';
import { InjectQueue } from '@nestjs/bull';
import { Inject, Injectable, Logger } from '@nestjs/common';
import { Prisma } from '@prisma/client';
import { Job, JobOptions, Queue } from 'bull';
-import {
- format,
- isBefore,
- min,
- subDays,
- subMilliseconds,
- subYears
-} from 'date-fns';
+import { format, min, subDays, subMilliseconds, subYears } from 'date-fns';
import { isEmpty } from 'lodash';
import ms, { StringValue } from 'ms';
@@ -269,15 +263,21 @@ export class DataGatheringService {
age: GATHER_HISTORICAL_MARKET_DATA_COOLDOWN_IN_MS / 1000
};
+ const assetProfileIdentifiersWithRecentMarketData =
+ await this.getAssetProfileIdentifiersWithRecentMarketData();
+
await this.gatherSymbols({
removeOnComplete,
- dataGatheringItems: await this.getCurrencies7D(),
+ dataGatheringItems: this.getCurrencies7D({
+ assetProfileIdentifiersWithRecentMarketData
+ }),
priority: DATA_GATHERING_QUEUE_PRIORITY_HIGH
});
await this.gatherSymbols({
removeOnComplete,
dataGatheringItems: await this.getSymbols7D({
+ assetProfileIdentifiersWithRecentMarketData,
withUserSubscription: true
}),
priority: DATA_GATHERING_QUEUE_PRIORITY_MEDIUM
@@ -286,6 +286,7 @@ export class DataGatheringService {
await this.gatherSymbols({
removeOnComplete,
dataGatheringItems: await this.getSymbols7D({
+ assetProfileIdentifiersWithRecentMarketData,
withUserSubscription: false
}),
priority: DATA_GATHERING_QUEUE_PRIORITY_LOW
@@ -322,29 +323,37 @@ export class DataGatheringService {
date,
symbol
}: { date: Date } & AssetProfileIdentifier) {
+ const startOfUtcDate = getStartOfUtcDate(date);
+
try {
const historicalData = await this.dataProviderService.getHistoricalRaw({
assetProfileIdentifiers: [{ dataSource, symbol }],
- from: date,
- to: date
+ from: startOfUtcDate,
+ to: startOfUtcDate
});
const marketPrice =
historicalData[getAssetProfileIdentifier({ dataSource, symbol })]?.[
- format(date, DATE_FORMAT)
+ format(startOfUtcDate, DATE_FORMAT, { in: utc })
]?.marketPrice;
if (marketPrice) {
return await this.prismaService.marketData.upsert({
create: {
dataSource,
- date,
marketPrice,
symbol,
+ date: startOfUtcDate,
isCarriedForward: false
},
update: { marketPrice, isCarriedForward: false },
- where: { dataSource_date_symbol: { dataSource, date, symbol } }
+ where: {
+ dataSource_date_symbol: {
+ dataSource,
+ symbol,
+ date: startOfUtcDate
+ }
+ }
});
}
} catch (error) {
@@ -382,7 +391,7 @@ export class DataGatheringService {
jobId: `${getAssetProfileIdentifier({
dataSource,
symbol
- })}-${format(date, DATE_FORMAT)}`
+ })}-${format(date, DATE_FORMAT, { in: utc })}`
}
};
})
@@ -426,28 +435,25 @@ export class DataGatheringService {
> {
return (
await this.prismaService.marketData.groupBy({
- _max: { date: true },
by: ['dataSource', 'symbol'],
- orderBy: [{ symbol: 'asc' }],
where: {
- date: { gt: subDays(resetHours(new Date()), 7) },
+ date: {
+ gte: getStartOfUtcDateOfYesterday()
+ },
isCarriedForward: false,
state: 'CLOSE'
}
})
- )
- .filter(({ _max }) => {
- return !isBefore(_max.date, getStartOfUtcDate(subDays(new Date(), 1)));
- })
- .map(({ dataSource, symbol }) => {
- return { dataSource, symbol };
- });
+ ).map(({ dataSource, symbol }) => {
+ return { dataSource, symbol };
+ });
}
- private async getCurrencies7D(): Promise<DataGatheringItem[]> {
- const assetProfileIdentifiersWithRecentMarketData =
- await this.getAssetProfileIdentifiersWithRecentMarketData();
-
+ private getCurrencies7D({
+ assetProfileIdentifiersWithRecentMarketData
+ }: {
+ assetProfileIdentifiersWithRecentMarketData: AssetProfileIdentifier[];
+ }): DataGatheringItem[] {
return this.exchangeRateDataService
.getCurrencyPairs()
.filter(({ dataSource, symbol }) => {
@@ -459,7 +465,7 @@ export class DataGatheringService {
return {
dataSource,
symbol,
- date: subDays(resetHours(new Date()), 7)
+ date: subDays(getStartOfUtcDate(new Date()), 7, { in: utc })
};
});
}
@@ -499,8 +505,10 @@ export class DataGatheringService {
}
private async getSymbols7D({
+ assetProfileIdentifiersWithRecentMarketData,
withUserSubscription = false
}: {
+ assetProfileIdentifiersWithRecentMarketData: AssetProfileIdentifier[];
withUserSubscription?: boolean;
}): Promise<DataGatheringItem[]> {
const symbolProfiles =
@@ -510,9 +518,6 @@ export class DataGatheringService {
}
);
- const assetProfileIdentifiersWithRecentMarketData =
- await this.getAssetProfileIdentifiersWithRecentMarketData();
-
return symbolProfiles
.filter(({ dataSource, scraperConfiguration, symbol }) => {
const manualDataSourceWithScraperConfiguration =
@@ -528,7 +533,7 @@ export class DataGatheringService {
.map((symbolProfile) => {
return {
...symbolProfile,
- date: subDays(resetHours(new Date()), 7)
+ date: subDays(getStartOfUtcDate(new Date()), 7, { in: utc })
};
});
}
diff --git a/libs/common/src/lib/helper.ts b/libs/common/src/lib/helper.ts
index eb1693b9fd..260b0f3430 100644
--- a/libs/common/src/lib/helper.ts
+++ b/libs/common/src/lib/helper.ts
@@ -1,3 +1,4 @@
+import { utc } from '@date-fns/utc';
import { NumberParser } from '@internationalized/number';
import {
AccessType,
@@ -465,6 +466,10 @@ export function getStartOfUtcDate(aDate: Date) {
return date;
}
+export function getStartOfUtcDateOfYesterday() {
+ return subDays(getStartOfUtcDate(new Date()), 1, { in: utc });
+}
+
export function getStringOrNull(aString: string | null | undefined) {
const trimmedString = aString?.trim();
diff --git a/package-lock.json b/package-lock.json
index 79917d90ed..14957e0f11 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -160,6 +160,7 @@
"husky": "9.1.7",
"jest": "30.3.0",
"jest-environment-jsdom": "30.2.0",
+ "jest-environment-node": "30.3.0",
"jest-preset-angular": "17.0.0",
"nx": "23.1.1",
"prettier": "3.9.6",
diff --git a/package.json b/package.json
index db37399d61..c1399e15f9 100644
--- a/package.json
+++ b/package.json
@@ -204,6 +204,7 @@
"husky": "9.1.7",
"jest": "30.3.0",
"jest-environment-jsdom": "30.2.0",
+ "jest-environment-node": "30.3.0",
"jest-preset-angular": "17.0.0",
"nx": "23.1.1",
"prettier": "3.9.6",