Skip to content

Open Source Wealth Management Software. Angular + NestJS + Prisma + Nx + TypeScript 🤍

Feature/increase robustness if live data is missing (#1884)

Thomas Kaulaafedd5f7508
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 526e0d194b..0256f3a561 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -7,6 +7,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
## Unreleased
+### Added
+
+- Added a fallback to historical market data if a data provider does not provide live data
+
+### Changed
+
+- Persisted today's market data continuously
+
### Fixed
- Fixed the alignment of the performance column header in the holdings table
diff --git a/apps/api/src/app/portfolio/current-rate.service.mock.ts b/apps/api/src/app/portfolio/current-rate.service.mock.ts
index cac110f05e..9fe6b2ec2b 100644
--- a/apps/api/src/app/portfolio/current-rate.service.mock.ts
+++ b/apps/api/src/app/portfolio/current-rate.service.mock.ts
@@ -1,9 +1,9 @@
import { parseDate, resetHours } from '@ghostfolio/common/helper';
-import { DataProviderInfo } from '@ghostfolio/common/interfaces';
import { addDays, endOfDay, isBefore, isSameDay } from 'date-fns';
import { GetValueObject } from './interfaces/get-value-object.interface';
import { GetValuesParams } from './interfaces/get-values-params.interface';
+import { GetValuesObject } from './interfaces/get-values-object.interface';
function mockGetValue(symbol: string, date: Date) {
switch (symbol) {
@@ -49,11 +49,9 @@ export const CurrentRateServiceMock = {
getValues: ({
dataGatheringItems,
dateQuery
- }: GetValuesParams): Promise<{
- dataProviderInfos: DataProviderInfo[];
- values: GetValueObject[];
- }> => {
+ }: GetValuesParams): Promise<GetValuesObject> => {
const values: GetValueObject[] = [];
+
if (dateQuery.lt) {
for (
let date = resetHours(dateQuery.gte);
@@ -85,6 +83,7 @@ export const CurrentRateServiceMock = {
}
}
}
- return Promise.resolve({ values, dataProviderInfos: [] });
+
+ return Promise.resolve({ values, dataProviderInfos: [], errors: [] });
}
};
diff --git a/apps/api/src/app/portfolio/current-rate.service.spec.ts b/apps/api/src/app/portfolio/current-rate.service.spec.ts
index 226d9e7dba..a4eb101560 100644
--- a/apps/api/src/app/portfolio/current-rate.service.spec.ts
+++ b/apps/api/src/app/portfolio/current-rate.service.spec.ts
@@ -1,12 +1,11 @@
import { DataProviderService } from '@ghostfolio/api/services/data-provider/data-provider.service';
import { ExchangeRateDataService } from '@ghostfolio/api/services/exchange-rate-data.service';
import { MarketDataService } from '@ghostfolio/api/services/market-data.service';
-import { DataProviderInfo } from '@ghostfolio/common/interfaces';
+import { PropertyService } from '@ghostfolio/api/services/property/property.service';
import { DataSource, MarketData } from '@prisma/client';
import { CurrentRateService } from './current-rate.service';
-import { GetValueObject } from './interfaces/get-value-object.interface';
-import { PropertyService } from '@ghostfolio/api/services/property/property.service';
+import { GetValuesObject } from './interfaces/get-values-object.interface';
jest.mock('@ghostfolio/api/services/market-data.service', () => {
return {
@@ -123,21 +122,14 @@ describe('CurrentRateService', () => {
},
userCurrency: 'CHF'
})
- ).toMatchObject<{
- dataProviderInfos: DataProviderInfo[];
- values: GetValueObject[];
- }>({
+ ).toMatchObject<GetValuesObject>({
dataProviderInfos: [],
+ errors: [],
values: [
{
date: undefined,
marketPriceInBaseCurrency: 1841.823902,
symbol: 'AMZN'
- },
- {
- date: undefined,
- marketPriceInBaseCurrency: 1847.839966,
- symbol: 'AMZN'
}
]
});
diff --git a/apps/api/src/app/portfolio/current-rate.service.ts b/apps/api/src/app/portfolio/current-rate.service.ts
index b3a432521a..9ca95d7791 100644
--- a/apps/api/src/app/portfolio/current-rate.service.ts
+++ b/apps/api/src/app/portfolio/current-rate.service.ts
@@ -2,13 +2,14 @@ import { DataProviderService } from '@ghostfolio/api/services/data-provider/data
import { ExchangeRateDataService } from '@ghostfolio/api/services/exchange-rate-data.service';
import { MarketDataService } from '@ghostfolio/api/services/market-data.service';
import { resetHours } from '@ghostfolio/common/helper';
-import { DataProviderInfo } from '@ghostfolio/common/interfaces';
+import { DataProviderInfo, ResponseError } from '@ghostfolio/common/interfaces';
import { Injectable } from '@nestjs/common';
import { isBefore, isToday } from 'date-fns';
-import { flatten } from 'lodash';
+import { flatten, isEmpty, uniqBy } from 'lodash';
import { GetValueObject } from './interfaces/get-value-object.interface';
import { GetValuesParams } from './interfaces/get-values-params.interface';
+import { GetValuesObject } from './interfaces/get-values-object.interface';
@Injectable()
export class CurrentRateService {
@@ -23,10 +24,7 @@ export class CurrentRateService {
dataGatheringItems,
dateQuery,
userCurrency
- }: GetValuesParams): Promise<{
- dataProviderInfos: DataProviderInfo[];
- values: GetValueObject[];
- }> {
+ }: GetValuesParams): Promise<GetValuesObject> {
const dataProviderInfos: DataProviderInfo[] = [];
const includeToday =
(!dateQuery.lt || isBefore(new Date(), dateQuery.lt)) &&
@@ -34,9 +32,10 @@ export class CurrentRateService {
(!dateQuery.in || this.containsToday(dateQuery.in));
const promises: Promise<GetValueObject[]>[] = [];
+ const quoteErrors: ResponseError['errors'] = [];
+ const today = resetHours(new Date());
if (includeToday) {
- const today = resetHours(new Date());
promises.push(
this.dataProviderService
.getQuotes(dataGatheringItems)
@@ -51,18 +50,26 @@ export class CurrentRateService {
);
}
- result.push({
- date: today,
- marketPriceInBaseCurrency:
- this.exchangeRateDataService.toCurrency(
- dataResultProvider?.[dataGatheringItem.symbol]
- ?.marketPrice ?? 0,
- dataResultProvider?.[dataGatheringItem.symbol]?.currency,
- userCurrency
- ),
- symbol: dataGatheringItem.symbol
- });
+ if (dataResultProvider?.[dataGatheringItem.symbol]?.marketPrice) {
+ result.push({
+ date: today,
+ marketPriceInBaseCurrency:
+ this.exchangeRateDataService.toCurrency(
+ dataResultProvider?.[dataGatheringItem.symbol]
+ ?.marketPrice,
+ dataResultProvider?.[dataGatheringItem.symbol]?.currency,
+ userCurrency
+ ),
+ symbol: dataGatheringItem.symbol
+ });
+ } else {
+ quoteErrors.push({
+ dataSource: dataGatheringItem.dataSource,
+ symbol: dataGatheringItem.symbol
+ });
+ }
}
+
return result;
})
);
@@ -94,10 +101,60 @@ export class CurrentRateService {
})
);
- return {
+ const values = flatten(await Promise.all(promises));
+
+ const response: GetValuesObject = {
dataProviderInfos,
- values: flatten(await Promise.all(promises))
+ errors: quoteErrors.map(({ dataSource, symbol }) => {
+ return { dataSource, symbol };
+ }),
+ values: uniqBy(values, ({ date, symbol }) => `${date}-${symbol}`)
};
+
+ if (!isEmpty(quoteErrors)) {
+ for (const { symbol } of quoteErrors) {
+ try {
+ // If missing quote, fallback to the latest available historical market price
+ let value: GetValueObject = response.values.find((currentValue) => {
+ return currentValue.symbol === symbol && isToday(currentValue.date);
+ });
+
+ if (!value) {
+ value = {
+ symbol,
+ date: today,
+ marketPriceInBaseCurrency: 0
+ };
+
+ response.values.push(value);
+ }
+
+ const [latestValue] = response.values
+ .filter((currentValue) => {
+ return (
+ currentValue.symbol === symbol &&
+ currentValue.marketPriceInBaseCurrency
+ );
+ })
+ .sort((a, b) => {
+ if (a.date < b.date) {
+ return 1;
+ }
+
+ if (a.date > b.date) {
+ return -1;
+ }
+
+ return 0;
+ });
+
+ value.marketPriceInBaseCurrency =
+ latestValue.marketPriceInBaseCurrency;
+ } catch {}
+ }
+ }
+
+ return response;
}
private containsToday(dates: Date[]): boolean {
diff --git a/apps/api/src/app/portfolio/interfaces/get-values-object.interface.ts b/apps/api/src/app/portfolio/interfaces/get-values-object.interface.ts
new file mode 100644
index 0000000000..ef6cb8f96c
--- /dev/null
+++ b/apps/api/src/app/portfolio/interfaces/get-values-object.interface.ts
@@ -0,0 +1,9 @@
+import { DataProviderInfo, ResponseError } from '@ghostfolio/common/interfaces';
+
+import { GetValueObject } from './get-value-object.interface';
+
+export interface GetValuesObject {
+ dataProviderInfos: DataProviderInfo[];
+ errors: ResponseError['errors'];
+ values: GetValueObject[];
+}
diff --git a/apps/api/src/app/portfolio/portfolio-calculator.ts b/apps/api/src/app/portfolio/portfolio-calculator.ts
index 952b816775..f71597e143 100644
--- a/apps/api/src/app/portfolio/portfolio-calculator.ts
+++ b/apps/api/src/app/portfolio/portfolio-calculator.ts
@@ -24,9 +24,10 @@ import {
isSameYear,
max,
min,
- set
+ set,
+ subDays
} from 'date-fns';
-import { first, flatten, isNumber, last, sortBy } from 'lodash';
+import { first, flatten, isNumber, last, sortBy, uniq } from 'lodash';
import { CurrentRateService } from './current-rate.service';
import { CurrentPositions } from './interfaces/current-positions.interface';
@@ -360,7 +361,7 @@ export class PortfolioCalculator {
let firstTransactionPoint: TransactionPoint = null;
let firstIndex = transactionPointsBeforeEndDate.length;
- const dates = [];
+ let dates = [];
const dataGatheringItems: IDataGatheringItem[] = [];
const currencies: { [symbol: string]: string } = {};
@@ -389,15 +390,37 @@ export class PortfolioCalculator {
dates.push(resetHours(end));
- const { dataProviderInfos, values: marketSymbols } =
- await this.currentRateService.getValues({
- currencies,
- dataGatheringItems,
- dateQuery: {
- in: dates
- },
- userCurrency: this.currency
- });
+ // Add dates of last week for fallback
+ dates.push(subDays(resetHours(new Date()), 7));
+ dates.push(subDays(resetHours(new Date()), 6));
+ dates.push(subDays(resetHours(new Date()), 5));
+ dates.push(subDays(resetHours(new Date()), 4));
+ dates.push(subDays(resetHours(new Date()), 3));
+ dates.push(subDays(resetHours(new Date()), 2));
+ dates.push(subDays(resetHours(new Date()), 1));
+ dates.push(resetHours(new Date()));
+
+ dates = uniq(
+ dates.map((date) => {
+ return date.getTime();
+ })
+ ).map((timestamp) => {
+ return new Date(timestamp);
+ });
+ dates.sort((a, b) => a.getTime() - b.getTime());
+
+ const {
+ dataProviderInfos,
+ errors: currentRateErrors,
+ values: marketSymbols
+ } = await this.currentRateService.getValues({
+ currencies,
+ dataGatheringItems,
+ dateQuery: {
+ in: dates
+ },
+ userCurrency: this.currency
+ });
this.dataProviderInfos = dataProviderInfos;
@@ -472,7 +495,13 @@ export class PortfolioCalculator {
transactionCount: item.transactionCount
});
- if (hasErrors && item.investment.gt(0)) {
+ if (
+ (hasErrors ||
+ currentRateErrors.find(({ dataSource, symbol }) => {
+ return dataSource === item.dataSource && symbol === item.symbol;
+ })) &&
+ item.investment.gt(0)
+ ) {
errors.push({ dataSource: item.dataSource, symbol: item.symbol });
}
}
diff --git a/apps/api/src/services/data-provider/data-provider.service.ts b/apps/api/src/services/data-provider/data-provider.service.ts
index d35e01b6ed..0fd7cceba1 100644
--- a/apps/api/src/services/data-provider/data-provider.service.ts
+++ b/apps/api/src/services/data-provider/data-provider.service.ts
@@ -7,13 +7,13 @@ import {
IDataProviderResponse
} from '@ghostfolio/api/services/interfaces/interfaces';
import { PrismaService } from '@ghostfolio/api/services/prisma.service';
-import { DATE_FORMAT } from '@ghostfolio/common/helper';
+import { DATE_FORMAT, getStartOfUtcDate } from '@ghostfolio/common/helper';
import { UserWithSettings } from '@ghostfolio/common/types';
import { Granularity } from '@ghostfolio/common/types';
import { Inject, Injectable, Logger } from '@nestjs/common';
import { DataSource, MarketData, SymbolProfile } from '@prisma/client';
import { format, isValid } from 'date-fns';
-import { groupBy, isEmpty } from 'lodash';
+import { groupBy, isEmpty, isNumber } from 'lodash';
import { PropertyService } from '@ghostfolio/api/services/property/property.service';
import { PROPERTY_DATA_SOURCE_MAPPING } from '@ghostfolio/common/config';
@@ -241,7 +241,7 @@ export class DataProviderService {
const promise = Promise.resolve(dataProvider.getQuotes(symbolsChunk));
promises.push(
- promise.then((result) => {
+ promise.then(async (result) => {
for (const [symbol, dataProviderResponse] of Object.entries(
result
)) {
@@ -256,6 +256,38 @@ export class DataProviderService {
1000
).toFixed(3)} seconds`
);
+
+ try {
+ const date = getStartOfUtcDate(new Date());
+
+ // Upsert quotes by imitating missing upsertMany functionality
+ // with $transaction
+ const upsertPromises = Object.keys(response)
+ .filter((symbol) => {
+ return (
+ isNumber(response[symbol].marketPrice) &&
+ response[symbol].marketPrice > 0
+ );
+ })
+ .map((symbol) =>
+ this.prismaService.marketData.upsert({
+ create: {
+ date,
+ symbol,
+ dataSource: response[symbol].dataSource,
+ marketPrice: response[symbol].marketPrice
+ },
+ update: {
+ marketPrice: response[symbol].marketPrice
+ },
+ where: {
+ date_symbol: { date, symbol }
+ }
+ })
+ );
+
+ await this.prismaService.$transaction(upsertPromises);
+ } catch {}
})
);
}
diff --git a/libs/common/src/lib/helper.ts b/libs/common/src/lib/helper.ts
index 0bfefb729c..d6da4ed945 100644
--- a/libs/common/src/lib/helper.ts
+++ b/libs/common/src/lib/helper.ts
@@ -152,6 +152,13 @@ export function getNumberFormatGroup(aLocale?: string) {
}).value;
}
+export function getStartOfUtcDate(aDate: Date) {
+ const date = new Date(aDate);
+ date.setUTCHours(0, 0, 0, 0);
+
+ return date;
+}
+
export function getSum(aArray: Big[]) {
if (aArray?.length > 0) {
return aArray.reduce((a, b) => a.plus(b), new Big(0));