Open Source Wealth Management Software. Angular + NestJS + Prisma + Nx + TypeScript 🤍
TypeScript angularetffinancefintechghostfoliohacktoberfestinvestingnestjsosspersonal-financeportfolioprismasoftwarestocktrackertradingtypescriptwealth-managementweb
Feature/increase robustness if live data is missing (#1884)
Thomas Kaulaafedd5f7508
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 526e0d194b..0256f3a561 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -7,6 +7,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0## Unreleased+### Added++- Added a fallback to historical market data if a data provider does not provide live data++### Changed++- Persisted today's market data continuously+### Fixed- Fixed the alignment of the performance column header in the holdings tablediff --git a/apps/api/src/app/portfolio/current-rate.service.mock.ts b/apps/api/src/app/portfolio/current-rate.service.mock.tsindex cac110f05e..9fe6b2ec2b 100644--- a/apps/api/src/app/portfolio/current-rate.service.mock.ts+++ b/apps/api/src/app/portfolio/current-rate.service.mock.ts@@ -1,9 +1,9 @@import { parseDate, resetHours } from '@ghostfolio/common/helper';-import { DataProviderInfo } from '@ghostfolio/common/interfaces';import { addDays, endOfDay, isBefore, isSameDay } from 'date-fns';import { GetValueObject } from './interfaces/get-value-object.interface';import { GetValuesParams } from './interfaces/get-values-params.interface';+import { GetValuesObject } from './interfaces/get-values-object.interface';function mockGetValue(symbol: string, date: Date) {switch (symbol) {@@ -49,11 +49,9 @@ export const CurrentRateServiceMock = {getValues: ({dataGatheringItems,dateQuery- }: GetValuesParams): Promise<{- dataProviderInfos: DataProviderInfo[];- values: GetValueObject[];- }> => {+ }: GetValuesParams): Promise<GetValuesObject> => {const values: GetValueObject[] = [];+if (dateQuery.lt) {for (let date = resetHours(dateQuery.gte);@@ -85,6 +83,7 @@ export const CurrentRateServiceMock = {}}}- return Promise.resolve({ values, dataProviderInfos: [] });++ return Promise.resolve({ values, dataProviderInfos: [], errors: [] });}};diff --git a/apps/api/src/app/portfolio/current-rate.service.spec.ts b/apps/api/src/app/portfolio/current-rate.service.spec.tsindex 226d9e7dba..a4eb101560 100644--- a/apps/api/src/app/portfolio/current-rate.service.spec.ts+++ b/apps/api/src/app/portfolio/current-rate.service.spec.ts@@ -1,12 +1,11 @@import { DataProviderService } from '@ghostfolio/api/services/data-provider/data-provider.service';import { ExchangeRateDataService } from '@ghostfolio/api/services/exchange-rate-data.service';import { MarketDataService } from '@ghostfolio/api/services/market-data.service';-import { DataProviderInfo } from '@ghostfolio/common/interfaces';+import { PropertyService } from '@ghostfolio/api/services/property/property.service';import { DataSource, MarketData } from '@prisma/client';import { CurrentRateService } from './current-rate.service';-import { GetValueObject } from './interfaces/get-value-object.interface';-import { PropertyService } from '@ghostfolio/api/services/property/property.service';+import { GetValuesObject } from './interfaces/get-values-object.interface';jest.mock('@ghostfolio/api/services/market-data.service', () => {return {@@ -123,21 +122,14 @@ describe('CurrentRateService', () => {},userCurrency: 'CHF'})- ).toMatchObject<{- dataProviderInfos: DataProviderInfo[];- values: GetValueObject[];- }>({+ ).toMatchObject<GetValuesObject>({dataProviderInfos: [],+ errors: [],values: [{date: undefined,marketPriceInBaseCurrency: 1841.823902,symbol: 'AMZN'- },- {- date: undefined,- marketPriceInBaseCurrency: 1847.839966,- symbol: 'AMZN'}]});diff --git a/apps/api/src/app/portfolio/current-rate.service.ts b/apps/api/src/app/portfolio/current-rate.service.tsindex b3a432521a..9ca95d7791 100644--- a/apps/api/src/app/portfolio/current-rate.service.ts+++ b/apps/api/src/app/portfolio/current-rate.service.ts@@ -2,13 +2,14 @@ import { DataProviderService } from '@ghostfolio/api/services/data-provider/dataimport { ExchangeRateDataService } from '@ghostfolio/api/services/exchange-rate-data.service';import { MarketDataService } from '@ghostfolio/api/services/market-data.service';import { resetHours } from '@ghostfolio/common/helper';-import { DataProviderInfo } from '@ghostfolio/common/interfaces';+import { DataProviderInfo, ResponseError } from '@ghostfolio/common/interfaces';import { Injectable } from '@nestjs/common';import { isBefore, isToday } from 'date-fns';-import { flatten } from 'lodash';+import { flatten, isEmpty, uniqBy } from 'lodash';import { GetValueObject } from './interfaces/get-value-object.interface';import { GetValuesParams } from './interfaces/get-values-params.interface';+import { GetValuesObject } from './interfaces/get-values-object.interface';@Injectable()export class CurrentRateService {@@ -23,10 +24,7 @@ export class CurrentRateService {dataGatheringItems,dateQuery,userCurrency- }: GetValuesParams): Promise<{- dataProviderInfos: DataProviderInfo[];- values: GetValueObject[];- }> {+ }: GetValuesParams): Promise<GetValuesObject> {const dataProviderInfos: DataProviderInfo[] = [];const includeToday =(!dateQuery.lt || isBefore(new Date(), dateQuery.lt)) &&@@ -34,9 +32,10 @@ export class CurrentRateService {(!dateQuery.in || this.containsToday(dateQuery.in));const promises: Promise<GetValueObject[]>[] = [];+ const quoteErrors: ResponseError['errors'] = [];+ const today = resetHours(new Date());if (includeToday) {- const today = resetHours(new Date());promises.push(this.dataProviderService.getQuotes(dataGatheringItems)@@ -51,18 +50,26 @@ export class CurrentRateService {);}- result.push({- date: today,- marketPriceInBaseCurrency:- this.exchangeRateDataService.toCurrency(- dataResultProvider?.[dataGatheringItem.symbol]- ?.marketPrice ?? 0,- dataResultProvider?.[dataGatheringItem.symbol]?.currency,- userCurrency- ),- symbol: dataGatheringItem.symbol- });+ if (dataResultProvider?.[dataGatheringItem.symbol]?.marketPrice) {+ result.push({+ date: today,+ marketPriceInBaseCurrency:+ this.exchangeRateDataService.toCurrency(+ dataResultProvider?.[dataGatheringItem.symbol]+ ?.marketPrice,+ dataResultProvider?.[dataGatheringItem.symbol]?.currency,+ userCurrency+ ),+ symbol: dataGatheringItem.symbol+ });+ } else {+ quoteErrors.push({+ dataSource: dataGatheringItem.dataSource,+ symbol: dataGatheringItem.symbol+ });+ }}+return result;}));@@ -94,10 +101,60 @@ export class CurrentRateService {}));- return {+ const values = flatten(await Promise.all(promises));++ const response: GetValuesObject = {dataProviderInfos,- values: flatten(await Promise.all(promises))+ errors: quoteErrors.map(({ dataSource, symbol }) => {+ return { dataSource, symbol };+ }),+ values: uniqBy(values, ({ date, symbol }) => `${date}-${symbol}`)};++ if (!isEmpty(quoteErrors)) {+ for (const { symbol } of quoteErrors) {+ try {+ // If missing quote, fallback to the latest available historical market price+ let value: GetValueObject = response.values.find((currentValue) => {+ return currentValue.symbol === symbol && isToday(currentValue.date);+ });++ if (!value) {+ value = {+ symbol,+ date: today,+ marketPriceInBaseCurrency: 0+ };++ response.values.push(value);+ }++ const [latestValue] = response.values+ .filter((currentValue) => {+ return (+ currentValue.symbol === symbol &&+ currentValue.marketPriceInBaseCurrency+ );+ })+ .sort((a, b) => {+ if (a.date < b.date) {+ return 1;+ }++ if (a.date > b.date) {+ return -1;+ }++ return 0;+ });++ value.marketPriceInBaseCurrency =+ latestValue.marketPriceInBaseCurrency;+ } catch {}+ }+ }++ return response;}private containsToday(dates: Date[]): boolean {diff --git a/apps/api/src/app/portfolio/interfaces/get-values-object.interface.ts b/apps/api/src/app/portfolio/interfaces/get-values-object.interface.tsnew file mode 100644index 0000000000..ef6cb8f96c--- /dev/null+++ b/apps/api/src/app/portfolio/interfaces/get-values-object.interface.ts@@ -0,0 +1,9 @@+import { DataProviderInfo, ResponseError } from '@ghostfolio/common/interfaces';++import { GetValueObject } from './get-value-object.interface';++export interface GetValuesObject {+ dataProviderInfos: DataProviderInfo[];+ errors: ResponseError['errors'];+ values: GetValueObject[];+}diff --git a/apps/api/src/app/portfolio/portfolio-calculator.ts b/apps/api/src/app/portfolio/portfolio-calculator.tsindex 952b816775..f71597e143 100644--- a/apps/api/src/app/portfolio/portfolio-calculator.ts+++ b/apps/api/src/app/portfolio/portfolio-calculator.ts@@ -24,9 +24,10 @@ import {isSameYear,max,min,- set+ set,+ subDays} from 'date-fns';-import { first, flatten, isNumber, last, sortBy } from 'lodash';+import { first, flatten, isNumber, last, sortBy, uniq } from 'lodash';import { CurrentRateService } from './current-rate.service';import { CurrentPositions } from './interfaces/current-positions.interface';@@ -360,7 +361,7 @@ export class PortfolioCalculator {let firstTransactionPoint: TransactionPoint = null;let firstIndex = transactionPointsBeforeEndDate.length;- const dates = [];+ let dates = [];const dataGatheringItems: IDataGatheringItem[] = [];const currencies: { [symbol: string]: string } = {};@@ -389,15 +390,37 @@ export class PortfolioCalculator {dates.push(resetHours(end));- const { dataProviderInfos, values: marketSymbols } =- await this.currentRateService.getValues({- currencies,- dataGatheringItems,- dateQuery: {- in: dates- },- userCurrency: this.currency- });+ // Add dates of last week for fallback+ dates.push(subDays(resetHours(new Date()), 7));+ dates.push(subDays(resetHours(new Date()), 6));+ dates.push(subDays(resetHours(new Date()), 5));+ dates.push(subDays(resetHours(new Date()), 4));+ dates.push(subDays(resetHours(new Date()), 3));+ dates.push(subDays(resetHours(new Date()), 2));+ dates.push(subDays(resetHours(new Date()), 1));+ dates.push(resetHours(new Date()));++ dates = uniq(+ dates.map((date) => {+ return date.getTime();+ })+ ).map((timestamp) => {+ return new Date(timestamp);+ });+ dates.sort((a, b) => a.getTime() - b.getTime());++ const {+ dataProviderInfos,+ errors: currentRateErrors,+ values: marketSymbols+ } = await this.currentRateService.getValues({+ currencies,+ dataGatheringItems,+ dateQuery: {+ in: dates+ },+ userCurrency: this.currency+ });this.dataProviderInfos = dataProviderInfos;@@ -472,7 +495,13 @@ export class PortfolioCalculator {transactionCount: item.transactionCount});- if (hasErrors && item.investment.gt(0)) {+ if (+ (hasErrors ||+ currentRateErrors.find(({ dataSource, symbol }) => {+ return dataSource === item.dataSource && symbol === item.symbol;+ })) &&+ item.investment.gt(0)+ ) {errors.push({ dataSource: item.dataSource, symbol: item.symbol });}}diff --git a/apps/api/src/services/data-provider/data-provider.service.ts b/apps/api/src/services/data-provider/data-provider.service.tsindex d35e01b6ed..0fd7cceba1 100644--- a/apps/api/src/services/data-provider/data-provider.service.ts+++ b/apps/api/src/services/data-provider/data-provider.service.ts@@ -7,13 +7,13 @@ import {IDataProviderResponse} from '@ghostfolio/api/services/interfaces/interfaces';import { PrismaService } from '@ghostfolio/api/services/prisma.service';-import { DATE_FORMAT } from '@ghostfolio/common/helper';+import { DATE_FORMAT, getStartOfUtcDate } from '@ghostfolio/common/helper';import { UserWithSettings } from '@ghostfolio/common/types';import { Granularity } from '@ghostfolio/common/types';import { Inject, Injectable, Logger } from '@nestjs/common';import { DataSource, MarketData, SymbolProfile } from '@prisma/client';import { format, isValid } from 'date-fns';-import { groupBy, isEmpty } from 'lodash';+import { groupBy, isEmpty, isNumber } from 'lodash';import { PropertyService } from '@ghostfolio/api/services/property/property.service';import { PROPERTY_DATA_SOURCE_MAPPING } from '@ghostfolio/common/config';@@ -241,7 +241,7 @@ export class DataProviderService {const promise = Promise.resolve(dataProvider.getQuotes(symbolsChunk));promises.push(- promise.then((result) => {+ promise.then(async (result) => {for (const [symbol, dataProviderResponse] of Object.entries(result)) {@@ -256,6 +256,38 @@ export class DataProviderService {1000).toFixed(3)} seconds`);++ try {+ const date = getStartOfUtcDate(new Date());++ // Upsert quotes by imitating missing upsertMany functionality+ // with $transaction+ const upsertPromises = Object.keys(response)+ .filter((symbol) => {+ return (+ isNumber(response[symbol].marketPrice) &&+ response[symbol].marketPrice > 0+ );+ })+ .map((symbol) =>+ this.prismaService.marketData.upsert({+ create: {+ date,+ symbol,+ dataSource: response[symbol].dataSource,+ marketPrice: response[symbol].marketPrice+ },+ update: {+ marketPrice: response[symbol].marketPrice+ },+ where: {+ date_symbol: { date, symbol }+ }+ })+ );++ await this.prismaService.$transaction(upsertPromises);+ } catch {}}));}diff --git a/libs/common/src/lib/helper.ts b/libs/common/src/lib/helper.tsindex 0bfefb729c..d6da4ed945 100644--- a/libs/common/src/lib/helper.ts+++ b/libs/common/src/lib/helper.ts@@ -152,6 +152,13 @@ export function getNumberFormatGroup(aLocale?: string) {}).value;}+export function getStartOfUtcDate(aDate: Date) {+ const date = new Date(aDate);+ date.setUTCHours(0, 0, 0, 0);++ return date;+}+export function getSum(aArray: Big[]) {if (aArray?.length > 0) {return aArray.reduce((a, b) => a.plus(b), new Big(0));