Open Source Wealth Management Software. Angular + NestJS + Prisma + Nx + TypeScript 🤍
TypeScript angularetffinancefintechghostfoliohacktoberfestinvestingnestjsosspersonal-financeportfolioprismasoftwarestocktrackertradingtypescriptwealth-managementweb
Feature/add data source eod historical data (#974)
Thomas Kaulf1e06347d348
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex dca00fbc20..040b771a6e 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -7,10 +7,18 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0## Unreleased+### Added++- Added `EOD_HISTORICAL_DATA` as a new data source type+### Changed- Exposed the environment variable `REDIS_PASSWORD`+### Todo++- Apply data migration (`yarn database:migrate`)+## 1.154.0 - 28.05.2022### Addeddiff --git a/apps/api/src/services/configuration.service.ts b/apps/api/src/services/configuration.service.tsindex 9666e7ab71..525951a70f 100644--- a/apps/api/src/services/configuration.service.ts+++ b/apps/api/src/services/configuration.service.ts@@ -25,6 +25,7 @@ export class ConfigurationService {ENABLE_FEATURE_STATISTICS: bool({ default: false }),ENABLE_FEATURE_SUBSCRIPTION: bool({ default: false }),ENABLE_FEATURE_SYSTEM_MESSAGE: bool({ default: false }),+ EOD_HISTORICAL_DATA_API_KEY: str({ default: '' }),GOOGLE_CLIENT_ID: str({ default: 'dummyClientId' }),GOOGLE_SECRET: str({ default: 'dummySecret' }),GOOGLE_SHEETS_ACCOUNT: str({ default: '' }),diff --git a/apps/api/src/services/data-provider/data-provider.module.ts b/apps/api/src/services/data-provider/data-provider.module.tsindex e2a77af4ae..dcdb7acbd8 100644--- a/apps/api/src/services/data-provider/data-provider.module.ts+++ b/apps/api/src/services/data-provider/data-provider.module.ts@@ -1,5 +1,7 @@import { ConfigurationModule } from '@ghostfolio/api/services/configuration.module';import { CryptocurrencyModule } from '@ghostfolio/api/services/cryptocurrency/cryptocurrency.module';+import { AlphaVantageService } from '@ghostfolio/api/services/data-provider/alpha-vantage/alpha-vantage.service';+import { EodHistoricalDataService } from '@ghostfolio/api/services/data-provider/eod-historical-data/eod-historical-data.service';import { GhostfolioScraperApiService } from '@ghostfolio/api/services/data-provider/ghostfolio-scraper-api/ghostfolio-scraper-api.service';import { GoogleSheetsService } from '@ghostfolio/api/services/data-provider/google-sheets/google-sheets.service';import { ManualService } from '@ghostfolio/api/services/data-provider/manual/manual.service';@@ -9,7 +11,6 @@ import { PrismaModule } from '@ghostfolio/api/services/prisma.module';import { SymbolProfileModule } from '@ghostfolio/api/services/symbol-profile.module';import { Module } from '@nestjs/common';-import { AlphaVantageService } from './alpha-vantage/alpha-vantage.service';import { DataProviderService } from './data-provider.service';@Module({@@ -22,6 +23,7 @@ import { DataProviderService } from './data-provider.service';providers: [AlphaVantageService,DataProviderService,+ EodHistoricalDataService,GhostfolioScraperApiService,GoogleSheetsService,ManualService,@@ -30,6 +32,7 @@ import { DataProviderService } from './data-provider.service';{inject: [AlphaVantageService,+ EodHistoricalDataService,GhostfolioScraperApiService,GoogleSheetsService,ManualService,@@ -39,6 +42,7 @@ import { DataProviderService } from './data-provider.service';provide: 'DataProviderInterfaces',useFactory: (alphaVantageService,+ eodHistoricalDataService,ghostfolioScraperApiService,googleSheetsService,manualService,@@ -46,6 +50,7 @@ import { DataProviderService } from './data-provider.service';yahooFinanceService) => [alphaVantageService,+ eodHistoricalDataService,ghostfolioScraperApiService,googleSheetsService,manualService,diff --git a/apps/api/src/services/data-provider/eod-historical-data/eod-historical-data.service.ts b/apps/api/src/services/data-provider/eod-historical-data/eod-historical-data.service.tsnew file mode 100644index 0000000000..bb0401f002--- /dev/null+++ b/apps/api/src/services/data-provider/eod-historical-data/eod-historical-data.service.ts@@ -0,0 +1,138 @@+import { LookupItem } from '@ghostfolio/api/app/symbol/interfaces/lookup-item.interface';+import { ConfigurationService } from '@ghostfolio/api/services/configuration.service';+import { DataProviderInterface } from '@ghostfolio/api/services/data-provider/interfaces/data-provider.interface';+import {+ IDataProviderHistoricalResponse,+ IDataProviderResponse+} from '@ghostfolio/api/services/interfaces/interfaces';+import { SymbolProfileService } from '@ghostfolio/api/services/symbol-profile.service';+import { DATE_FORMAT } from '@ghostfolio/common/helper';+import { Granularity } from '@ghostfolio/common/types';+import { Injectable, Logger } from '@nestjs/common';+import { DataSource, SymbolProfile } from '@prisma/client';+import bent from 'bent';+import { format } from 'date-fns';++@Injectable()+export class EodHistoricalDataService implements DataProviderInterface {+ private apiKey: string;+ private readonly URL = 'https://eodhistoricaldata.com/api';++ public constructor(+ private readonly configurationService: ConfigurationService,+ private readonly symbolProfileService: SymbolProfileService+ ) {+ this.apiKey = this.configurationService.get('EOD_HISTORICAL_DATA_API_KEY');+ }++ public canHandle(symbol: string) {+ return true;+ }++ public async getAssetProfile(+ aSymbol: string+ ): Promise<Partial<SymbolProfile>> {+ return {+ dataSource: this.getName()+ };+ }++ public async getHistorical(+ aSymbol: string,+ aGranularity: Granularity = 'day',+ from: Date,+ to: Date+ ): Promise<{+ [symbol: string]: { [date: string]: IDataProviderHistoricalResponse };+ }> {+ try {+ const get = bent(+ `${this.URL}/eod/${aSymbol}?api_token=${+ this.apiKey+ }&fmt=json&from=${format(from, DATE_FORMAT)}&to=${format(+ to,+ DATE_FORMAT+ )}&period={aGranularity}`,+ 'GET',+ 'json',+ 200+ );++ const response = await get();++ return response.reduce(+ (result, historicalItem, index, array) => {+ result[aSymbol][historicalItem.date] = {+ marketPrice: historicalItem.close,+ performance: historicalItem.open - historicalItem.close+ };++ return result;+ },+ { [aSymbol]: {} }+ );+ } catch (error) {+ Logger.error(error, 'EodHistoricalDataService');+ }++ return {};+ }++ public getName(): DataSource {+ return DataSource.EOD_HISTORICAL_DATA;+ }++ public async getQuotes(+ aSymbols: string[]+ ): Promise<{ [symbol: string]: IDataProviderResponse }> {+ if (aSymbols.length <= 0) {+ return {};+ }++ try {+ const get = bent(+ `${this.URL}/real-time/${aSymbols[0]}?api_token=${+ this.apiKey+ }&fmt=json&s=${aSymbols.join(',')}`,+ 'GET',+ 'json',+ 200+ );++ const [response, symbolProfiles] = await Promise.all([+ get(),+ this.symbolProfileService.getSymbolProfiles(+ aSymbols.map((symbol) => {+ return {+ symbol,+ dataSource: DataSource.EOD_HISTORICAL_DATA+ };+ })+ )+ ]);++ const quotes = aSymbols.length === 1 ? [response] : response;++ return quotes.reduce((result, item, index, array) => {+ result[item.code] = {+ currency: symbolProfiles.find((symbolProfile) => {+ return symbolProfile.symbol === item.code;+ })?.currency,+ dataSource: DataSource.EOD_HISTORICAL_DATA,+ marketPrice: item.close,+ marketState: 'delayed'+ };++ return result;+ }, {});+ } catch (error) {+ Logger.error(error, 'EodHistoricalDataService');+ }++ return {};+ }++ public async search(aQuery: string): Promise<{ items: LookupItem[] }> {+ return { items: [] };+ }+}diff --git a/apps/api/src/services/data-provider/ghostfolio-scraper-api/ghostfolio-scraper-api.service.ts b/apps/api/src/services/data-provider/ghostfolio-scraper-api/ghostfolio-scraper-api.service.tsindex 26437bcdf9..7186ea7ec6 100644--- a/apps/api/src/services/data-provider/ghostfolio-scraper-api/ghostfolio-scraper-api.service.ts+++ b/apps/api/src/services/data-provider/ghostfolio-scraper-api/ghostfolio-scraper-api.service.ts@@ -10,7 +10,7 @@ import { DATE_FORMAT, getYesterday } from '@ghostfolio/common/helper';import { Granularity } from '@ghostfolio/common/types';import { Injectable, Logger } from '@nestjs/common';import { DataSource, SymbolProfile } from '@prisma/client';-import * as bent from 'bent';+import bent from 'bent';import * as cheerio from 'cheerio';import { addDays, format, isBefore } from 'date-fns';diff --git a/apps/api/src/services/data-provider/rakuten-rapid-api/rakuten-rapid-api.service.ts b/apps/api/src/services/data-provider/rakuten-rapid-api/rakuten-rapid-api.service.tsindex baa6591f46..2a516c5ef5 100644--- a/apps/api/src/services/data-provider/rakuten-rapid-api/rakuten-rapid-api.service.ts+++ b/apps/api/src/services/data-provider/rakuten-rapid-api/rakuten-rapid-api.service.ts@@ -11,7 +11,7 @@ import { DATE_FORMAT, getToday, getYesterday } from '@ghostfolio/common/helper';import { Granularity } from '@ghostfolio/common/types';import { Injectable, Logger } from '@nestjs/common';import { DataSource, SymbolProfile } from '@prisma/client';-import * as bent from 'bent';+import bent from 'bent';import { format, subMonths, subWeeks, subYears } from 'date-fns';@Injectable()diff --git a/apps/api/src/services/interfaces/environment.interface.ts b/apps/api/src/services/interfaces/environment.interface.tsindex 79db93f54e..36e9c7261b 100644--- a/apps/api/src/services/interfaces/environment.interface.ts+++ b/apps/api/src/services/interfaces/environment.interface.ts@@ -16,6 +16,7 @@ export interface Environment extends CleanedEnvAccessors {ENABLE_FEATURE_STATISTICS: boolean;ENABLE_FEATURE_SUBSCRIPTION: boolean;ENABLE_FEATURE_SYSTEM_MESSAGE: boolean;+ EOD_HISTORICAL_DATA_API_KEY: string;GOOGLE_CLIENT_ID: string;GOOGLE_SECRET: string;GOOGLE_SHEETS_ACCOUNT: string;diff --git a/prisma/migrations/20220529071429_added_eod_historical_data_to_data_source/migration.sql b/prisma/migrations/20220529071429_added_eod_historical_data_to_data_source/migration.sqlnew file mode 100644index 0000000000..4add280219--- /dev/null+++ b/prisma/migrations/20220529071429_added_eod_historical_data_to_data_source/migration.sql@@ -0,0 +1,2 @@+-- AlterEnum+ALTER TYPE "DataSource" ADD VALUE 'EOD_HISTORICAL_DATA';diff --git a/prisma/schema.prisma b/prisma/schema.prismaindex c5a03dc934..16249d51fe 100644--- a/prisma/schema.prisma+++ b/prisma/schema.prisma@@ -201,6 +201,7 @@ enum AssetSubClass {enum DataSource {ALPHA_VANTAGE+ EOD_HISTORICAL_DATAGHOSTFOLIOGOOGLE_SHEETSMANUAL