Skip to content

Open Source Wealth Management Software. Angular + NestJS + Prisma + Nx + TypeScript 🤍

Feature/add data source eod historical data (#974)

Thomas Kaulf1e06347d348
diff --git a/CHANGELOG.md b/CHANGELOG.md
index dca00fbc20..040b771a6e 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -7,10 +7,18 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
## Unreleased
+### Added
+
+- Added `EOD_HISTORICAL_DATA` as a new data source type
+
### Changed
- Exposed the environment variable `REDIS_PASSWORD`
+### Todo
+
+- Apply data migration (`yarn database:migrate`)
+
## 1.154.0 - 28.05.2022
### Added
diff --git a/apps/api/src/services/configuration.service.ts b/apps/api/src/services/configuration.service.ts
index 9666e7ab71..525951a70f 100644
--- a/apps/api/src/services/configuration.service.ts
+++ b/apps/api/src/services/configuration.service.ts
@@ -25,6 +25,7 @@ export class ConfigurationService {
ENABLE_FEATURE_STATISTICS: bool({ default: false }),
ENABLE_FEATURE_SUBSCRIPTION: bool({ default: false }),
ENABLE_FEATURE_SYSTEM_MESSAGE: bool({ default: false }),
+ EOD_HISTORICAL_DATA_API_KEY: str({ default: '' }),
GOOGLE_CLIENT_ID: str({ default: 'dummyClientId' }),
GOOGLE_SECRET: str({ default: 'dummySecret' }),
GOOGLE_SHEETS_ACCOUNT: str({ default: '' }),
diff --git a/apps/api/src/services/data-provider/data-provider.module.ts b/apps/api/src/services/data-provider/data-provider.module.ts
index e2a77af4ae..dcdb7acbd8 100644
--- a/apps/api/src/services/data-provider/data-provider.module.ts
+++ b/apps/api/src/services/data-provider/data-provider.module.ts
@@ -1,5 +1,7 @@
import { ConfigurationModule } from '@ghostfolio/api/services/configuration.module';
import { CryptocurrencyModule } from '@ghostfolio/api/services/cryptocurrency/cryptocurrency.module';
+import { AlphaVantageService } from '@ghostfolio/api/services/data-provider/alpha-vantage/alpha-vantage.service';
+import { EodHistoricalDataService } from '@ghostfolio/api/services/data-provider/eod-historical-data/eod-historical-data.service';
import { GhostfolioScraperApiService } from '@ghostfolio/api/services/data-provider/ghostfolio-scraper-api/ghostfolio-scraper-api.service';
import { GoogleSheetsService } from '@ghostfolio/api/services/data-provider/google-sheets/google-sheets.service';
import { ManualService } from '@ghostfolio/api/services/data-provider/manual/manual.service';
@@ -9,7 +11,6 @@ import { PrismaModule } from '@ghostfolio/api/services/prisma.module';
import { SymbolProfileModule } from '@ghostfolio/api/services/symbol-profile.module';
import { Module } from '@nestjs/common';
-import { AlphaVantageService } from './alpha-vantage/alpha-vantage.service';
import { DataProviderService } from './data-provider.service';
@Module({
@@ -22,6 +23,7 @@ import { DataProviderService } from './data-provider.service';
providers: [
AlphaVantageService,
DataProviderService,
+ EodHistoricalDataService,
GhostfolioScraperApiService,
GoogleSheetsService,
ManualService,
@@ -30,6 +32,7 @@ import { DataProviderService } from './data-provider.service';
{
inject: [
AlphaVantageService,
+ EodHistoricalDataService,
GhostfolioScraperApiService,
GoogleSheetsService,
ManualService,
@@ -39,6 +42,7 @@ import { DataProviderService } from './data-provider.service';
provide: 'DataProviderInterfaces',
useFactory: (
alphaVantageService,
+ eodHistoricalDataService,
ghostfolioScraperApiService,
googleSheetsService,
manualService,
@@ -46,6 +50,7 @@ import { DataProviderService } from './data-provider.service';
yahooFinanceService
) => [
alphaVantageService,
+ eodHistoricalDataService,
ghostfolioScraperApiService,
googleSheetsService,
manualService,
diff --git a/apps/api/src/services/data-provider/eod-historical-data/eod-historical-data.service.ts b/apps/api/src/services/data-provider/eod-historical-data/eod-historical-data.service.ts
new file mode 100644
index 0000000000..bb0401f002
--- /dev/null
+++ b/apps/api/src/services/data-provider/eod-historical-data/eod-historical-data.service.ts
@@ -0,0 +1,138 @@
+import { LookupItem } from '@ghostfolio/api/app/symbol/interfaces/lookup-item.interface';
+import { ConfigurationService } from '@ghostfolio/api/services/configuration.service';
+import { DataProviderInterface } from '@ghostfolio/api/services/data-provider/interfaces/data-provider.interface';
+import {
+ IDataProviderHistoricalResponse,
+ IDataProviderResponse
+} from '@ghostfolio/api/services/interfaces/interfaces';
+import { SymbolProfileService } from '@ghostfolio/api/services/symbol-profile.service';
+import { DATE_FORMAT } from '@ghostfolio/common/helper';
+import { Granularity } from '@ghostfolio/common/types';
+import { Injectable, Logger } from '@nestjs/common';
+import { DataSource, SymbolProfile } from '@prisma/client';
+import bent from 'bent';
+import { format } from 'date-fns';
+
+@Injectable()
+export class EodHistoricalDataService implements DataProviderInterface {
+ private apiKey: string;
+ private readonly URL = 'https://eodhistoricaldata.com/api';
+
+ public constructor(
+ private readonly configurationService: ConfigurationService,
+ private readonly symbolProfileService: SymbolProfileService
+ ) {
+ this.apiKey = this.configurationService.get('EOD_HISTORICAL_DATA_API_KEY');
+ }
+
+ public canHandle(symbol: string) {
+ return true;
+ }
+
+ public async getAssetProfile(
+ aSymbol: string
+ ): Promise<Partial<SymbolProfile>> {
+ return {
+ dataSource: this.getName()
+ };
+ }
+
+ public async getHistorical(
+ aSymbol: string,
+ aGranularity: Granularity = 'day',
+ from: Date,
+ to: Date
+ ): Promise<{
+ [symbol: string]: { [date: string]: IDataProviderHistoricalResponse };
+ }> {
+ try {
+ const get = bent(
+ `${this.URL}/eod/${aSymbol}?api_token=${
+ this.apiKey
+ }&fmt=json&from=${format(from, DATE_FORMAT)}&to=${format(
+ to,
+ DATE_FORMAT
+ )}&period={aGranularity}`,
+ 'GET',
+ 'json',
+ 200
+ );
+
+ const response = await get();
+
+ return response.reduce(
+ (result, historicalItem, index, array) => {
+ result[aSymbol][historicalItem.date] = {
+ marketPrice: historicalItem.close,
+ performance: historicalItem.open - historicalItem.close
+ };
+
+ return result;
+ },
+ { [aSymbol]: {} }
+ );
+ } catch (error) {
+ Logger.error(error, 'EodHistoricalDataService');
+ }
+
+ return {};
+ }
+
+ public getName(): DataSource {
+ return DataSource.EOD_HISTORICAL_DATA;
+ }
+
+ public async getQuotes(
+ aSymbols: string[]
+ ): Promise<{ [symbol: string]: IDataProviderResponse }> {
+ if (aSymbols.length <= 0) {
+ return {};
+ }
+
+ try {
+ const get = bent(
+ `${this.URL}/real-time/${aSymbols[0]}?api_token=${
+ this.apiKey
+ }&fmt=json&s=${aSymbols.join(',')}`,
+ 'GET',
+ 'json',
+ 200
+ );
+
+ const [response, symbolProfiles] = await Promise.all([
+ get(),
+ this.symbolProfileService.getSymbolProfiles(
+ aSymbols.map((symbol) => {
+ return {
+ symbol,
+ dataSource: DataSource.EOD_HISTORICAL_DATA
+ };
+ })
+ )
+ ]);
+
+ const quotes = aSymbols.length === 1 ? [response] : response;
+
+ return quotes.reduce((result, item, index, array) => {
+ result[item.code] = {
+ currency: symbolProfiles.find((symbolProfile) => {
+ return symbolProfile.symbol === item.code;
+ })?.currency,
+ dataSource: DataSource.EOD_HISTORICAL_DATA,
+ marketPrice: item.close,
+ marketState: 'delayed'
+ };
+
+ return result;
+ }, {});
+ } catch (error) {
+ Logger.error(error, 'EodHistoricalDataService');
+ }
+
+ return {};
+ }
+
+ public async search(aQuery: string): Promise<{ items: LookupItem[] }> {
+ return { items: [] };
+ }
+}
diff --git a/apps/api/src/services/data-provider/ghostfolio-scraper-api/ghostfolio-scraper-api.service.ts b/apps/api/src/services/data-provider/ghostfolio-scraper-api/ghostfolio-scraper-api.service.ts
index 26437bcdf9..7186ea7ec6 100644
--- a/apps/api/src/services/data-provider/ghostfolio-scraper-api/ghostfolio-scraper-api.service.ts
+++ b/apps/api/src/services/data-provider/ghostfolio-scraper-api/ghostfolio-scraper-api.service.ts
@@ -10,7 +10,7 @@ import { DATE_FORMAT, getYesterday } from '@ghostfolio/common/helper';
import { Granularity } from '@ghostfolio/common/types';
import { Injectable, Logger } from '@nestjs/common';
import { DataSource, SymbolProfile } from '@prisma/client';
-import * as bent from 'bent';
+import bent from 'bent';
import * as cheerio from 'cheerio';
import { addDays, format, isBefore } from 'date-fns';
diff --git a/apps/api/src/services/data-provider/rakuten-rapid-api/rakuten-rapid-api.service.ts b/apps/api/src/services/data-provider/rakuten-rapid-api/rakuten-rapid-api.service.ts
index baa6591f46..2a516c5ef5 100644
--- a/apps/api/src/services/data-provider/rakuten-rapid-api/rakuten-rapid-api.service.ts
+++ b/apps/api/src/services/data-provider/rakuten-rapid-api/rakuten-rapid-api.service.ts
@@ -11,7 +11,7 @@ import { DATE_FORMAT, getToday, getYesterday } from '@ghostfolio/common/helper';
import { Granularity } from '@ghostfolio/common/types';
import { Injectable, Logger } from '@nestjs/common';
import { DataSource, SymbolProfile } from '@prisma/client';
-import * as bent from 'bent';
+import bent from 'bent';
import { format, subMonths, subWeeks, subYears } from 'date-fns';
@Injectable()
diff --git a/apps/api/src/services/interfaces/environment.interface.ts b/apps/api/src/services/interfaces/environment.interface.ts
index 79db93f54e..36e9c7261b 100644
--- a/apps/api/src/services/interfaces/environment.interface.ts
+++ b/apps/api/src/services/interfaces/environment.interface.ts
@@ -16,6 +16,7 @@ export interface Environment extends CleanedEnvAccessors {
ENABLE_FEATURE_STATISTICS: boolean;
ENABLE_FEATURE_SUBSCRIPTION: boolean;
ENABLE_FEATURE_SYSTEM_MESSAGE: boolean;
+ EOD_HISTORICAL_DATA_API_KEY: string;
GOOGLE_CLIENT_ID: string;
GOOGLE_SECRET: string;
GOOGLE_SHEETS_ACCOUNT: string;
diff --git a/prisma/migrations/20220529071429_added_eod_historical_data_to_data_source/migration.sql b/prisma/migrations/20220529071429_added_eod_historical_data_to_data_source/migration.sql
new file mode 100644
index 0000000000..4add280219
--- /dev/null
+++ b/prisma/migrations/20220529071429_added_eod_historical_data_to_data_source/migration.sql
@@ -0,0 +1,2 @@
+-- AlterEnum
+ALTER TYPE "DataSource" ADD VALUE 'EOD_HISTORICAL_DATA';
diff --git a/prisma/schema.prisma b/prisma/schema.prisma
index c5a03dc934..16249d51fe 100644
--- a/prisma/schema.prisma
+++ b/prisma/schema.prisma
@@ -201,6 +201,7 @@ enum AssetSubClass {
enum DataSource {
ALPHA_VANTAGE
+ EOD_HISTORICAL_DATA
GHOSTFOLIO
GOOGLE_SHEETS
MANUAL