Skip to content

Open Source Wealth Management Software. Angular + NestJS + Prisma + Nx + TypeScript 🤍

Feature/extend data source eod historical data by asset class and isin (#1791)

Thomas Kaulc8ca82b8037d
diff --git a/CHANGELOG.md b/CHANGELOG.md
index d39c31058e..a5c8890c6b 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -7,6 +7,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
## Unreleased
+### Added
+
+- Added support for asset and asset sub class to the `EOD_HISTORICAL_DATA` data source type
+- Added `isin` to the asset profile model
+
### Changed
- Improved the language localization for _Gather Data_
diff --git a/apps/api/src/app/import/import.service.ts b/apps/api/src/app/import/import.service.ts
index 37693f1607..5b10e381d8 100644
--- a/apps/api/src/app/import/import.service.ts
+++ b/apps/api/src/app/import/import.service.ts
@@ -254,6 +254,7 @@ export class ImportService {
countries: null,
createdAt: undefined,
id: undefined,
+ isin: null,
name: null,
scraperConfiguration: null,
sectors: null,
diff --git a/apps/api/src/app/symbol/symbol.controller.ts b/apps/api/src/app/symbol/symbol.controller.ts
index 249284ea80..6003c50d3f 100644
--- a/apps/api/src/app/symbol/symbol.controller.ts
+++ b/apps/api/src/app/symbol/symbol.controller.ts
@@ -1,15 +1,18 @@
import { TransformDataSourceInRequestInterceptor } from '@ghostfolio/api/interceptors/transform-data-source-in-request.interceptor';
import { TransformDataSourceInResponseInterceptor } from '@ghostfolio/api/interceptors/transform-data-source-in-response.interceptor';
import { IDataProviderHistoricalResponse } from '@ghostfolio/api/services/interfaces/interfaces';
+import type { RequestWithUser } from '@ghostfolio/common/types';
import {
Controller,
Get,
HttpException,
+ Inject,
Param,
Query,
UseGuards,
UseInterceptors
} from '@nestjs/common';
+import { REQUEST } from '@nestjs/core';
import { AuthGuard } from '@nestjs/passport';
import { DataSource } from '@prisma/client';
import { StatusCodes, getReasonPhrase } from 'http-status-codes';
@@ -21,7 +24,10 @@ import { SymbolService } from './symbol.service';
@Controller('symbol')
export class SymbolController {
- public constructor(private readonly symbolService: SymbolService) {}
+ public constructor(
+ @Inject(REQUEST) private readonly request: RequestWithUser,
+ private readonly symbolService: SymbolService
+ ) {}
/**
* Must be before /:symbol
@@ -33,7 +39,10 @@ export class SymbolController {
@Query() { query = '' }
): Promise<{ items: LookupItem[] }> {
try {
- return this.symbolService.lookup(query.toLowerCase());
+ return this.symbolService.lookup({
+ query: query.toLowerCase(),
+ user: this.request.user
+ });
} catch {
throw new HttpException(
getReasonPhrase(StatusCodes.INTERNAL_SERVER_ERROR),
diff --git a/apps/api/src/app/symbol/symbol.service.ts b/apps/api/src/app/symbol/symbol.service.ts
index 7a5f5586dc..718387b3b1 100644
--- a/apps/api/src/app/symbol/symbol.service.ts
+++ b/apps/api/src/app/symbol/symbol.service.ts
@@ -5,7 +5,10 @@ import {
} from '@ghostfolio/api/services/interfaces/interfaces';
import { MarketDataService } from '@ghostfolio/api/services/market-data.service';
import { DATE_FORMAT } from '@ghostfolio/common/helper';
-import { HistoricalDataItem } from '@ghostfolio/common/interfaces';
+import {
+ HistoricalDataItem,
+ UserWithSettings
+} from '@ghostfolio/common/interfaces';
import { Injectable, Logger } from '@nestjs/common';
import { format, subDays } from 'date-fns';
@@ -79,15 +82,24 @@ export class SymbolService {
};
}
- public async lookup(aQuery: string): Promise<{ items: LookupItem[] }> {
+ public async lookup({
+ query,
+ user
+ }: {
+ query: string;
+ user: UserWithSettings;
+ }): Promise<{ items: LookupItem[] }> {
const results: { items: LookupItem[] } = { items: [] };
- if (!aQuery) {
+ if (!query) {
return results;
}
try {
- const { items } = await this.dataProviderService.search(aQuery);
+ const { items } = await this.dataProviderService.search({
+ query,
+ user
+ });
results.items = items;
return results;
} catch (error) {
diff --git a/apps/api/src/services/data-gathering.service.ts b/apps/api/src/services/data-gathering.service.ts
index d3843456d7..aef198e5b9 100644
--- a/apps/api/src/services/data-gathering.service.ts
+++ b/apps/api/src/services/data-gathering.service.ts
@@ -152,10 +152,11 @@ export class DataGatheringService {
countries,
currency,
dataSource,
+ isin,
name,
sectors,
url
- } = assetProfiles[symbol];
+ } = assetProfile;
try {
await this.prismaService.symbolProfile.upsert({
@@ -165,6 +166,7 @@ export class DataGatheringService {
countries,
currency,
dataSource,
+ isin,
name,
sectors,
symbol,
@@ -175,6 +177,7 @@ export class DataGatheringService {
assetSubClass,
countries,
currency,
+ isin,
name,
sectors,
url
diff --git a/apps/api/src/services/data-provider/data-provider.service.ts b/apps/api/src/services/data-provider/data-provider.service.ts
index 9edeedc1c6..ad8929eee6 100644
--- a/apps/api/src/services/data-provider/data-provider.service.ts
+++ b/apps/api/src/services/data-provider/data-provider.service.ts
@@ -8,6 +8,7 @@ import {
} from '@ghostfolio/api/services/interfaces/interfaces';
import { PrismaService } from '@ghostfolio/api/services/prisma.service';
import { DATE_FORMAT } from '@ghostfolio/common/helper';
+import { UserWithSettings } from '@ghostfolio/common/interfaces';
import { Granularity } from '@ghostfolio/common/types';
import { Inject, Injectable, Logger } from '@nestjs/common';
import { DataSource, MarketData, SymbolProfile } from '@prisma/client';
@@ -260,18 +261,33 @@ export class DataProviderService {
return response;
}
- public async search(aQuery: string): Promise<{ items: LookupItem[] }> {
+ public async search({
+ query,
+ user
+ }: {
+ query: string;
+ user: UserWithSettings;
+ }): Promise<{ items: LookupItem[] }> {
const promises: Promise<{ items: LookupItem[] }>[] = [];
let lookupItems: LookupItem[] = [];
- if (aQuery?.length < 2) {
+ if (query?.length < 2) {
return { items: lookupItems };
}
- for (const dataSource of this.configurationService.get('DATA_SOURCES')) {
- promises.push(
- this.getDataProvider(DataSource[dataSource]).search(aQuery)
- );
+ let dataSources = this.configurationService.get('DATA_SOURCES');
+
+ if (
+ this.configurationService.get('ENABLE_FEATURE_SUBSCRIPTION') &&
+ user.subscription.type === 'Basic'
+ ) {
+ dataSources = dataSources.filter((dataSource) => {
+ return !this.isPremiumDataSource(DataSource[dataSource]);
+ });
+ }
+
+ for (const dataSource of dataSources) {
+ promises.push(this.getDataProvider(DataSource[dataSource]).search(query));
}
const searchResults = await Promise.all(promises);
@@ -305,4 +321,9 @@ export class DataProviderService {
throw new Error('No data provider has been found.');
}
+
+ private isPremiumDataSource(aDataSource: DataSource) {
+ const premiumDataSources: DataSource[] = [DataSource.EOD_HISTORICAL_DATA];
+ return premiumDataSources.includes(aDataSource);
+ }
}
diff --git a/apps/api/src/services/data-provider/eod-historical-data/eod-historical-data.service.ts b/apps/api/src/services/data-provider/eod-historical-data/eod-historical-data.service.ts
index a39100d1fb..e2d1efd95a 100644
--- a/apps/api/src/services/data-provider/eod-historical-data/eod-historical-data.service.ts
+++ b/apps/api/src/services/data-provider/eod-historical-data/eod-historical-data.service.ts
@@ -8,7 +8,12 @@ import {
import { DATE_FORMAT } from '@ghostfolio/common/helper';
import { Granularity } from '@ghostfolio/common/types';
import { Injectable, Logger } from '@nestjs/common';
-import { DataSource, SymbolProfile } from '@prisma/client';
+import {
+ AssetClass,
+ AssetSubClass,
+ DataSource,
+ SymbolProfile
+} from '@prisma/client';
import bent from 'bent';
import { format, isToday } from 'date-fns';
@@ -30,12 +35,15 @@ export class EodHistoricalDataService implements DataProviderInterface {
public async getAssetProfile(
aSymbol: string
): Promise<Partial<SymbolProfile>> {
- const { items } = await this.search(aSymbol);
+ const [searchResult] = await this.getSearchResult(aSymbol);
return {
- currency: items[0]?.currency,
+ assetClass: searchResult?.assetClass,
+ assetSubClass: searchResult?.assetSubClass,
+ currency: searchResult?.currency,
dataSource: this.getName(),
- name: items[0]?.name
+ isin: searchResult?.isin,
+ name: searchResult?.name
};
}
@@ -156,7 +164,27 @@ export class EodHistoricalDataService implements DataProviderInterface {
}
public async search(aQuery: string): Promise<{ items: LookupItem[] }> {
- let items: LookupItem[] = [];
+ const searchResult = await this.getSearchResult(aQuery);
+
+ return {
+ items: searchResult
+ .filter(({ symbol }) => {
+ return !symbol.toLowerCase().endsWith('forex');
+ })
+ .map(({ currency, dataSource, name, symbol }) => {
+ return { currency, dataSource, name, symbol };
+ })
+ };
+ }
+
+ private async getSearchResult(aQuery: string): Promise<
+ (LookupItem & {
+ assetClass: AssetClass;
+ assetSubClass: AssetSubClass;
+ isin: string;
+ })[]
+ > {
+ let searchResult = [];
try {
const get = bent(
@@ -167,10 +195,25 @@ export class EodHistoricalDataService implements DataProviderInterface {
);
const response = await get();
- items = response.map(
- ({ Code, Currency: currency, Exchange, Name: name }) => {
+ searchResult = response.map(
+ ({
+ Code,
+ Currency: currency,
+ Exchange,
+ ISIN: isin,
+ Name: name,
+ Type
+ }) => {
+ const { assetClass, assetSubClass } = this.parseAssetClass({
+ Exchange,
+ Type
+ });
+
return {
+ assetClass,
+ assetSubClass,
currency,
+ isin,
name,
dataSource: this.getName(),
symbol: `${Code}.${Exchange}`
@@ -181,6 +224,41 @@ export class EodHistoricalDataService implements DataProviderInterface {
Logger.error(error, 'EodHistoricalDataService');
}
- return { items };
+ return searchResult;
+ }
+
+ private parseAssetClass({
+ Exchange,
+ Type
+ }: {
+ Exchange: string;
+ Type: string;
+ }): {
+ assetClass: AssetClass;
+ assetSubClass: AssetSubClass;
+ } {
+ let assetClass: AssetClass;
+ let assetSubClass: AssetSubClass;
+
+ switch (Type?.toLowerCase()) {
+ case 'common stock':
+ assetClass = AssetClass.EQUITY;
+ assetSubClass = AssetSubClass.STOCK;
+ break;
+ case 'currency':
+ assetClass = AssetClass.CASH;
+
+ if (Exchange?.toLowerCase() === 'cc') {
+ assetSubClass = AssetSubClass.CRYPTOCURRENCY;
+ }
+
+ break;
+ case 'etf':
+ assetClass = AssetClass.EQUITY;
+ assetSubClass = AssetSubClass.ETF;
+ break;
+ }
+
+ return { assetClass, assetSubClass };
}
}
diff --git a/apps/client/src/app/components/subscription-interstitial-dialog/subscription-interstitial-dialog.html b/apps/client/src/app/components/subscription-interstitial-dialog/subscription-interstitial-dialog.html
index 061b43e3d0..321ab1e135 100644
--- a/apps/client/src/app/components/subscription-interstitial-dialog/subscription-interstitial-dialog.html
+++ b/apps/client/src/app/components/subscription-interstitial-dialog/subscription-interstitial-dialog.html
@@ -19,16 +19,20 @@
</li>
<li class="align-items-center d-flex mb-1">
<ion-icon class="mr-1" name="checkmark-circle-outline"></ion-icon>
- <span i18n>Performance Benchmarks</span>
+ <span i18n>Portfolio Allocations</span>
</li>
<li class="align-items-center d-flex mb-1">
<ion-icon class="mr-1" name="checkmark-circle-outline"></ion-icon>
- <span i18n>Allocations</span>
+ <span i18n>Performance Benchmarks</span>
</li>
<li class="align-items-center d-flex mb-1">
<ion-icon class="mr-1" name="checkmark-circle-outline"></ion-icon>
<span i18n>FIRE Calculator</span>
</li>
+ <li class="align-items-center d-flex mb-1">
+ <ion-icon class="mr-1" name="checkmark-circle-outline"></ion-icon>
+ <span i18n>Professional Data Provider</span>
+ </li>
<li class="align-items-center d-flex mb-1">
<ion-icon class="mr-1" name="checkmark-circle-outline"></ion-icon>
<span i18n>and more Features...</span>
diff --git a/apps/client/src/app/pages/pricing/pricing-page.html b/apps/client/src/app/pages/pricing/pricing-page.html
index 8140822a64..1284cda005 100644
--- a/apps/client/src/app/pages/pricing/pricing-page.html
+++ b/apps/client/src/app/pages/pricing/pricing-page.html
@@ -280,6 +280,13 @@
<ion-icon name="information-circle-outline"></ion-icon>
</span>
</li>
+ <li class="align-items-center d-flex mb-1">
+ <ion-icon
+ class="mr-1"
+ name="checkmark-circle-outline"
+ ></ion-icon>
+ <span i18n>Professional Data Provider</span>
+ </li>
<li class="align-items-center d-flex mb-1">
<ion-icon
class="mr-1"
diff --git a/prisma/migrations/20230318081658_added_isin_to_symbol_profile/migration.sql b/prisma/migrations/20230318081658_added_isin_to_symbol_profile/migration.sql
new file mode 100644
index 0000000000..f7081354cf
--- /dev/null
+++ b/prisma/migrations/20230318081658_added_isin_to_symbol_profile/migration.sql
@@ -0,0 +1,2 @@
+-- AlterTable
+ALTER TABLE "SymbolProfile" ADD COLUMN "isin" TEXT;
diff --git a/prisma/schema.prisma b/prisma/schema.prisma
index dc4c1302da..dd46e52b67 100644
--- a/prisma/schema.prisma
+++ b/prisma/schema.prisma
@@ -119,6 +119,7 @@ model SymbolProfile {
currency String
dataSource DataSource
id String @id @default(uuid())
+ isin String?
name String?
updatedAt DateTime @updatedAt
scraperConfiguration Json?