Open Source Wealth Management Software. Angular + NestJS + Prisma + Nx + TypeScript 🤍
TypeScript angularetffinancefintechghostfoliohacktoberfestinvestingnestjsosspersonal-financeportfolioprismasoftwarestocktrackertradingtypescriptwealth-managementweb
Feature/eliminate ghostfolio scraper api service (#1706)
Thomas Kaul0cbf275a2e2c
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 8fd7d738bf..d80726239b 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -5,6 +5,12 @@ All notable changes to this project will be documented in this file.The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).+## Unreleased++### Changed++- Eliminated the `GhostfolioScraperApiService`+## 1.234.0 - 2023-02-15### Addeddiff --git a/apps/api/src/services/configuration.service.ts b/apps/api/src/services/configuration.service.tsindex 7ea284a1e5..60e36623eb 100644--- a/apps/api/src/services/configuration.service.ts+++ b/apps/api/src/services/configuration.service.ts@@ -19,7 +19,7 @@ export class ConfigurationService {CACHE_TTL: num({ default: 1 }),DATA_SOURCE_PRIMARY: str({ default: DataSource.YAHOO }),DATA_SOURCES: json({- default: [DataSource.GHOSTFOLIO, DataSource.MANUAL, DataSource.YAHOO]+ default: [DataSource.MANUAL, DataSource.YAHOO]}),ENABLE_FEATURE_BLOG: bool({ default: false }),ENABLE_FEATURE_CUSTOM_SYMBOLS: bool({ default: false }),diff --git a/apps/api/src/services/data-gathering.service.ts b/apps/api/src/services/data-gathering.service.tsindex e977c9c07c..6e8ceb8088 100644--- a/apps/api/src/services/data-gathering.service.ts+++ b/apps/api/src/services/data-gathering.service.ts@@ -278,7 +278,6 @@ export class DataGatheringService {return symbolProfiles.filter(({ dataSource }) => {return (- dataSource !== DataSource.GHOSTFOLIO &&dataSource !== DataSource.MANUAL &&dataSource !== DataSource.RAPID_API);diff --git a/apps/api/src/services/data-provider/data-provider.module.ts b/apps/api/src/services/data-provider/data-provider.module.tsindex 07982e6984..abafe61892 100644--- a/apps/api/src/services/data-provider/data-provider.module.ts+++ b/apps/api/src/services/data-provider/data-provider.module.ts@@ -2,7 +2,6 @@ import { ConfigurationModule } from '@ghostfolio/api/services/configuration.moduimport { CryptocurrencyModule } from '@ghostfolio/api/services/cryptocurrency/cryptocurrency.module';import { AlphaVantageService } from '@ghostfolio/api/services/data-provider/alpha-vantage/alpha-vantage.service';import { EodHistoricalDataService } from '@ghostfolio/api/services/data-provider/eod-historical-data/eod-historical-data.service';-import { GhostfolioScraperApiService } from '@ghostfolio/api/services/data-provider/ghostfolio-scraper-api/ghostfolio-scraper-api.service';import { GoogleSheetsService } from '@ghostfolio/api/services/data-provider/google-sheets/google-sheets.service';import { ManualService } from '@ghostfolio/api/services/data-provider/manual/manual.service';import { RapidApiService } from '@ghostfolio/api/services/data-provider/rapid-api/rapid-api.service';@@ -24,7 +23,6 @@ import { DataProviderService } from './data-provider.service';AlphaVantageService,DataProviderService,EodHistoricalDataService,- GhostfolioScraperApiService,GoogleSheetsService,ManualService,RapidApiService,@@ -33,7 +31,6 @@ import { DataProviderService } from './data-provider.service';inject: [AlphaVantageService,EodHistoricalDataService,- GhostfolioScraperApiService,GoogleSheetsService,ManualService,RapidApiService,@@ -43,7 +40,6 @@ import { DataProviderService } from './data-provider.service';useFactory: (alphaVantageService,eodHistoricalDataService,- ghostfolioScraperApiService,googleSheetsService,manualService,rapidApiService,@@ -51,7 +47,6 @@ import { DataProviderService } from './data-provider.service';) => [alphaVantageService,eodHistoricalDataService,- ghostfolioScraperApiService,googleSheetsService,manualService,rapidApiService,@@ -59,10 +54,6 @@ import { DataProviderService } from './data-provider.service';]}],- exports: [- DataProviderService,- GhostfolioScraperApiService,- YahooFinanceService- ]+ exports: [DataProviderService, YahooFinanceService]})export class DataProviderModule {}diff --git a/apps/api/src/services/data-provider/ghostfolio-scraper-api/ghostfolio-scraper-api.service.ts b/apps/api/src/services/data-provider/ghostfolio-scraper-api/ghostfolio-scraper-api.service.tsdeleted file mode 100644index d1049ffb15..0000000000--- a/apps/api/src/services/data-provider/ghostfolio-scraper-api/ghostfolio-scraper-api.service.ts+++ /dev/null@@ -1,194 +0,0 @@-import { LookupItem } from '@ghostfolio/api/app/symbol/interfaces/lookup-item.interface';-import { DataProviderInterface } from '@ghostfolio/api/services/data-provider/interfaces/data-provider.interface';-import {- IDataProviderHistoricalResponse,- IDataProviderResponse-} from '@ghostfolio/api/services/interfaces/interfaces';-import { PrismaService } from '@ghostfolio/api/services/prisma.service';-import { SymbolProfileService } from '@ghostfolio/api/services/symbol-profile.service';-import {- DATE_FORMAT,- extractNumberFromString,- getYesterday-} from '@ghostfolio/common/helper';-import { Granularity } from '@ghostfolio/common/types';-import { Injectable, Logger } from '@nestjs/common';-import { DataSource, SymbolProfile } from '@prisma/client';-import bent from 'bent';-import * as cheerio from 'cheerio';-import { addDays, format, isBefore } from 'date-fns';--@Injectable()-export class GhostfolioScraperApiService implements DataProviderInterface {- public constructor(- private readonly prismaService: PrismaService,- private readonly symbolProfileService: SymbolProfileService- ) {}-- public canHandle(symbol: string) {- return true;- }-- public async getAssetProfile(- aSymbol: string- ): Promise<Partial<SymbolProfile>> {- return {- dataSource: this.getName()- };- }-- public async getDividends({- from,- granularity = 'day',- symbol,- to- }: {- from: Date;- granularity: Granularity;- symbol: string;- to: Date;- }) {- return {};- }-- public async getHistorical(- aSymbol: string,- aGranularity: Granularity = 'day',- from: Date,- to: Date- ): Promise<{- [symbol: string]: { [date: string]: IDataProviderHistoricalResponse };- }> {- try {- const symbol = aSymbol;-- const [symbolProfile] =- await this.symbolProfileService.getSymbolProfilesBySymbols([symbol]);- const { defaultMarketPrice, selector, url } =- symbolProfile.scraperConfiguration ?? {};-- if (defaultMarketPrice) {- const historical: {- [symbol: string]: { [date: string]: IDataProviderHistoricalResponse };- } = {- [symbol]: {}- };- let date = from;-- while (isBefore(date, to)) {- historical[symbol][format(date, DATE_FORMAT)] = {- marketPrice: defaultMarketPrice- };-- date = addDays(date, 1);- }-- return historical;- } else if (selector === undefined || url === undefined) {- return {};- }-- const get = bent(url, 'GET', 'string', 200, {});-- const html = await get();- const $ = cheerio.load(html);-- const value = extractNumberFromString($(selector).text());-- return {- [symbol]: {- [format(getYesterday(), DATE_FORMAT)]: {- marketPrice: value- }- }- };- } catch (error) {- throw new Error(- `Could not get historical market data for ${aSymbol} (${this.getName()}) from ${format(- from,- DATE_FORMAT- )} to ${format(to, DATE_FORMAT)}: [${error.name}] ${error.message}`- );- }- }-- public getName(): DataSource {- return DataSource.GHOSTFOLIO;- }-- public async getQuotes(- aSymbols: string[]- ): Promise<{ [symbol: string]: IDataProviderResponse }> {- const response: { [symbol: string]: IDataProviderResponse } = {};-- if (aSymbols.length <= 0) {- return response;- }-- try {- const symbolProfiles =- await this.symbolProfileService.getSymbolProfilesBySymbols(aSymbols);-- const marketData = await this.prismaService.marketData.findMany({- distinct: ['symbol'],- orderBy: {- date: 'desc'- },- take: aSymbols.length,- where: {- symbol: {- in: aSymbols- }- }- });-- for (const symbolProfile of symbolProfiles) {- response[symbolProfile.symbol] = {- currency: symbolProfile.currency,- dataSource: this.getName(),- marketPrice: marketData.find((marketDataItem) => {- return marketDataItem.symbol === symbolProfile.symbol;- })?.marketPrice,- marketState: 'delayed'- };- }-- return response;- } catch (error) {- Logger.error(error, 'GhostfolioScraperApiService');- }-- return {};- }-- public async search(aQuery: string): Promise<{ items: LookupItem[] }> {- const items = await this.prismaService.symbolProfile.findMany({- select: {- currency: true,- dataSource: true,- name: true,- symbol: true- },- where: {- OR: [- {- dataSource: this.getName(),- name: {- mode: 'insensitive',- startsWith: aQuery- }- },- {- dataSource: this.getName(),- symbol: {- mode: 'insensitive',- startsWith: aQuery- }- }- ]- }- });-- return { items };- }-}diff --git a/prisma/migrations/20230215174320_changed_data_source_from_ghostfolio_to_manual/migration.sql b/prisma/migrations/20230215174320_changed_data_source_from_ghostfolio_to_manual/migration.sqlnew file mode 100644index 0000000000..fbf129be4f--- /dev/null+++ b/prisma/migrations/20230215174320_changed_data_source_from_ghostfolio_to_manual/migration.sql@@ -0,0 +1,2 @@+UPDATE "MarketData" SET "dataSource" = 'MANUAL' WHERE "dataSource" = 'GHOSTFOLIO';+UPDATE "SymbolProfile" SET "dataSource" = 'MANUAL' WHERE "dataSource" = 'GHOSTFOLIO';