Skip to content

Open Source Wealth Management Software. Angular + NestJS + Prisma + Nx + TypeScript 🤍

Feature/eliminate ghostfolio scraper api service (#1706)

Thomas Kaul0cbf275a2e2c
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 8fd7d738bf..d80726239b 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -5,6 +5,12 @@ All notable changes to this project will be documented in this file.
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+## Unreleased
+
+### Changed
+
+- Eliminated the `GhostfolioScraperApiService`
+
## 1.234.0 - 2023-02-15
### Added
diff --git a/apps/api/src/services/configuration.service.ts b/apps/api/src/services/configuration.service.ts
index 7ea284a1e5..60e36623eb 100644
--- a/apps/api/src/services/configuration.service.ts
+++ b/apps/api/src/services/configuration.service.ts
@@ -19,7 +19,7 @@ export class ConfigurationService {
CACHE_TTL: num({ default: 1 }),
DATA_SOURCE_PRIMARY: str({ default: DataSource.YAHOO }),
DATA_SOURCES: json({
- default: [DataSource.GHOSTFOLIO, DataSource.MANUAL, DataSource.YAHOO]
+ default: [DataSource.MANUAL, DataSource.YAHOO]
}),
ENABLE_FEATURE_BLOG: bool({ default: false }),
ENABLE_FEATURE_CUSTOM_SYMBOLS: bool({ default: false }),
diff --git a/apps/api/src/services/data-gathering.service.ts b/apps/api/src/services/data-gathering.service.ts
index e977c9c07c..6e8ceb8088 100644
--- a/apps/api/src/services/data-gathering.service.ts
+++ b/apps/api/src/services/data-gathering.service.ts
@@ -278,7 +278,6 @@ export class DataGatheringService {
return symbolProfiles
.filter(({ dataSource }) => {
return (
- dataSource !== DataSource.GHOSTFOLIO &&
dataSource !== DataSource.MANUAL &&
dataSource !== DataSource.RAPID_API
);
diff --git a/apps/api/src/services/data-provider/data-provider.module.ts b/apps/api/src/services/data-provider/data-provider.module.ts
index 07982e6984..abafe61892 100644
--- a/apps/api/src/services/data-provider/data-provider.module.ts
+++ b/apps/api/src/services/data-provider/data-provider.module.ts
@@ -2,7 +2,6 @@ import { ConfigurationModule } from '@ghostfolio/api/services/configuration.modu
import { CryptocurrencyModule } from '@ghostfolio/api/services/cryptocurrency/cryptocurrency.module';
import { AlphaVantageService } from '@ghostfolio/api/services/data-provider/alpha-vantage/alpha-vantage.service';
import { EodHistoricalDataService } from '@ghostfolio/api/services/data-provider/eod-historical-data/eod-historical-data.service';
-import { GhostfolioScraperApiService } from '@ghostfolio/api/services/data-provider/ghostfolio-scraper-api/ghostfolio-scraper-api.service';
import { GoogleSheetsService } from '@ghostfolio/api/services/data-provider/google-sheets/google-sheets.service';
import { ManualService } from '@ghostfolio/api/services/data-provider/manual/manual.service';
import { RapidApiService } from '@ghostfolio/api/services/data-provider/rapid-api/rapid-api.service';
@@ -24,7 +23,6 @@ import { DataProviderService } from './data-provider.service';
AlphaVantageService,
DataProviderService,
EodHistoricalDataService,
- GhostfolioScraperApiService,
GoogleSheetsService,
ManualService,
RapidApiService,
@@ -33,7 +31,6 @@ import { DataProviderService } from './data-provider.service';
inject: [
AlphaVantageService,
EodHistoricalDataService,
- GhostfolioScraperApiService,
GoogleSheetsService,
ManualService,
RapidApiService,
@@ -43,7 +40,6 @@ import { DataProviderService } from './data-provider.service';
useFactory: (
alphaVantageService,
eodHistoricalDataService,
- ghostfolioScraperApiService,
googleSheetsService,
manualService,
rapidApiService,
@@ -51,7 +47,6 @@ import { DataProviderService } from './data-provider.service';
) => [
alphaVantageService,
eodHistoricalDataService,
- ghostfolioScraperApiService,
googleSheetsService,
manualService,
rapidApiService,
@@ -59,10 +54,6 @@ import { DataProviderService } from './data-provider.service';
]
}
],
- exports: [
- DataProviderService,
- GhostfolioScraperApiService,
- YahooFinanceService
- ]
+ exports: [DataProviderService, YahooFinanceService]
})
export class DataProviderModule {}
diff --git a/apps/api/src/services/data-provider/ghostfolio-scraper-api/ghostfolio-scraper-api.service.ts b/apps/api/src/services/data-provider/ghostfolio-scraper-api/ghostfolio-scraper-api.service.ts
deleted file mode 100644
index d1049ffb15..0000000000
--- a/apps/api/src/services/data-provider/ghostfolio-scraper-api/ghostfolio-scraper-api.service.ts
+++ /dev/null
@@ -1,194 +0,0 @@
-import { LookupItem } from '@ghostfolio/api/app/symbol/interfaces/lookup-item.interface';
-import { DataProviderInterface } from '@ghostfolio/api/services/data-provider/interfaces/data-provider.interface';
-import {
- IDataProviderHistoricalResponse,
- IDataProviderResponse
-} from '@ghostfolio/api/services/interfaces/interfaces';
-import { PrismaService } from '@ghostfolio/api/services/prisma.service';
-import { SymbolProfileService } from '@ghostfolio/api/services/symbol-profile.service';
-import {
- DATE_FORMAT,
- extractNumberFromString,
- getYesterday
-} from '@ghostfolio/common/helper';
-import { Granularity } from '@ghostfolio/common/types';
-import { Injectable, Logger } from '@nestjs/common';
-import { DataSource, SymbolProfile } from '@prisma/client';
-import bent from 'bent';
-import * as cheerio from 'cheerio';
-import { addDays, format, isBefore } from 'date-fns';
-
-@Injectable()
-export class GhostfolioScraperApiService implements DataProviderInterface {
- public constructor(
- private readonly prismaService: PrismaService,
- private readonly symbolProfileService: SymbolProfileService
- ) {}
-
- public canHandle(symbol: string) {
- return true;
- }
-
- public async getAssetProfile(
- aSymbol: string
- ): Promise<Partial<SymbolProfile>> {
- return {
- dataSource: this.getName()
- };
- }
-
- public async getDividends({
- from,
- granularity = 'day',
- symbol,
- to
- }: {
- from: Date;
- granularity: Granularity;
- symbol: string;
- to: Date;
- }) {
- return {};
- }
-
- public async getHistorical(
- aSymbol: string,
- aGranularity: Granularity = 'day',
- from: Date,
- to: Date
- ): Promise<{
- [symbol: string]: { [date: string]: IDataProviderHistoricalResponse };
- }> {
- try {
- const symbol = aSymbol;
-
- const [symbolProfile] =
- await this.symbolProfileService.getSymbolProfilesBySymbols([symbol]);
- const { defaultMarketPrice, selector, url } =
- symbolProfile.scraperConfiguration ?? {};
-
- if (defaultMarketPrice) {
- const historical: {
- [symbol: string]: { [date: string]: IDataProviderHistoricalResponse };
- } = {
- [symbol]: {}
- };
- let date = from;
-
- while (isBefore(date, to)) {
- historical[symbol][format(date, DATE_FORMAT)] = {
- marketPrice: defaultMarketPrice
- };
-
- date = addDays(date, 1);
- }
-
- return historical;
- } else if (selector === undefined || url === undefined) {
- return {};
- }
-
- const get = bent(url, 'GET', 'string', 200, {});
-
- const html = await get();
- const $ = cheerio.load(html);
-
- const value = extractNumberFromString($(selector).text());
-
- return {
- [symbol]: {
- [format(getYesterday(), DATE_FORMAT)]: {
- marketPrice: value
- }
- }
- };
- } catch (error) {
- throw new Error(
- `Could not get historical market data for ${aSymbol} (${this.getName()}) from ${format(
- from,
- DATE_FORMAT
- )} to ${format(to, DATE_FORMAT)}: [${error.name}] ${error.message}`
- );
- }
- }
-
- public getName(): DataSource {
- return DataSource.GHOSTFOLIO;
- }
-
- public async getQuotes(
- aSymbols: string[]
- ): Promise<{ [symbol: string]: IDataProviderResponse }> {
- const response: { [symbol: string]: IDataProviderResponse } = {};
-
- if (aSymbols.length <= 0) {
- return response;
- }
-
- try {
- const symbolProfiles =
- await this.symbolProfileService.getSymbolProfilesBySymbols(aSymbols);
-
- const marketData = await this.prismaService.marketData.findMany({
- distinct: ['symbol'],
- orderBy: {
- date: 'desc'
- },
- take: aSymbols.length,
- where: {
- symbol: {
- in: aSymbols
- }
- }
- });
-
- for (const symbolProfile of symbolProfiles) {
- response[symbolProfile.symbol] = {
- currency: symbolProfile.currency,
- dataSource: this.getName(),
- marketPrice: marketData.find((marketDataItem) => {
- return marketDataItem.symbol === symbolProfile.symbol;
- })?.marketPrice,
- marketState: 'delayed'
- };
- }
-
- return response;
- } catch (error) {
- Logger.error(error, 'GhostfolioScraperApiService');
- }
-
- return {};
- }
-
- public async search(aQuery: string): Promise<{ items: LookupItem[] }> {
- const items = await this.prismaService.symbolProfile.findMany({
- select: {
- currency: true,
- dataSource: true,
- name: true,
- symbol: true
- },
- where: {
- OR: [
- {
- dataSource: this.getName(),
- name: {
- mode: 'insensitive',
- startsWith: aQuery
- }
- },
- {
- dataSource: this.getName(),
- symbol: {
- mode: 'insensitive',
- startsWith: aQuery
- }
- }
- ]
- }
- });
-
- return { items };
- }
-}
diff --git a/prisma/migrations/20230215174320_changed_data_source_from_ghostfolio_to_manual/migration.sql b/prisma/migrations/20230215174320_changed_data_source_from_ghostfolio_to_manual/migration.sql
new file mode 100644
index 0000000000..fbf129be4f
--- /dev/null
+++ b/prisma/migrations/20230215174320_changed_data_source_from_ghostfolio_to_manual/migration.sql
@@ -0,0 +1,2 @@
+UPDATE "MarketData" SET "dataSource" = 'MANUAL' WHERE "dataSource" = 'GHOSTFOLIO';
+UPDATE "SymbolProfile" SET "dataSource" = 'MANUAL' WHERE "dataSource" = 'GHOSTFOLIO';