Skip to content

Open Source Wealth Management Software. Angular + NestJS + Prisma + Nx + TypeScript 🤍

Feature/copy logic of ghostfolio scraper api service to manual service (#1691)

Thomas Kaule62989c9819b
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 00b1e42f15..799d99d999 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
### Changed
+- Copy logic of `GhostfolioScraperApiService` to `ManualService`
- Improved the content of the landing page
- Improved the content of the Frequently Asked Questions (FAQ) page
- Set the exposed port as an environment variable (`PORT`) in `Dockerfile`
@@ -26,6 +27,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
- Fixed an issue on the landing page caused by the global heat map of subscribers
- Fixed the links in the interstitial for the subscription
+### Todo
+
+- Rename the `dataSource` from `GHOSTFOLIO` to `MANUAL`
+- Eliminate `GhostfolioScraperApiService`
+
## 1.233.0 - 2023-02-09
### Added
diff --git a/apps/api/src/services/data-provider/ghostfolio-scraper-api/ghostfolio-scraper-api.service.ts b/apps/api/src/services/data-provider/ghostfolio-scraper-api/ghostfolio-scraper-api.service.ts
index 7412fec7bc..d1049ffb15 100644
--- a/apps/api/src/services/data-provider/ghostfolio-scraper-api/ghostfolio-scraper-api.service.ts
+++ b/apps/api/src/services/data-provider/ghostfolio-scraper-api/ghostfolio-scraper-api.service.ts
@@ -65,7 +65,7 @@ export class GhostfolioScraperApiService implements DataProviderInterface {
const [symbolProfile] =
await this.symbolProfileService.getSymbolProfilesBySymbols([symbol]);
const { defaultMarketPrice, selector, url } =
- symbolProfile.scraperConfiguration;
+ symbolProfile.scraperConfiguration ?? {};
if (defaultMarketPrice) {
const historical: {
@@ -148,7 +148,7 @@ export class GhostfolioScraperApiService implements DataProviderInterface {
dataSource: this.getName(),
marketPrice: marketData.find((marketDataItem) => {
return marketDataItem.symbol === symbolProfile.symbol;
- }).marketPrice,
+ })?.marketPrice,
marketState: 'delayed'
};
}
diff --git a/apps/api/src/services/data-provider/manual/manual.service.ts b/apps/api/src/services/data-provider/manual/manual.service.ts
index 7b60510873..9f95029689 100644
--- a/apps/api/src/services/data-provider/manual/manual.service.ts
+++ b/apps/api/src/services/data-provider/manual/manual.service.ts
@@ -6,9 +6,17 @@ import {
} from '@ghostfolio/api/services/interfaces/interfaces';
import { PrismaService } from '@ghostfolio/api/services/prisma.service';
import { SymbolProfileService } from '@ghostfolio/api/services/symbol-profile.service';
+import {
+ DATE_FORMAT,
+ extractNumberFromString,
+ getYesterday
+} from '@ghostfolio/common/helper';
import { Granularity } from '@ghostfolio/common/types';
import { Injectable, Logger } from '@nestjs/common';
import { DataSource, SymbolProfile } from '@prisma/client';
+import bent from 'bent';
+import * as cheerio from 'cheerio';
+import { addDays, format, isBefore } from 'date-fns';
@Injectable()
export class ManualService implements DataProviderInterface {
@@ -18,7 +26,7 @@ export class ManualService implements DataProviderInterface {
) {}
public canHandle(symbol: string) {
- return false;
+ return true;
}
public async getAssetProfile(
@@ -51,7 +59,57 @@ export class ManualService implements DataProviderInterface {
): Promise<{
[symbol: string]: { [date: string]: IDataProviderHistoricalResponse };
}> {
- return {};
+ try {
+ const symbol = aSymbol;
+
+ const [symbolProfile] =
+ await this.symbolProfileService.getSymbolProfilesBySymbols([symbol]);
+ const { defaultMarketPrice, selector, url } =
+ symbolProfile.scraperConfiguration ?? {};
+
+ if (defaultMarketPrice) {
+ const historical: {
+ [symbol: string]: { [date: string]: IDataProviderHistoricalResponse };
+ } = {
+ [symbol]: {}
+ };
+ let date = from;
+
+ while (isBefore(date, to)) {
+ historical[symbol][format(date, DATE_FORMAT)] = {
+ marketPrice: defaultMarketPrice
+ };
+
+ date = addDays(date, 1);
+ }
+
+ return historical;
+ } else if (selector === undefined || url === undefined) {
+ return {};
+ }
+
+ const get = bent(url, 'GET', 'string', 200, {});
+
+ const html = await get();
+ const $ = cheerio.load(html);
+
+ const value = extractNumberFromString($(selector).text());
+
+ return {
+ [symbol]: {
+ [format(getYesterday(), DATE_FORMAT)]: {
+ marketPrice: value
+ }
+ }
+ };
+ } catch (error) {
+ throw new Error(
+ `Could not get historical market data for ${aSymbol} (${this.getName()}) from ${format(
+ from,
+ DATE_FORMAT
+ )} to ${format(to, DATE_FORMAT)}: [${error.name}] ${error.message}`
+ );
+ }
}
public getName(): DataSource {
@@ -88,10 +146,9 @@ export class ManualService implements DataProviderInterface {
response[symbolProfile.symbol] = {
currency: symbolProfile.currency,
dataSource: this.getName(),
- marketPrice:
- marketData.find((marketDataItem) => {
- return marketDataItem.symbol === symbolProfile.symbol;
- })?.marketPrice ?? 0,
+ marketPrice: marketData.find((marketDataItem) => {
+ return marketDataItem.symbol === symbolProfile.symbol;
+ })?.marketPrice,
marketState: 'delayed'
};
}