Open Source Wealth Management Software. Angular + NestJS + Prisma + Nx + TypeScript 🤍
TypeScript angularetffinancefintechghostfoliohacktoberfestinvestingnestjsosspersonal-financeportfolioprismasoftwarestocktrackertradingtypescriptwealth-managementweb
Task/extend error messages in scraper configuration (#6322)
Thomas Kaul299bca278022
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 3d3c55eb3c..7c41c75bd5 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -10,6 +10,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0### Changed- Ignored nested ETFs when fetching top holdings for ETF and mutual fund assets from _Yahoo Finance_+- Improved the scraper configuration with more detailed error messages### Fixeddiff --git a/apps/api/src/app/admin/admin.controller.ts b/apps/api/src/app/admin/admin.controller.tsindex 24467c7324..8a202a9265 100644--- a/apps/api/src/app/admin/admin.controller.ts+++ b/apps/api/src/app/admin/admin.controller.ts@@ -247,14 +247,17 @@ export class AdminController {@Param('symbol') symbol: string): Promise<{ price: number }> {try {- const price = await this.manualService.test(data.scraperConfiguration);+ const price = await this.manualService.test({+ symbol,+ scraperConfiguration: data.scraperConfiguration+ });if (price) {return { price };}throw new Error(- `Could not parse the current market price for ${symbol} (${dataSource})`+ `Could not parse the market price for ${symbol} (${dataSource})`);} catch (error) {Logger.error(error, 'AdminController');diff --git a/apps/api/src/services/data-provider/manual/manual.service.ts b/apps/api/src/services/data-provider/manual/manual.service.tsindex 7392f0914b..51e65e631b 100644--- a/apps/api/src/services/data-provider/manual/manual.service.ts+++ b/apps/api/src/services/data-provider/manual/manual.service.ts@@ -105,7 +105,10 @@ export class ManualService implements DataProviderInterface {return {};}- const value = await this.scrape(symbolProfile.scraperConfiguration);+ const value = await this.scrape({+ symbol,+ scraperConfiguration: symbolProfile.scraperConfiguration+ });return {[symbol]: {@@ -170,7 +173,10 @@ export class ManualService implements DataProviderInterface {symbolProfilesWithScraperConfigurationAndInstantMode.map(async ({ scraperConfiguration, symbol }) => {try {- const marketPrice = await this.scrape(scraperConfiguration);+ const marketPrice = await this.scrape({+ scraperConfiguration,+ symbol+ });return { marketPrice, symbol };} catch (error) {Logger.error(@@ -267,13 +273,23 @@ export class ManualService implements DataProviderInterface {};}- public async test(scraperConfiguration: ScraperConfiguration) {- return this.scrape(scraperConfiguration);+ public async test({+ scraperConfiguration,+ symbol+ }: {+ scraperConfiguration: ScraperConfiguration;+ symbol: string;+ }) {+ return this.scrape({ scraperConfiguration, symbol });}- private async scrape(- scraperConfiguration: ScraperConfiguration- ): Promise<number> {+ private async scrape({+ scraperConfiguration,+ symbol+ }: {+ scraperConfiguration: ScraperConfiguration;+ symbol: string;+ }): Promise<number> {let locale = scraperConfiguration.locale;const response = await fetch(scraperConfiguration.url, {@@ -283,6 +299,12 @@ export class ManualService implements DataProviderInterface {)});+ if (!response.ok) {+ throw new Error(+ `Failed to scrape the market price for ${symbol} (${this.getName()}): ${response.status} ${response.statusText} at ${scraperConfiguration.url}`+ );+ }+let value: string;if (response.headers.get('content-type')?.includes('application/json')) {