Skip to content

Open Source Wealth Management Software. Angular + NestJS + Prisma + Nx + TypeScript 🤍

Task/extend error messages in scraper configuration (#6322)

Thomas Kaul299bca278022
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 3d3c55eb3c..7c41c75bd5 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -10,6 +10,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
### Changed
- Ignored nested ETFs when fetching top holdings for ETF and mutual fund assets from _Yahoo Finance_
+- Improved the scraper configuration with more detailed error messages
### Fixed
diff --git a/apps/api/src/app/admin/admin.controller.ts b/apps/api/src/app/admin/admin.controller.ts
index 24467c7324..8a202a9265 100644
--- a/apps/api/src/app/admin/admin.controller.ts
+++ b/apps/api/src/app/admin/admin.controller.ts
@@ -247,14 +247,17 @@ export class AdminController {
@Param('symbol') symbol: string
): Promise<{ price: number }> {
try {
- const price = await this.manualService.test(data.scraperConfiguration);
+ const price = await this.manualService.test({
+ symbol,
+ scraperConfiguration: data.scraperConfiguration
+ });
if (price) {
return { price };
}
throw new Error(
- `Could not parse the current market price for ${symbol} (${dataSource})`
+ `Could not parse the market price for ${symbol} (${dataSource})`
);
} catch (error) {
Logger.error(error, 'AdminController');
diff --git a/apps/api/src/services/data-provider/manual/manual.service.ts b/apps/api/src/services/data-provider/manual/manual.service.ts
index 7392f0914b..51e65e631b 100644
--- a/apps/api/src/services/data-provider/manual/manual.service.ts
+++ b/apps/api/src/services/data-provider/manual/manual.service.ts
@@ -105,7 +105,10 @@ export class ManualService implements DataProviderInterface {
return {};
}
- const value = await this.scrape(symbolProfile.scraperConfiguration);
+ const value = await this.scrape({
+ symbol,
+ scraperConfiguration: symbolProfile.scraperConfiguration
+ });
return {
[symbol]: {
@@ -170,7 +173,10 @@ export class ManualService implements DataProviderInterface {
symbolProfilesWithScraperConfigurationAndInstantMode.map(
async ({ scraperConfiguration, symbol }) => {
try {
- const marketPrice = await this.scrape(scraperConfiguration);
+ const marketPrice = await this.scrape({
+ scraperConfiguration,
+ symbol
+ });
return { marketPrice, symbol };
} catch (error) {
Logger.error(
@@ -267,13 +273,23 @@ export class ManualService implements DataProviderInterface {
};
}
- public async test(scraperConfiguration: ScraperConfiguration) {
- return this.scrape(scraperConfiguration);
+ public async test({
+ scraperConfiguration,
+ symbol
+ }: {
+ scraperConfiguration: ScraperConfiguration;
+ symbol: string;
+ }) {
+ return this.scrape({ scraperConfiguration, symbol });
}
- private async scrape(
- scraperConfiguration: ScraperConfiguration
- ): Promise<number> {
+ private async scrape({
+ scraperConfiguration,
+ symbol
+ }: {
+ scraperConfiguration: ScraperConfiguration;
+ symbol: string;
+ }): Promise<number> {
let locale = scraperConfiguration.locale;
const response = await fetch(scraperConfiguration.url, {
@@ -283,6 +299,12 @@ export class ManualService implements DataProviderInterface {
)
});
+ if (!response.ok) {
+ throw new Error(
+ `Failed to scrape the market price for ${symbol} (${this.getName()}): ${response.status} ${response.statusText} at ${scraperConfiguration.url}`
+ );
+ }
+
let value: string;
if (response.headers.get('content-type')?.includes('application/json')) {