Skip to content

Open Source Wealth Management Software. Angular + NestJS + Prisma + Nx + TypeScript 🤍

Feature/add button to test scraper configuration (#2808)

Thomas Kaul43b4f14acebb
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 9d9406281b..c985bcaca5 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -5,6 +5,12 @@ All notable changes to this project will be documented in this file.
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+## Unreleased
+
+### Added
+
+- Added a button to test the scraper configuration in the asset profile details dialog of the admin control
+
## 2.33.0 - 2023-12-31
### Added
diff --git a/apps/api/src/app/admin/admin.controller.ts b/apps/api/src/app/admin/admin.controller.ts
index cc4da298ba..634bbbfdc4 100644
--- a/apps/api/src/app/admin/admin.controller.ts
+++ b/apps/api/src/app/admin/admin.controller.ts
@@ -3,6 +3,7 @@ import { HasPermissionGuard } from '@ghostfolio/api/guards/has-permission.guard'
import { TransformDataSourceInRequestInterceptor } from '@ghostfolio/api/interceptors/transform-data-source-in-request.interceptor';
import { ApiService } from '@ghostfolio/api/services/api/api.service';
import { DataGatheringService } from '@ghostfolio/api/services/data-gathering/data-gathering.service';
+import { ManualService } from '@ghostfolio/api/services/data-provider/manual/manual.service';
import { MarketDataService } from '@ghostfolio/api/services/market-data/market-data.service';
import { PropertyDto } from '@ghostfolio/api/services/property/property.dto';
import {
@@ -31,6 +32,7 @@ import {
Get,
HttpException,
Inject,
+ Logger,
Param,
Patch,
Post,
@@ -56,6 +58,7 @@ export class AdminController {
private readonly adminService: AdminService,
private readonly apiService: ApiService,
private readonly dataGatheringService: DataGatheringService,
+ private readonly manualService: ManualService,
private readonly marketDataService: MarketDataService,
@Inject(REQUEST) private readonly request: RequestWithUser
) {}
@@ -179,8 +182,8 @@ export class AdminController {
}
@Get('market-data')
- @UseGuards(AuthGuard('jwt'), HasPermissionGuard)
@HasPermission(permissions.accessAdminControl)
+ @UseGuards(AuthGuard('jwt'), HasPermissionGuard)
public async getMarketData(
@Query('assetSubClasses') filterByAssetSubClasses?: string,
@Query('presetId') presetId?: MarketDataPreset,
@@ -215,6 +218,30 @@ export class AdminController {
return this.adminService.getMarketDataBySymbol({ dataSource, symbol });
}
+ @HasPermission(permissions.accessAdminControl)
+ @Post('market-data/:dataSource/:symbol/test')
+ @UseGuards(AuthGuard('jwt'), HasPermissionGuard)
+ public async testMarketData(
+ @Body() data: { scraperConfiguration: string },
+ @Param('dataSource') dataSource: DataSource,
+ @Param('symbol') symbol: string
+ ): Promise<{ price: number }> {
+ try {
+ const { headers, selector, url } = JSON.parse(data.scraperConfiguration);
+ const price = await this.manualService.test({ headers, selector, url });
+
+ if (price) {
+ return { price };
+ }
+
+ throw new Error('Could not parse the current market price');
+ } catch (error) {
+ Logger.error(error);
+
+ throw new HttpException(error.message, StatusCodes.BAD_REQUEST);
+ }
+ }
+
@HasPermission(permissions.accessAdminControl)
@Post('market-data/:dataSource/:symbol')
@UseGuards(AuthGuard('jwt'), HasPermissionGuard)
diff --git a/apps/api/src/services/data-provider/data-provider.module.ts b/apps/api/src/services/data-provider/data-provider.module.ts
index b3a219a50a..e63ec08070 100644
--- a/apps/api/src/services/data-provider/data-provider.module.ts
+++ b/apps/api/src/services/data-provider/data-provider.module.ts
@@ -74,6 +74,6 @@ import { DataProviderService } from './data-provider.service';
},
YahooFinanceDataEnhancerService
],
- exports: [DataProviderService, YahooFinanceService]
+ exports: [DataProviderService, ManualService, YahooFinanceService]
})
export class DataProviderModule {}
diff --git a/apps/api/src/services/data-provider/manual/manual.service.ts b/apps/api/src/services/data-provider/manual/manual.service.ts
index 9a0ff82d96..d5a5c7eb3d 100644
--- a/apps/api/src/services/data-provider/manual/manual.service.ts
+++ b/apps/api/src/services/data-provider/manual/manual.service.ts
@@ -18,7 +18,7 @@ import { DataSource, SymbolProfile } from '@prisma/client';
import * as cheerio from 'cheerio';
import { isUUID } from 'class-validator';
import { addDays, format, isBefore } from 'date-fns';
-import got from 'got';
+import got, { Headers } from 'got';
@Injectable()
export class ManualService implements DataProviderInterface {
@@ -97,21 +97,7 @@ export class ManualService implements DataProviderInterface {
return {};
}
- const abortController = new AbortController();
-
- setTimeout(() => {
- abortController.abort();
- }, this.configurationService.get('REQUEST_TIMEOUT'));
-
- const { body } = await got(url, {
- headers,
- // @ts-ignore
- signal: abortController.signal
- });
-
- const $ = cheerio.load(body);
-
- const value = extractNumberFromString($(selector).text());
+ const value = await this.scrape({ headers, selector, url });
return {
[symbol]: {
@@ -233,4 +219,42 @@ export class ManualService implements DataProviderInterface {
return { items };
}
+
+ public async test(params: any) {
+ return this.scrape({
+ headers: params.headers,
+ selector: params.selector,
+ url: params.url
+ });
+ }
+
+ private async scrape({
+ headers = {},
+ selector,
+ url
+ }: {
+ headers?: Headers;
+ selector: string;
+ url: string;
+ }): Promise<number> {
+ try {
+ const abortController = new AbortController();
+
+ setTimeout(() => {
+ abortController.abort();
+ }, this.configurationService.get('REQUEST_TIMEOUT'));
+
+ const { body } = await got(url, {
+ headers,
+ // @ts-ignore
+ signal: abortController.signal
+ });
+
+ const $ = cheerio.load(body);
+
+ return extractNumberFromString($(selector).first().text());
+ } catch (error) {
+ throw error;
+ }
+ }
}
diff --git a/apps/client/src/app/components/admin-market-data/asset-profile-dialog/asset-profile-dialog.component.ts b/apps/client/src/app/components/admin-market-data/asset-profile-dialog/asset-profile-dialog.component.ts
index 6421e888b5..6a6e4e64ad 100644
--- a/apps/client/src/app/components/admin-market-data/asset-profile-dialog/asset-profile-dialog.component.ts
+++ b/apps/client/src/app/components/admin-market-data/asset-profile-dialog/asset-profile-dialog.component.ts
@@ -277,6 +277,34 @@ export class AssetProfileDialog implements OnDestroy, OnInit {
});
}
+ public onTestMarketData() {
+ this.adminService
+ .testMarketData({
+ dataSource: this.data.dataSource,
+ scraperConfiguration:
+ this.assetProfileForm.controls['scraperConfiguration'].value,
+ symbol: this.data.symbol
+ })
+ .pipe(
+ catchError(({ error }) => {
+ alert(`Error: ${error?.message}`);
+ return EMPTY;
+ }),
+ takeUntil(this.unsubscribeSubject)
+ )
+ .subscribe(({ price }) => {
+ alert(
+ $localize`The current market price is` +
+ ' ' +
+ price +
+ ' ' +
+ (<Currency>(
+ (<unknown>this.assetProfileForm.controls['currency'].value)
+ ))?.value
+ );
+ });
+ }
+
public onUnsetBenchmark({ dataSource, symbol }: UniqueAsset) {
this.dataService
.deleteBenchmark({ dataSource, symbol })
diff --git a/apps/client/src/app/components/admin-market-data/asset-profile-dialog/asset-profile-dialog.html b/apps/client/src/app/components/admin-market-data/asset-profile-dialog/asset-profile-dialog.html
index 08051a0efc..3ff71154f2 100644
--- a/apps/client/src/app/components/admin-market-data/asset-profile-dialog/asset-profile-dialog.html
+++ b/apps/client/src/app/components/admin-market-data/asset-profile-dialog/asset-profile-dialog.html
@@ -243,12 +243,24 @@
<div *ngIf="assetProfile?.dataSource === 'MANUAL'">
<mat-form-field appearance="outline" class="w-100">
<mat-label i18n>Scraper Configuration</mat-label>
- <textarea
- cdkTextareaAutosize
- formControlName="scraperConfiguration"
- matInput
- type="text"
- ></textarea>
+ <div class="align-items-end d-flex">
+ <textarea
+ cdkTextareaAutosize
+ formControlName="scraperConfiguration"
+ matInput
+ type="text"
+ (keyup.enter)="$event.stopPropagation()"
+ ></textarea>
+ <button
+ color="accent"
+ mat-flat-button
+ type="button"
+ [disabled]="assetProfileForm.controls['scraperConfiguration'].value === '{}'"
+ (click)="onTestMarketData()"
+ >
+ <ng-container i18n>Test</ng-container>
+ </button>
+ </div>
</mat-form-field>
</div>
<div>
diff --git a/apps/client/src/app/services/admin.service.ts b/apps/client/src/app/services/admin.service.ts
index 9f8fc9d13b..a1c7d626a1 100644
--- a/apps/client/src/app/services/admin.service.ts
+++ b/apps/client/src/app/services/admin.service.ts
@@ -259,4 +259,17 @@ export class AdminService {
public putTag(aTag: UpdateTagDto) {
return this.http.put<Tag>(`/api/v1/tag/${aTag.id}`, aTag);
}
+
+ public testMarketData({
+ dataSource,
+ scraperConfiguration,
+ symbol
+ }: UniqueAsset & UpdateAssetProfileDto['scraperConfiguration']) {
+ return this.http.post<any>(
+ `/api/v1/admin/market-data/${dataSource}/${symbol}/test`,
+ {
+ scraperConfiguration
+ }
+ );
+ }
}