Open Source Wealth Management Software. Angular + NestJS + Prisma + Nx + TypeScript 🤍
TypeScript angularetffinancefintechghostfoliohacktoberfestinvestingnestjsosspersonal-financeportfolioprismasoftwarestocktrackertradingtypescriptwealth-managementweb
Feature/add button to test scraper configuration (#2808)
Thomas Kaul43b4f14acebb
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 9d9406281b..c985bcaca5 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -5,6 +5,12 @@ All notable changes to this project will be documented in this file.The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).+## Unreleased++### Added++- Added a button to test the scraper configuration in the asset profile details dialog of the admin control+## 2.33.0 - 2023-12-31### Addeddiff --git a/apps/api/src/app/admin/admin.controller.ts b/apps/api/src/app/admin/admin.controller.tsindex cc4da298ba..634bbbfdc4 100644--- a/apps/api/src/app/admin/admin.controller.ts+++ b/apps/api/src/app/admin/admin.controller.ts@@ -3,6 +3,7 @@ import { HasPermissionGuard } from '@ghostfolio/api/guards/has-permission.guard'import { TransformDataSourceInRequestInterceptor } from '@ghostfolio/api/interceptors/transform-data-source-in-request.interceptor';import { ApiService } from '@ghostfolio/api/services/api/api.service';import { DataGatheringService } from '@ghostfolio/api/services/data-gathering/data-gathering.service';+import { ManualService } from '@ghostfolio/api/services/data-provider/manual/manual.service';import { MarketDataService } from '@ghostfolio/api/services/market-data/market-data.service';import { PropertyDto } from '@ghostfolio/api/services/property/property.dto';import {@@ -31,6 +32,7 @@ import {Get,HttpException,Inject,+ Logger,Param,Patch,Post,@@ -56,6 +58,7 @@ export class AdminController {private readonly adminService: AdminService,private readonly apiService: ApiService,private readonly dataGatheringService: DataGatheringService,+ private readonly manualService: ManualService,private readonly marketDataService: MarketDataService,@Inject(REQUEST) private readonly request: RequestWithUser) {}@@ -179,8 +182,8 @@ export class AdminController {}@Get('market-data')- @UseGuards(AuthGuard('jwt'), HasPermissionGuard)@HasPermission(permissions.accessAdminControl)+ @UseGuards(AuthGuard('jwt'), HasPermissionGuard)public async getMarketData(@Query('assetSubClasses') filterByAssetSubClasses?: string,@Query('presetId') presetId?: MarketDataPreset,@@ -215,6 +218,30 @@ export class AdminController {return this.adminService.getMarketDataBySymbol({ dataSource, symbol });}+ @HasPermission(permissions.accessAdminControl)+ @Post('market-data/:dataSource/:symbol/test')+ @UseGuards(AuthGuard('jwt'), HasPermissionGuard)+ public async testMarketData(+ @Body() data: { scraperConfiguration: string },+ @Param('dataSource') dataSource: DataSource,+ @Param('symbol') symbol: string+ ): Promise<{ price: number }> {+ try {+ const { headers, selector, url } = JSON.parse(data.scraperConfiguration);+ const price = await this.manualService.test({ headers, selector, url });++ if (price) {+ return { price };+ }++ throw new Error('Could not parse the current market price');+ } catch (error) {+ Logger.error(error);++ throw new HttpException(error.message, StatusCodes.BAD_REQUEST);+ }+ }+@HasPermission(permissions.accessAdminControl)@Post('market-data/:dataSource/:symbol')@UseGuards(AuthGuard('jwt'), HasPermissionGuard)diff --git a/apps/api/src/services/data-provider/data-provider.module.ts b/apps/api/src/services/data-provider/data-provider.module.tsindex b3a219a50a..e63ec08070 100644--- a/apps/api/src/services/data-provider/data-provider.module.ts+++ b/apps/api/src/services/data-provider/data-provider.module.ts@@ -74,6 +74,6 @@ import { DataProviderService } from './data-provider.service';},YahooFinanceDataEnhancerService],- exports: [DataProviderService, YahooFinanceService]+ exports: [DataProviderService, ManualService, YahooFinanceService]})export class DataProviderModule {}diff --git a/apps/api/src/services/data-provider/manual/manual.service.ts b/apps/api/src/services/data-provider/manual/manual.service.tsindex 9a0ff82d96..d5a5c7eb3d 100644--- a/apps/api/src/services/data-provider/manual/manual.service.ts+++ b/apps/api/src/services/data-provider/manual/manual.service.ts@@ -18,7 +18,7 @@ import { DataSource, SymbolProfile } from '@prisma/client';import * as cheerio from 'cheerio';import { isUUID } from 'class-validator';import { addDays, format, isBefore } from 'date-fns';-import got from 'got';+import got, { Headers } from 'got';@Injectable()export class ManualService implements DataProviderInterface {@@ -97,21 +97,7 @@ export class ManualService implements DataProviderInterface {return {};}- const abortController = new AbortController();-- setTimeout(() => {- abortController.abort();- }, this.configurationService.get('REQUEST_TIMEOUT'));-- const { body } = await got(url, {- headers,- // @ts-ignore- signal: abortController.signal- });-- const $ = cheerio.load(body);-- const value = extractNumberFromString($(selector).text());+ const value = await this.scrape({ headers, selector, url });return {[symbol]: {@@ -233,4 +219,42 @@ export class ManualService implements DataProviderInterface {return { items };}++ public async test(params: any) {+ return this.scrape({+ headers: params.headers,+ selector: params.selector,+ url: params.url+ });+ }++ private async scrape({+ headers = {},+ selector,+ url+ }: {+ headers?: Headers;+ selector: string;+ url: string;+ }): Promise<number> {+ try {+ const abortController = new AbortController();++ setTimeout(() => {+ abortController.abort();+ }, this.configurationService.get('REQUEST_TIMEOUT'));++ const { body } = await got(url, {+ headers,+ // @ts-ignore+ signal: abortController.signal+ });++ const $ = cheerio.load(body);++ return extractNumberFromString($(selector).first().text());+ } catch (error) {+ throw error;+ }+ }}diff --git a/apps/client/src/app/components/admin-market-data/asset-profile-dialog/asset-profile-dialog.component.ts b/apps/client/src/app/components/admin-market-data/asset-profile-dialog/asset-profile-dialog.component.tsindex 6421e888b5..6a6e4e64ad 100644--- a/apps/client/src/app/components/admin-market-data/asset-profile-dialog/asset-profile-dialog.component.ts+++ b/apps/client/src/app/components/admin-market-data/asset-profile-dialog/asset-profile-dialog.component.ts@@ -277,6 +277,34 @@ export class AssetProfileDialog implements OnDestroy, OnInit {});}+ public onTestMarketData() {+ this.adminService+ .testMarketData({+ dataSource: this.data.dataSource,+ scraperConfiguration:+ this.assetProfileForm.controls['scraperConfiguration'].value,+ symbol: this.data.symbol+ })+ .pipe(+ catchError(({ error }) => {+ alert(`Error: ${error?.message}`);+ return EMPTY;+ }),+ takeUntil(this.unsubscribeSubject)+ )+ .subscribe(({ price }) => {+ alert(+ $localize`The current market price is` ++ ' ' ++ price ++ ' ' ++ (<Currency>(+ (<unknown>this.assetProfileForm.controls['currency'].value)+ ))?.value+ );+ });+ }+public onUnsetBenchmark({ dataSource, symbol }: UniqueAsset) {this.dataService.deleteBenchmark({ dataSource, symbol })diff --git a/apps/client/src/app/components/admin-market-data/asset-profile-dialog/asset-profile-dialog.html b/apps/client/src/app/components/admin-market-data/asset-profile-dialog/asset-profile-dialog.htmlindex 08051a0efc..3ff71154f2 100644--- a/apps/client/src/app/components/admin-market-data/asset-profile-dialog/asset-profile-dialog.html+++ b/apps/client/src/app/components/admin-market-data/asset-profile-dialog/asset-profile-dialog.html@@ -243,12 +243,24 @@<div *ngIf="assetProfile?.dataSource === 'MANUAL'"><mat-form-field appearance="outline" class="w-100"><mat-label i18n>Scraper Configuration</mat-label>- <textarea- cdkTextareaAutosize- formControlName="scraperConfiguration"- matInput- type="text"- ></textarea>+ <div class="align-items-end d-flex">+ <textarea+ cdkTextareaAutosize+ formControlName="scraperConfiguration"+ matInput+ type="text"+ (keyup.enter)="$event.stopPropagation()"+ ></textarea>+ <button+ color="accent"+ mat-flat-button+ type="button"+ [disabled]="assetProfileForm.controls['scraperConfiguration'].value === '{}'"+ (click)="onTestMarketData()"+ >+ <ng-container i18n>Test</ng-container>+ </button>+ </div></mat-form-field></div><div>diff --git a/apps/client/src/app/services/admin.service.ts b/apps/client/src/app/services/admin.service.tsindex 9f8fc9d13b..a1c7d626a1 100644--- a/apps/client/src/app/services/admin.service.ts+++ b/apps/client/src/app/services/admin.service.ts@@ -259,4 +259,17 @@ export class AdminService {public putTag(aTag: UpdateTagDto) {return this.http.put<Tag>(`/api/v1/tag/${aTag.id}`, aTag);}++ public testMarketData({+ dataSource,+ scraperConfiguration,+ symbol+ }: UniqueAsset & UpdateAssetProfileDto['scraperConfiguration']) {+ return this.http.post<any>(+ `/api/v1/admin/market-data/${dataSource}/${symbol}/test`,+ {+ scraperConfiguration+ }+ );+ }}