Open Source Wealth Management Software. Angular + NestJS + Prisma + Nx + TypeScript 🤍
TypeScript angularetffinancefintechghostfoliohacktoberfestinvestingnestjsosspersonal-financeportfolioprismasoftwarestocktrackertradingtypescriptwealth-managementweb
Feature/extend scraper configuration support (#2110)
Thomas Kaule62da06c5cfe
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 6c7f58d03e..72d0928765 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -10,6 +10,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0### Added- Added pagination to the historical market data table of the admin control panel+- Added the attribute `headers` to the scraper configuration++### Changed++- Extended the asset profile details dialog in the admin control panel by the scraper configuration## 1.284.0 - 2023-06-27diff --git a/apps/api/src/app/admin/admin.service.ts b/apps/api/src/app/admin/admin.service.tsindex 861cca0191..4dce779827 100644--- a/apps/api/src/app/admin/admin.service.ts+++ b/apps/api/src/app/admin/admin.service.ts@@ -249,12 +249,14 @@ export class AdminService {public async patchAssetProfileData({comment,dataSource,+ scraperConfiguration,symbol,symbolMapping}: Prisma.SymbolProfileUpdateInput & UniqueAsset) {await this.symbolProfileService.updateSymbolProfile({comment,dataSource,+ scraperConfiguration,symbol,symbolMapping});diff --git a/apps/api/src/app/admin/update-asset-profile.dto.ts b/apps/api/src/app/admin/update-asset-profile.dto.tsindex 228aa6f2ce..54f2d8f250 100644--- a/apps/api/src/app/admin/update-asset-profile.dto.ts+++ b/apps/api/src/app/admin/update-asset-profile.dto.ts@@ -1,3 +1,4 @@+import { Prisma } from '@prisma/client';import { IsObject, IsOptional, IsString } from 'class-validator';export class UpdateAssetProfileDto {@@ -5,6 +6,10 @@ export class UpdateAssetProfileDto {@IsOptional()comment?: string;+ @IsObject()+ @IsOptional()+ scraperConfiguration?: Prisma.InputJsonObject;+@IsObject()@IsOptional()symbolMapping?: {diff --git a/apps/api/src/services/data-provider/manual/manual.service.ts b/apps/api/src/services/data-provider/manual/manual.service.tsindex 14df4b7f40..7b39335321 100644--- a/apps/api/src/services/data-provider/manual/manual.service.ts+++ b/apps/api/src/services/data-provider/manual/manual.service.ts@@ -67,8 +67,12 @@ export class ManualService implements DataProviderInterface {const [symbolProfile] = await this.symbolProfileService.getSymbolProfiles([{ symbol, dataSource: this.getName() }]);- const { defaultMarketPrice, selector, url } =- symbolProfile.scraperConfiguration ?? {};+ const {+ defaultMarketPrice,+ headers = {},+ selector,+ url+ } = symbolProfile.scraperConfiguration ?? {};if (defaultMarketPrice) {const historical: {@@ -91,7 +95,7 @@ export class ManualService implements DataProviderInterface {return {};}- const get = bent(url, 'GET', 'string', 200, {});+ const get = bent(url, 'GET', 'string', 200, headers);const html = await get();const $ = cheerio.load(html);diff --git a/apps/api/src/services/symbol-profile/symbol-profile.service.ts b/apps/api/src/services/symbol-profile/symbol-profile.service.tsindex 2f90771bd4..99244c352d 100644--- a/apps/api/src/services/symbol-profile/symbol-profile.service.ts+++ b/apps/api/src/services/symbol-profile/symbol-profile.service.ts@@ -96,11 +96,12 @@ export class SymbolProfileService {public updateSymbolProfile({comment,dataSource,+ scraperConfiguration,symbol,symbolMapping}: Prisma.SymbolProfileUpdateInput & UniqueAsset) {return this.prismaService.symbolProfile.update({- data: { comment, symbolMapping },+ data: { comment, scraperConfiguration, symbolMapping },where: { dataSource_symbol: { dataSource, symbol } }});}@@ -195,6 +196,8 @@ export class SymbolProfileService {if (scraperConfiguration) {return {defaultMarketPrice: scraperConfiguration.defaultMarketPrice as number,+ headers:+ scraperConfiguration.headers as ScraperConfiguration['headers'],selector: scraperConfiguration.selector as string,url: scraperConfiguration.url as string};diff --git a/apps/client/src/app/components/admin-market-data/asset-profile-dialog/asset-profile-dialog.component.ts b/apps/client/src/app/components/admin-market-data/asset-profile-dialog/asset-profile-dialog.component.tsindex fac98982b2..be1892e911 100644--- a/apps/client/src/app/components/admin-market-data/asset-profile-dialog/asset-profile-dialog.component.ts+++ b/apps/client/src/app/components/admin-market-data/asset-profile-dialog/asset-profile-dialog.component.ts@@ -13,6 +13,7 @@ import { AdminService } from '@ghostfolio/client/services/admin.service';import { DataService } from '@ghostfolio/client/services/data.service';import {AdminMarketDataDetails,+ ScraperConfiguration,UniqueAsset} from '@ghostfolio/common/interfaces';import { translate } from '@ghostfolio/ui/i18n';@@ -34,6 +35,7 @@ export class AssetProfileDialog implements OnDestroy, OnInit {public assetProfile: AdminMarketDataDetails['assetProfile'];public assetProfileForm = this.formBuilder.group({comment: '',+ scraperConfiguration: '',symbolMapping: ''});public assetSubClass: string;@@ -103,6 +105,9 @@ export class AssetProfileDialog implements OnDestroy, OnInit {this.assetProfileForm.setValue({comment: this.assetProfile?.comment ?? '',+ scraperConfiguration: JSON.stringify(+ this.assetProfile?.scraperConfiguration ?? {}+ ),symbolMapping: JSON.stringify(this.assetProfile?.symbolMapping ?? {})});@@ -148,8 +153,15 @@ export class AssetProfileDialog implements OnDestroy, OnInit {}public onSubmit() {+ let scraperConfiguration = {};let symbolMapping = {};+ try {+ scraperConfiguration = JSON.parse(+ this.assetProfileForm.controls['scraperConfiguration'].value+ );+ } catch {}+try {symbolMapping = JSON.parse(this.assetProfileForm.controls['symbolMapping'].value@@ -157,6 +169,7 @@ export class AssetProfileDialog implements OnDestroy, OnInit {} catch {}const assetProfileData: UpdateAssetProfileDto = {+ scraperConfiguration,symbolMapping,comment: this.assetProfileForm.controls['comment'].value ?? null};diff --git a/apps/client/src/app/components/admin-market-data/asset-profile-dialog/asset-profile-dialog.html b/apps/client/src/app/components/admin-market-data/asset-profile-dialog/asset-profile-dialog.htmlindex a6f782b509..be99df7cb8 100644--- a/apps/client/src/app/components/admin-market-data/asset-profile-dialog/asset-profile-dialog.html+++ b/apps/client/src/app/components/admin-market-data/asset-profile-dialog/asset-profile-dialog.html@@ -162,6 +162,17 @@></textarea></mat-form-field></div>+ <div *ngIf="assetProfile?.dataSource === 'MANUAL'">+ <mat-form-field appearance="outline" class="w-100">+ <mat-label i18n>Scraper Configuration</mat-label>+ <textarea+ cdkTextareaAutosize+ formControlName="scraperConfiguration"+ matInput+ type="text"+ ></textarea>+ </mat-form-field>+ </div><div><mat-form-field appearance="outline" class="w-100"><mat-label i18n>Note</mat-label>diff --git a/apps/client/src/app/services/admin.service.ts b/apps/client/src/app/services/admin.service.tsindex 856fe9341b..2ffb59b658 100644--- a/apps/client/src/app/services/admin.service.ts+++ b/apps/client/src/app/services/admin.service.ts@@ -191,12 +191,13 @@ export class AdminService {public patchAssetProfile({comment,dataSource,+ scraperConfiguration,symbol,symbolMapping}: UniqueAsset & UpdateAssetProfileDto) {return this.http.patch<EnhancedSymbolProfile>(`/api/v1/admin/profile-data/${dataSource}/${symbol}`,- { comment, symbolMapping }+ { comment, scraperConfiguration, symbolMapping });}diff --git a/libs/common/src/lib/interfaces/scraper-configuration.interface.ts b/libs/common/src/lib/interfaces/scraper-configuration.interface.tsindex 9bae194183..0446459adf 100644--- a/libs/common/src/lib/interfaces/scraper-configuration.interface.ts+++ b/libs/common/src/lib/interfaces/scraper-configuration.interface.ts@@ -1,5 +1,6 @@export interface ScraperConfiguration {defaultMarketPrice?: number;+ headers?: { [key: string]: string };selector: string;url: string;}