Skip to content

Open Source Wealth Management Software. Angular + NestJS + Prisma + Nx + TypeScript 🤍

Feature/extend scraper configuration support (#2110)

Thomas Kaule62da06c5cfe
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 6c7f58d03e..72d0928765 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -10,6 +10,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
### Added
- Added pagination to the historical market data table of the admin control panel
+- Added the attribute `headers` to the scraper configuration
+
+### Changed
+
+- Extended the asset profile details dialog in the admin control panel by the scraper configuration
## 1.284.0 - 2023-06-27
diff --git a/apps/api/src/app/admin/admin.service.ts b/apps/api/src/app/admin/admin.service.ts
index 861cca0191..4dce779827 100644
--- a/apps/api/src/app/admin/admin.service.ts
+++ b/apps/api/src/app/admin/admin.service.ts
@@ -249,12 +249,14 @@ export class AdminService {
public async patchAssetProfileData({
comment,
dataSource,
+ scraperConfiguration,
symbol,
symbolMapping
}: Prisma.SymbolProfileUpdateInput & UniqueAsset) {
await this.symbolProfileService.updateSymbolProfile({
comment,
dataSource,
+ scraperConfiguration,
symbol,
symbolMapping
});
diff --git a/apps/api/src/app/admin/update-asset-profile.dto.ts b/apps/api/src/app/admin/update-asset-profile.dto.ts
index 228aa6f2ce..54f2d8f250 100644
--- a/apps/api/src/app/admin/update-asset-profile.dto.ts
+++ b/apps/api/src/app/admin/update-asset-profile.dto.ts
@@ -1,3 +1,4 @@
+import { Prisma } from '@prisma/client';
import { IsObject, IsOptional, IsString } from 'class-validator';
export class UpdateAssetProfileDto {
@@ -5,6 +6,10 @@ export class UpdateAssetProfileDto {
@IsOptional()
comment?: string;
+ @IsObject()
+ @IsOptional()
+ scraperConfiguration?: Prisma.InputJsonObject;
+
@IsObject()
@IsOptional()
symbolMapping?: {
diff --git a/apps/api/src/services/data-provider/manual/manual.service.ts b/apps/api/src/services/data-provider/manual/manual.service.ts
index 14df4b7f40..7b39335321 100644
--- a/apps/api/src/services/data-provider/manual/manual.service.ts
+++ b/apps/api/src/services/data-provider/manual/manual.service.ts
@@ -67,8 +67,12 @@ export class ManualService implements DataProviderInterface {
const [symbolProfile] = await this.symbolProfileService.getSymbolProfiles(
[{ symbol, dataSource: this.getName() }]
);
- const { defaultMarketPrice, selector, url } =
- symbolProfile.scraperConfiguration ?? {};
+ const {
+ defaultMarketPrice,
+ headers = {},
+ selector,
+ url
+ } = symbolProfile.scraperConfiguration ?? {};
if (defaultMarketPrice) {
const historical: {
@@ -91,7 +95,7 @@ export class ManualService implements DataProviderInterface {
return {};
}
- const get = bent(url, 'GET', 'string', 200, {});
+ const get = bent(url, 'GET', 'string', 200, headers);
const html = await get();
const $ = cheerio.load(html);
diff --git a/apps/api/src/services/symbol-profile/symbol-profile.service.ts b/apps/api/src/services/symbol-profile/symbol-profile.service.ts
index 2f90771bd4..99244c352d 100644
--- a/apps/api/src/services/symbol-profile/symbol-profile.service.ts
+++ b/apps/api/src/services/symbol-profile/symbol-profile.service.ts
@@ -96,11 +96,12 @@ export class SymbolProfileService {
public updateSymbolProfile({
comment,
dataSource,
+ scraperConfiguration,
symbol,
symbolMapping
}: Prisma.SymbolProfileUpdateInput & UniqueAsset) {
return this.prismaService.symbolProfile.update({
- data: { comment, symbolMapping },
+ data: { comment, scraperConfiguration, symbolMapping },
where: { dataSource_symbol: { dataSource, symbol } }
});
}
@@ -195,6 +196,8 @@ export class SymbolProfileService {
if (scraperConfiguration) {
return {
defaultMarketPrice: scraperConfiguration.defaultMarketPrice as number,
+ headers:
+ scraperConfiguration.headers as ScraperConfiguration['headers'],
selector: scraperConfiguration.selector as string,
url: scraperConfiguration.url as string
};
diff --git a/apps/client/src/app/components/admin-market-data/asset-profile-dialog/asset-profile-dialog.component.ts b/apps/client/src/app/components/admin-market-data/asset-profile-dialog/asset-profile-dialog.component.ts
index fac98982b2..be1892e911 100644
--- a/apps/client/src/app/components/admin-market-data/asset-profile-dialog/asset-profile-dialog.component.ts
+++ b/apps/client/src/app/components/admin-market-data/asset-profile-dialog/asset-profile-dialog.component.ts
@@ -13,6 +13,7 @@ import { AdminService } from '@ghostfolio/client/services/admin.service';
import { DataService } from '@ghostfolio/client/services/data.service';
import {
AdminMarketDataDetails,
+ ScraperConfiguration,
UniqueAsset
} from '@ghostfolio/common/interfaces';
import { translate } from '@ghostfolio/ui/i18n';
@@ -34,6 +35,7 @@ export class AssetProfileDialog implements OnDestroy, OnInit {
public assetProfile: AdminMarketDataDetails['assetProfile'];
public assetProfileForm = this.formBuilder.group({
comment: '',
+ scraperConfiguration: '',
symbolMapping: ''
});
public assetSubClass: string;
@@ -103,6 +105,9 @@ export class AssetProfileDialog implements OnDestroy, OnInit {
this.assetProfileForm.setValue({
comment: this.assetProfile?.comment ?? '',
+ scraperConfiguration: JSON.stringify(
+ this.assetProfile?.scraperConfiguration ?? {}
+ ),
symbolMapping: JSON.stringify(this.assetProfile?.symbolMapping ?? {})
});
@@ -148,8 +153,15 @@ export class AssetProfileDialog implements OnDestroy, OnInit {
}
public onSubmit() {
+ let scraperConfiguration = {};
let symbolMapping = {};
+ try {
+ scraperConfiguration = JSON.parse(
+ this.assetProfileForm.controls['scraperConfiguration'].value
+ );
+ } catch {}
+
try {
symbolMapping = JSON.parse(
this.assetProfileForm.controls['symbolMapping'].value
@@ -157,6 +169,7 @@ export class AssetProfileDialog implements OnDestroy, OnInit {
} catch {}
const assetProfileData: UpdateAssetProfileDto = {
+ scraperConfiguration,
symbolMapping,
comment: this.assetProfileForm.controls['comment'].value ?? null
};
diff --git a/apps/client/src/app/components/admin-market-data/asset-profile-dialog/asset-profile-dialog.html b/apps/client/src/app/components/admin-market-data/asset-profile-dialog/asset-profile-dialog.html
index a6f782b509..be99df7cb8 100644
--- a/apps/client/src/app/components/admin-market-data/asset-profile-dialog/asset-profile-dialog.html
+++ b/apps/client/src/app/components/admin-market-data/asset-profile-dialog/asset-profile-dialog.html
@@ -162,6 +162,17 @@
></textarea>
</mat-form-field>
</div>
+ <div *ngIf="assetProfile?.dataSource === 'MANUAL'">
+ <mat-form-field appearance="outline" class="w-100">
+ <mat-label i18n>Scraper Configuration</mat-label>
+ <textarea
+ cdkTextareaAutosize
+ formControlName="scraperConfiguration"
+ matInput
+ type="text"
+ ></textarea>
+ </mat-form-field>
+ </div>
<div>
<mat-form-field appearance="outline" class="w-100">
<mat-label i18n>Note</mat-label>
diff --git a/apps/client/src/app/services/admin.service.ts b/apps/client/src/app/services/admin.service.ts
index 856fe9341b..2ffb59b658 100644
--- a/apps/client/src/app/services/admin.service.ts
+++ b/apps/client/src/app/services/admin.service.ts
@@ -191,12 +191,13 @@ export class AdminService {
public patchAssetProfile({
comment,
dataSource,
+ scraperConfiguration,
symbol,
symbolMapping
}: UniqueAsset & UpdateAssetProfileDto) {
return this.http.patch<EnhancedSymbolProfile>(
`/api/v1/admin/profile-data/${dataSource}/${symbol}`,
- { comment, symbolMapping }
+ { comment, scraperConfiguration, symbolMapping }
);
}
diff --git a/libs/common/src/lib/interfaces/scraper-configuration.interface.ts b/libs/common/src/lib/interfaces/scraper-configuration.interface.ts
index 9bae194183..0446459adf 100644
--- a/libs/common/src/lib/interfaces/scraper-configuration.interface.ts
+++ b/libs/common/src/lib/interfaces/scraper-configuration.interface.ts
@@ -1,5 +1,6 @@
export interface ScraperConfiguration {
defaultMarketPrice?: number;
+ headers?: { [key: string]: string };
selector: string;
url: string;
}