Skip to content

Open Source Wealth Management Software. Angular + NestJS + Prisma + Nx + TypeScript 🤍

Improve historical market data import (#2581)

Aldrin4a1e05b8cd68
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 48b0cb190f..d42cccdc35 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
### Changed
+- Improved the import of historical market data in the admin control panel
- Removed the account type from the `Account` database schema
## 2.19.0 - 2023-11-06
diff --git a/apps/client/src/app/components/admin-market-data/asset-profile-dialog/asset-profile-dialog.component.ts b/apps/client/src/app/components/admin-market-data/asset-profile-dialog/asset-profile-dialog.component.ts
index 5ccfda5036..03dc717d55 100644
--- a/apps/client/src/app/components/admin-market-data/asset-profile-dialog/asset-profile-dialog.component.ts
+++ b/apps/client/src/app/components/admin-market-data/asset-profile-dialog/asset-profile-dialog.component.ts
@@ -8,6 +8,7 @@ import {
} from '@angular/core';
import { FormBuilder, FormControl, Validators } from '@angular/forms';
import { MAT_DIALOG_DATA, MatDialogRef } from '@angular/material/dialog';
+import { MatSnackBar } from '@angular/material/snack-bar';
import { UpdateAssetProfileDto } from '@ghostfolio/api/app/admin/update-asset-profile.dto';
import { AdminService } from '@ghostfolio/client/services/admin.service';
import { DataService } from '@ghostfolio/client/services/data.service';
@@ -25,8 +26,8 @@ import {
} from '@prisma/client';
import { format } from 'date-fns';
import { parse as csvToJson } from 'papaparse';
-import { Subject } from 'rxjs';
-import { takeUntil } from 'rxjs/operators';
+import { EMPTY, Subject } from 'rxjs';
+import { catchError, takeUntil } from 'rxjs/operators';
import { AssetProfileDialogParams } from './interfaces/interfaces';
@@ -50,6 +51,9 @@ export class AssetProfileDialog implements OnDestroy, OnInit {
assetClass: new FormControl<AssetClass>(undefined),
assetSubClass: new FormControl<AssetSubClass>(undefined),
comment: '',
+ historicalData: this.formBuilder.group({
+ csvString: ''
+ }),
name: ['', Validators.required],
scraperConfiguration: '',
symbolMapping: ''
@@ -59,7 +63,6 @@ export class AssetProfileDialog implements OnDestroy, OnInit {
public countries: {
[code: string]: { name: string; value: number };
};
- public historicalDataAsCsvString: string;
public isBenchmark = false;
public marketDataDetails: MarketData[] = [];
public sectors: {
@@ -78,7 +81,8 @@ export class AssetProfileDialog implements OnDestroy, OnInit {
@Inject(MAT_DIALOG_DATA) public data: AssetProfileDialogParams,
private dataService: DataService,
public dialogRef: MatDialogRef<AssetProfileDialog>,
- private formBuilder: FormBuilder
+ private formBuilder: FormBuilder,
+ private snackBar: MatSnackBar
) {}
public ngOnInit(): void {
@@ -88,9 +92,6 @@ export class AssetProfileDialog implements OnDestroy, OnInit {
}
public initialize() {
- this.historicalDataAsCsvString =
- AssetProfileDialog.HISTORICAL_DATA_TEMPLATE;
-
this.adminService
.fetchAdminMarketDataBySymbol({
dataSource: this.data.dataSource,
@@ -131,6 +132,9 @@ export class AssetProfileDialog implements OnDestroy, OnInit {
assetClass: this.assetProfile.assetClass ?? null,
assetSubClass: this.assetProfile.assetSubClass ?? null,
comment: this.assetProfile?.comment ?? '',
+ historicalData: {
+ csvString: AssetProfileDialog.HISTORICAL_DATA_TEMPLATE
+ },
name: this.assetProfile.name ?? this.assetProfile.symbol,
scraperConfiguration: JSON.stringify(
this.assetProfile?.scraperConfiguration ?? {}
@@ -163,26 +167,46 @@ export class AssetProfileDialog implements OnDestroy, OnInit {
}
public onImportHistoricalData() {
- const marketData = csvToJson(this.historicalDataAsCsvString, {
- dynamicTyping: true,
- header: true,
- skipEmptyLines: true
- }).data;
-
- this.adminService
- .postMarketData({
- dataSource: this.data.dataSource,
- marketData: {
- marketData: marketData.map(({ date, marketPrice }) => {
- return { marketPrice, date: parseDate(date).toISOString() };
- })
- },
- symbol: this.data.symbol
- })
- .pipe(takeUntil(this.unsubscribeSubject))
- .subscribe(() => {
- this.initialize();
- });
+ try {
+ const marketData = csvToJson(
+ this.assetProfileForm.controls['historicalData'].controls['csvString']
+ .value,
+ {
+ dynamicTyping: true,
+ header: true,
+ skipEmptyLines: true
+ }
+ ).data;
+
+ this.adminService
+ .postMarketData({
+ dataSource: this.data.dataSource,
+ marketData: {
+ marketData: marketData.map(({ date, marketPrice }) => {
+ return { marketPrice, date: parseDate(date).toISOString() };
+ })
+ },
+ symbol: this.data.symbol
+ })
+ .pipe(
+ catchError(({ error, message }) => {
+ this.snackBar.open(`${error}: ${message[0]}`, undefined, {
+ duration: 3000
+ });
+ return EMPTY;
+ }),
+ takeUntil(this.unsubscribeSubject)
+ )
+ .subscribe(() => {
+ this.initialize();
+ });
+ } catch {
+ this.snackBar.open(
+ $localize`Oops! Could not parse historical data.`,
+ undefined,
+ { duration: 3000 }
+ );
+ }
}
public onMarketDataChanged(withRefresh: boolean = false) {
diff --git a/apps/client/src/app/components/admin-market-data/asset-profile-dialog/asset-profile-dialog.html b/apps/client/src/app/components/admin-market-data/asset-profile-dialog/asset-profile-dialog.html
index 7557682097..72d673776f 100644
--- a/apps/client/src/app/components/admin-market-data/asset-profile-dialog/asset-profile-dialog.html
+++ b/apps/client/src/app/components/admin-market-data/asset-profile-dialog/asset-profile-dialog.html
@@ -52,7 +52,7 @@
(marketDataChanged)="onMarketDataChanged($event)"
></gf-admin-market-data-detail>
- <div class="mt-3">
+ <div class="mt-3" formGroupName="historicalData">
<mat-form-field appearance="outline" class="w-100 without-hint">
<mat-label>
<ng-container i18n>Historical Data</ng-container> (CSV)
@@ -60,11 +60,9 @@
<textarea
cdkAutosizeMaxRows="5"
cdkTextareaAutosize
+ formControlName="csvString"
matInput
- placeholder="e.g. 20230601;1.61"
type="text"
- [ngModelOptions]="{standalone: true}"
- [(ngModel)]="historicalDataAsCsvString"
(keyup.enter)="$event.stopPropagation()"
></textarea>
</mat-form-field>
@@ -75,6 +73,7 @@
color="accent"
mat-flat-button
type="button"
+ [disabled]="!assetProfileForm.controls['historicalData']?.controls['csvString'].touched || assetProfileForm.controls['historicalData']?.controls['csvString']?.value === ''"
(click)="onImportHistoricalData()"
>
<ng-container i18n>Import</ng-container>
@@ -179,13 +178,13 @@
</ng-container>
</div>
<div *ngIf="assetProfile?.dataSource === 'MANUAL'" class="mt-3">
- <mat-form-field appearance="outline" class="w-100">
+ <mat-form-field appearance="outline" class="w-100 without-hint">
<mat-label i18n>Name</mat-label>
<input formControlName="name" matInput type="text" />
</mat-form-field>
</div>
<div *ngIf="assetProfile?.dataSource === 'MANUAL'" class="mt-3">
- <mat-form-field appearance="outline" class="w-100">
+ <mat-form-field appearance="outline" class="w-100 without-hint">
<mat-label i18n>Asset Class</mat-label>
<mat-select formControlName="assetClass">
<mat-option [value]="null"></mat-option>
@@ -198,7 +197,7 @@
</mat-form-field>
</div>
<div *ngIf="assetProfile?.dataSource === 'MANUAL'" class="mt-3">
- <mat-form-field appearance="outline" class="w-100">
+ <mat-form-field appearance="outline" class="w-100 without-hint">
<mat-label i18n>Asset Sub Class</mat-label>
<mat-select formControlName="assetSubClass">
<mat-option [value]="null"></mat-option>
diff --git a/apps/client/src/app/components/admin-market-data/asset-profile-dialog/asset-profile-dialog.module.ts b/apps/client/src/app/components/admin-market-data/asset-profile-dialog/asset-profile-dialog.module.ts
index b836e40661..0f9fdaacab 100644
--- a/apps/client/src/app/components/admin-market-data/asset-profile-dialog/asset-profile-dialog.module.ts
+++ b/apps/client/src/app/components/admin-market-data/asset-profile-dialog/asset-profile-dialog.module.ts
@@ -8,6 +8,7 @@ import { MatDialogModule } from '@angular/material/dialog';
import { MatInputModule } from '@angular/material/input';
import { MatMenuModule } from '@angular/material/menu';
import { MatSelectModule } from '@angular/material/select';
+import { MatSnackBarModule } from '@angular/material/snack-bar';
import { GfAdminMarketDataDetailModule } from '@ghostfolio/client/components/admin-market-data-detail/admin-market-data-detail.module';
import { GfPortfolioProportionChartModule } from '@ghostfolio/ui/portfolio-proportion-chart/portfolio-proportion-chart.module';
import { GfValueModule } from '@ghostfolio/ui/value';
@@ -28,6 +29,7 @@ import { AssetProfileDialog } from './asset-profile-dialog.component';
MatInputModule,
MatMenuModule,
MatSelectModule,
+ MatSnackBarModule,
ReactiveFormsModule,
TextFieldModule
],