Skip to content

Open Source Wealth Management Software. Angular + NestJS + Prisma + Nx + TypeScript 🤍

Feature/improve data provider service (#637)

Thomas Kaul38f2930ec68e
diff --git a/CHANGELOG.md b/CHANGELOG.md
index c1ff0c73d8..2ca1317fc7 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -7,9 +7,18 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
## Unreleased
+### Added
+
+- Added support for fetching multiple symbols in the `GOOGLE_SHEETS` data provider
+
+### Changed
+
+- Improved the data provider with grouping by data source and thereby reducing the number of requests
+
### Fixed
- Fixed the unresolved account names in the _X-ray_ section
+- Fixed the date conversion in the `GOOGLE_SHEETS` data provider
## 1.104.0 - 16.01.2022
diff --git a/apps/api/src/services/data-provider/data-provider.service.ts b/apps/api/src/services/data-provider/data-provider.service.ts
index 98237af8bb..f051927f99 100644
--- a/apps/api/src/services/data-provider/data-provider.service.ts
+++ b/apps/api/src/services/data-provider/data-provider.service.ts
@@ -12,7 +12,7 @@ import { Granularity } from '@ghostfolio/common/types';
import { Inject, Injectable, Logger } from '@nestjs/common';
import { DataSource, MarketData } from '@prisma/client';
import { format, isValid } from 'date-fns';
-import { isEmpty } from 'lodash';
+import { groupBy, isEmpty } from 'lodash';
@Injectable()
export class DataProviderService {
@@ -30,18 +30,27 @@ export class DataProviderService {
[symbol: string]: IDataProviderResponse;
} = {};
- for (const item of items) {
- const dataProvider = this.getDataProvider(item.dataSource);
- response[item.symbol] = (await dataProvider.get([item.symbol]))[
- item.symbol
- ];
- }
+ const itemsGroupedByDataSource = groupBy(items, (item) => item.dataSource);
const promises = [];
- for (const symbol of Object.keys(response)) {
- const promise = Promise.resolve(response[symbol]);
+
+ for (const [dataSource, dataGatheringItems] of Object.entries(
+ itemsGroupedByDataSource
+ )) {
+ const symbols = dataGatheringItems.map((dataGatheringItem) => {
+ return dataGatheringItem.symbol;
+ });
+
+ const promise = Promise.resolve(
+ this.getDataProvider(DataSource[dataSource]).get(symbols)
+ );
+
promises.push(
- promise.then((currentResponse) => (response[symbol] = currentResponse))
+ promise.then((result) => {
+ for (const [symbol, dataProviderResponse] of Object.entries(result)) {
+ response[symbol] = dataProviderResponse;
+ }
+ })
);
}
diff --git a/apps/api/src/services/data-provider/google-sheets/google-sheets.service.ts b/apps/api/src/services/data-provider/google-sheets/google-sheets.service.ts
index 81b369279a..fff9db21e8 100644
--- a/apps/api/src/services/data-provider/google-sheets/google-sheets.service.ts
+++ b/apps/api/src/services/data-provider/google-sheets/google-sheets.service.ts
@@ -8,7 +8,7 @@ import {
} from '@ghostfolio/api/services/interfaces/interfaces';
import { PrismaService } from '@ghostfolio/api/services/prisma.service';
import { SymbolProfileService } from '@ghostfolio/api/services/symbol-profile.service';
-import { DATE_FORMAT } from '@ghostfolio/common/helper';
+import { DATE_FORMAT, parseDate } from '@ghostfolio/common/helper';
import { Granularity } from '@ghostfolio/common/types';
import { Injectable, Logger } from '@nestjs/common';
import { DataSource } from '@prisma/client';
@@ -35,27 +35,36 @@ export class GoogleSheetsService implements DataProviderInterface {
}
try {
- const [symbol] = aSymbols;
- const [symbolProfile] = await this.symbolProfileService.getSymbolProfiles(
- [symbol]
+ const response: { [symbol: string]: IDataProviderResponse } = {};
+
+ const symbolProfiles = await this.symbolProfileService.getSymbolProfiles(
+ aSymbols
);
const sheet = await this.getSheet({
sheetId: this.configurationService.get('GOOGLE_SHEETS_ID'),
- symbol
+ symbol: 'Overview'
});
- const marketPrice = parseFloat(
- (await sheet.getCellByA1('B1').value) as string
- );
- return {
- [symbol]: {
- marketPrice,
- currency: symbolProfile?.currency,
- dataSource: this.getName(),
- marketState: MarketState.delayed
+ const rows = await sheet.getRows();
+
+ for (const row of rows) {
+ const marketPrice = parseFloat(row['marketPrice']);
+ const symbol = row['symbol'];
+
+ if (aSymbols.includes(symbol)) {
+ response[symbol] = {
+ marketPrice,
+ currency: symbolProfiles.find((symbolProfile) => {
+ return symbolProfile.symbol === symbol;
+ })?.currency,
+ dataSource: this.getName(),
+ marketState: MarketState.delayed
+ };
}
- };
+ }
+
+ return response;
} catch (error) {
Logger.error(error);
}
@@ -94,7 +103,7 @@ export class GoogleSheetsService implements DataProviderInterface {
return index >= 1;
})
.forEach((row) => {
- const date = new Date(row._rawData[0]);
+ const date = parseDate(row._rawData[0]);
const close = parseFloat(row._rawData[1]);
historicalData[format(date, DATE_FORMAT)] = { marketPrice: close };
diff --git a/apps/client/src/app/components/admin-market-data/admin-market-data.html b/apps/client/src/app/components/admin-market-data/admin-market-data.html
index 3410e99edf..50d6790a5f 100644
--- a/apps/client/src/app/components/admin-market-data/admin-market-data.html
+++ b/apps/client/src/app/components/admin-market-data/admin-market-data.html
@@ -16,8 +16,8 @@
class="cursor-pointer mat-row"
(click)="setCurrentSymbol(item.symbol)"
>
- <td class="mat-cell px-1 py-2">{{ item.symbol }}</td>
- <td class="mat-cell px-1 py-2">{{ item.dataSource}}</td>
+ <td class="mat-cell px-1 py-2">{{ item.symbol }}</td>
+ <td class="mat-cell px-1 py-2">{{ item.dataSource }}</td>
<td class="mat-cell px-1 py-2">
{{ (item.date | date: defaultDateFormat) ?? '' }}
</td>