Open Source Wealth Management Software. Angular + NestJS + Prisma + Nx + TypeScript 🤍
TypeScript angularetffinancefintechghostfoliohacktoberfestinvestingnestjsosspersonal-financeportfolioprismasoftwarestocktrackertradingtypescriptwealth-managementweb
Feature/add data source as unique constraint to market data schema (#1889)
Thomas Kaulcddea0401f7d
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex b6238cf3d0..b727ca33dd 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0## Unreleased+### Added++- Added `dataSource` as a unique constraint to the `MarketData` database schema+### Fixed- Removed the unnecessary sort header of the comment column in the historical market data table of the admin control paneldiff --git a/apps/api/src/app/admin/admin.controller.ts b/apps/api/src/app/admin/admin.controller.tsindex bcd96a2f1a..81fd25308a 100644--- a/apps/api/src/app/admin/admin.controller.ts+++ b/apps/api/src/app/admin/admin.controller.ts@@ -319,7 +319,8 @@ export class AdminController {return this.marketDataService.updateMarketData({data: { ...data, dataSource },where: {- date_symbol: {+ dataSource_date_symbol: {+ dataSource,date,symbol}diff --git a/apps/api/src/app/portfolio/current-rate.service.spec.ts b/apps/api/src/app/portfolio/current-rate.service.spec.tsindex a4eb101560..f0e4ea4a9f 100644--- a/apps/api/src/app/portfolio/current-rate.service.spec.ts+++ b/apps/api/src/app/portfolio/current-rate.service.spec.ts@@ -91,6 +91,7 @@ describe('CurrentRateService', () => {null,[],null,+ null,propertyService);exchangeRateDataService = new ExchangeRateDataService(diff --git a/apps/api/src/services/data-gathering.service.ts b/apps/api/src/services/data-gathering.service.tsindex 5de70c9251..e4f8c0b38c 100644--- a/apps/api/src/services/data-gathering.service.ts+++ b/apps/api/src/services/data-gathering.service.ts@@ -102,7 +102,7 @@ export class DataGatheringService {symbol},update: { marketPrice },- where: { date_symbol: { date, symbol } }+ where: { dataSource_date_symbol: { dataSource, date, symbol } }});}} catch (error) {diff --git a/apps/api/src/services/data-provider/data-provider.module.ts b/apps/api/src/services/data-provider/data-provider.module.tsindex cfcd70d6b2..b27d5faed8 100644--- a/apps/api/src/services/data-provider/data-provider.module.ts+++ b/apps/api/src/services/data-provider/data-provider.module.ts@@ -7,20 +7,22 @@ import { GoogleSheetsService } from '@ghostfolio/api/services/data-provider/googimport { ManualService } from '@ghostfolio/api/services/data-provider/manual/manual.service';import { RapidApiService } from '@ghostfolio/api/services/data-provider/rapid-api/rapid-api.service';import { YahooFinanceService } from '@ghostfolio/api/services/data-provider/yahoo-finance/yahoo-finance.service';+import { MarketDataModule } from '@ghostfolio/api/services/market-data.module';import { PrismaModule } from '@ghostfolio/api/services/prisma.module';+import { PropertyModule } from '@ghostfolio/api/services/property/property.module';import { SymbolProfileModule } from '@ghostfolio/api/services/symbol-profile.module';import { Module } from '@nestjs/common';import { DataEnhancerModule } from './data-enhancer/data-enhancer.module';import { YahooFinanceDataEnhancerService } from './data-enhancer/yahoo-finance/yahoo-finance.service';import { DataProviderService } from './data-provider.service';-import { PropertyModule } from '@ghostfolio/api/services/property/property.module';@Module({imports: [ConfigurationModule,CryptocurrencyModule,DataEnhancerModule,+ MarketDataModule,PrismaModule,PropertyModule,SymbolProfileModulediff --git a/apps/api/src/services/data-provider/data-provider.service.ts b/apps/api/src/services/data-provider/data-provider.service.tsindex 2076fafd62..0823af704b 100644--- a/apps/api/src/services/data-provider/data-provider.service.ts+++ b/apps/api/src/services/data-provider/data-provider.service.ts@@ -6,6 +6,7 @@ import {IDataProviderHistoricalResponse,IDataProviderResponse} from '@ghostfolio/api/services/interfaces/interfaces';+import { MarketDataService } from '@ghostfolio/api/services/market-data.service';import { PrismaService } from '@ghostfolio/api/services/prisma.service';import { DATE_FORMAT, getStartOfUtcDate } from '@ghostfolio/common/helper';import { UserWithSettings } from '@ghostfolio/common/types';@@ -25,6 +26,7 @@ export class DataProviderService {private readonly configurationService: ConfigurationService,@Inject('DataProviderInterfaces')private readonly dataProviderInterfaces: DataProviderInterface[],+ private readonly marketDataService: MarketDataService,private readonly prismaService: PrismaService,private readonly propertyService: PropertyService) {@@ -276,35 +278,23 @@ export class DataProviderService {);try {- const date = getStartOfUtcDate(new Date());-- // Upsert quotes by imitating missing upsertMany functionality- // with $transaction- const upsertPromises = Object.keys(response)- .filter((symbol) => {- return (- isNumber(response[symbol].marketPrice) &&- response[symbol].marketPrice > 0- );- })- .map((symbol) =>- this.prismaService.marketData.upsert({- create: {- date,+ await this.marketDataService.updateMany({+ data: Object.keys(response)+ .filter((symbol) => {+ return (+ isNumber(response[symbol].marketPrice) &&+ response[symbol].marketPrice > 0+ );+ })+ .map((symbol) => {+ return {symbol,dataSource: response[symbol].dataSource,+ date: getStartOfUtcDate(new Date()),marketPrice: response[symbol].marketPrice- },- update: {- marketPrice: response[symbol].marketPrice- },- where: {- date_symbol: { date, symbol }- }+ };})- );-- await this.prismaService.$transaction(upsertPromises);+ });} catch {}}));diff --git a/apps/api/src/services/market-data.service.ts b/apps/api/src/services/market-data.service.tsindex fbea8e1043..ced802a3f7 100644--- a/apps/api/src/services/market-data.service.ts+++ b/apps/api/src/services/market-data.service.ts@@ -102,11 +102,46 @@ export class MarketDataService {where,create: {dataSource: data.dataSource,- date: where.date_symbol.date,+ date: where.dataSource_date_symbol.date,marketPrice: data.marketPrice,- symbol: where.date_symbol.symbol+ symbol: where.dataSource_date_symbol.symbol},update: { marketPrice: data.marketPrice }});}++ /**+ * Upsert market data by imitating missing upsertMany functionality+ * with $transaction+ */+ public async updateMany({+ data+ }: {+ data: Prisma.MarketDataUpdateInput[];+ }): Promise<MarketData[]> {+ const upsertPromises = data.map(+ ({ dataSource, date, marketPrice, symbol }) => {+ return this.prismaService.marketData.upsert({+ create: {+ dataSource: <DataSource>dataSource,+ date: <Date>date,+ marketPrice: <number>marketPrice,+ symbol: <string>symbol+ },+ update: {+ marketPrice: <number>marketPrice+ },+ where: {+ dataSource_date_symbol: {+ dataSource: <DataSource>dataSource,+ date: <Date>date,+ symbol: <string>symbol+ }+ }+ });+ }+ );++ return this.prismaService.$transaction(upsertPromises);+ }}diff --git a/prisma/migrations/20230422180309_added_data_source_to_market_data_as_unique_constraint/migration.sql b/prisma/migrations/20230422180309_added_data_source_to_market_data_as_unique_constraint/migration.sqlnew file mode 100644index 0000000000..7d4210bae1--- /dev/null+++ b/prisma/migrations/20230422180309_added_data_source_to_market_data_as_unique_constraint/migration.sql@@ -0,0 +1,5 @@+-- DropIndex+DROP INDEX "MarketData_date_symbol_key";++-- CreateIndex+CREATE UNIQUE INDEX "MarketData_dataSource_date_symbol_key" ON "MarketData"("dataSource", "date", "symbol");diff --git a/prisma/schema.prisma b/prisma/schema.prismaindex dd46e52b67..6e099d5ee1 100644--- a/prisma/schema.prisma+++ b/prisma/schema.prisma@@ -66,7 +66,7 @@ model MarketData {symbol StringmarketPrice Float- @@unique([date, symbol])+ @@unique([dataSource, date, symbol])@@index([symbol])}