Skip to content

Open Source Wealth Management Software. Angular + NestJS + Prisma + Nx + TypeScript 🤍

Feature/add data source as unique constraint to market data schema (#1889)

Thomas Kaulcddea0401f7d
diff --git a/CHANGELOG.md b/CHANGELOG.md
index b6238cf3d0..b727ca33dd 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
## Unreleased
+### Added
+
+- Added `dataSource` as a unique constraint to the `MarketData` database schema
+
### Fixed
- Removed the unnecessary sort header of the comment column in the historical market data table of the admin control panel
diff --git a/apps/api/src/app/admin/admin.controller.ts b/apps/api/src/app/admin/admin.controller.ts
index bcd96a2f1a..81fd25308a 100644
--- a/apps/api/src/app/admin/admin.controller.ts
+++ b/apps/api/src/app/admin/admin.controller.ts
@@ -319,7 +319,8 @@ export class AdminController {
return this.marketDataService.updateMarketData({
data: { ...data, dataSource },
where: {
- date_symbol: {
+ dataSource_date_symbol: {
+ dataSource,
date,
symbol
}
diff --git a/apps/api/src/app/portfolio/current-rate.service.spec.ts b/apps/api/src/app/portfolio/current-rate.service.spec.ts
index a4eb101560..f0e4ea4a9f 100644
--- a/apps/api/src/app/portfolio/current-rate.service.spec.ts
+++ b/apps/api/src/app/portfolio/current-rate.service.spec.ts
@@ -91,6 +91,7 @@ describe('CurrentRateService', () => {
null,
[],
null,
+ null,
propertyService
);
exchangeRateDataService = new ExchangeRateDataService(
diff --git a/apps/api/src/services/data-gathering.service.ts b/apps/api/src/services/data-gathering.service.ts
index 5de70c9251..e4f8c0b38c 100644
--- a/apps/api/src/services/data-gathering.service.ts
+++ b/apps/api/src/services/data-gathering.service.ts
@@ -102,7 +102,7 @@ export class DataGatheringService {
symbol
},
update: { marketPrice },
- where: { date_symbol: { date, symbol } }
+ where: { dataSource_date_symbol: { dataSource, date, symbol } }
});
}
} catch (error) {
diff --git a/apps/api/src/services/data-provider/data-provider.module.ts b/apps/api/src/services/data-provider/data-provider.module.ts
index cfcd70d6b2..b27d5faed8 100644
--- a/apps/api/src/services/data-provider/data-provider.module.ts
+++ b/apps/api/src/services/data-provider/data-provider.module.ts
@@ -7,20 +7,22 @@ import { GoogleSheetsService } from '@ghostfolio/api/services/data-provider/goog
import { ManualService } from '@ghostfolio/api/services/data-provider/manual/manual.service';
import { RapidApiService } from '@ghostfolio/api/services/data-provider/rapid-api/rapid-api.service';
import { YahooFinanceService } from '@ghostfolio/api/services/data-provider/yahoo-finance/yahoo-finance.service';
+import { MarketDataModule } from '@ghostfolio/api/services/market-data.module';
import { PrismaModule } from '@ghostfolio/api/services/prisma.module';
+import { PropertyModule } from '@ghostfolio/api/services/property/property.module';
import { SymbolProfileModule } from '@ghostfolio/api/services/symbol-profile.module';
import { Module } from '@nestjs/common';
import { DataEnhancerModule } from './data-enhancer/data-enhancer.module';
import { YahooFinanceDataEnhancerService } from './data-enhancer/yahoo-finance/yahoo-finance.service';
import { DataProviderService } from './data-provider.service';
-import { PropertyModule } from '@ghostfolio/api/services/property/property.module';
@Module({
imports: [
ConfigurationModule,
CryptocurrencyModule,
DataEnhancerModule,
+ MarketDataModule,
PrismaModule,
PropertyModule,
SymbolProfileModule
diff --git a/apps/api/src/services/data-provider/data-provider.service.ts b/apps/api/src/services/data-provider/data-provider.service.ts
index 2076fafd62..0823af704b 100644
--- a/apps/api/src/services/data-provider/data-provider.service.ts
+++ b/apps/api/src/services/data-provider/data-provider.service.ts
@@ -6,6 +6,7 @@ import {
IDataProviderHistoricalResponse,
IDataProviderResponse
} from '@ghostfolio/api/services/interfaces/interfaces';
+import { MarketDataService } from '@ghostfolio/api/services/market-data.service';
import { PrismaService } from '@ghostfolio/api/services/prisma.service';
import { DATE_FORMAT, getStartOfUtcDate } from '@ghostfolio/common/helper';
import { UserWithSettings } from '@ghostfolio/common/types';
@@ -25,6 +26,7 @@ export class DataProviderService {
private readonly configurationService: ConfigurationService,
@Inject('DataProviderInterfaces')
private readonly dataProviderInterfaces: DataProviderInterface[],
+ private readonly marketDataService: MarketDataService,
private readonly prismaService: PrismaService,
private readonly propertyService: PropertyService
) {
@@ -276,35 +278,23 @@ export class DataProviderService {
);
try {
- const date = getStartOfUtcDate(new Date());
-
- // Upsert quotes by imitating missing upsertMany functionality
- // with $transaction
- const upsertPromises = Object.keys(response)
- .filter((symbol) => {
- return (
- isNumber(response[symbol].marketPrice) &&
- response[symbol].marketPrice > 0
- );
- })
- .map((symbol) =>
- this.prismaService.marketData.upsert({
- create: {
- date,
+ await this.marketDataService.updateMany({
+ data: Object.keys(response)
+ .filter((symbol) => {
+ return (
+ isNumber(response[symbol].marketPrice) &&
+ response[symbol].marketPrice > 0
+ );
+ })
+ .map((symbol) => {
+ return {
symbol,
dataSource: response[symbol].dataSource,
+ date: getStartOfUtcDate(new Date()),
marketPrice: response[symbol].marketPrice
- },
- update: {
- marketPrice: response[symbol].marketPrice
- },
- where: {
- date_symbol: { date, symbol }
- }
+ };
})
- );
-
- await this.prismaService.$transaction(upsertPromises);
+ });
} catch {}
})
);
diff --git a/apps/api/src/services/market-data.service.ts b/apps/api/src/services/market-data.service.ts
index fbea8e1043..ced802a3f7 100644
--- a/apps/api/src/services/market-data.service.ts
+++ b/apps/api/src/services/market-data.service.ts
@@ -102,11 +102,46 @@ export class MarketDataService {
where,
create: {
dataSource: data.dataSource,
- date: where.date_symbol.date,
+ date: where.dataSource_date_symbol.date,
marketPrice: data.marketPrice,
- symbol: where.date_symbol.symbol
+ symbol: where.dataSource_date_symbol.symbol
},
update: { marketPrice: data.marketPrice }
});
}
+
+ /**
+ * Upsert market data by imitating missing upsertMany functionality
+ * with $transaction
+ */
+ public async updateMany({
+ data
+ }: {
+ data: Prisma.MarketDataUpdateInput[];
+ }): Promise<MarketData[]> {
+ const upsertPromises = data.map(
+ ({ dataSource, date, marketPrice, symbol }) => {
+ return this.prismaService.marketData.upsert({
+ create: {
+ dataSource: <DataSource>dataSource,
+ date: <Date>date,
+ marketPrice: <number>marketPrice,
+ symbol: <string>symbol
+ },
+ update: {
+ marketPrice: <number>marketPrice
+ },
+ where: {
+ dataSource_date_symbol: {
+ dataSource: <DataSource>dataSource,
+ date: <Date>date,
+ symbol: <string>symbol
+ }
+ }
+ });
+ }
+ );
+
+ return this.prismaService.$transaction(upsertPromises);
+ }
}
diff --git a/prisma/migrations/20230422180309_added_data_source_to_market_data_as_unique_constraint/migration.sql b/prisma/migrations/20230422180309_added_data_source_to_market_data_as_unique_constraint/migration.sql
new file mode 100644
index 0000000000..7d4210bae1
--- /dev/null
+++ b/prisma/migrations/20230422180309_added_data_source_to_market_data_as_unique_constraint/migration.sql
@@ -0,0 +1,5 @@
+-- DropIndex
+DROP INDEX "MarketData_date_symbol_key";
+
+-- CreateIndex
+CREATE UNIQUE INDEX "MarketData_dataSource_date_symbol_key" ON "MarketData"("dataSource", "date", "symbol");
diff --git a/prisma/schema.prisma b/prisma/schema.prisma
index dd46e52b67..6e099d5ee1 100644
--- a/prisma/schema.prisma
+++ b/prisma/schema.prisma
@@ -66,7 +66,7 @@ model MarketData {
symbol String
marketPrice Float
- @@unique([date, symbol])
+ @@unique([dataSource, date, symbol])
@@index([symbol])
}