Skip to content

Open Source Wealth Management Software. Angular + NestJS + Prisma + Nx + TypeScript 🤍

Feature/improve check for duplicates in import #1932 (#1940)

Visrut7d6a74a67df1
diff --git a/CHANGELOG.md b/CHANGELOG.md
index a9e05dfcb8..035366a4b8 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -5,6 +5,12 @@ All notable changes to this project will be documented in this file.
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+## Unreleased
+
+### Changed
+
+- Improved the preview step of the activities import by unchecking duplicates
+
## 1.267.0 - 2023-05-07
### Added
diff --git a/apps/api/src/app/import/import.service.ts b/apps/api/src/app/import/import.service.ts
index faf71784c3..14bdd4c507 100644
--- a/apps/api/src/app/import/import.service.ts
+++ b/apps/api/src/app/import/import.service.ts
@@ -84,6 +84,7 @@ export class ImportService {
feeInBaseCurrency: 0,
id: assetProfile.id,
isDraft: false,
+ isDuplicate: false, // TODO: Use evaluated state
SymbolProfile: <SymbolProfile>(<unknown>assetProfile),
symbolProfileId: assetProfile.id,
type: 'DIVIDEND',
@@ -204,9 +205,14 @@ export class ImportService {
userId
});
+ const activitiesMarkedAsDuplicates = await this.markActivitiesAsDuplicates({
+ activitiesDto,
+ userId
+ });
+
const accounts = (await this.accountService.getAccounts(userId)).map(
- (account) => {
- return { id: account.id, name: account.name };
+ ({ id, name }) => {
+ return { id, name };
}
);
@@ -221,16 +227,14 @@ export class ImportService {
for (const {
accountId,
comment,
- currency,
- dataSource,
- date: dateString,
+ date,
fee,
+ isDuplicate,
quantity,
- symbol,
+ SymbolProfile: assetProfile,
type,
unitPrice
- } of activitiesDto) {
- const date = parseISO(<string>(<unknown>dateString));
+ } of activitiesMarkedAsDuplicates) {
const validatedAccount = accounts.find(({ id }) => {
return id === accountId;
});
@@ -256,29 +260,33 @@ export class ImportService {
id: uuidv4(),
isDraft: isAfter(date, endOfToday()),
SymbolProfile: {
- currency,
- dataSource,
- symbol,
- assetClass: null,
- assetSubClass: null,
- comment: null,
- countries: null,
- createdAt: undefined,
- id: undefined,
- isin: null,
- name: null,
- scraperConfiguration: null,
- sectors: null,
- symbolMapping: null,
- updatedAt: undefined,
- url: null,
- ...assetProfiles[symbol]
+ assetClass: assetProfile.assetClass,
+ assetSubClass: assetProfile.assetSubClass,
+ comment: assetProfile.comment,
+ countries: assetProfile.countries,
+ createdAt: assetProfile.createdAt,
+ currency: assetProfile.currency,
+ dataSource: assetProfile.dataSource,
+ id: assetProfile.id,
+ isin: assetProfile.isin,
+ name: assetProfile.name,
+ scraperConfiguration: assetProfile.scraperConfiguration,
+ sectors: assetProfile.sectors,
+ symbol: assetProfile.currency,
+ symbolMapping: assetProfile.symbolMapping,
+ updatedAt: assetProfile.updatedAt,
+ url: assetProfile.url,
+ ...assetProfiles[assetProfile.symbol]
},
Account: validatedAccount,
symbolProfileId: undefined,
updatedAt: new Date()
};
} else {
+ if (isDuplicate) {
+ continue;
+ }
+
order = await this.orderService.createOrder({
comment,
date,
@@ -291,14 +299,14 @@ export class ImportService {
SymbolProfile: {
connectOrCreate: {
create: {
- currency,
- dataSource,
- symbol
+ currency: assetProfile.currency,
+ dataSource: assetProfile.dataSource,
+ symbol: assetProfile.symbol
},
where: {
dataSource_symbol: {
- dataSource,
- symbol
+ dataSource: assetProfile.dataSource,
+ symbol: assetProfile.symbol
}
}
}
@@ -313,15 +321,16 @@ export class ImportService {
//@ts-ignore
activities.push({
...order,
+ isDuplicate,
value,
feeInBaseCurrency: this.exchangeRateDataService.toCurrency(
fee,
- currency,
+ assetProfile.currency,
userCurrency
),
valueInBaseCurrency: this.exchangeRateDataService.toCurrency(
value,
- currency,
+ assetProfile.currency,
userCurrency
)
});
@@ -340,6 +349,78 @@ export class ImportService {
return uniqueAccountIds.size === 1;
}
+ private async markActivitiesAsDuplicates({
+ activitiesDto,
+ userId
+ }: {
+ activitiesDto: Partial<CreateOrderDto>[];
+ userId: string;
+ }): Promise<Partial<Activity>[]> {
+ const existingActivities = await this.orderService.orders({
+ include: { SymbolProfile: true },
+ orderBy: { date: 'desc' },
+ where: { userId }
+ });
+
+ return activitiesDto.map(
+ ({
+ accountId,
+ comment,
+ currency,
+ dataSource,
+ date: dateString,
+ fee,
+ quantity,
+ symbol,
+ type,
+ unitPrice
+ }) => {
+ const date = parseISO(<string>(<unknown>dateString));
+ const isDuplicate = existingActivities.some((activity) => {
+ return (
+ activity.SymbolProfile.currency === currency &&
+ activity.SymbolProfile.dataSource === dataSource &&
+ isSameDay(activity.date, date) &&
+ activity.fee === fee &&
+ activity.quantity === quantity &&
+ activity.SymbolProfile.symbol === symbol &&
+ activity.type === type &&
+ activity.unitPrice === unitPrice
+ );
+ });
+
+ return {
+ accountId,
+ comment,
+ date,
+ fee,
+ isDuplicate,
+ quantity,
+ type,
+ unitPrice,
+ SymbolProfile: {
+ currency,
+ dataSource,
+ symbol,
+ assetClass: null,
+ assetSubClass: null,
+ comment: null,
+ countries: null,
+ createdAt: undefined,
+ id: undefined,
+ isin: null,
+ name: null,
+ scraperConfiguration: null,
+ sectors: null,
+ symbolMapping: null,
+ updatedAt: undefined,
+ url: null
+ }
+ };
+ }
+ );
+ }
+
private async validateActivities({
activitiesDto,
maxActivitiesToImport,
@@ -356,33 +437,11 @@ export class ImportService {
const assetProfiles: {
[symbol: string]: Partial<SymbolProfile>;
} = {};
- const existingActivities = await this.orderService.orders({
- include: { SymbolProfile: true },
- orderBy: { date: 'desc' },
- where: { userId }
- });
for (const [
index,
- { currency, dataSource, date, fee, quantity, symbol, type, unitPrice }
+ { currency, dataSource, symbol }
] of activitiesDto.entries()) {
- const duplicateActivity = existingActivities.find((activity) => {
- return (
- activity.SymbolProfile.currency === currency &&
- activity.SymbolProfile.dataSource === dataSource &&
- isSameDay(activity.date, parseISO(<string>(<unknown>date))) &&
- activity.fee === fee &&
- activity.quantity === quantity &&
- activity.SymbolProfile.symbol === symbol &&
- activity.type === type &&
- activity.unitPrice === unitPrice
- );
- });
-
- if (duplicateActivity) {
- throw new Error(`activities.${index} is a duplicate activity`);
- }
-
if (dataSource !== 'MANUAL') {
const assetProfile = (
await this.dataProviderService.getAssetProfiles([
diff --git a/apps/api/src/app/order/interfaces/activities.interface.ts b/apps/api/src/app/order/interfaces/activities.interface.ts
index 33fe8b40d1..f10417f076 100644
--- a/apps/api/src/app/order/interfaces/activities.interface.ts
+++ b/apps/api/src/app/order/interfaces/activities.interface.ts
@@ -6,6 +6,7 @@ export interface Activities {
export interface Activity extends OrderWithAccount {
feeInBaseCurrency: number;
+ isDuplicate: boolean;
updateAccountBalance?: boolean;
value: number;
valueInBaseCurrency: number;
diff --git a/apps/api/src/app/order/order.service.ts b/apps/api/src/app/order/order.service.ts
index 696f5442e7..b3f27c2215 100644
--- a/apps/api/src/app/order/order.service.ts
+++ b/apps/api/src/app/order/order.service.ts
@@ -333,6 +333,7 @@ export class OrderService {
order.SymbolProfile.currency,
userCurrency
),
+ isDuplicate: false,
valueInBaseCurrency: this.exchangeRateDataService.toCurrency(
value,
order.SymbolProfile.currency,
diff --git a/libs/ui/src/lib/activities-table/activities-table.component.html b/libs/ui/src/lib/activities-table/activities-table.component.html
index 4ac853436f..45551305b8 100644
--- a/libs/ui/src/lib/activities-table/activities-table.component.html
+++ b/libs/ui/src/lib/activities-table/activities-table.component.html
@@ -76,7 +76,6 @@
<ng-container matColumnDef="select">
<th *matHeaderCellDef class="px-1" mat-header-cell>
<mat-checkbox
- class="mt-2"
color="primary"
[checked]="selectedRows.hasValue() && areAllRowsSelected()"
[indeterminate]="selectedRows.hasValue() && !areAllRowsSelected()"
@@ -85,9 +84,11 @@
</th>
<td *matCellDef="let element" class="px-1" mat-cell>
<mat-checkbox
- class="mt-2"
color="primary"
- [checked]="selectedRows.isSelected(element)"
+ [checked]="
+ element.isDuplicate ? false : selectedRows.isSelected(element)
+ "
+ [disabled]="element.isDuplicate"
(change)="$event ? selectedRows.toggle(element) : null"
(click)="$event.stopPropagation()"
></mat-checkbox>
diff --git a/libs/ui/src/lib/activities-table/activities-table.component.ts b/libs/ui/src/lib/activities-table/activities-table.component.ts
index 1c3fb03670..0ba6e4f589 100644
--- a/libs/ui/src/lib/activities-table/activities-table.component.ts
+++ b/libs/ui/src/lib/activities-table/activities-table.component.ts
@@ -177,7 +177,9 @@ export class ActivitiesTableComponent implements OnChanges, OnDestroy, OnInit {
public onClickActivity(activity: Activity) {
if (this.showCheckbox) {
- this.selectedRows.toggle(activity);
+ if (!activity.isDuplicate) {
+ this.selectedRows.toggle(activity);
+ }
} else if (
this.hasPermissionToOpenDetails &&
!activity.isDraft &&