Open Source Wealth Management Software. Angular + NestJS + Prisma + Nx + TypeScript 🤍
TypeScript angularetffinancefintechghostfoliohacktoberfestinvestingnestjsosspersonal-financeportfolioprismasoftwarestocktrackertradingtypescriptwealth-managementweb
Feature/Premium data source of holdings in import (#8196)
Thomas Kaul6ef4988d867e
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 7391a2e2b8..cc53dee0e2 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0### Changed+- Extended the creation and the import of activities to support the asset profiles of a premium data provider which the user already has- Improved the performance of the historical market data table of the admin control panel by counting the market data per asset profile- Improved the performance of the historical market data table of the admin control panel by querying subscription usage in a single database query- Hardened the `POST api/v1/import` endpoint by hiding internal error messagesdiff --git a/apps/api/src/app/activities/activities.controller.ts b/apps/api/src/app/activities/activities.controller.tsindex 98c3f674bc..84f62395a7 100644--- a/apps/api/src/app/activities/activities.controller.ts+++ b/apps/api/src/app/activities/activities.controller.ts@@ -233,6 +233,7 @@ export class ActivitiesController {try {await this.dataProviderService.validateActivities({subscription,+ userId,activitiesDto: [{currency: data.currency,diff --git a/apps/api/src/app/endpoints/ai/ai.service.spec.ts b/apps/api/src/app/endpoints/ai/ai.service.spec.tsindex 69636a19fb..bc9f6cd6ce 100644--- a/apps/api/src/app/endpoints/ai/ai.service.spec.ts+++ b/apps/api/src/app/endpoints/ai/ai.service.spec.ts@@ -12,7 +12,6 @@ import {PROPERTY_OPENROUTER_MODEL_AGENT,PROPERTY_OPENROUTER_MODEL_VISION} from '@ghostfolio/common/config';-import { SubscriptionType } from '@ghostfolio/common/enums';import { isValidCustomAssetProfileSymbol } from '@ghostfolio/common/helper';import type { UserWithSettings } from '@ghostfolio/common/types';@@ -100,8 +99,7 @@ describe('AiService', () => {dataProviderService = {getDataSources: jest.fn()- .mockResolvedValue([DataSource.MANUAL, DataSource.YAHOO]),- isPremiumRequired: jest.fn().mockReturnValue(false)+ .mockResolvedValue([DataSource.MANUAL, DataSource.YAHOO])} as unknown as DataProviderService;mcpService = {@@ -357,8 +355,7 @@ describe('AiService', () => {const user = {accounts: [{ currency: 'CHF', id: 'account-id', name: 'Brokerage' }],id: 'user-id',- settings: { settings: { baseCurrency: 'EUR', locale: 'de-CH' } },- subscription: { type: SubscriptionType.Basic }+ settings: { settings: { baseCurrency: 'EUR', locale: 'de-CH' } }} as unknown as UserWithSettings;it('Reads the rows of the file and matches them with the accounts, the base currency and the id of the user', async () => {@@ -402,7 +399,6 @@ describe('AiService', () => {rows,accounts: user.accounts,broker: 'Swissquote',- subscription: user.subscription,userCurrency: 'EUR',userId: 'user-id'});@@ -975,55 +971,6 @@ describe('AiService', () => {],rowsWithoutMatch: []});-- expect(dataProviderService.isPremiumRequired).toHaveBeenCalledWith({- dataSource: DataSource.YAHOO,- subscription: undefined- });- });-- it('Passes no holding with a data source which requires Premium to the model and gives a row whose asset match is such a holding as a row without a match', async () => {- jest.spyOn(Logger.prototype, 'warn').mockImplementation();-- jest.mocked(dataProviderService.isPremiumRequired).mockReturnValue(true);-- const row = createActivityRow({- accountName: null,- isin: 'IE00BK5BQT80',- symbol: null- });-- const subscription = {- type: SubscriptionType.Basic- } as UserWithSettings['subscription'];-- mockRowMatches({- assetProfilesOfSearch: [],- assets: [- { currency: 'USD', dataSource: 'YAHOO', id: 0, symbol: 'VWRA.SW' }- ]- });-- mockHoldings();-- await expect(- aiService.getActivitiesFromRows({- accounts,- subscription,- userCurrency,- userId,- rows: [row]- })- ).resolves.toEqual({ activities: [], rowsWithoutMatch: [row] });-- expect(dataProviderService.isPremiumRequired).toHaveBeenCalledWith({- subscription,- dataSource: DataSource.YAHOO- });-- const { prompt } = jest.mocked(generateText).mock.calls[0][0];-- expect(JSON.parse(prompt as string)).toHaveProperty('holdings', []);});it('Passes no holding with a data source which is not configured to the model and gives a row whose asset match is such a holding as a row without a match', async () => {diff --git a/apps/api/src/app/endpoints/ai/ai.service.ts b/apps/api/src/app/endpoints/ai/ai.service.tsindex 516e93ad81..02af5d8d0a 100644--- a/apps/api/src/app/endpoints/ai/ai.service.ts+++ b/apps/api/src/app/endpoints/ai/ai.service.ts@@ -124,7 +124,6 @@ export class AiService {broker,rows,accounts: user.accounts,- subscription: user.subscription,userCurrency: user.settings.settings.baseCurrency,userId: user.id});@@ -152,7 +151,6 @@ export class AiService {accounts,broker,rows,- subscription,userCurrency,userId}: {@@ -160,7 +158,6 @@ export class AiService {accounts: AccountToMatch[];broker?: string;rows: ActivityRow[];- subscription?: UserWithSettings['subscription'];userCurrency: string;userId: string;}) {@@ -211,7 +208,6 @@ export class AiService {abortSignal,accountHints,assets,- subscription,userId}): { accounts: [], assets: [] };@@ -524,15 +520,9 @@ export class AiService {/*** Gives the holdings of the user with the tool get-portfolio of the server* of the Model Context Protocol, without the holdings whose data source the- * import refuses for the user+ * import refuses*/- private async getHoldingsToMatch({- subscription,- userId- }: {- subscription?: UserWithSettings['subscription'];- userId: string;- }) {+ private async getHoldingsToMatch({ userId }: { userId: string }) {const [{ content }, dataSources] = await Promise.all([this.mcpService.getPortfolio({ userId, withValues: false }),this.dataProviderService.getDataSources()@@ -540,11 +530,7 @@ export class AiService {return this.getHoldingsFromTable(content[0].text).filter(({ dataSource }) => {- return this.isValidDataSourceForImport({- dataSource,- dataSources,- subscription- });+ return this.isValidDataSourceForImport({ dataSource, dataSources });});}@@ -608,8 +594,8 @@ export class AiService {* symbol only. The holdings let the model select an asset profile which the* user already has, also if no search gives it, for example as no data* provider finds its ISIN. A holding whose data source the import refuses- * for the user is not given to the model, so that the model selects an- * asset profile of the search instead and the whole import does not fail.+ * is not given to the model, so that the model selects an asset profile of+ * the search instead and the whole import does not fail.* If get-accounts fails, the model gets no accounts, so that only the* account hints have no match. If get-portfolio or the data sources fail,* the model gets no holdings. The reasoning is disabled, as it can add@@ -625,13 +611,11 @@ export class AiService {abortSignal,accountHints,assets,- subscription,userId}: {abortSignal?: AbortSignal;accountHints: AccountHint[];assets: AssetToMatch[];- subscription?: UserWithSettings['subscription'];userId: string;}) {const startTime = performance.now();@@ -650,7 +634,7 @@ export class AiService {}): undefined,assets.length > 0- ? this.getHoldingsToMatch({ subscription, userId }).catch(+ ? this.getHoldingsToMatch({ userId }).catch((error: Error): undefined => {this.logger.warn(`Could not get the holdings: ${error.message}`);@@ -751,31 +735,24 @@ export class AiService {/*** Checks without a request to a data provider if the import takes a data- * source for the user: it is configured and does not require Ghostfolio- * Premium, for example after the Premium subscription has expired. The data- * source is first transformed as POST api/v1/import transforms it. A symbol- * which the data source cannot resolve, for example a delisted one, is not- * found, as this needs a request to the data provider.+ * source of a holding: it is configured. The data source is first+ * transformed as POST api/v1/import transforms it. A data source which+ * requires Ghostfolio Premium is taken, as the import takes it for an asset+ * profile which the user already has. A symbol which the data source cannot+ * resolve, for example a delisted one, is not found, as this needs a+ * request to the data provider.*/private isValidDataSourceForImport({dataSource,- dataSources,- subscription+ dataSources}: {dataSource: string;dataSources: DataSource[];- subscription?: UserWithSettings['subscription'];}) {- const dataSourceOfImport = transformDataSourceInRequest({- dataSource,- configurationService: this.configurationService- });-- return (- dataSources.includes(dataSourceOfImport) &&- !this.dataProviderService.isPremiumRequired({- subscription,- dataSource: dataSourceOfImport+ return dataSources.includes(+ transformDataSourceInRequest({+ dataSource,+ configurationService: this.configurationService}));}diff --git a/apps/api/src/services/data-provider/data-provider.service.spec.ts b/apps/api/src/services/data-provider/data-provider.service.spec.tsindex 815ab2743e..d2e6b513f5 100644--- a/apps/api/src/services/data-provider/data-provider.service.spec.ts+++ b/apps/api/src/services/data-provider/data-provider.service.spec.ts@@ -4,8 +4,11 @@ import { MarketDataService } from '@ghostfolio/api/services/market-data/market-dimport { PrismaService } from '@ghostfolio/api/services/prisma/prisma.service';import { PropertyService } from '@ghostfolio/api/services/property/property.service';import { RedisCacheService } from '@ghostfolio/api/services/redis-cache/redis-cache.service';+import { SubscriptionType } from '@ghostfolio/common/enums';+import { getAssetProfileIdentifier } from '@ghostfolio/common/helper';+import type { UserWithSettings } from '@ghostfolio/common/types';-import { DataSource } from '@prisma/client';+import { AssetClass, AssetSubClass, DataSource } from '@prisma/client';import { DataProviderService } from './data-provider.service';@@ -109,4 +112,139 @@ describe('DataProviderService', () => {}).toThrow('No data provider has been found.');});});++ describe('validateActivities', () => {+ const assetProfileOfUser = {+ assetClass: AssetClass.EQUITY,+ assetSubClass: AssetSubClass.ETF,+ currency: 'USD',+ dataSource: DataSource.EOD_HISTORICAL_DATA,+ isin: 'IE00BK5BQT80',+ name: 'Vanguard FTSE All-World UCITS ETF',+ symbol: 'VWRA.LSE'+ };++ const activityOfAssetProfileOfUser = {+ currency: 'USD',+ dataSource: DataSource.EOD_HISTORICAL_DATA,+ symbol: 'VWRA.LSE',+ type: 'BUY' as const+ };++ const basicSubscription = {+ type: SubscriptionType.Basic+ } as UserWithSettings['subscription'];++ const userId = 'user-id';++ let dataProviderService: DataProviderService;+ let eodHistoricalDataService: DataProviderInterface;+ let findFirstSymbolProfile: jest.Mock;++ beforeEach(() => {+ const configuration = {+ DATA_SOURCES: [DataSource.EOD_HISTORICAL_DATA, DataSource.YAHOO],+ DATA_SOURCES_GHOSTFOLIO_DATA_PROVIDER: [],+ ENABLE_FEATURE_SUBSCRIPTION: true+ };++ eodHistoricalDataService = {+ getAssetProfile: jest.fn().mockResolvedValue(assetProfileOfUser),+ getDataProviderInfo: jest.fn().mockReturnValue({ isPremium: true }),+ getName: jest.fn().mockReturnValue(DataSource.EOD_HISTORICAL_DATA)+ } as unknown as DataProviderInterface;++ findFirstSymbolProfile = jest.fn().mockResolvedValue(assetProfileOfUser);++ dataProviderService = new DataProviderService(+ {+ get: (key: keyof typeof configuration) => {+ return configuration[key];+ }+ } as unknown as ConfigurationService,+ [eodHistoricalDataService],+ {} as MarketDataService,+ {+ symbolProfile: { findFirst: findFirstSymbolProfile }+ } as unknown as PrismaService,+ {+ getByKey: jest.fn().mockResolvedValue(undefined)+ } as unknown as PropertyService,+ {} as RedisCacheService+ );+ });++ it('takes the stored asset profile of a premium data provider without a request to the data provider for the Basic subscription if the user has an activity with it', async () => {+ await expect(+ dataProviderService.validateActivities({+ userId,+ activitiesDto: [activityOfAssetProfileOfUser],+ maxActivitiesToImport: 1,+ subscription: basicSubscription+ })+ ).resolves.toEqual({+ [getAssetProfileIdentifier(assetProfileOfUser)]: assetProfileOfUser+ });++ expect(findFirstSymbolProfile).toHaveBeenCalledWith(+ expect.objectContaining({+ where: {+ activities: { some: { userId } },+ dataSource: DataSource.EOD_HISTORICAL_DATA,+ symbol: 'VWRA.LSE'+ }+ })+ );++ expect(eodHistoricalDataService.getAssetProfile).not.toHaveBeenCalled();+ });++ it('gets the stored asset profile once for several activities with it', async () => {+ await dataProviderService.validateActivities({+ userId,+ activitiesDto: [+ activityOfAssetProfileOfUser,+ { ...activityOfAssetProfileOfUser, type: 'SELL' }+ ],+ maxActivitiesToImport: 2,+ subscription: basicSubscription+ });++ expect(findFirstSymbolProfile).toHaveBeenCalledTimes(1);+ });++ it('throws an error for a premium data provider for the Basic subscription if the user has no activity with the asset profile', async () => {+ findFirstSymbolProfile.mockResolvedValue(null);++ await expect(+ dataProviderService.validateActivities({+ userId,+ activitiesDto: [activityOfAssetProfileOfUser],+ maxActivitiesToImport: 1,+ subscription: basicSubscription+ })+ ).rejects.toThrow(+ 'activity.dataSource ("EOD_HISTORICAL_DATA") requires Ghostfolio Premium'+ );++ expect(eodHistoricalDataService.getAssetProfile).not.toHaveBeenCalled();+ });++ it('resolves the asset profile of a premium data provider with the data provider for the Premium subscription', async () => {+ await dataProviderService.validateActivities({+ userId,+ activitiesDto: [activityOfAssetProfileOfUser],+ maxActivitiesToImport: 1,+ subscription: {+ type: SubscriptionType.Premium+ } as UserWithSettings['subscription']+ });++ expect(findFirstSymbolProfile).not.toHaveBeenCalled();++ expect(eodHistoricalDataService.getAssetProfile).toHaveBeenCalledWith({+ symbol: 'VWRA.LSE'+ });+ });+ });});diff --git a/apps/api/src/services/data-provider/data-provider.service.ts b/apps/api/src/services/data-provider/data-provider.service.tsindex 526637891e..2fe65c2678 100644--- a/apps/api/src/services/data-provider/data-provider.service.ts+++ b/apps/api/src/services/data-provider/data-provider.service.ts@@ -222,7 +222,8 @@ export class DataProviderService implements OnModuleInit {activitiesDto,assetProfilesWithMarketDataDto,maxActivitiesToImport,- subscription+ subscription,+ userId}: {activitiesDto: Pick<Partial<CreateOrderDto>,@@ -231,6 +232,7 @@ export class DataProviderService implements OnModuleInit {assetProfilesWithMarketDataDto?: ImportDataDto['assetProfiles'];maxActivitiesToImport: number;subscription: UserWithSettings['subscription'];+ userId: string;}) {if (activitiesDto?.length > maxActivitiesToImport) {throw new ImportValidationError(@@ -273,17 +275,31 @@ export class DataProviderService implements OnModuleInit {);}- if (this.isPremiumRequired({ dataSource, subscription })) {- throw new ImportValidationError(- `${activityPath}.dataSource ("${maskedDataSource}") requires Ghostfolio Premium`- );- }-const assetProfileIdentifier = getAssetProfileIdentifier({dataSource,symbol});+ if (this.isPremiumRequired({ dataSource, subscription })) {+ // The user can add activities to an asset profile of a premium data+ // provider which the user already has, for example after the Premium+ // subscription has expired. The stored asset profile is taken, so+ // that the validation sends no request to the data provider.+ const assetProfileOfUser =+ assetProfiles[assetProfileIdentifier] ??+ (await this.getAssetProfileOfUser({ dataSource, symbol, userId }));++ if (!assetProfileOfUser) {+ throw new ImportValidationError(+ `${activityPath}.dataSource ("${maskedDataSource}") requires Ghostfolio Premium`+ );+ }++ assetProfiles[assetProfileIdentifier] = assetProfileOfUser;++ continue;+ }+if (!assetProfiles[assetProfileIdentifier]) {const assetProfileInImport = assetProfilesWithMarketDataDto?.find((assetProfileWithMarketData) => {@@ -948,6 +964,35 @@ export class DataProviderService implements OnModuleInit {};}+ /**+ * Gives the stored asset profile if the user has an activity with it, with+ * the fields which a data provider gives+ */+ private async getAssetProfileOfUser({+ dataSource,+ symbol,+ userId+ }: AssetProfileIdentifier & { userId: string }): Promise<+ Partial<SymbolProfile>+ > {+ return this.prismaService.symbolProfile.findFirst({+ select: {+ assetClass: true,+ assetSubClass: true,+ currency: true,+ dataSource: true,+ isin: true,+ name: true,+ symbol: true+ },+ where: {+ dataSource,+ symbol,+ activities: { some: { userId } }+ }+ });+ }+private hasCurrency({assetProfileIdentifiers,currencydiff --git a/apps/api/src/services/import/import.service.ts b/apps/api/src/services/import/import.service.tsindex 2d09cb78df..57132691bd 100644--- a/apps/api/src/services/import/import.service.ts+++ b/apps/api/src/services/import/import.service.ts@@ -761,7 +761,8 @@ export class ImportService {activitiesDto,assetProfilesWithMarketDataDto,maxActivitiesToImport,- subscription: user.subscription+ subscription: user.subscription,+ userId: user.id});const activitiesExtendedWithErrors = await this.extendActivitiesWithErrors({