Skip to content

Open Source Wealth Management Software. Angular + NestJS + Prisma + Nx + TypeScript 🤍

Task/improve data source encoding (#7138)

Thomas Kaul830c1e229573
diff --git a/apps/api/src/app/info/info.service.ts b/apps/api/src/app/info/info.service.ts
index 86630db53f..4441036a2d 100644
--- a/apps/api/src/app/info/info.service.ts
+++ b/apps/api/src/app/info/info.service.ts
@@ -1,6 +1,7 @@
import { RedisCacheService } from '@ghostfolio/api/app/redis-cache/redis-cache.service';
import { SubscriptionService } from '@ghostfolio/api/app/subscription/subscription.service';
import { UserService } from '@ghostfolio/api/app/user/user.service';
+import { encodeDataSource } from '@ghostfolio/api/helper/data-source.helper';
import { BenchmarkService } from '@ghostfolio/api/services/benchmark/benchmark.service';
import { ConfigurationService } from '@ghostfolio/api/services/configuration/configuration.service';
import { ExchangeRateDataService } from '@ghostfolio/api/services/exchange-rate-data/exchange-rate-data.service';
@@ -17,7 +18,6 @@ import {
PROPERTY_UPTIME,
ghostfolioFearAndGreedIndexDataSourceStocks
} from '@ghostfolio/common/config';
-import { encodeDataSource } from '@ghostfolio/common/helper';
import { InfoItem, Statistics } from '@ghostfolio/common/interfaces';
import { permissions } from '@ghostfolio/common/permissions';
diff --git a/apps/api/src/helper/data-source.helper.ts b/apps/api/src/helper/data-source.helper.ts
index c4a55c9dad..f3ed752295 100644
--- a/apps/api/src/helper/data-source.helper.ts
+++ b/apps/api/src/helper/data-source.helper.ts
@@ -1,4 +1,58 @@
import { DataSource } from '@prisma/client';
+import { createHash } from 'node:crypto';
+
+const encodedDataSourceByDataSource = new Map<DataSource, string>(
+ Object.values(DataSource).map((dataSource) => {
+ return [dataSource, hashDataSource(dataSource)];
+ })
+);
+
+const dataSourceByEncodedDataSource = new Map<string, DataSource>(
+ [...encodedDataSourceByDataSource].map(([dataSource, encodedDataSource]) => {
+ return [encodedDataSource, dataSource];
+ })
+);
+
+/**
+ * @deprecated Backward compatibility to support importing data that was
+ * exported using the previous data source encoding
+ */
+const dataSourceByDeprecatedEncodedDataSource = new Map<string, DataSource>(
+ Object.values(DataSource).map((dataSource) => {
+ return [deprecatedHashDataSource(dataSource), dataSource];
+ })
+);
+
+/**
+ * @deprecated Backward compatibility (see above)
+ */
+function deprecatedHashDataSource(dataSource: DataSource) {
+ return Buffer.from(dataSource, 'utf-8').toString('hex');
+}
+
+function hashDataSource(dataSource: DataSource) {
+ return createHash('sha256').update(dataSource).digest('hex').slice(0, 8);
+}
+
+export function decodeDataSource(encodedDataSource: string) {
+ if (!encodedDataSource) {
+ return undefined;
+ }
+
+ return (
+ dataSourceByEncodedDataSource.get(encodedDataSource) ??
+ dataSourceByDeprecatedEncodedDataSource.get(encodedDataSource) ??
+ encodedDataSource
+ );
+}
+
+export function encodeDataSource(dataSource: DataSource) {
+ if (!dataSource) {
+ return undefined;
+ }
+
+ return encodedDataSourceByDataSource.get(dataSource);
+}
export function getMaskedGhostfolioDataSource({
dataSource,
diff --git a/apps/api/src/interceptors/transform-data-source-in-request/transform-data-source-in-request.interceptor.ts b/apps/api/src/interceptors/transform-data-source-in-request/transform-data-source-in-request.interceptor.ts
index 17c5ebe57c..4ab0794ec1 100644
--- a/apps/api/src/interceptors/transform-data-source-in-request/transform-data-source-in-request.interceptor.ts
+++ b/apps/api/src/interceptors/transform-data-source-in-request/transform-data-source-in-request.interceptor.ts
@@ -1,5 +1,5 @@
+import { decodeDataSource } from '@ghostfolio/api/helper/data-source.helper';
import { ConfigurationService } from '@ghostfolio/api/services/configuration/configuration.service';
-import { decodeDataSource } from '@ghostfolio/common/helper';
import {
CallHandler,
diff --git a/apps/api/src/interceptors/transform-data-source-in-response/transform-data-source-in-response.interceptor.ts b/apps/api/src/interceptors/transform-data-source-in-response/transform-data-source-in-response.interceptor.ts
index ddcf30b0cf..56d260d4d3 100644
--- a/apps/api/src/interceptors/transform-data-source-in-response/transform-data-source-in-response.interceptor.ts
+++ b/apps/api/src/interceptors/transform-data-source-in-response/transform-data-source-in-response.interceptor.ts
@@ -1,7 +1,10 @@
-import { getMaskedGhostfolioDataSource } from '@ghostfolio/api/helper/data-source.helper';
+import {
+ encodeDataSource,
+ getMaskedGhostfolioDataSource
+} from '@ghostfolio/api/helper/data-source.helper';
import { redactPaths } from '@ghostfolio/api/helper/object.helper';
import { ConfigurationService } from '@ghostfolio/api/services/configuration/configuration.service';
-import { encodeDataSource } from '@ghostfolio/common/helper';
+import { hasRole } from '@ghostfolio/common/permissions';
import {
CallHandler,
@@ -45,11 +48,14 @@ export class TransformDataSourceInResponseInterceptor<
next: CallHandler<T>
): Observable<any> {
const isExportMode = context.getClass().name === 'ExportController';
+ const { user } = context.switchToHttp().getRequest();
return next.handle().pipe(
map((data: any) => {
if (this.configurationService.get('ENABLE_FEATURE_SUBSCRIPTION')) {
- const valueMap = this.encodedDataSourceMap;
+ const valueMap = hasRole(user, 'ADMIN')
+ ? {}
+ : { ...this.encodedDataSourceMap };
if (isExportMode) {
const ghostfolioDataSources = this.configurationService.get(
@@ -64,6 +70,10 @@ export class TransformDataSourceInResponseInterceptor<
}
}
+ if (Object.keys(valueMap).length === 0) {
+ return data;
+ }
+
data = redactPaths({
valueMap,
object: data,
diff --git a/libs/common/src/lib/helper.ts b/libs/common/src/lib/helper.ts
index 3219ed553c..58e62618bf 100644
--- a/libs/common/src/lib/helper.ts
+++ b/libs/common/src/lib/helper.ts
@@ -1,7 +1,6 @@
import { NumberParser } from '@internationalized/number';
import {
Type as ActivityType,
- DataSource,
MarketData,
Prisma,
SymbolProfile,
@@ -166,14 +165,6 @@ export function capitalize(aString: string) {
return aString.charAt(0).toUpperCase() + aString.slice(1).toLowerCase();
}
-export function decodeDataSource(encodedDataSource: string) {
- if (encodedDataSource) {
- return Buffer.from(encodedDataSource, 'hex').toString();
- }
-
- return undefined;
-}
-
export function downloadAsFile({
content,
contentType = 'text/plain',
@@ -199,14 +190,6 @@ export function downloadAsFile({
a.click();
}
-export function encodeDataSource(aDataSource: DataSource) {
- if (aDataSource) {
- return Buffer.from(aDataSource, 'utf-8').toString('hex');
- }
-
- return undefined;
-}
-
export function extractNumberFromString({
locale = 'en-US',
value