Skip to content

Open Source Wealth Management Software. Angular + NestJS + Prisma + Nx + TypeScript 🤍

Feature/execute scraper configuration instantly (#3723)

Shaunak Das7ea906185219
diff --git a/CHANGELOG.md b/CHANGELOG.md
index d598db05c3..4639d24857 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -10,6 +10,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
### Added
- Set up a performance logging service
+- Added the attribute `mode` to the scraper configuration to get quotes instantly
### Changed
diff --git a/apps/api/src/services/data-provider/manual/manual.service.ts b/apps/api/src/services/data-provider/manual/manual.service.ts
index 0655d2318c..02e50a145d 100644
--- a/apps/api/src/services/data-provider/manual/manual.service.ts
+++ b/apps/api/src/services/data-provider/manual/manual.service.ts
@@ -166,11 +166,42 @@ export class ManualService implements DataProviderInterface {
}
});
+ const symbolProfilesWithScraperConfigurationAndInstantMode =
+ symbolProfiles.filter(({ scraperConfiguration }) => {
+ return scraperConfiguration?.mode === 'instant';
+ });
+
+ const scraperResultPromises =
+ symbolProfilesWithScraperConfigurationAndInstantMode.map(
+ async ({ scraperConfiguration, symbol }) => {
+ try {
+ const marketPrice = await this.scrape(scraperConfiguration);
+ return { marketPrice, symbol };
+ } catch (error) {
+ Logger.error(
+ `Could not get quote for ${symbol} (${this.getName()}): [${error.name}] ${error.message}`,
+ 'ManualService'
+ );
+ return { symbol, marketPrice: undefined };
+ }
+ }
+ );
+
+ // Wait for all scraping requests to complete concurrently
+ const scraperResults = await Promise.all(scraperResultPromises);
+
for (const { currency, symbol } of symbolProfiles) {
- let marketPrice =
+ let { marketPrice } =
+ scraperResults.find((result) => {
+ return result.symbol === symbol;
+ }) ?? {};
+
+ marketPrice =
+ marketPrice ??
marketData.find((marketDataItem) => {
return marketDataItem.symbol === symbol;
- })?.marketPrice ?? 0;
+ })?.marketPrice ??
+ 0;
response[symbol] = {
currency,
diff --git a/apps/api/src/services/symbol-profile/symbol-profile.service.ts b/apps/api/src/services/symbol-profile/symbol-profile.service.ts
index 50cb25000a..283da7b525 100644
--- a/apps/api/src/services/symbol-profile/symbol-profile.service.ts
+++ b/apps/api/src/services/symbol-profile/symbol-profile.service.ts
@@ -275,6 +275,8 @@ export class SymbolProfileService {
headers:
scraperConfiguration.headers as ScraperConfiguration['headers'],
locale: scraperConfiguration.locale as string,
+ mode:
+ (scraperConfiguration.mode as ScraperConfiguration['mode']) ?? 'lazy',
selector: scraperConfiguration.selector as string,
url: scraperConfiguration.url as string
};
diff --git a/libs/common/src/lib/interfaces/scraper-configuration.interface.ts b/libs/common/src/lib/interfaces/scraper-configuration.interface.ts
index ef55063280..70fcd939d0 100644
--- a/libs/common/src/lib/interfaces/scraper-configuration.interface.ts
+++ b/libs/common/src/lib/interfaces/scraper-configuration.interface.ts
@@ -2,6 +2,7 @@ export interface ScraperConfiguration {
defaultMarketPrice?: number;
headers?: { [key: string]: string };
locale?: string;
+ mode?: 'instant' | 'lazy';
selector: string;
url: string;
}