Open Source Wealth Management Software. Angular + NestJS + Prisma + Nx + TypeScript 🤍
TypeScript angularetffinancefintechghostfoliohacktoberfestinvestingnestjsosspersonal-financeportfolioprismasoftwarestocktrackertradingtypescriptwealth-managementweb
Feature/support rest api in scraper (#2810)
Hugo Persson371f1dc4510f
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 8017e3d0eb..c8fe750660 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -9,6 +9,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0### Added+- Added support for REST APIs (`JSON`) via the scraper configuration++### Changed+- Improved the user interface of the access table to share the portfolio## 2.34.0 - 2024-01-02diff --git a/apps/api/src/app/admin/admin.controller.ts b/apps/api/src/app/admin/admin.controller.tsindex e769325ef4..456550110c 100644--- a/apps/api/src/app/admin/admin.controller.ts+++ b/apps/api/src/app/admin/admin.controller.ts@@ -227,8 +227,8 @@ export class AdminController {@Param('symbol') symbol: string): Promise<{ price: number }> {try {- const { headers, selector, url } = JSON.parse(data.scraperConfiguration);- const price = await this.manualService.test({ headers, selector, url });+ const scraperConfiguration = JSON.parse(data.scraperConfiguration);+ const price = await this.manualService.test(scraperConfiguration);if (price) {return { price };diff --git a/apps/api/src/services/data-provider/manual/manual.service.ts b/apps/api/src/services/data-provider/manual/manual.service.tsindex d5a5c7eb3d..4ac8bd668a 100644--- a/apps/api/src/services/data-provider/manual/manual.service.ts+++ b/apps/api/src/services/data-provider/manual/manual.service.ts@@ -12,6 +12,7 @@ import {extractNumberFromString,getYesterday} from '@ghostfolio/common/helper';+import { ScraperConfiguration } from '@ghostfolio/common/interfaces';import { Granularity } from '@ghostfolio/common/types';import { Injectable, Logger } from '@nestjs/common';import { DataSource, SymbolProfile } from '@prisma/client';@@ -19,6 +20,7 @@ import * as cheerio from 'cheerio';import { isUUID } from 'class-validator';import { addDays, format, isBefore } from 'date-fns';import got, { Headers } from 'got';+import jsonpath from 'jsonpath';@Injectable()export class ManualService implements DataProviderInterface {@@ -97,7 +99,7 @@ export class ManualService implements DataProviderInterface {return {};}- const value = await this.scrape({ headers, selector, url });+ const value = await this.scrape(symbolProfile.scraperConfiguration);return {[symbol]: {@@ -220,23 +222,13 @@ export class ManualService implements DataProviderInterface {return { items };}- public async test(params: any) {- return this.scrape({- headers: params.headers,- selector: params.selector,- url: params.url- });+ public async test(scraperConfiguration: ScraperConfiguration) {+ return this.scrape(scraperConfiguration);}- private async scrape({- headers = {},- selector,- url- }: {- headers?: Headers;- selector: string;- url: string;- }): Promise<number> {+ private async scrape(+ scraperConfiguration: ScraperConfiguration+ ): Promise<number> {try {const abortController = new AbortController();@@ -244,15 +236,26 @@ export class ManualService implements DataProviderInterface {abortController.abort();}, this.configurationService.get('REQUEST_TIMEOUT'));- const { body } = await got(url, {- headers,+ const { body, headers } = await got(scraperConfiguration.url, {+ headers: scraperConfiguration.headers as Headers,// @ts-ignoresignal: abortController.signal});- const $ = cheerio.load(body);+ if (headers['content-type'] === 'application/json') {+ const data = JSON.parse(body);+ const value = String(+ jsonpath.query(data, scraperConfiguration.selector)[0]+ );- return extractNumberFromString($(selector).first().text());+ return extractNumberFromString(value);+ } else {+ const $ = cheerio.load(body);++ return extractNumberFromString(+ $(scraperConfiguration.selector).first().text()+ );+ }} catch (error) {throw error;}diff --git a/package.json b/package.jsonindex 8b761b2ac0..d9379fb343 100644--- a/package.json+++ b/package.json@@ -110,6 +110,7 @@"helmet": "7.0.0","http-status-codes": "2.3.0","ionicons": "7.1.0",+ "jsonpath": "1.1.1","lodash": "4.17.21","marked": "9.1.6","ms": "3.0.0-canary.1",diff --git a/yarn.lock b/yarn.lockindex ce4304e000..7854b53ee6 100644--- a/yarn.lock+++ b/yarn.lock@@ -10959,6 +10959,18 @@ escape-string-regexp@^5.0.0:resolved "https://registry.yarnpkg.com/escape-string-regexp/-/escape-string-regexp-5.0.0.tgz#4683126b500b61762f2dbebace1806e8be31b1c8"integrity sha512-/veY75JbMK4j1yjvuUxuVsiS/hr/4iHs9FTT6cgTexxdE0Ly/glccBAkloH/DofkjRbZU3bnoj38mOmhkZ0lHw==+escodegen@^1.8.1:+ version "1.14.3"+ resolved "https://registry.yarnpkg.com/escodegen/-/escodegen-1.14.3.tgz#4e7b81fba61581dc97582ed78cab7f0e8d63f503"+ integrity sha512-qFcX0XJkdg+PB3xjZZG/wKSuT1PnQWx57+TVSjIMmILd2yC/6ByYElPwJnslDsuWuSAp4AwJGumarAAmJch5Kw==+ dependencies:+ esprima "^4.0.1"+ estraverse "^4.2.0"+ esutils "^2.0.2"+ optionator "^0.8.1"+ optionalDependencies:+ source-map "~0.6.1"+escodegen@^2.0.0:version "2.1.0"resolved "https://registry.yarnpkg.com/escodegen/-/escodegen-2.1.0.tgz#ba93bbb7a43986d29d6041f99f5262da773e2e17"@@ -11187,6 +11199,11 @@ espree@^9.0.0, espree@^9.4.0:acorn-jsx "^5.3.2"eslint-visitor-keys "^3.4.1"+ version "1.2.2"+ resolved "https://registry.yarnpkg.com/esprima/-/esprima-1.2.2.tgz#76a0fd66fcfe154fd292667dc264019750b1657b"+ integrity sha512-+JpPZam9w5DuJ3Q67SqsMGtiHKENSMRVoxvArfJZK01/BfLEObtZ6orJa/MtoGNR/rfMgp5837T41PAmTwAv/A==+esprima@^4.0.0, esprima@^4.0.1, esprima@~4.0.0:version "4.0.1"resolved "https://registry.yarnpkg.com/esprima/-/esprima-4.0.1.tgz#13b04cdb3e6c5d19df91ab6987a8695619b0aa71"@@ -11206,7 +11223,7 @@ esrecurse@^4.1.0, esrecurse@^4.3.0:dependencies:estraverse "^5.2.0"-estraverse@^4.1.1:+estraverse@^4.1.1, estraverse@^4.2.0:version "4.3.0"resolved "https://registry.yarnpkg.com/estraverse/-/estraverse-4.3.0.tgz#398ad3f3c5a24948be7725e83d11a7de28cdbd1d"integrity sha512-39nnKffWz8xN1BU/2c79n9nB9HDzo0niYUqx6xyqUnyoAnQyyWpOTdZEeiCch8BBu515t4wp9ZmgVfVhn9EBpw==@@ -14245,6 +14262,15 @@ jsonparse@^1.3.1:resolved "https://registry.yarnpkg.com/jsonparse/-/jsonparse-1.3.1.tgz#3f4dae4a91fac315f71062f8521cc239f1366280"integrity sha512-POQXvpdL69+CluYsillJ7SUhKvytYjW9vG/GKpnf+xP8UWgYEM/RaMzHHofbALDiKbbP1W8UEYmgGl39WkPZsg==+ version "1.1.1"+ resolved "https://registry.yarnpkg.com/jsonpath/-/jsonpath-1.1.1.tgz#0ca1ed8fb65bb3309248cc9d5466d12d5b0b9901"+ integrity sha512-l6Cg7jRpixfbgoWgkrl77dgEj8RPvND0wMH6TwQmi9Qs4TFfS9u5cUFnbeKTwj5ga5Y3BTGGNI28k117LJ009w==+ dependencies:+ esprima "1.2.2"+ static-eval "2.0.2"+ underscore "1.12.1"+version "9.0.0"resolved "https://registry.yarnpkg.com/jsonwebtoken/-/jsonwebtoken-9.0.0.tgz#d0faf9ba1cc3a56255fe49c0961a67e520c1926d"@@ -16051,7 +16077,7 @@ [email protected]:dependencies:is-wsl "^1.1.0"-optionator@^0.8.2:+optionator@^0.8.1, optionator@^0.8.2:version "0.8.3"resolved "https://registry.yarnpkg.com/optionator/-/optionator-0.8.3.tgz#84fa1d036fe9d3c7e21d99884b601167ec8fb495"integrity sha512-+IW9pACdk3XWmmTXG8m3upGUJst5XRGzxMRjXzAuJ1XnIFNvfhjjIuYkDvysnPQ7qzqVzLt78BCruntqRhWQbA==@@ -18446,6 +18472,13 @@ standard-as-callback@^2.1.0:resolved "https://registry.yarnpkg.com/standard-as-callback/-/standard-as-callback-2.1.0.tgz#8953fc05359868a77b5b9739a665c5977bb7df45"integrity sha512-qoRRSyROncaz1z0mvYqIE4lCd9p2R90i6GxW3uZv5ucSu8tU7B5HXUP1gG8pVZsYNVaXjk8ClXHPttLyxAL48A==+ version "2.0.2"+ resolved "https://registry.yarnpkg.com/static-eval/-/static-eval-2.0.2.tgz#2d1759306b1befa688938454c546b7871f806a42"+ integrity sha512-N/D219Hcr2bPjLxPiV+TQE++Tsmrady7TqAJugLy7Xk1EumfDWS/f5dtBbkRCGE7wKKXuYockQoj8Rm2/pVKyg==+ dependencies:+ escodegen "^1.8.1"+static-extend@^0.1.1:version "0.1.2"resolved "https://registry.yarnpkg.com/static-extend/-/static-extend-0.1.2.tgz#60809c39cbff55337226fd5e0b520f341f1fb5c6"@@ -19380,6 +19413,11 @@ unbox-primitive@^1.0.2:has-symbols "^1.0.3"which-boxed-primitive "^1.0.2"+ version "1.12.1"+ resolved "https://registry.yarnpkg.com/underscore/-/underscore-1.12.1.tgz#7bb8cc9b3d397e201cf8553336d262544ead829e"+ integrity sha512-hEQt0+ZLDVUMhebKxL4x1BTtDY7bavVofhZ9KZ4aI26X9SRaE+Y3m83XUL1UP2jn8ynjndwCCpEHdUG+9pP1Tw==+undici-types@~5.26.4:version "5.26.5"resolved "https://registry.yarnpkg.com/undici-types/-/undici-types-5.26.5.tgz#bcd539893d00b56e964fd2657a4866b221a65617"