Open Source Wealth Management Software. Angular + NestJS + Prisma + Nx + TypeScript 🤍
TypeScript angularetffinancefintechghostfoliohacktoberfestinvestingnestjsosspersonal-financeportfolioprismasoftwarestocktrackertradingtypescriptwealth-managementweb
Task/improve web_fetch tool in fetch service (#7730)
Thomas Kaul1355098bdb3c
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex b54041fb08..e2cd64b0e5 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -11,6 +11,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0- Added an expiration date to the access to share the portfolio- Added the date of the last usage to the access to share the portfolio+- Added support for a dedicated _OpenRouter_ engine for the `web_fetch` tool in the `FetchService`++### Changed++- Improved the logging of the `web_fetch` tool in the `FetchService`## 3.61.0 - 2026-08-25diff --git a/apps/api/src/services/data-provider/data-enhancer/trackinsight/trackinsight.service.ts b/apps/api/src/services/data-provider/data-enhancer/trackinsight/trackinsight.service.tsindex 23a8ce33cc..aa61c048c7 100644--- a/apps/api/src/services/data-provider/data-enhancer/trackinsight/trackinsight.service.ts+++ b/apps/api/src/services/data-provider/data-enhancer/trackinsight/trackinsight.service.ts@@ -64,10 +64,10 @@ export class TrackinsightDataEnhancerService implements DataEnhancerInterface {symbol});- if (!trackinsightSymbol) {+ if (!trackinsightSymbol && symbol.includes('.')) {trackinsightSymbol = await this.searchTrackinsightSymbol({requestTimeout,- symbol: symbol.split('.')?.[0]+ symbol: symbol.split('.')[0]});}@@ -213,6 +213,12 @@ export class TrackinsightDataEnhancerService implements DataEnhancerInterface {return jsonRes['results']['docs'][0]['ticker'];}+ this.logger.debug(+ `Could not match a Trackinsight symbol for "${symbol}": ${JSON.stringify(+ jsonRes+ )}`+ );+return undefined;}).catch(({ message }) => {diff --git a/apps/api/src/services/fetch/fetch.service.ts b/apps/api/src/services/fetch/fetch.service.tsindex 1f53203785..1cf260ed3e 100644--- a/apps/api/src/services/fetch/fetch.service.ts+++ b/apps/api/src/services/fetch/fetch.service.ts@@ -1,7 +1,9 @@import { redactPaths } from '@ghostfolio/api/helper/object.helper';import { PropertyService } from '@ghostfolio/api/services/property/property.service';import {+ DEFAULT_OPENROUTER_ENGINE_WEB_FETCH,PROPERTY_API_KEY_OPENROUTER,+ PROPERTY_OPENROUTER_ENGINE_WEB_FETCH,PROPERTY_OPENROUTER_MODEL,PROPERTY_OPENROUTER_MODEL_WEB_FETCH,PROPERTY_PROXY_ROUTES,@@ -20,6 +22,7 @@ import { WebFetchRoute } from './interfaces/web-fetch-route.interface';export class FetchService implements OnModuleInit {private readonly logger = new Logger(FetchService.name);+ private static readonly BODY_PREVIEW_LENGTH = 500;private static readonly REDACTED_QUERY_PARAM_NAMES = ['apikey', 'api_token'];private static readonly WEB_FETCH_TIMEOUT = ms('30 seconds');@@ -91,15 +94,22 @@ export class FetchService implements OnModuleInit {url: string;webFetchRoute: WebFetchRoute;}) {- const [openRouterApiKey, openRouterModel, openRouterModelWebFetch] =- await Promise.all([- this.propertyService.getByKey<string>(PROPERTY_API_KEY_OPENROUTER),- this.propertyService.getByKey<string>(PROPERTY_OPENROUTER_MODEL),- this.propertyService.getByKey<string>(- PROPERTY_OPENROUTER_MODEL_WEB_FETCH- )- ]);-+ const [+ openRouterApiKey,+ openRouterEngineWebFetch,+ openRouterModel,+ openRouterModelWebFetch+ ] = await Promise.all([+ this.propertyService.getByKey<string>(PROPERTY_API_KEY_OPENROUTER),+ this.propertyService.getByKey<string>(+ PROPERTY_OPENROUTER_ENGINE_WEB_FETCH+ ),+ this.propertyService.getByKey<string>(PROPERTY_OPENROUTER_MODEL),+ this.propertyService.getByKey<string>(PROPERTY_OPENROUTER_MODEL_WEB_FETCH)+ ]);++ const engine =+ openRouterEngineWebFetch || DEFAULT_OPENROUTER_ENGINE_WEB_FETCH;const model = openRouterModelWebFetch || openRouterModel;if (!model || !openRouterApiKey) {@@ -119,11 +129,15 @@ export class FetchService implements OnModuleInit {timeout: FetchService.WEB_FETCH_TIMEOUT,tools: {// Provider-executed tool: lets OpenRouter perform the actual web- // request server-side via its `web_fetch` engine. `id` and `args`- // are the OpenRouter-specific identifiers. The input schema is left- // open as the arguments are supplied by the model.+ // request server-side via its `web_fetch` engine. `args` and `id`+ // are the OpenRouter-specific identifiers. The engine must stay+ // nested in `parameters`, because the provider spreads `args` on the+ // top level of the tool object, where OpenRouter ignores it. Nested+ // keys need snake case, as only the top level gets converted. The+ // input schema is left open as the arguments are supplied by the+ // model.web_fetch: tool({- args: { engine: 'openrouter' },+ args: { parameters: { engine } },id: 'openrouter.web_fetch',inputSchema: jsonSchema({additionalProperties: true,@@ -142,6 +156,8 @@ export class FetchService implements OnModuleInit {text];+ let rejectedBody: string;+for (const candidate of candidates) {if (typeof candidate !== 'string') {continue;@@ -157,6 +173,8 @@ export class FetchService implements OnModuleInit {try {JSON.parse(body);} catch {+ rejectedBody = body;+continue;}}@@ -170,6 +188,16 @@ export class FetchService implements OnModuleInit {});}+ this.logger.warn(+ `Web fetch tool returned no usable body for ${this.redactUrl(url)}`+ );++ this.logger.debug(+ `Web fetch tool response for ${this.redactUrl(url)} (${+ sources?.length ?? 0+ } sources): ${this.getBodyPreview(rejectedBody || text)}`+ );+return undefined;} catch (error) {this.logger.error(@@ -227,6 +255,22 @@ export class FetchService implements OnModuleInit {: requestUrl.toString();}+ /**+ * Makes a short single-line preview of a response body for the log. Gives+ * back a placeholder if the body is empty.+ */+ private getBodyPreview(body: string): string {+ if (!body) {+ return '(empty)';+ }++ const singleLineBody = body.replace(/\s+/g, ' ').trim();++ return singleLineBody.length > FetchService.BODY_PREVIEW_LENGTH+ ? `${singleLineBody.slice(0, FetchService.BODY_PREVIEW_LENGTH)}…`+ : singleLineBody;+ }+private getMatchingWebFetchRoute(url: string) {try {const { hostname } = new URL(url);diff --git a/libs/common/src/lib/config.ts b/libs/common/src/lib/config.tsindex 3adddf39a8..2285030429 100644--- a/libs/common/src/lib/config.ts+++ b/libs/common/src/lib/config.ts@@ -99,6 +99,7 @@ export const DEFAULT_DATE_RANGE: DateRange = 'max';export const DEFAULT_HOST = '0.0.0.0';export const DEFAULT_LANGUAGE_CODE = 'en';export const DEFAULT_LOCALE = 'en-US';+export const DEFAULT_OPENROUTER_ENGINE_WEB_FETCH = 'openrouter';export const DEFAULT_PAGE_SIZE = 50;export const DEFAULT_PORT = 3333;export const DEFAULT_PROCESSOR_GATHER_ASSET_PROFILE_CONCURRENCY = 1;@@ -295,6 +296,8 @@ export const PROPERTY_IS_DATA_GATHERING_ENABLED = 'IS_DATA_GATHERING_ENABLED';export const PROPERTY_IS_READ_ONLY_MODE = 'IS_READ_ONLY_MODE';export const PROPERTY_IS_USER_SIGNUP_ENABLED = 'IS_USER_SIGNUP_ENABLED';export const PROPERTY_MAX_DAILY_REQUESTS = 'MAX_DAILY_REQUESTS';+export const PROPERTY_OPENROUTER_ENGINE_WEB_FETCH =+ 'OPENROUTER_ENGINE_WEB_FETCH';export const PROPERTY_OPENROUTER_MODEL = 'OPENROUTER_MODEL';export const PROPERTY_OPENROUTER_MODEL_WEB_FETCH = 'OPENROUTER_MODEL_WEB_FETCH';export const PROPERTY_PROXY_ROUTES = 'PROXY_ROUTES';