Skip to content

Open Source Wealth Management Software. Angular + NestJS + Prisma + Nx + TypeScript 🤍

Task/improve web_fetch tool in fetch service (#7730)

Thomas Kaul1355098bdb3c
diff --git a/CHANGELOG.md b/CHANGELOG.md
index b54041fb08..e2cd64b0e5 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -11,6 +11,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
- Added an expiration date to the access to share the portfolio
- Added the date of the last usage to the access to share the portfolio
+- Added support for a dedicated _OpenRouter_ engine for the `web_fetch` tool in the `FetchService`
+
+### Changed
+
+- Improved the logging of the `web_fetch` tool in the `FetchService`
## 3.61.0 - 2026-08-25
diff --git a/apps/api/src/services/data-provider/data-enhancer/trackinsight/trackinsight.service.ts b/apps/api/src/services/data-provider/data-enhancer/trackinsight/trackinsight.service.ts
index 23a8ce33cc..aa61c048c7 100644
--- a/apps/api/src/services/data-provider/data-enhancer/trackinsight/trackinsight.service.ts
+++ b/apps/api/src/services/data-provider/data-enhancer/trackinsight/trackinsight.service.ts
@@ -64,10 +64,10 @@ export class TrackinsightDataEnhancerService implements DataEnhancerInterface {
symbol
});
- if (!trackinsightSymbol) {
+ if (!trackinsightSymbol && symbol.includes('.')) {
trackinsightSymbol = await this.searchTrackinsightSymbol({
requestTimeout,
- symbol: symbol.split('.')?.[0]
+ symbol: symbol.split('.')[0]
});
}
@@ -213,6 +213,12 @@ export class TrackinsightDataEnhancerService implements DataEnhancerInterface {
return jsonRes['results']['docs'][0]['ticker'];
}
+ this.logger.debug(
+ `Could not match a Trackinsight symbol for "${symbol}": ${JSON.stringify(
+ jsonRes
+ )}`
+ );
+
return undefined;
})
.catch(({ message }) => {
diff --git a/apps/api/src/services/fetch/fetch.service.ts b/apps/api/src/services/fetch/fetch.service.ts
index 1f53203785..1cf260ed3e 100644
--- a/apps/api/src/services/fetch/fetch.service.ts
+++ b/apps/api/src/services/fetch/fetch.service.ts
@@ -1,7 +1,9 @@
import { redactPaths } from '@ghostfolio/api/helper/object.helper';
import { PropertyService } from '@ghostfolio/api/services/property/property.service';
import {
+ DEFAULT_OPENROUTER_ENGINE_WEB_FETCH,
PROPERTY_API_KEY_OPENROUTER,
+ PROPERTY_OPENROUTER_ENGINE_WEB_FETCH,
PROPERTY_OPENROUTER_MODEL,
PROPERTY_OPENROUTER_MODEL_WEB_FETCH,
PROPERTY_PROXY_ROUTES,
@@ -20,6 +22,7 @@ import { WebFetchRoute } from './interfaces/web-fetch-route.interface';
export class FetchService implements OnModuleInit {
private readonly logger = new Logger(FetchService.name);
+ private static readonly BODY_PREVIEW_LENGTH = 500;
private static readonly REDACTED_QUERY_PARAM_NAMES = ['apikey', 'api_token'];
private static readonly WEB_FETCH_TIMEOUT = ms('30 seconds');
@@ -91,15 +94,22 @@ export class FetchService implements OnModuleInit {
url: string;
webFetchRoute: WebFetchRoute;
}) {
- const [openRouterApiKey, openRouterModel, openRouterModelWebFetch] =
- await Promise.all([
- this.propertyService.getByKey<string>(PROPERTY_API_KEY_OPENROUTER),
- this.propertyService.getByKey<string>(PROPERTY_OPENROUTER_MODEL),
- this.propertyService.getByKey<string>(
- PROPERTY_OPENROUTER_MODEL_WEB_FETCH
- )
- ]);
-
+ const [
+ openRouterApiKey,
+ openRouterEngineWebFetch,
+ openRouterModel,
+ openRouterModelWebFetch
+ ] = await Promise.all([
+ this.propertyService.getByKey<string>(PROPERTY_API_KEY_OPENROUTER),
+ this.propertyService.getByKey<string>(
+ PROPERTY_OPENROUTER_ENGINE_WEB_FETCH
+ ),
+ this.propertyService.getByKey<string>(PROPERTY_OPENROUTER_MODEL),
+ this.propertyService.getByKey<string>(PROPERTY_OPENROUTER_MODEL_WEB_FETCH)
+ ]);
+
+ const engine =
+ openRouterEngineWebFetch || DEFAULT_OPENROUTER_ENGINE_WEB_FETCH;
const model = openRouterModelWebFetch || openRouterModel;
if (!model || !openRouterApiKey) {
@@ -119,11 +129,15 @@ export class FetchService implements OnModuleInit {
timeout: FetchService.WEB_FETCH_TIMEOUT,
tools: {
// Provider-executed tool: lets OpenRouter perform the actual web
- // request server-side via its `web_fetch` engine. `id` and `args`
- // are the OpenRouter-specific identifiers. The input schema is left
- // open as the arguments are supplied by the model.
+ // request server-side via its `web_fetch` engine. `args` and `id`
+ // are the OpenRouter-specific identifiers. The engine must stay
+ // nested in `parameters`, because the provider spreads `args` on the
+ // top level of the tool object, where OpenRouter ignores it. Nested
+ // keys need snake case, as only the top level gets converted. The
+ // input schema is left open as the arguments are supplied by the
+ // model.
web_fetch: tool({
- args: { engine: 'openrouter' },
+ args: { parameters: { engine } },
id: 'openrouter.web_fetch',
inputSchema: jsonSchema({
additionalProperties: true,
@@ -142,6 +156,8 @@ export class FetchService implements OnModuleInit {
text
];
+ let rejectedBody: string;
+
for (const candidate of candidates) {
if (typeof candidate !== 'string') {
continue;
@@ -157,6 +173,8 @@ export class FetchService implements OnModuleInit {
try {
JSON.parse(body);
} catch {
+ rejectedBody = body;
+
continue;
}
}
@@ -170,6 +188,16 @@ export class FetchService implements OnModuleInit {
});
}
+ this.logger.warn(
+ `Web fetch tool returned no usable body for ${this.redactUrl(url)}`
+ );
+
+ this.logger.debug(
+ `Web fetch tool response for ${this.redactUrl(url)} (${
+ sources?.length ?? 0
+ } sources): ${this.getBodyPreview(rejectedBody || text)}`
+ );
+
return undefined;
} catch (error) {
this.logger.error(
@@ -227,6 +255,22 @@ export class FetchService implements OnModuleInit {
: requestUrl.toString();
}
+ /**
+ * Makes a short single-line preview of a response body for the log. Gives
+ * back a placeholder if the body is empty.
+ */
+ private getBodyPreview(body: string): string {
+ if (!body) {
+ return '(empty)';
+ }
+
+ const singleLineBody = body.replace(/\s+/g, ' ').trim();
+
+ return singleLineBody.length > FetchService.BODY_PREVIEW_LENGTH
+ ? `${singleLineBody.slice(0, FetchService.BODY_PREVIEW_LENGTH)}…`
+ : singleLineBody;
+ }
+
private getMatchingWebFetchRoute(url: string) {
try {
const { hostname } = new URL(url);
diff --git a/libs/common/src/lib/config.ts b/libs/common/src/lib/config.ts
index 3adddf39a8..2285030429 100644
--- a/libs/common/src/lib/config.ts
+++ b/libs/common/src/lib/config.ts
@@ -99,6 +99,7 @@ export const DEFAULT_DATE_RANGE: DateRange = 'max';
export const DEFAULT_HOST = '0.0.0.0';
export const DEFAULT_LANGUAGE_CODE = 'en';
export const DEFAULT_LOCALE = 'en-US';
+export const DEFAULT_OPENROUTER_ENGINE_WEB_FETCH = 'openrouter';
export const DEFAULT_PAGE_SIZE = 50;
export const DEFAULT_PORT = 3333;
export const DEFAULT_PROCESSOR_GATHER_ASSET_PROFILE_CONCURRENCY = 1;
@@ -295,6 +296,8 @@ export const PROPERTY_IS_DATA_GATHERING_ENABLED = 'IS_DATA_GATHERING_ENABLED';
export const PROPERTY_IS_READ_ONLY_MODE = 'IS_READ_ONLY_MODE';
export const PROPERTY_IS_USER_SIGNUP_ENABLED = 'IS_USER_SIGNUP_ENABLED';
export const PROPERTY_MAX_DAILY_REQUESTS = 'MAX_DAILY_REQUESTS';
+export const PROPERTY_OPENROUTER_ENGINE_WEB_FETCH =
+ 'OPENROUTER_ENGINE_WEB_FETCH';
export const PROPERTY_OPENROUTER_MODEL = 'OPENROUTER_MODEL';
export const PROPERTY_OPENROUTER_MODEL_WEB_FETCH = 'OPENROUTER_MODEL_WEB_FETCH';
export const PROPERTY_PROXY_ROUTES = 'PROXY_ROUTES';