Skip to content

Open Source Wealth Management Software. Angular + NestJS + Prisma + Nx + TypeScript 🤍

Feature/add support for web fetch in fetch service (#6960)

Thomas Kaul697ef59e3b58
diff --git a/CHANGELOG.md b/CHANGELOG.md
index e0c416fb0b..d094f394b2 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
## Unreleased
+### Added
+
+- Added support for routing selected requests through the _OpenRouter_ `web_fetch` tool in the `FetchService`
+
### Changed
- Extended the countries mapping in the data enhancer for asset profile data via _Trackinsight_
diff --git a/apps/api/src/services/fetch/fetch.module.ts b/apps/api/src/services/fetch/fetch.module.ts
index f98f2f45ce..16e6f5f5d0 100644
--- a/apps/api/src/services/fetch/fetch.module.ts
+++ b/apps/api/src/services/fetch/fetch.module.ts
@@ -1,9 +1,11 @@
import { FetchService } from '@ghostfolio/api/services/fetch/fetch.service';
+import { PropertyModule } from '@ghostfolio/api/services/property/property.module';
import { Module } from '@nestjs/common';
@Module({
exports: [FetchService],
+ imports: [PropertyModule],
providers: [FetchService]
})
export class FetchModule {}
diff --git a/apps/api/src/services/fetch/fetch.service.ts b/apps/api/src/services/fetch/fetch.service.ts
index b3bd022d91..f32e56a1c8 100644
--- a/apps/api/src/services/fetch/fetch.service.ts
+++ b/apps/api/src/services/fetch/fetch.service.ts
@@ -1,15 +1,35 @@
import { redactPaths } from '@ghostfolio/api/helper/object.helper';
+import { PropertyService } from '@ghostfolio/api/services/property/property.service';
+import {
+ PROPERTY_API_KEY_OPENROUTER,
+ PROPERTY_OPENROUTER_MODEL,
+ PROPERTY_WEB_FETCH_ROUTES
+} from '@ghostfolio/common/config';
-import { Injectable, Logger } from '@nestjs/common';
+import { Injectable, Logger, OnModuleInit } from '@nestjs/common';
+import { createOpenRouter } from '@openrouter/ai-sdk-provider';
+import { generateText, jsonSchema, tool } from 'ai';
+import ms from 'ms';
+
+import { WebFetchRoute } from './interfaces/web-fetch-route.interface';
@Injectable()
-export class FetchService {
+export class FetchService implements OnModuleInit {
private static readonly REDACTED_QUERY_PARAM_NAMES = ['apikey', 'api_token'];
+ private static readonly WEB_FETCH_TIMEOUT = ms('30 seconds');
+
+ private webFetchRoutes: WebFetchRoute[] = [];
+
+ public constructor(private readonly propertyService: PropertyService) {}
- public async fetch(
- input: RequestInfo | URL,
- init?: RequestInit
- ): Promise<Response> {
+ public async onModuleInit() {
+ this.webFetchRoutes =
+ (await this.propertyService.getByKey<WebFetchRoute[]>(
+ PROPERTY_WEB_FETCH_ROUTES
+ )) ?? [];
+ }
+
+ public async fetch(input: RequestInfo | URL, init?: RequestInit) {
const method = (
init?.method ??
(input instanceof Request ? input.method : undefined) ??
@@ -21,6 +41,21 @@ export class FetchService {
Logger.debug(`${method} ${urlRedacted}`, 'FetchService');
+ if (method === 'GET') {
+ const webFetchRoute = this.getMatchingWebFetchRoute(url);
+
+ if (webFetchRoute) {
+ const response = await this.fetchViaWebFetchTool({
+ url,
+ webFetchRoute
+ });
+
+ if (response) {
+ return response;
+ }
+ }
+ }
+
try {
return await globalThis.fetch(input, init);
} catch (error) {
@@ -40,6 +75,113 @@ export class FetchService {
}
}
+ private async fetchViaWebFetchTool({
+ url,
+ webFetchRoute
+ }: {
+ url: string;
+ webFetchRoute: WebFetchRoute;
+ }) {
+ const [openRouterApiKey, openRouterModel] = await Promise.all([
+ this.propertyService.getByKey<string>(PROPERTY_API_KEY_OPENROUTER),
+ this.propertyService.getByKey<string>(PROPERTY_OPENROUTER_MODEL)
+ ]);
+
+ if (!openRouterApiKey || !openRouterModel) {
+ return undefined;
+ }
+
+ try {
+ const openRouterService = createOpenRouter({ apiKey: openRouterApiKey });
+
+ const { sources, text } = await generateText({
+ model: openRouterService.chat(openRouterModel),
+ prompt: [
+ 'You have access to a web_fetch tool. You MUST call it to retrieve the URL below, do not answer from prior knowledge.',
+ 'Return the fetched response body exactly as received: raw body only, no commentary, no Markdown, and no code fences.',
+ `URL: ${url}`
+ ].join('\n'),
+ timeout: FetchService.WEB_FETCH_TIMEOUT,
+ tools: {
+ // Provider-defined tool: lets OpenRouter perform the actual web
+ // request server-side via its `web_fetch` engine. `id` and `args`
+ // are the OpenRouter-specific identifiers; the input schema is left
+ // open as the arguments are supplied by the model.
+ web_fetch: tool({
+ args: { engine: 'openrouter' },
+ id: 'openrouter.web_fetch',
+ inputSchema: jsonSchema({
+ additionalProperties: true,
+ type: 'object'
+ }),
+ type: 'provider'
+ })
+ }
+ });
+
+ const candidates = [
+ ...(sources ?? []).map((source) => {
+ return source.providerMetadata?.openrouter?.content;
+ }),
+ text
+ ];
+
+ for (const candidate of candidates) {
+ if (typeof candidate !== 'string') {
+ continue;
+ }
+
+ const body = candidate.trim();
+
+ if (!body) {
+ continue;
+ }
+
+ if (webFetchRoute.responseContentType?.includes('application/json')) {
+ try {
+ JSON.parse(body);
+ } catch {
+ continue;
+ }
+ }
+
+ Logger.debug(
+ `Routed ${this.redactUrl(url)} via web fetch tool`,
+ 'FetchService'
+ );
+
+ return new Response(body, {
+ headers: webFetchRoute.responseContentType
+ ? { 'content-type': webFetchRoute.responseContentType }
+ : undefined
+ });
+ }
+
+ return undefined;
+ } catch (error) {
+ Logger.error(
+ `Web fetch tool failed for ${this.redactUrl(url)}: ${
+ error instanceof Error ? error.message : String(error)
+ }`,
+ 'FetchService'
+ );
+
+ return undefined;
+ }
+ }
+
+ private getMatchingWebFetchRoute(url: string) {
+ try {
+ const { hostname } = new URL(url);
+
+ return this.webFetchRoutes.find(({ domain }) => {
+ return hostname === domain || hostname.endsWith(`.${domain}`);
+ });
+ } catch {
+ return undefined;
+ }
+ }
+
private redactUrl(rawUrl: string): string {
try {
const url = new URL(rawUrl);
diff --git a/apps/api/src/services/fetch/interfaces/web-fetch-route.interface.ts b/apps/api/src/services/fetch/interfaces/web-fetch-route.interface.ts
new file mode 100644
index 0000000000..efff093987
--- /dev/null
+++ b/apps/api/src/services/fetch/interfaces/web-fetch-route.interface.ts
@@ -0,0 +1,19 @@
+/**
+ * Routes outgoing GET requests for a given domain through the OpenRouter
+ * `web_fetch` tool instead of a direct network request.
+ *
+ * Configured via the `WEB_FETCH_ROUTES` property as a JSON array, e.g.
+ *
+ * [
+ * {
+ * "domain": "example.com",
+ * "responseContentType": "application/json"
+ * }
+ * ]
+ *
+ * Matches the domain itself and its subdomains (e.g. `api.example.com`).
+ */
+export interface WebFetchRoute {
+ domain: string;
+ responseContentType?: string;
+}
diff --git a/libs/common/src/lib/config.ts b/libs/common/src/lib/config.ts
index 113dffe4ab..28d902d715 100644
--- a/libs/common/src/lib/config.ts
+++ b/libs/common/src/lib/config.ts
@@ -256,6 +256,7 @@ export const PROPERTY_SLACK_COMMUNITY_USERS = 'SLACK_COMMUNITY_USERS';
export const PROPERTY_STRIPE_CONFIG = 'STRIPE_CONFIG';
export const PROPERTY_SYSTEM_MESSAGE = 'SYSTEM_MESSAGE';
export const PROPERTY_UPTIME = 'UPTIME';
+export const PROPERTY_WEB_FETCH_ROUTES = 'WEB_FETCH_ROUTES';
export const QUEUE_JOB_STATUS_LIST = [
'active',