Open Source Wealth Management Software. Angular + NestJS + Prisma + Nx + TypeScript 🤍
TypeScript angularetffinancefintechghostfoliohacktoberfestinvestingnestjsosspersonal-financeportfolioprismasoftwarestocktrackertradingtypescriptwealth-managementweb
Feature/add support for web fetch in fetch service (#6960)
Thomas Kaul697ef59e3b58
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex e0c416fb0b..d094f394b2 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0## Unreleased+### Added++- Added support for routing selected requests through the _OpenRouter_ `web_fetch` tool in the `FetchService`+### Changed- Extended the countries mapping in the data enhancer for asset profile data via _Trackinsight_diff --git a/apps/api/src/services/fetch/fetch.module.ts b/apps/api/src/services/fetch/fetch.module.tsindex f98f2f45ce..16e6f5f5d0 100644--- a/apps/api/src/services/fetch/fetch.module.ts+++ b/apps/api/src/services/fetch/fetch.module.ts@@ -1,9 +1,11 @@import { FetchService } from '@ghostfolio/api/services/fetch/fetch.service';+import { PropertyModule } from '@ghostfolio/api/services/property/property.module';import { Module } from '@nestjs/common';@Module({exports: [FetchService],+ imports: [PropertyModule],providers: [FetchService]})export class FetchModule {}diff --git a/apps/api/src/services/fetch/fetch.service.ts b/apps/api/src/services/fetch/fetch.service.tsindex b3bd022d91..f32e56a1c8 100644--- a/apps/api/src/services/fetch/fetch.service.ts+++ b/apps/api/src/services/fetch/fetch.service.ts@@ -1,15 +1,35 @@import { redactPaths } from '@ghostfolio/api/helper/object.helper';+import { PropertyService } from '@ghostfolio/api/services/property/property.service';+import {+ PROPERTY_API_KEY_OPENROUTER,+ PROPERTY_OPENROUTER_MODEL,+ PROPERTY_WEB_FETCH_ROUTES+} from '@ghostfolio/common/config';-import { Injectable, Logger } from '@nestjs/common';+import { Injectable, Logger, OnModuleInit } from '@nestjs/common';+import { createOpenRouter } from '@openrouter/ai-sdk-provider';+import { generateText, jsonSchema, tool } from 'ai';+import ms from 'ms';++import { WebFetchRoute } from './interfaces/web-fetch-route.interface';@Injectable()-export class FetchService {+export class FetchService implements OnModuleInit {private static readonly REDACTED_QUERY_PARAM_NAMES = ['apikey', 'api_token'];+ private static readonly WEB_FETCH_TIMEOUT = ms('30 seconds');++ private webFetchRoutes: WebFetchRoute[] = [];++ public constructor(private readonly propertyService: PropertyService) {}- public async fetch(- input: RequestInfo | URL,- init?: RequestInit- ): Promise<Response> {+ public async onModuleInit() {+ this.webFetchRoutes =+ (await this.propertyService.getByKey<WebFetchRoute[]>(+ PROPERTY_WEB_FETCH_ROUTES+ )) ?? [];+ }++ public async fetch(input: RequestInfo | URL, init?: RequestInit) {const method = (init?.method ??(input instanceof Request ? input.method : undefined) ??@@ -21,6 +41,21 @@ export class FetchService {Logger.debug(`${method} ${urlRedacted}`, 'FetchService');+ if (method === 'GET') {+ const webFetchRoute = this.getMatchingWebFetchRoute(url);++ if (webFetchRoute) {+ const response = await this.fetchViaWebFetchTool({+ url,+ webFetchRoute+ });++ if (response) {+ return response;+ }+ }+ }+try {return await globalThis.fetch(input, init);} catch (error) {@@ -40,6 +75,113 @@ export class FetchService {}}+ private async fetchViaWebFetchTool({+ url,+ webFetchRoute+ }: {+ url: string;+ webFetchRoute: WebFetchRoute;+ }) {+ const [openRouterApiKey, openRouterModel] = await Promise.all([+ this.propertyService.getByKey<string>(PROPERTY_API_KEY_OPENROUTER),+ this.propertyService.getByKey<string>(PROPERTY_OPENROUTER_MODEL)+ ]);++ if (!openRouterApiKey || !openRouterModel) {+ return undefined;+ }++ try {+ const openRouterService = createOpenRouter({ apiKey: openRouterApiKey });++ const { sources, text } = await generateText({+ model: openRouterService.chat(openRouterModel),+ prompt: [+ 'You have access to a web_fetch tool. You MUST call it to retrieve the URL below, do not answer from prior knowledge.',+ 'Return the fetched response body exactly as received: raw body only, no commentary, no Markdown, and no code fences.',+ `URL: ${url}`+ ].join('\n'),+ timeout: FetchService.WEB_FETCH_TIMEOUT,+ tools: {+ // Provider-defined tool: lets OpenRouter perform the actual web+ // request server-side via its `web_fetch` engine. `id` and `args`+ // are the OpenRouter-specific identifiers; the input schema is left+ // open as the arguments are supplied by the model.+ web_fetch: tool({+ args: { engine: 'openrouter' },+ id: 'openrouter.web_fetch',+ inputSchema: jsonSchema({+ additionalProperties: true,+ type: 'object'+ }),+ type: 'provider'+ })+ }+ });++ const candidates = [+ ...(sources ?? []).map((source) => {+ return source.providerMetadata?.openrouter?.content;+ }),+ text+ ];++ for (const candidate of candidates) {+ if (typeof candidate !== 'string') {+ continue;+ }++ const body = candidate.trim();++ if (!body) {+ continue;+ }++ if (webFetchRoute.responseContentType?.includes('application/json')) {+ try {+ JSON.parse(body);+ } catch {+ continue;+ }+ }++ Logger.debug(+ `Routed ${this.redactUrl(url)} via web fetch tool`,+ 'FetchService'+ );++ return new Response(body, {+ headers: webFetchRoute.responseContentType+ ? { 'content-type': webFetchRoute.responseContentType }+ : undefined+ });+ }++ return undefined;+ } catch (error) {+ Logger.error(+ `Web fetch tool failed for ${this.redactUrl(url)}: ${+ error instanceof Error ? error.message : String(error)+ }`,+ 'FetchService'+ );++ return undefined;+ }+ }++ private getMatchingWebFetchRoute(url: string) {+ try {+ const { hostname } = new URL(url);++ return this.webFetchRoutes.find(({ domain }) => {+ return hostname === domain || hostname.endsWith(`.${domain}`);+ });+ } catch {+ return undefined;+ }+ }+private redactUrl(rawUrl: string): string {try {const url = new URL(rawUrl);diff --git a/apps/api/src/services/fetch/interfaces/web-fetch-route.interface.ts b/apps/api/src/services/fetch/interfaces/web-fetch-route.interface.tsnew file mode 100644index 0000000000..efff093987--- /dev/null+++ b/apps/api/src/services/fetch/interfaces/web-fetch-route.interface.ts@@ -0,0 +1,19 @@+/**+ * Routes outgoing GET requests for a given domain through the OpenRouter+ * `web_fetch` tool instead of a direct network request.+ *+ * Configured via the `WEB_FETCH_ROUTES` property as a JSON array, e.g.+ *+ * [+ * {+ * "domain": "example.com",+ * "responseContentType": "application/json"+ * }+ * ]+ *+ * Matches the domain itself and its subdomains (e.g. `api.example.com`).+ */+export interface WebFetchRoute {+ domain: string;+ responseContentType?: string;+}diff --git a/libs/common/src/lib/config.ts b/libs/common/src/lib/config.tsindex 113dffe4ab..28d902d715 100644--- a/libs/common/src/lib/config.ts+++ b/libs/common/src/lib/config.ts@@ -256,6 +256,7 @@ export const PROPERTY_SLACK_COMMUNITY_USERS = 'SLACK_COMMUNITY_USERS';export const PROPERTY_STRIPE_CONFIG = 'STRIPE_CONFIG';export const PROPERTY_SYSTEM_MESSAGE = 'SYSTEM_MESSAGE';export const PROPERTY_UPTIME = 'UPTIME';+export const PROPERTY_WEB_FETCH_ROUTES = 'WEB_FETCH_ROUTES';export const QUEUE_JOB_STATUS_LIST = ['active',