Open Source Wealth Management Software. Angular + NestJS + Prisma + Nx + TypeScript 🤍
TypeScript angularetffinancefintechghostfoliohacktoberfestinvestingnestjsosspersonal-financeportfolioprismasoftwarestocktrackertradingtypescriptwealth-managementweb
Task/improve logging of web fetch tool (#8365)
Thomas Kaulfc3cd9477053
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 392d03c480..8200209ec6 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -9,6 +9,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0### Changed+- Improved the logging of the `web_fetch` tool in the `FetchService`+- Improved the validation of the `WEB_FETCH_ROUTES` setting in the `FetchService`- Improved the validation of the response of the `web_fetch` tool in the `FetchService`- Deprecated the `api/v1/admin/user`, `api/v1/user/:id` and `api/v1/user/:id/access-token` endpoints in favor of the `api/v1/users` endpoints- Improved the language localization for German (`de`)diff --git a/apps/api/src/helper/web-fetch.helper.spec.ts b/apps/api/src/helper/web-fetch.helper.spec.tsindex 0759b65005..aa0f756897 100644--- a/apps/api/src/helper/web-fetch.helper.spec.ts+++ b/apps/api/src/helper/web-fetch.helper.spec.ts@@ -1,4 +1,4 @@-import { getWebFetchBody } from './web-fetch.helper';+import { getWebFetchBody, getWebFetchError } from './web-fetch.helper';const FAILURE_MESSAGE ='{"url":"https://www.example.com/data.json","status":"failed","error":"HTTP 405: Method Not Allowed","_failure_reason":"http_error","http_status":405}';@@ -110,3 +110,23 @@ describe('getWebFetchBody', () => {});});});++describe('getWebFetchError', () => {+ it.each([+ ['the raw message', FAILURE_MESSAGE],+ ['a text around the message', `Failed:\n\n${FAILURE_MESSAGE}\n\nSorry.`]+ ])('should get the error from %s', (_, candidate) => {+ expect(getWebFetchError({ candidate })).toBe(+ 'HTTP 405: Method Not Allowed'+ );+ });++ it.each([+ ['a body', BODY],+ ['a text', 'The fetch did not succeed.'],+ ['an empty text', ''],+ ['broken JSON', '{"status":"failed",']+ ])('should get no error from %s', (_, candidate) => {+ expect(getWebFetchError({ candidate })).toBeUndefined();+ });+});diff --git a/apps/api/src/helper/web-fetch.helper.ts b/apps/api/src/helper/web-fetch.helper.tsindex 530c41a5d9..d54e185373 100644--- a/apps/api/src/helper/web-fetch.helper.ts+++ b/apps/api/src/helper/web-fetch.helper.ts@@ -29,6 +29,22 @@ export function getWebFetchBody({return json?.isWhole ? json.text : undefined;}+/**+ * Gets the error of a failed fetch from a candidate of the `web_fetch` tool.+ * Gives back `undefined` if the candidate has no failure message.+ */+export function getWebFetchError({+ candidate+}: {+ candidate: string;+}): string | undefined {+ const value = extractJson(candidate.trim())?.value;++ return isWebFetchFailureMessage(value)+ ? (value as { error: string }).error+ : undefined;+}+/*** Extracts the JSON object or array from a text. The JSON must start at the* beginning of a line and goes to the last matching closing bracket, so that adiff --git a/apps/api/src/services/fetch/fetch.service.spec.ts b/apps/api/src/services/fetch/fetch.service.spec.tsindex 9659a038f0..e53fe4587b 100644--- a/apps/api/src/services/fetch/fetch.service.spec.ts+++ b/apps/api/src/services/fetch/fetch.service.spec.ts@@ -2,6 +2,7 @@ import { PropertyService } from '@ghostfolio/api/services/property/property.servimport {PROPERTY_API_KEY_OPENROUTER,PROPERTY_OPENROUTER_ENGINE_WEB_FETCH,+ PROPERTY_OPENROUTER_MODEL,PROPERTY_OPENROUTER_MODEL_WEB_FETCH,PROPERTY_WEB_FETCH_ROUTES} from '@ghostfolio/common/config';@@ -250,6 +251,279 @@ describe('FetchService', () => {expect(directFetch).toHaveBeenCalledTimes(1);});+ describe('logging', () => {+ let error: jest.SpyInstance;+ let log: jest.SpyInstance;+ let warn: jest.SpyInstance;++ beforeEach(() => {+ error = jest.spyOn(Logger.prototype, 'error').mockImplementation();+ log = jest.spyOn(Logger.prototype, 'log').mockImplementation();+ warn = jest.spyOn(Logger.prototype, 'warn').mockImplementation();+ });++ afterEach(() => {+ error.mockRestore();+ log.mockRestore();+ warn.mockRestore();+ });++ it('should log a successful web fetch', async () => {+ mockGenerateText({ text: BODY });++ await fetchService.fetch(URL_WITH_ROUTE);++ expect(log).toHaveBeenCalledTimes(1);+ expect(log.mock.calls[0][0]).toMatch(+ /\(engine=parallel finishReason=stop model=model duration=\d+ms\)$/+ );+ expect(warn).not.toHaveBeenCalled();+ });++ it('should log the finish reason of a body from a source', async () => {+ mockGenerateText({+ finishReason: 'length',+ sources: [{ providerMetadata: { openrouter: { content: BODY } } }],+ text: '{"items":['+ });++ await fetchService.fetch(URL_WITH_ROUTE);++ expect(log.mock.calls[0][0]).toContain('finishReason=length');+ expect(warn).not.toHaveBeenCalled();+ });++ it.each(['apikey', 'APIKEY', 'api_token', 'key', 'token'])(+ 'should redact the query parameter %s in the log of a successful web fetch',+ async (name) => {+ mockGenerateText({ text: BODY });++ await fetchService.fetch(`${URL_WITH_ROUTE}?${name}=secret&a=b`);++ expect(log.mock.calls[0][0]).toContain(`${name}=*******&a=b`);+ expect(log.mock.calls[0][0]).not.toContain('secret');+ }+ );++ it.each([+ [+ 'a failure message',+ { text: FAILURE_MESSAGE },+ 'reason=failure_message error="HTTP 405: Method Not Allowed"'+ ],+ [+ 'a failure message with an empty error',+ { text: '{"url":"u","status":"failed","error":""}' },+ 'reason=failure_message error="(empty)"'+ ],+ [+ 'a failure message of a source before a text without JSON',+ {+ sources: [+ { providerMetadata: { openrouter: { content: FAILURE_MESSAGE } } }+ ],+ text: 'Sorry.'+ },+ 'reason=failure_message error="HTTP 405: Method Not Allowed"'+ ],+ [+ 'a failure message after a source without JSON',+ {+ sources: [+ { providerMetadata: { openrouter: { content: 'Sorry.' } } }+ ],+ text: FAILURE_MESSAGE+ },+ 'reason=failure_message error="HTTP 405: Method Not Allowed"'+ ],+ ['a text without JSON', { text: 'Sorry.' }, 'reason=invalid_body'],+ ['an empty text', { text: '' }, 'reason=empty_body'],+ [+ 'a truncated body',+ { finishReason: 'length', text: BODY },+ 'finishReason=length model=model duration='+ ],+ [+ 'the reason of a truncated body',+ { finishReason: 'length', text: BODY },+ 'reason=finish_reason'+ ]+ ])('should log one warning on %s', async (_, result, expected) => {+ mockGenerateText(result);++ await fetchService.fetch(URL_WITH_ROUTE);++ expect(warn).toHaveBeenCalledTimes(1);+ expect(warn.mock.calls[0][0]).toContain(expected);+ expect(log).not.toHaveBeenCalled();+ });++ it.each([+ ['https://www.example.com?unknown=secret', 'No access to [url]'],+ [+ 'https://www.example.com?u=https%3A%2F%2Fx%3Fapikey%3Dsecret',+ 'No access to [url]'+ ],+ ['HTTPS://www.example.com?apikey=secret', 'No access to [url]']+ ])(+ 'should replace the URL %s in the error of a failure message',+ async (url, expected) => {+ mockGenerateText({+ text: JSON.stringify({+ error: `No access to ${url}`,+ status: 'failed',+ url: 'https://www.example.com'+ })+ });++ await fetchService.fetch(URL_WITH_ROUTE);++ expect(warn.mock.calls[0][0]).toContain(`error="${expected}"`);+ expect(warn.mock.calls[0][0]).not.toContain('secret');+ }+ );++ it('should redact the API keys outside of a URL in the error of a failure message', async () => {+ mockGenerateText({+ text: '{"url":"https://www.example.com","status":"failed","error":"Invalid apikey=secret"}'+ });++ await fetchService.fetch(URL_WITH_ROUTE);++ expect(warn.mock.calls[0][0]).toContain('apikey=*******');+ expect(warn.mock.calls[0][0]).not.toContain('secret');+ });++ it('should log an error of the model', async () => {+ (generateText as jest.Mock).mockRejectedValue(new Error('Timeout'));++ await fetchService.fetch(URL_WITH_ROUTE);++ expect(error).toHaveBeenCalledTimes(1);+ expect(error.mock.calls[0][0]).toMatch(+ /\(engine=parallel model=model duration=\d+ms reason=request_error\): Timeout$/+ );+ });++ it('should log the name of a missing model', async () => {+ delete properties[PROPERTY_OPENROUTER_MODEL_WEB_FETCH];++ await fetchService.fetch(URL_WITH_ROUTE);++ expect(warn).toHaveBeenCalledTimes(1);+ expect(warn.mock.calls[0][0]).toContain(+ `reason=missing_configuration missing=${PROPERTY_OPENROUTER_MODEL})`+ );+ expect(warn.mock.calls[0][0]).not.toContain('api-key');+ });++ it('should log the name of a missing API key', async () => {+ delete properties[PROPERTY_API_KEY_OPENROUTER];++ await fetchService.fetch(URL_WITH_ROUTE);++ expect(warn).toHaveBeenCalledTimes(1);+ expect(warn.mock.calls[0][0]).toContain(+ `reason=missing_configuration missing=${PROPERTY_API_KEY_OPENROUTER})`+ );+ });++ it('should log the names of a missing API key and model', async () => {+ delete properties[PROPERTY_API_KEY_OPENROUTER];+ delete properties[PROPERTY_OPENROUTER_MODEL_WEB_FETCH];++ await fetchService.fetch(URL_WITH_ROUTE);++ expect(warn.mock.calls[0][0]).toContain(+ `missing=${PROPERTY_API_KEY_OPENROUTER},${PROPERTY_OPENROUTER_MODEL})`+ );+ });++ it('should log a missing configuration only one time', async () => {+ delete properties[PROPERTY_API_KEY_OPENROUTER];++ await fetchService.fetch(URL_WITH_ROUTE);+ await fetchService.fetch(URL_WITH_ROUTE);++ expect(warn).toHaveBeenCalledTimes(1);+ expect(directFetch).toHaveBeenCalledTimes(2);+ });+ });++ describe('validation of the routes', () => {+ let warn: jest.SpyInstance;++ beforeEach(() => {+ warn = jest.spyOn(Logger.prototype, 'warn').mockImplementation();+ });++ afterEach(() => {+ warn.mockRestore();+ });++ it('should accept valid routes without a warning', async () => {+ await fetchService.onModuleInit();++ expect(warn).not.toHaveBeenCalled();+ });++ it('should accept a missing property without a warning', async () => {+ delete properties[PROPERTY_WEB_FETCH_ROUTES];++ await fetchService.onModuleInit();++ expect(warn).not.toHaveBeenCalled();+ });++ it.each([+ ['an object', { domain: 'example.com' }],+ ['a string', 'example.com'],+ ['a number', 1]+ ])('should ignore %s as the value of the property', async (_, value) => {+ properties[PROPERTY_WEB_FETCH_ROUTES] = value;++ await fetchService.onModuleInit();++ expect(warn).toHaveBeenCalledTimes(1);+ expect(warn.mock.calls[0][0]).toContain('It must be an array');++ await fetchService.fetch(URL_WITH_ROUTE);++ expect(generateText).not.toHaveBeenCalled();+ });++ it.each([+ ['a route which is not an object', 'example.com'],+ ['a route without a domain', {}],+ ['a domain with a protocol', { domain: 'https://example.com' }],+ ['a domain with a path', { domain: 'example.com/data' }],+ [+ 'a content type which is not a string',+ { domain: 'example.com', responseContentType: 1 }+ ]+ ])('should ignore %s and keep the valid routes', async (_, route) => {+ mockGenerateText({ text: 'Plain text' });++ properties[PROPERTY_WEB_FETCH_ROUTES] = [+ route,+ { domain: 'example.org' }+ ];++ await fetchService.onModuleInit();++ expect(warn).toHaveBeenCalledTimes(1);+ expect(warn.mock.calls[0][0]).toContain('Ignoring an invalid route');++ await fetchService.fetch(URL_WITH_ROUTE);++ expect(generateText).not.toHaveBeenCalled();++ await fetchService.fetch('https://example.org/page');++ expect(generateText).toHaveBeenCalledTimes(1);+ });+ });+it('should not use the model for a domain without a route', async () => {await fetchService.fetch('https://www.ghostfol.io');diff --git a/apps/api/src/services/fetch/fetch.service.ts b/apps/api/src/services/fetch/fetch.service.tsindex d8648154e5..9e1791cf3c 100644--- a/apps/api/src/services/fetch/fetch.service.ts+++ b/apps/api/src/services/fetch/fetch.service.ts@@ -1,5 +1,7 @@-import { redactPaths } from '@ghostfolio/api/helper/object.helper';-import { getWebFetchBody } from '@ghostfolio/api/helper/web-fetch.helper';+import {+ getWebFetchBody,+ getWebFetchError+} from '@ghostfolio/api/helper/web-fetch.helper';import { PropertyService } from '@ghostfolio/api/services/property/property.service';import {DEFAULT_OPENROUTER_ENGINE_WEB_FETCH,@@ -14,6 +16,8 @@ import {import { Injectable, Logger, OnModuleInit } from '@nestjs/common';import { createOpenRouter } from '@openrouter/ai-sdk-provider';import { generateText, jsonSchema, tool } from 'ai';+import { isFQDN } from 'class-validator';+import { isPlainObject } from 'lodash-es';import ms from 'ms';import { ProxyRoute } from './interfaces/proxy-route.interface';@@ -24,9 +28,15 @@ export class FetchService implements OnModuleInit {private readonly logger = new Logger(FetchService.name);private static readonly BODY_PREVIEW_LENGTH = 500;- private static readonly REDACTED_QUERY_PARAM_NAMES = ['apikey', 'api_token'];+ private static readonly REDACTED_QUERY_PARAM_NAMES = [+ 'apikey',+ 'api_token',+ 'key',+ 'token'+ ];private static readonly WEB_FETCH_TIMEOUT = ms('30 seconds');+ private hasLoggedMissingConfiguration = false;private proxyRoutes: ProxyRoute[] = [];private webFetchRoutes: WebFetchRoute[] = [];@@ -38,10 +48,11 @@ export class FetchService implements OnModuleInit {PROPERTY_PROXY_ROUTES)) ?? [];- this.webFetchRoutes =- (await this.propertyService.getByKey<WebFetchRoute[]>(+ this.webFetchRoutes = this.getValidWebFetchRoutes(+ await this.propertyService.getByKey<WebFetchRoute[]>(PROPERTY_WEB_FETCH_ROUTES- )) ?? [];+ )+ );}public async fetch(input: RequestInfo | URL, init?: RequestInit) {@@ -114,9 +125,27 @@ export class FetchService implements OnModuleInit {const model = openRouterModelWebFetch || openRouterModel;if (!model || !openRouterApiKey) {+ if (!this.hasLoggedMissingConfiguration) {+ const missing = [+ ...(openRouterApiKey ? [] : [PROPERTY_API_KEY_OPENROUTER]),+ ...(model ? [] : [PROPERTY_OPENROUTER_MODEL])+ ].join(',');++ this.logger.warn(+ `Web fetch tool is not available (${this.getWebFetchLogDetails({+ engine,+ reason: `missing_configuration missing=${missing}`+ })})`+ );++ this.hasLoggedMissingConfiguration = true;+ }+return undefined;}+ const startTime = performance.now();+try {const openRouterService = createOpenRouter({ apiKey: openRouterApiKey });@@ -151,12 +180,6 @@ export class FetchService implements OnModuleInit {}});- if (finishReason !== 'stop') {- this.logger.warn(- `Web fetch tool stopped with the reason "${finishReason}" for ${this.redactUrl(url)}`- );- }-const candidates = [...(sources ?? []).map((source) => {return source.providerMetadata?.openrouter?.content;@@ -178,12 +201,22 @@ export class FetchService implements OnModuleInit {const body = getWebFetchBody({ candidate, isJson });if (!body) {- rejectedBody = candidate.trim() || rejectedBody;+ // A failure message tells more than the text of a later candidate+ if (+ !rejectedBody ||+ getWebFetchError({ candidate: rejectedBody }) === undefined+ ) {+ rejectedBody = candidate.trim() || rejectedBody;+ }continue;}- this.logger.debug(`Routed ${this.redactUrl(url)} via web fetch tool`);+ this.logger.log(+ `Routed ${this.redactUrl(url)} via web fetch tool (${this.getWebFetchLogDetails(+ { engine, finishReason, model, startTime }+ )})`+ );return new Response(body, {headers: webFetchRoute.responseContentType@@ -193,7 +226,15 @@ export class FetchService implements OnModuleInit {}this.logger.warn(- `Web fetch tool returned no usable body for ${this.redactUrl(url)}`+ `Web fetch tool returned no usable body for ${this.redactUrl(url)} (${this.getWebFetchLogDetails(+ {+ engine,+ finishReason,+ model,+ startTime,+ reason: this.getRejectionReason({ finishReason, rejectedBody })+ }+ )})`);this.logger.debug(@@ -205,9 +246,9 @@ export class FetchService implements OnModuleInit {return undefined;} catch (error) {this.logger.error(- `Web fetch tool failed for ${this.redactUrl(url)}: ${- error instanceof Error ? error.message : String(error)- }`+ `Web fetch tool failed for ${this.redactUrl(url)} (${this.getWebFetchLogDetails(+ { engine, model, startTime, reason: 'request_error' }+ )}): ${error instanceof Error ? error.message : String(error)}`);return undefined;@@ -296,6 +337,89 @@ export class FetchService implements OnModuleInit {}}+ private getRejectionReason({+ finishReason,+ rejectedBody+ }: {+ finishReason: string;+ rejectedBody?: string;+ }) {+ if (!rejectedBody) {+ return finishReason === 'stop' ? 'empty_body' : 'finish_reason';+ }++ const error = getWebFetchError({ candidate: rejectedBody });++ // The model can repeat the URL of the prompt, which can contain a secret+ return error !== undefined+ ? `failure_message error="${this.getBodyPreview(error.replace(/https?:\/\/\S+/gi, '[url]'))}"`+ : 'invalid_body';+ }++ /**+ * Gives back the valid routes of the property. A route which is not valid+ * would never match, so it gets a warning instead of a silent direct request.+ */+ private getValidWebFetchRoutes(routes: unknown): WebFetchRoute[] {+ if (routes === undefined || routes === null) {+ return [];+ }++ if (!Array.isArray(routes)) {+ this.logger.warn(+ `Ignoring the property ${PROPERTY_WEB_FETCH_ROUTES}: It must be an array`+ );++ return [];+ }++ return routes.filter((route): route is WebFetchRoute => {+ const isValid = this.isValidWebFetchRoute(route);++ if (!isValid) {+ this.logger.warn(+ `Ignoring an invalid route of the property ${PROPERTY_WEB_FETCH_ROUTES}: ${JSON.stringify(route)}`+ );+ }++ return isValid;+ });+ }++ private getWebFetchLogDetails({+ engine,+ finishReason,+ model,+ reason,+ startTime+ }: {+ engine: string;+ finishReason?: string;+ model?: string;+ reason?: string;+ startTime?: number;+ }) {+ const details = {+ engine,+ finishReason,+ model,+ duration:+ startTime === undefined+ ? undefined+ : `${Math.round(performance.now() - startTime)}ms`,+ reason+ };++ return Object.entries(details)+ .filter(([, value]) => {+ return value !== undefined;+ })+ .map(([key, value]) => {+ return `${key}=${value}`;+ })+ .join(' ');+ }+private hostnameMatchesDomain({domain,hostname@@ -306,17 +430,28 @@ export class FetchService implements OnModuleInit {return hostname === domain || hostname.endsWith(`.${domain}`);}+ private isValidWebFetchRoute(route: unknown) {+ if (!isPlainObject(route)) {+ return false;+ }++ const { domain, responseContentType } = route as Record<string, unknown>;++ return (+ isFQDN(domain) &&+ (responseContentType === undefined ||+ typeof responseContentType === 'string')+ );+ }+private redactUrl(rawUrl: string): string {try {const url = new URL(rawUrl);- const redacted = redactPaths({- object: Object.fromEntries(url.searchParams),- paths: FetchService.REDACTED_QUERY_PARAM_NAMES- });-- for (const [key, value] of Object.entries(redacted)) {- if (value === null) {+ for (const key of [...url.searchParams.keys()]) {+ if (+ FetchService.REDACTED_QUERY_PARAM_NAMES.includes(key.toLowerCase())+ ) {url.searchParams.set(key, '*******');}}