A self-hostable bookmark-everything app (links, notes and images) with AI-based automatic tagging and full text search
fix(workers): fix headers timeout for AI interference jobs. (#2888)
Erik Tews7ef82473d924
diff --git a/docs/docs/03-configuration/01-environment-variables.md b/docs/docs/03-configuration/01-environment-variables.mdindex bec66794..c0e332ba 100644--- a/docs/docs/03-configuration/01-environment-variables.md+++ b/docs/docs/03-configuration/01-environment-variables.md@@ -97,7 +97,7 @@ Either `OPENAI_API_KEY` or `OLLAMA_BASE_URL` need to be set for automatic taggin| OPENAI_API_KEY | No | Not set | The OpenAI key used for automatic tagging. More on that in [here](../06-administration/03-openai.md). || OPENAI_BASE_URL | No | Not set | If you just want to use OpenAI you don't need to pass this variable. If, however, you want to use some other openai compatible API (e.g. azure openai service), set this to the url of the API. || OPENAI_PROXY_URL | No | Not set | HTTP proxy server URL for OpenAI API requests (e.g., `http://proxy.example.com:8080`). |-| OPENAI_TIMEOUT_SEC | No | Not set | Timeout for OpenAI API requests in seconds. If unset, the OpenAI SDK default of 10 minutes is used. Increase this alongside `INFERENCE_JOB_TIMEOUT_SEC` when slow inference providers need more time. |+| OPENAI_TIMEOUT_SEC | No | Not set | Timeout for OpenAI API requests in seconds. If unset, the OpenAI SDK default of 10 minutes is used. The HTTP headers/body timeouts are set to the same value. Note that the OpenAI SDK retries timed-out requests up to 2 times by default. Increase this alongside `INFERENCE_JOB_TIMEOUT_SEC` when slow inference providers need more time. || OPENAI_SERVICE_TIER | No | Not set | Set to `auto`, `default`, or `flex`. Flex processing provides lower costs in exchange for slower response times and occasional resource unavailability. See [OpenAI Flex Processing](https://platform.openai.com/docs/guides/flex-processing) and [Chat Service Tier](https://platform.openai.com/docs/api-reference/chat/object#chat-object-service_tier) for more details. || OPENAI_REASONING_EFFORT | No | Not set | Set to `none`, `minimal`, `low`, `medium`, `high` or `xhigh`. Controls the reasoning effort in reasoning models. See [OpenAI: How Reasoning Works](https://developers.openai.com/api/docs/guides/reasoning#how-reasoning-works). || OLLAMA_BASE_URL | No | Not set | If you want to use ollama for local inference, set the address of ollama API here. |@@ -122,7 +122,7 @@ Either `OPENAI_API_KEY` or `OLLAMA_BASE_URL` need to be set for automatic taggin| INFERENCE_ENABLE_AUTO_TAGGING | No | true | Whether automatic AI tagging is enabled or disabled. || INFERENCE_ENABLE_AUTO_SUMMARIZATION | No | false | Whether automatic AI summarization is enabled or disabled. || INFERENCE_JOB_TIMEOUT_SEC | No | 30 | How long to wait for the inference job to finish before timing out. If you're running ollama without powerful GPUs, you might want to increase the timeout a bit. |-| INFERENCE_FETCH_TIMEOUT_SEC | No | 300 | \[Ollama Only\] The timeout of the fetch request to the ollama server. If your inference requests take longer than the default 5mins, you might want to increase this timeout. |+| INFERENCE_FETCH_TIMEOUT_SEC | No | 300 | \[Ollama Only\] How long to wait for the ollama server to start responding, and between streamed response chunks. If your ollama server takes longer than the default 5mins to start responding, you might want to increase this timeout. || INFERENCE_SUPPORTS_STRUCTURED_OUTPUT | No | Not set | \[DEPRECATED\] Whether the inference model supports structured output or not. Use INFERENCE_OUTPUT_SCHEMA instead. Setting this to true translates to INFERENCE_OUTPUT_SCHEMA=structured, and to false translates to INFERENCE_OUTPUT_SCHEMA=plain. || INFERENCE_OUTPUT_SCHEMA | No | structured | Possible values are "structured", "json", "plain". Structured is the preferred option, but if your model doesn't support it, you can use "json" if your model supports JSON mode, otherwise "plain" which should be supported by all the models but the model might not output the data in the correct format. |diff --git a/packages/shared/customFetch.ts b/packages/shared/customFetch.tsindex 18b4e133..0619b6bd 100644--- a/packages/shared/customFetch.ts+++ b/packages/shared/customFetch.ts@@ -1,24 +1,35 @@-import serverConfig from "./config";+import { Agent, Dispatcher, ProxyAgent, fetch as undiciFetch } from "undici";-// Generic fetch function type that works across environments-type FetchFunction = (- input: RequestInfo | URL | string,- init?: RequestInit,-) => Promise<Response>;+// Dispatchers are reused across calls so that clients built per job (e.g. the+// inference, embeddings and asset-preprocessing workers) share a connection+// pool instead of opening fresh sockets each time.+const dispatchers = new Map<string, Dispatcher>();-// Factory function to create a custom fetch with timeout for any fetch implementation-export function createCustomFetch(fetchImpl: FetchFunction = globalThis.fetch) {- return function customFetch(- input: Parameters<typeof fetchImpl>[0],- init?: Parameters<typeof fetchImpl>[1],- ): ReturnType<typeof fetchImpl> {- const timeout = serverConfig.inference.fetchTimeoutSec * 1000; // Convert to milliseconds- return fetchImpl(input, {- signal: AbortSignal.timeout(timeout),- ...init,- });- };+function getDispatcher(timeoutMs: number, proxyUrl?: string): Dispatcher {+ const key = `${proxyUrl ?? ""}:${timeoutMs}`;+ let dispatcher = dispatchers.get(key);+ if (!dispatcher) {+ const opts = { headersTimeout: timeoutMs, bodyTimeout: timeoutMs };+ dispatcher = proxyUrl+ ? new ProxyAgent({ uri: proxyUrl, ...opts })+ : new Agent(opts);+ dispatchers.set(key, dispatcher);+ }+ return dispatcher;}-// Default export for backward compatibility - uses global fetch-export const customFetch = createCustomFetch();+// Creates a fetch whose undici headers/body timeouts match the given timeout.+// Without this, undici's defaults (5 mins) cut off slow inference requests+// regardless of the configured timeout. We use undici's own fetch alongside+// its Agent so that the fetch and the dispatcher come from the same undici copy.+export function createCustomFetch(+ timeoutMs: number,+ proxyUrl?: string,+): typeof fetch {+ const dispatcher = getDispatcher(timeoutMs, proxyUrl);+ return ((input: RequestInfo | URL, init?: RequestInit) =>+ undiciFetch(+ input as Parameters<typeof undiciFetch>[0],+ { ...init, dispatcher } as Parameters<typeof undiciFetch>[1],+ )) as unknown as typeof fetch;+}diff --git a/packages/shared/inference.ts b/packages/shared/inference.tsindex 420bedc8..1ab83dfd 100644--- a/packages/shared/inference.ts+++ b/packages/shared/inference.ts@@ -1,11 +1,10 @@import { Ollama } from "ollama";import OpenAI from "openai";import { zodResponseFormat } from "openai/helpers/zod";-import * as undici from "undici";import { z } from "zod";import serverConfig from "./config";-import { customFetch } from "./customFetch";+import { createCustomFetch } from "./customFetch";import logger from "./logger";export interface InferenceResponse {@@ -200,9 +199,12 @@ const buildOpenAIClient = (config: OpenAIEmbeddingConfig) =>"X-Title": "Karakeep","HTTP-Referer": "https://karakeep.app",},- fetchOptions: config.proxyUrl- ? { dispatcher: new undici.ProxyAgent(config.proxyUrl) }- : undefined,+ fetch: createCustomFetch(+ config.timeoutSec !== undefined+ ? config.timeoutSec * 1000+ : OpenAI.DEFAULT_TIMEOUT,+ config.proxyUrl,+ ),});export class InferenceClientFactory {@@ -427,7 +429,7 @@ class OllamaInferenceClient implements InferenceClient {this.config = config;this.ollama = new Ollama({host: config.baseUrl,- fetch: customFetch, // Use the custom fetch with configurable timeout+ fetch: createCustomFetch(serverConfig.inference.fetchTimeoutSec * 1000),});}