The minimalistic document archiving platform.
docs(content-extraction): add request timeout configuration (#1365)
Corentin Thomasset881796edd4e3
diff --git a/.changeset/lovely-readers-clean.md b/.changeset/lovely-readers-clean.mdnew file mode 100644index 00000000..300de13a--- /dev/null+++ b/.changeset/lovely-readers-clean.md@@ -0,0 +1,6 @@+---+'@papra/app': patch+---++Increased and made configurable the timeout of content extraction external service.+Docling, Mistral OCR, and Azure DI can now take up to 5 minutes to process a document, instead of the previous 30 seconds, configurable via `DOCLING_REQUEST_TIMEOUT_MS`, `MISTRAL_OCR_REQUEST_TIMEOUT_MS` and `AZURE_DI_REQUEST_TIMEOUT_MS` respectively.diff --git a/apps/docs/src/content/docs/03-guides/13-content-extraction.mdx b/apps/docs/src/content/docs/03-guides/13-content-extraction.mdxindex 059345e4..0f5bad00 100644--- a/apps/docs/src/content/docs/03-guides/13-content-extraction.mdx+++ b/apps/docs/src/content/docs/03-guides/13-content-extraction.mdx@@ -62,6 +62,7 @@ MISTRAL_API_KEY=your-mistral-api-key| --- | --- | --- || `MISTRAL_API_KEY` | – | API key for the Mistral API (shared with the Mistral LLM adapter). || `MISTRAL_OCR_MODEL_NAME` | `mistral-ocr-latest` | The Mistral OCR model to use. |+| `MISTRAL_OCR_REQUEST_TIMEOUT_MS` | `300000` | Request timeout in milliseconds. || `MISTRAL_OCR_MIME_TYPES_ALLOW_LIST` | `application/pdf,image/*` | Mime types this strategy handles. See [mime type filtering](#mime-type-filtering). |### Docling@@ -77,6 +78,7 @@ DOCLING_BASE_URL=http://docling:5001| --- | --- | --- || `DOCLING_BASE_URL` | `http://localhost:5001` | Base URL of the Docling service. || `DOCLING_API_KEY` | – | Optional API key, sent as the `X-API-Key` header. On the Docling side this maps to `DOCLING_SERVE_API_KEY`. |+| `DOCLING_REQUEST_TIMEOUT_MS` | `300000` | Request timeout in milliseconds. || `DOCLING_MIME_TYPES_ALLOW_LIST` | `*` | Mime types this strategy handles. See [mime type filtering](#mime-type-filtering). |### Azure Document Intelligence@@ -93,6 +95,7 @@ AZURE_DI_API_KEY=your-azure-api-key| --- | --- | --- || `AZURE_DI_ENDPOINT` | – | Endpoint of the Azure Document Intelligence resource. || `AZURE_DI_API_KEY` | – | API key, sent as the `Ocp-Apim-Subscription-Key` header. |+| `AZURE_DI_REQUEST_TIMEOUT_MS` | `300000` | Request timeout in milliseconds, applied to the job creation and polling requests. || `AZURE_DI_MIME_TYPES_ALLOW_LIST` | `*` | Mime types this strategy handles. See [mime type filtering](#mime-type-filtering). || `AZURE_DI_POLLING_ATTEMPTS` | `10` | Number of times to poll for the result of a processing job. || `AZURE_DI_POLLING_DELAY_MS` | `2000` | Delay in milliseconds between polling attempts. |diff --git a/apps/papra-server/src/modules/documents/content-extraction/content-extraction-strategies/azure-di/azure-di.content-extraction-strategy.config.ts b/apps/papra-server/src/modules/documents/content-extraction/content-extraction-strategies/azure-di/azure-di.content-extraction-strategy.config.tsindex 9536d9c0..bed35fbd 100644--- a/apps/papra-server/src/modules/documents/content-extraction/content-extraction-strategies/azure-di/azure-di.content-extraction-strategy.config.ts+++ b/apps/papra-server/src/modules/documents/content-extraction/content-extraction-strategies/azure-di/azure-di.content-extraction-strategy.config.ts@@ -2,6 +2,7 @@ import * as v from 'valibot';import type { AppConfigDefinition } from '../../../../config/config.types';import { urlSchema } from '../../../../config/config.schemas';import { coercedStrictlyPositiveIntegerSchema } from '../../../../shared/schemas/number.schemas';+import { IN_MS } from '../../../../shared/units';export const azureDiConfig = {endpoint: {@@ -16,6 +17,12 @@ export const azureDiConfig = {schema: v.string(),default: '',},+ timeoutMs: {+ doc: 'The timeout in milliseconds for requests to the Azure Document Intelligence service.',+ env: 'AZURE_DI_REQUEST_TIMEOUT_MS',+ schema: coercedStrictlyPositiveIntegerSchema,+ default: 5 * IN_MS.MINUTE,+ },mimeTypesAllowList: {doc: 'The list of mime types that the Azure Document Inteligence strategy will be used for. If the document mime type is not in this list, the strategy will be skipped. Comma separated list. Supports wildcards, e.g. "image/*" matches all image mime types, and "*" matches all formats. Prefix an entry with "!" to negate it, e.g. "*,!image/png" allows everything except PNG. Negations always take precedence over allows, even more specific ones (e.g. "image/png,!image/*" rejects PNG).',env: 'AZURE_DI_MIME_TYPES_ALLOW_LIST',diff --git a/apps/papra-server/src/modules/documents/content-extraction/content-extraction-strategies/azure-di/azure-di.content-extraction-strategy.ts b/apps/papra-server/src/modules/documents/content-extraction/content-extraction-strategies/azure-di/azure-di.content-extraction-strategy.tsindex 32d2b974..e1de0178 100644--- a/apps/papra-server/src/modules/documents/content-extraction/content-extraction-strategies/azure-di/azure-di.content-extraction-strategy.ts+++ b/apps/papra-server/src/modules/documents/content-extraction/content-extraction-strategies/azure-di/azure-di.content-extraction-strategy.ts@@ -9,7 +9,7 @@ export function buildAzureDiContentExtractionStrategy({}: {config: Config;}): ContentExtractionStrategy {- const { endpoint, apiKey, mimeTypesAllowList, pollingAttempts, pollingDelayMs } =+ const { endpoint, apiKey, mimeTypesAllowList, pollingAttempts, pollingDelayMs, timeoutMs } =config.documentContentExtraction.strategy.azureDi;return {@@ -35,6 +35,7 @@ export function buildAzureDiContentExtractionStrategy({apiKey,pollingAttempts,pollingDelayMs,+ timeoutMs,});return { text };diff --git a/apps/papra-server/src/modules/documents/content-extraction/content-extraction-strategies/azure-di/azure-di.content-extraction-strategy.usecases.ts b/apps/papra-server/src/modules/documents/content-extraction/content-extraction-strategies/azure-di/azure-di.content-extraction-strategy.usecases.tsindex 7397fc84..4db6d0d7 100644--- a/apps/papra-server/src/modules/documents/content-extraction/content-extraction-strategies/azure-di/azure-di.content-extraction-strategy.usecases.ts+++ b/apps/papra-server/src/modules/documents/content-extraction/content-extraction-strategies/azure-di/azure-di.content-extraction-strategy.usecases.ts@@ -1,5 +1,4 @@import { ofetch } from 'ofetch';-import { IN_MS } from '../../../../shared/units';import * as v from 'valibot';import { buildUrl } from '@corentinth/chisels';import { sleep } from '@papra/std';@@ -26,6 +25,7 @@ export async function extractTextWithAzureDi({apiKey,pollingAttempts,pollingDelayMs,+ timeoutMs,logger = createLogger({ namespace: 'azure-di.content-extraction-strategy.usecases' }),}: {file: File;@@ -33,6 +33,7 @@ export async function extractTextWithAzureDi({apiKey: string;pollingAttempts: number;pollingDelayMs: number;+ timeoutMs: number;logger?: Logger;}) {const url = buildUrl({@@ -50,7 +51,7 @@ export async function extractTextWithAzureDi({'Content-Type': 'application/octet-stream','Ocp-Apim-Subscription-Key': apiKey,},- timeout: 10 * IN_MS.SECOND,+ timeout: timeoutMs,body: file,});@@ -67,6 +68,7 @@ export async function extractTextWithAzureDi({apiKey,pollingAttempts,pollingDelayMs,+ timeoutMs,logger,});@@ -78,12 +80,14 @@ export async function pollJobResult({apiKey,pollingAttempts,pollingDelayMs,+ timeoutMs,logger,}: {operationLocation: string;apiKey: string;pollingAttempts: number;pollingDelayMs: number;+ timeoutMs: number;logger: Logger;}) {for (let attempt = 0; attempt < pollingAttempts; attempt++) {@@ -92,7 +96,7 @@ export async function pollJobResult({headers: {'Ocp-Apim-Subscription-Key': apiKey,},- timeout: 10 * IN_MS.SECOND,+ timeout: timeoutMs,});const result = v.parse(jobResultResponseSchema, response);diff --git a/apps/papra-server/src/modules/documents/content-extraction/content-extraction-strategies/docling/docling.content-extraction-strategy.config.ts b/apps/papra-server/src/modules/documents/content-extraction/content-extraction-strategies/docling/docling.content-extraction-strategy.config.tsindex f7823f88..a154bfed 100644--- a/apps/papra-server/src/modules/documents/content-extraction/content-extraction-strategies/docling/docling.content-extraction-strategy.config.ts+++ b/apps/papra-server/src/modules/documents/content-extraction/content-extraction-strategies/docling/docling.content-extraction-strategy.config.ts@@ -1,6 +1,8 @@import * as v from 'valibot';import type { AppConfigDefinition } from '../../../../config/config.types';import { urlSchema } from '../../../../config/config.schemas';+import { coercedStrictlyPositiveIntegerSchema } from '../../../../shared/schemas/number.schemas';+import { IN_MS } from '../../../../shared/units';export const doclingConfig = {baseUrl: {@@ -15,6 +17,12 @@ export const doclingConfig = {schema: v.optional(v.string()),default: undefined,},+ timeoutMs: {+ doc: 'The timeout in milliseconds for requests to the Docling service.',+ env: 'DOCLING_REQUEST_TIMEOUT_MS',+ schema: coercedStrictlyPositiveIntegerSchema,+ default: 5 * IN_MS.MINUTE,+ },mimeTypesAllowList: {doc: 'The list of mime types that the Docling strategy will be used for. If the document mime type is not in this list, the strategy will be skipped. Comma separated list. Supports wildcards, e.g. "image/*" matches all image mime types, and "*" matches all formats. Prefix an entry with "!" to negate it, e.g. "*,!image/png" allows everything except PNG. Negations always take precedence over allows, even more specific ones (e.g. "image/png,!image/*" rejects PNG).',env: 'DOCLING_MIME_TYPES_ALLOW_LIST',diff --git a/apps/papra-server/src/modules/documents/content-extraction/content-extraction-strategies/docling/docling.content-extraction-strategy.ts b/apps/papra-server/src/modules/documents/content-extraction/content-extraction-strategies/docling/docling.content-extraction-strategy.tsindex 7330372b..da6dacdb 100644--- a/apps/papra-server/src/modules/documents/content-extraction/content-extraction-strategies/docling/docling.content-extraction-strategy.ts+++ b/apps/papra-server/src/modules/documents/content-extraction/content-extraction-strategies/docling/docling.content-extraction-strategy.ts@@ -8,7 +8,8 @@ export function buildDoclingContentExtractionStrategy({}: {config: Config;}): ContentExtractionStrategy {- const { baseUrl, apiKey, mimeTypesAllowList } = config.documentContentExtraction.strategy.docling;+ const { baseUrl, apiKey, mimeTypesAllowList, timeoutMs } =+ config.documentContentExtraction.strategy.docling;return {canExtractTextFromDocument: async ({ file }) => {@@ -23,6 +24,7 @@ export function buildDoclingContentExtractionStrategy({file,baseUrl,apiKey,+ timeoutMs,});return { text };diff --git a/apps/papra-server/src/modules/documents/content-extraction/content-extraction-strategies/docling/docling.content-extraction-strategy.usecases.ts b/apps/papra-server/src/modules/documents/content-extraction/content-extraction-strategies/docling/docling.content-extraction-strategy.usecases.tsindex 2db982fd..17b7f56a 100644--- a/apps/papra-server/src/modules/documents/content-extraction/content-extraction-strategies/docling/docling.content-extraction-strategy.usecases.ts+++ b/apps/papra-server/src/modules/documents/content-extraction/content-extraction-strategies/docling/docling.content-extraction-strategy.usecases.ts@@ -1,5 +1,4 @@import { ofetch } from 'ofetch';-import { IN_MS } from '../../../../shared/units';import * as v from 'valibot';import { stripDoclingImagePlaceholders } from './docling.content-extraction-strategy.models';@@ -14,10 +13,12 @@ export async function extractTextWithDoclingServer({file,baseUrl,apiKey,+ timeoutMs,}: {file: File;baseUrl: string;apiKey?: string;+ timeoutMs: number;}) {const url = `${baseUrl.replace(/\/$/, '')}/v1/convert/file`;@@ -32,7 +33,7 @@ export async function extractTextWithDoclingServer({// 'Content-Type': 'multipart/form-data',...(apiKey ? { 'X-Api-Key': apiKey } : {}),},- timeout: 30 * IN_MS.SECOND,+ timeout: timeoutMs,body,});diff --git a/apps/papra-server/src/modules/documents/content-extraction/content-extraction-strategies/mistral-ocr/mistral-ocr.content-extraction-strategy.config.ts b/apps/papra-server/src/modules/documents/content-extraction/content-extraction-strategies/mistral-ocr/mistral-ocr.content-extraction-strategy.config.tsindex a1c75fe2..0f1a0493 100644--- a/apps/papra-server/src/modules/documents/content-extraction/content-extraction-strategies/mistral-ocr/mistral-ocr.content-extraction-strategy.config.ts+++ b/apps/papra-server/src/modules/documents/content-extraction/content-extraction-strategies/mistral-ocr/mistral-ocr.content-extraction-strategy.config.ts@@ -1,5 +1,7 @@import * as v from 'valibot';import type { AppConfigDefinition } from '../../../../config/config.types';+import { coercedStrictlyPositiveIntegerSchema } from '../../../../shared/schemas/number.schemas';+import { IN_MS } from '../../../../shared/units';export const mistralOcrConfig = {modelName: {@@ -8,6 +10,12 @@ export const mistralOcrConfig = {schema: v.string(),default: 'mistral-ocr-latest',},+ timeoutMs: {+ doc: 'The timeout in milliseconds for requests to the Mistral OCR service.',+ env: 'MISTRAL_OCR_REQUEST_TIMEOUT_MS',+ schema: coercedStrictlyPositiveIntegerSchema,+ default: 5 * IN_MS.MINUTE,+ },mimeTypesAllowList: {doc: 'The list of mime types that the Mistral OCR strategy will be used for. If the document mime type is not in this list, the strategy will be skipped. Comma separated list. Supports wildcards, e.g. "image/*" matches all image mime types, and "*" matches all formats. Prefix an entry with "!" to negate it, e.g. "*,!image/png" allows everything except PNG. Negations always take precedence over allows, even more specific ones (e.g. "image/png,!image/*" rejects PNG).',env: 'MISTRAL_OCR_MIME_TYPES_ALLOW_LIST',diff --git a/apps/papra-server/src/modules/documents/content-extraction/content-extraction-strategies/mistral-ocr/mistral-ocr.content-extraction-strategy.ts b/apps/papra-server/src/modules/documents/content-extraction/content-extraction-strategies/mistral-ocr/mistral-ocr.content-extraction-strategy.tsindex 17c88ec6..b78d4b73 100644--- a/apps/papra-server/src/modules/documents/content-extraction/content-extraction-strategies/mistral-ocr/mistral-ocr.content-extraction-strategy.ts+++ b/apps/papra-server/src/modules/documents/content-extraction/content-extraction-strategies/mistral-ocr/mistral-ocr.content-extraction-strategy.ts@@ -9,7 +9,8 @@ export function buildMistralOcrContentExtractionStrategy({}: {config: Config;}): ContentExtractionStrategy {- const { modelName, mimeTypesAllowList } = config.documentContentExtraction.strategy.mistralOcr;+ const { modelName, mimeTypesAllowList, timeoutMs } =+ config.documentContentExtraction.strategy.mistralOcr;const { baseUrl, apiKey } = config.ai.adapters.mistral;return {@@ -27,6 +28,7 @@ export function buildMistralOcrContentExtractionStrategy({modelName,baseUrl,apiKey,+ timeoutMs,});return { text, extractionContext: { processedPagesCount } };diff --git a/apps/papra-server/src/modules/documents/content-extraction/content-extraction-strategies/mistral-ocr/mistral-ocr.content-extraction-strategy.usecases.ts b/apps/papra-server/src/modules/documents/content-extraction/content-extraction-strategies/mistral-ocr/mistral-ocr.content-extraction-strategy.usecases.tsindex 306db0db..2d01ec5a 100644--- a/apps/papra-server/src/modules/documents/content-extraction/content-extraction-strategies/mistral-ocr/mistral-ocr.content-extraction-strategy.usecases.ts+++ b/apps/papra-server/src/modules/documents/content-extraction/content-extraction-strategies/mistral-ocr/mistral-ocr.content-extraction-strategy.usecases.ts@@ -1,7 +1,6 @@import { ofetch } from 'ofetch';import * as v from 'valibot';import { buildDocumentInfo } from './mistral-ocr.content-extraction-strategy.models';-import { IN_MS } from '../../../../shared/units';const ocrResponseSchema = v.object({pages: v.array(@@ -19,12 +18,14 @@ export async function extractTextWithMistralOcr({modelName,baseUrl,apiKey,+ timeoutMs,}: {file: File;modelName: string;baseUrl: string;apiKey: string;+ timeoutMs: number;}) {const url = `${baseUrl.replace(/\/$/, '')}/ocr`;@@ -34,7 +35,7 @@ export async function extractTextWithMistralOcr({'Content-Type': 'application/json','Authorization': `Bearer ${apiKey}`,},- timeout: 30 * IN_MS.SECOND,+ timeout: timeoutMs,body: JSON.stringify({model: modelName,include_image_base64: false,