Skip to content

armgit / papra

Public Imported from GitHub
0

The minimalistic document archiving platform.

docs(content-extraction): add request timeout configuration (#1365)

Corentin Thomasset881796edd4e3
diff --git a/.changeset/lovely-readers-clean.md b/.changeset/lovely-readers-clean.md
new file mode 100644
index 00000000..300de13a
--- /dev/null
+++ b/.changeset/lovely-readers-clean.md
@@ -0,0 +1,6 @@
+---
+'@papra/app': patch
+---
+
+Increased and made configurable the timeout of content extraction external service.
+Docling, Mistral OCR, and Azure DI can now take up to 5 minutes to process a document, instead of the previous 30 seconds, configurable via `DOCLING_REQUEST_TIMEOUT_MS`, `MISTRAL_OCR_REQUEST_TIMEOUT_MS` and `AZURE_DI_REQUEST_TIMEOUT_MS` respectively.
diff --git a/apps/docs/src/content/docs/03-guides/13-content-extraction.mdx b/apps/docs/src/content/docs/03-guides/13-content-extraction.mdx
index 059345e4..0f5bad00 100644
--- a/apps/docs/src/content/docs/03-guides/13-content-extraction.mdx
+++ b/apps/docs/src/content/docs/03-guides/13-content-extraction.mdx
@@ -62,6 +62,7 @@ MISTRAL_API_KEY=your-mistral-api-key
| --- | --- | --- |
| `MISTRAL_API_KEY` | – | API key for the Mistral API (shared with the Mistral LLM adapter). |
| `MISTRAL_OCR_MODEL_NAME` | `mistral-ocr-latest` | The Mistral OCR model to use. |
+| `MISTRAL_OCR_REQUEST_TIMEOUT_MS` | `300000` | Request timeout in milliseconds. |
| `MISTRAL_OCR_MIME_TYPES_ALLOW_LIST` | `application/pdf,image/*` | Mime types this strategy handles. See [mime type filtering](#mime-type-filtering). |
### Docling
@@ -77,6 +78,7 @@ DOCLING_BASE_URL=http://docling:5001
| --- | --- | --- |
| `DOCLING_BASE_URL` | `http://localhost:5001` | Base URL of the Docling service. |
| `DOCLING_API_KEY` | – | Optional API key, sent as the `X-API-Key` header. On the Docling side this maps to `DOCLING_SERVE_API_KEY`. |
+| `DOCLING_REQUEST_TIMEOUT_MS` | `300000` | Request timeout in milliseconds. |
| `DOCLING_MIME_TYPES_ALLOW_LIST` | `*` | Mime types this strategy handles. See [mime type filtering](#mime-type-filtering). |
### Azure Document Intelligence
@@ -93,6 +95,7 @@ AZURE_DI_API_KEY=your-azure-api-key
| --- | --- | --- |
| `AZURE_DI_ENDPOINT` | – | Endpoint of the Azure Document Intelligence resource. |
| `AZURE_DI_API_KEY` | – | API key, sent as the `Ocp-Apim-Subscription-Key` header. |
+| `AZURE_DI_REQUEST_TIMEOUT_MS` | `300000` | Request timeout in milliseconds, applied to the job creation and polling requests. |
| `AZURE_DI_MIME_TYPES_ALLOW_LIST` | `*` | Mime types this strategy handles. See [mime type filtering](#mime-type-filtering). |
| `AZURE_DI_POLLING_ATTEMPTS` | `10` | Number of times to poll for the result of a processing job. |
| `AZURE_DI_POLLING_DELAY_MS` | `2000` | Delay in milliseconds between polling attempts. |
diff --git a/apps/papra-server/src/modules/documents/content-extraction/content-extraction-strategies/azure-di/azure-di.content-extraction-strategy.config.ts b/apps/papra-server/src/modules/documents/content-extraction/content-extraction-strategies/azure-di/azure-di.content-extraction-strategy.config.ts
index 9536d9c0..bed35fbd 100644
--- a/apps/papra-server/src/modules/documents/content-extraction/content-extraction-strategies/azure-di/azure-di.content-extraction-strategy.config.ts
+++ b/apps/papra-server/src/modules/documents/content-extraction/content-extraction-strategies/azure-di/azure-di.content-extraction-strategy.config.ts
@@ -2,6 +2,7 @@ import * as v from 'valibot';
import type { AppConfigDefinition } from '../../../../config/config.types';
import { urlSchema } from '../../../../config/config.schemas';
import { coercedStrictlyPositiveIntegerSchema } from '../../../../shared/schemas/number.schemas';
+import { IN_MS } from '../../../../shared/units';
export const azureDiConfig = {
endpoint: {
@@ -16,6 +17,12 @@ export const azureDiConfig = {
schema: v.string(),
default: '',
},
+ timeoutMs: {
+ doc: 'The timeout in milliseconds for requests to the Azure Document Intelligence service.',
+ env: 'AZURE_DI_REQUEST_TIMEOUT_MS',
+ schema: coercedStrictlyPositiveIntegerSchema,
+ default: 5 * IN_MS.MINUTE,
+ },
mimeTypesAllowList: {
doc: 'The list of mime types that the Azure Document Inteligence strategy will be used for. If the document mime type is not in this list, the strategy will be skipped. Comma separated list. Supports wildcards, e.g. "image/*" matches all image mime types, and "*" matches all formats. Prefix an entry with "!" to negate it, e.g. "*,!image/png" allows everything except PNG. Negations always take precedence over allows, even more specific ones (e.g. "image/png,!image/*" rejects PNG).',
env: 'AZURE_DI_MIME_TYPES_ALLOW_LIST',
diff --git a/apps/papra-server/src/modules/documents/content-extraction/content-extraction-strategies/azure-di/azure-di.content-extraction-strategy.ts b/apps/papra-server/src/modules/documents/content-extraction/content-extraction-strategies/azure-di/azure-di.content-extraction-strategy.ts
index 32d2b974..e1de0178 100644
--- a/apps/papra-server/src/modules/documents/content-extraction/content-extraction-strategies/azure-di/azure-di.content-extraction-strategy.ts
+++ b/apps/papra-server/src/modules/documents/content-extraction/content-extraction-strategies/azure-di/azure-di.content-extraction-strategy.ts
@@ -9,7 +9,7 @@ export function buildAzureDiContentExtractionStrategy({
}: {
config: Config;
}): ContentExtractionStrategy {
- const { endpoint, apiKey, mimeTypesAllowList, pollingAttempts, pollingDelayMs } =
+ const { endpoint, apiKey, mimeTypesAllowList, pollingAttempts, pollingDelayMs, timeoutMs } =
config.documentContentExtraction.strategy.azureDi;
return {
@@ -35,6 +35,7 @@ export function buildAzureDiContentExtractionStrategy({
apiKey,
pollingAttempts,
pollingDelayMs,
+ timeoutMs,
});
return { text };
diff --git a/apps/papra-server/src/modules/documents/content-extraction/content-extraction-strategies/azure-di/azure-di.content-extraction-strategy.usecases.ts b/apps/papra-server/src/modules/documents/content-extraction/content-extraction-strategies/azure-di/azure-di.content-extraction-strategy.usecases.ts
index 7397fc84..4db6d0d7 100644
--- a/apps/papra-server/src/modules/documents/content-extraction/content-extraction-strategies/azure-di/azure-di.content-extraction-strategy.usecases.ts
+++ b/apps/papra-server/src/modules/documents/content-extraction/content-extraction-strategies/azure-di/azure-di.content-extraction-strategy.usecases.ts
@@ -1,5 +1,4 @@
import { ofetch } from 'ofetch';
-import { IN_MS } from '../../../../shared/units';
import * as v from 'valibot';
import { buildUrl } from '@corentinth/chisels';
import { sleep } from '@papra/std';
@@ -26,6 +25,7 @@ export async function extractTextWithAzureDi({
apiKey,
pollingAttempts,
pollingDelayMs,
+ timeoutMs,
logger = createLogger({ namespace: 'azure-di.content-extraction-strategy.usecases' }),
}: {
file: File;
@@ -33,6 +33,7 @@ export async function extractTextWithAzureDi({
apiKey: string;
pollingAttempts: number;
pollingDelayMs: number;
+ timeoutMs: number;
logger?: Logger;
}) {
const url = buildUrl({
@@ -50,7 +51,7 @@ export async function extractTextWithAzureDi({
'Content-Type': 'application/octet-stream',
'Ocp-Apim-Subscription-Key': apiKey,
},
- timeout: 10 * IN_MS.SECOND,
+ timeout: timeoutMs,
body: file,
});
@@ -67,6 +68,7 @@ export async function extractTextWithAzureDi({
apiKey,
pollingAttempts,
pollingDelayMs,
+ timeoutMs,
logger,
});
@@ -78,12 +80,14 @@ export async function pollJobResult({
apiKey,
pollingAttempts,
pollingDelayMs,
+ timeoutMs,
logger,
}: {
operationLocation: string;
apiKey: string;
pollingAttempts: number;
pollingDelayMs: number;
+ timeoutMs: number;
logger: Logger;
}) {
for (let attempt = 0; attempt < pollingAttempts; attempt++) {
@@ -92,7 +96,7 @@ export async function pollJobResult({
headers: {
'Ocp-Apim-Subscription-Key': apiKey,
},
- timeout: 10 * IN_MS.SECOND,
+ timeout: timeoutMs,
});
const result = v.parse(jobResultResponseSchema, response);
diff --git a/apps/papra-server/src/modules/documents/content-extraction/content-extraction-strategies/docling/docling.content-extraction-strategy.config.ts b/apps/papra-server/src/modules/documents/content-extraction/content-extraction-strategies/docling/docling.content-extraction-strategy.config.ts
index f7823f88..a154bfed 100644
--- a/apps/papra-server/src/modules/documents/content-extraction/content-extraction-strategies/docling/docling.content-extraction-strategy.config.ts
+++ b/apps/papra-server/src/modules/documents/content-extraction/content-extraction-strategies/docling/docling.content-extraction-strategy.config.ts
@@ -1,6 +1,8 @@
import * as v from 'valibot';
import type { AppConfigDefinition } from '../../../../config/config.types';
import { urlSchema } from '../../../../config/config.schemas';
+import { coercedStrictlyPositiveIntegerSchema } from '../../../../shared/schemas/number.schemas';
+import { IN_MS } from '../../../../shared/units';
export const doclingConfig = {
baseUrl: {
@@ -15,6 +17,12 @@ export const doclingConfig = {
schema: v.optional(v.string()),
default: undefined,
},
+ timeoutMs: {
+ doc: 'The timeout in milliseconds for requests to the Docling service.',
+ env: 'DOCLING_REQUEST_TIMEOUT_MS',
+ schema: coercedStrictlyPositiveIntegerSchema,
+ default: 5 * IN_MS.MINUTE,
+ },
mimeTypesAllowList: {
doc: 'The list of mime types that the Docling strategy will be used for. If the document mime type is not in this list, the strategy will be skipped. Comma separated list. Supports wildcards, e.g. "image/*" matches all image mime types, and "*" matches all formats. Prefix an entry with "!" to negate it, e.g. "*,!image/png" allows everything except PNG. Negations always take precedence over allows, even more specific ones (e.g. "image/png,!image/*" rejects PNG).',
env: 'DOCLING_MIME_TYPES_ALLOW_LIST',
diff --git a/apps/papra-server/src/modules/documents/content-extraction/content-extraction-strategies/docling/docling.content-extraction-strategy.ts b/apps/papra-server/src/modules/documents/content-extraction/content-extraction-strategies/docling/docling.content-extraction-strategy.ts
index 7330372b..da6dacdb 100644
--- a/apps/papra-server/src/modules/documents/content-extraction/content-extraction-strategies/docling/docling.content-extraction-strategy.ts
+++ b/apps/papra-server/src/modules/documents/content-extraction/content-extraction-strategies/docling/docling.content-extraction-strategy.ts
@@ -8,7 +8,8 @@ export function buildDoclingContentExtractionStrategy({
}: {
config: Config;
}): ContentExtractionStrategy {
- const { baseUrl, apiKey, mimeTypesAllowList } = config.documentContentExtraction.strategy.docling;
+ const { baseUrl, apiKey, mimeTypesAllowList, timeoutMs } =
+ config.documentContentExtraction.strategy.docling;
return {
canExtractTextFromDocument: async ({ file }) => {
@@ -23,6 +24,7 @@ export function buildDoclingContentExtractionStrategy({
file,
baseUrl,
apiKey,
+ timeoutMs,
});
return { text };
diff --git a/apps/papra-server/src/modules/documents/content-extraction/content-extraction-strategies/docling/docling.content-extraction-strategy.usecases.ts b/apps/papra-server/src/modules/documents/content-extraction/content-extraction-strategies/docling/docling.content-extraction-strategy.usecases.ts
index 2db982fd..17b7f56a 100644
--- a/apps/papra-server/src/modules/documents/content-extraction/content-extraction-strategies/docling/docling.content-extraction-strategy.usecases.ts
+++ b/apps/papra-server/src/modules/documents/content-extraction/content-extraction-strategies/docling/docling.content-extraction-strategy.usecases.ts
@@ -1,5 +1,4 @@
import { ofetch } from 'ofetch';
-import { IN_MS } from '../../../../shared/units';
import * as v from 'valibot';
import { stripDoclingImagePlaceholders } from './docling.content-extraction-strategy.models';
@@ -14,10 +13,12 @@ export async function extractTextWithDoclingServer({
file,
baseUrl,
apiKey,
+ timeoutMs,
}: {
file: File;
baseUrl: string;
apiKey?: string;
+ timeoutMs: number;
}) {
const url = `${baseUrl.replace(/\/$/, '')}/v1/convert/file`;
@@ -32,7 +33,7 @@ export async function extractTextWithDoclingServer({
// 'Content-Type': 'multipart/form-data',
...(apiKey ? { 'X-Api-Key': apiKey } : {}),
},
- timeout: 30 * IN_MS.SECOND,
+ timeout: timeoutMs,
body,
});
diff --git a/apps/papra-server/src/modules/documents/content-extraction/content-extraction-strategies/mistral-ocr/mistral-ocr.content-extraction-strategy.config.ts b/apps/papra-server/src/modules/documents/content-extraction/content-extraction-strategies/mistral-ocr/mistral-ocr.content-extraction-strategy.config.ts
index a1c75fe2..0f1a0493 100644
--- a/apps/papra-server/src/modules/documents/content-extraction/content-extraction-strategies/mistral-ocr/mistral-ocr.content-extraction-strategy.config.ts
+++ b/apps/papra-server/src/modules/documents/content-extraction/content-extraction-strategies/mistral-ocr/mistral-ocr.content-extraction-strategy.config.ts
@@ -1,5 +1,7 @@
import * as v from 'valibot';
import type { AppConfigDefinition } from '../../../../config/config.types';
+import { coercedStrictlyPositiveIntegerSchema } from '../../../../shared/schemas/number.schemas';
+import { IN_MS } from '../../../../shared/units';
export const mistralOcrConfig = {
modelName: {
@@ -8,6 +10,12 @@ export const mistralOcrConfig = {
schema: v.string(),
default: 'mistral-ocr-latest',
},
+ timeoutMs: {
+ doc: 'The timeout in milliseconds for requests to the Mistral OCR service.',
+ env: 'MISTRAL_OCR_REQUEST_TIMEOUT_MS',
+ schema: coercedStrictlyPositiveIntegerSchema,
+ default: 5 * IN_MS.MINUTE,
+ },
mimeTypesAllowList: {
doc: 'The list of mime types that the Mistral OCR strategy will be used for. If the document mime type is not in this list, the strategy will be skipped. Comma separated list. Supports wildcards, e.g. "image/*" matches all image mime types, and "*" matches all formats. Prefix an entry with "!" to negate it, e.g. "*,!image/png" allows everything except PNG. Negations always take precedence over allows, even more specific ones (e.g. "image/png,!image/*" rejects PNG).',
env: 'MISTRAL_OCR_MIME_TYPES_ALLOW_LIST',
diff --git a/apps/papra-server/src/modules/documents/content-extraction/content-extraction-strategies/mistral-ocr/mistral-ocr.content-extraction-strategy.ts b/apps/papra-server/src/modules/documents/content-extraction/content-extraction-strategies/mistral-ocr/mistral-ocr.content-extraction-strategy.ts
index 17c88ec6..b78d4b73 100644
--- a/apps/papra-server/src/modules/documents/content-extraction/content-extraction-strategies/mistral-ocr/mistral-ocr.content-extraction-strategy.ts
+++ b/apps/papra-server/src/modules/documents/content-extraction/content-extraction-strategies/mistral-ocr/mistral-ocr.content-extraction-strategy.ts
@@ -9,7 +9,8 @@ export function buildMistralOcrContentExtractionStrategy({
}: {
config: Config;
}): ContentExtractionStrategy {
- const { modelName, mimeTypesAllowList } = config.documentContentExtraction.strategy.mistralOcr;
+ const { modelName, mimeTypesAllowList, timeoutMs } =
+ config.documentContentExtraction.strategy.mistralOcr;
const { baseUrl, apiKey } = config.ai.adapters.mistral;
return {
@@ -27,6 +28,7 @@ export function buildMistralOcrContentExtractionStrategy({
modelName,
baseUrl,
apiKey,
+ timeoutMs,
});
return { text, extractionContext: { processedPagesCount } };
diff --git a/apps/papra-server/src/modules/documents/content-extraction/content-extraction-strategies/mistral-ocr/mistral-ocr.content-extraction-strategy.usecases.ts b/apps/papra-server/src/modules/documents/content-extraction/content-extraction-strategies/mistral-ocr/mistral-ocr.content-extraction-strategy.usecases.ts
index 306db0db..2d01ec5a 100644
--- a/apps/papra-server/src/modules/documents/content-extraction/content-extraction-strategies/mistral-ocr/mistral-ocr.content-extraction-strategy.usecases.ts
+++ b/apps/papra-server/src/modules/documents/content-extraction/content-extraction-strategies/mistral-ocr/mistral-ocr.content-extraction-strategy.usecases.ts
@@ -1,7 +1,6 @@
import { ofetch } from 'ofetch';
import * as v from 'valibot';
import { buildDocumentInfo } from './mistral-ocr.content-extraction-strategy.models';
-import { IN_MS } from '../../../../shared/units';
const ocrResponseSchema = v.object({
pages: v.array(
@@ -19,12 +18,14 @@ export async function extractTextWithMistralOcr({
modelName,
baseUrl,
apiKey,
+ timeoutMs,
}: {
file: File;
modelName: string;
baseUrl: string;
apiKey: string;
+ timeoutMs: number;
}) {
const url = `${baseUrl.replace(/\/$/, '')}/ocr`;
@@ -34,7 +35,7 @@ export async function extractTextWithMistralOcr({
'Content-Type': 'application/json',
'Authorization': `Bearer ${apiKey}`,
},
- timeout: 30 * IN_MS.SECOND,
+ timeout: timeoutMs,
body: JSON.stringify({
model: modelName,
include_image_base64: false,