The minimalistic document archiving platform.
feat(content-extraction): add custom HTTP extraction strategy (#1294)
Corentin Thomasset6e59ed401128
diff --git a/apps/papra-server/src/modules/documents/content-extraction/content-extraction-strategies/content-extraction-strategies.constants.ts b/apps/papra-server/src/modules/documents/content-extraction/content-extraction-strategies/content-extraction-strategies.constants.tsindex c3e1c436..7762769a 100644--- a/apps/papra-server/src/modules/documents/content-extraction/content-extraction-strategies/content-extraction-strategies.constants.ts+++ b/apps/papra-server/src/modules/documents/content-extraction/content-extraction-strategies/content-extraction-strategies.constants.ts@@ -3,6 +3,7 @@ export const CONTENT_EXTRACTION_STRATEGIES = {mistralOcr: 'mistral-ocr',docling: 'docling',azureDi: 'azure-di',+ customHttp: 'custom-http',} as const;export const CONTENT_EXTRACTION_STRATEGY_NAMES = Object.values(CONTENT_EXTRACTION_STRATEGIES);diff --git a/apps/papra-server/src/modules/documents/content-extraction/content-extraction-strategies/content-extraction.strategies.ts b/apps/papra-server/src/modules/documents/content-extraction/content-extraction-strategies/content-extraction.strategies.tsindex ef026257..589da179 100644--- a/apps/papra-server/src/modules/documents/content-extraction/content-extraction-strategies/content-extraction.strategies.ts+++ b/apps/papra-server/src/modules/documents/content-extraction/content-extraction-strategies/content-extraction.strategies.ts@@ -1,6 +1,7 @@import { buildAzureDiContentExtractionStrategy } from './azure-di/azure-di.content-extraction-strategy';import type { ContentExtractionStrategyName } from './content-extraction-strategies.constants';import type { ContentExtractionStrategyFactory } from './content-extraction-strategies.types';+import { buildCustomHttpContentExtractionStrategy } from './custom-http/custom-http.content-extraction-strategy';import { buildDoclingContentExtractionStrategy } from './docling/docling.content-extraction-strategy';import { buildLectureContentExtractionStrategy } from './lecture/lecture.content-extraction-strategy';import { buildMistralOcrContentExtractionStrategy } from './mistral-ocr/mistral-ocr.content-extraction-strategy';@@ -10,6 +11,7 @@ export const strategiesRegistry = {'mistral-ocr': buildMistralOcrContentExtractionStrategy,'docling': buildDoclingContentExtractionStrategy,'azure-di': buildAzureDiContentExtractionStrategy,+ 'custom-http': buildCustomHttpContentExtractionStrategy,} satisfies Record<ContentExtractionStrategyName, ContentExtractionStrategyFactory>;export function getContentExtractionStrategy({ strategyName }: { strategyName: string }) {diff --git a/apps/papra-server/src/modules/documents/content-extraction/content-extraction-strategies/custom-http/custom-http.content-extraction-strategy.config.ts b/apps/papra-server/src/modules/documents/content-extraction/content-extraction-strategies/custom-http/custom-http.content-extraction-strategy.config.tsnew file mode 100644index 00000000..30068a81--- /dev/null+++ b/apps/papra-server/src/modules/documents/content-extraction/content-extraction-strategies/custom-http/custom-http.content-extraction-strategy.config.ts@@ -0,0 +1,95 @@+import * as v from 'valibot';+import type { AppConfigDefinition } from '../../../../config/config.types';+import { urlSchema } from '../../../../config/config.schemas';+import { headersDefinitionStringSchema } from '../../../../shared/schemas/headers.schemas';+import {+ RESPONSE_FORMAT,+ RESPONSE_FORMATS,+ UPLOAD_FORMAT,+ UPLOAD_FORMATS,+} from './custom-http.content-extraction-strategy.constants';+import { coercedStrictlyPositiveIntegerSchema } from '../../../../shared/schemas/number.schemas';+import { IN_MS } from '../../../../shared/units';++export const customHttpConfig = {+ url: {+ doc: 'The URL of the Custom HTTP service.',+ env: 'CONTENT_EXTRACTION_CUSTOM_HTTP_URL',+ schema: v.optional(urlSchema),+ default: undefined,+ showInDocumentation: false,+ },+ headers: {+ doc: 'Custom headers to include in requests to the Custom HTTP service. Content-type header will be automatically set based on the uploadFormat, so it should not be included in this list. The headers should be specified as a JSON object, e.g. `{"Authorization": "Bearer <token>"}`.',+ env: 'CONTENT_EXTRACTION_CUSTOM_HTTP_HEADERS',+ schema: headersDefinitionStringSchema,+ default: '{}',+ showInDocumentation: false,+ },+ uploadFormat: {+ doc: [+ 'The format in which the document will be uploaded to the Custom HTTP service. Available formats are:',+ `- \`${UPLOAD_FORMAT.FORM_DATA}\`: The document will be uploaded as a multipart/form-data request, with the document file in the "file" field.`,+ `- \`${UPLOAD_FORMAT.JSON}\`: The document will be uploaded as a JSON request, with the document file base64-encoded: \`{ document: { "filename": "file.pdf", "type": "application/pdf", "content": "<base64-encoded-content>" } }\`.`,+ ].join('\n'),+ schema: v.picklist(UPLOAD_FORMATS),+ default: UPLOAD_FORMAT.FORM_DATA,+ env: 'CONTENT_EXTRACTION_CUSTOM_HTTP_UPLOAD_FORMAT',+ showInDocumentation: false,+ },+ requestTimeout: {+ doc: 'The timeout in milliseconds for requests to the Custom HTTP service.',+ env: 'CONTENT_EXTRACTION_CUSTOM_HTTP_REQUEST_TIMEOUT_MS',+ schema: coercedStrictlyPositiveIntegerSchema,+ default: 30 * IN_MS.SECOND,+ showInDocumentation: false,+ },+ responseFormat: {+ doc: [+ 'The format in which the response from the Custom HTTP service will be interpreted. Available formats are:',+ `- \`${RESPONSE_FORMAT.JSON}\`: The response will be interpreted as JSON, and the text will be extracted from the path specified in \`jsonResponseTextPath\`.`,+ `- \`${RESPONSE_FORMAT.TEXT}\`: The response will be interpreted as plain text, and the entire response body will be used as the extracted text.`,+ ].join('\n'),+ schema: v.picklist(RESPONSE_FORMATS),+ default: RESPONSE_FORMAT.JSON,+ env: 'CONTENT_EXTRACTION_CUSTOM_HTTP_RESPONSE_FORMAT',+ showInDocumentation: false,+ },+ jsonResponseTextPath: {+ doc: '',+ schema: v.pipe(+ v.string(),+ v.transform((value) =>+ value+ .split('.')+ .map((s) => s.trim())+ .filter(Boolean),+ ),+ v.pipe(+ v.array(v.string()),+ v.minLength(1, 'The jsonResponseTextPath must contain at least one path segment.'),+ ),+ ),+ default: 'text',+ env: 'CONTENT_EXTRACTION_CUSTOM_HTTP_JSON_RESPONSE_TEXT_PATH',+ showInDocumentation: false,+ },+ mimeTypesAllowList: {+ doc: 'The list of mime types that the Custom HTTP strategy will be used for. If the document mime type is not in this list, the strategy will be skipped. Comma separated list. Supports wildcards, e.g. "image/*" matches all image mime types, and "*" matches all formats. Prefix an entry with "!" to negate it, e.g. "*,!image/png" allows everything except PNG. Negations always take precedence over allows, even more specific ones (e.g. "image/png,!image/*" rejects PNG).',+ env: 'CONTENT_EXTRACTION_CUSTOM_HTTP_MIME_TYPES_ALLOW_LIST',+ schema: v.pipe(+ v.string(),+ v.transform(+ (value) =>+ new Set(+ value+ .split(',')+ .map((s) => s.trim().toLowerCase())+ .filter(Boolean),+ ),+ ),+ ),+ default: '*',+ showInDocumentation: false,+ },+} as const satisfies AppConfigDefinition;diff --git a/apps/papra-server/src/modules/documents/content-extraction/content-extraction-strategies/custom-http/custom-http.content-extraction-strategy.constants.ts b/apps/papra-server/src/modules/documents/content-extraction/content-extraction-strategies/custom-http/custom-http.content-extraction-strategy.constants.tsnew file mode 100644index 00000000..a3b0a13e--- /dev/null+++ b/apps/papra-server/src/modules/documents/content-extraction/content-extraction-strategies/custom-http/custom-http.content-extraction-strategy.constants.ts@@ -0,0 +1,16 @@+export const UPLOAD_FORMAT = {+ JSON: 'json',+ FORM_DATA: 'form-data',+} as const;++export const UPLOAD_FORMATS = Object.values(UPLOAD_FORMAT);+export type UploadFormat = (typeof UPLOAD_FORMATS)[number];++export const RESPONSE_FORMAT = {+ JSON: 'json',+ TEXT: 'text',+} as const;++export const RESPONSE_FORMATS = Object.values(RESPONSE_FORMAT);++export type ResponseFormat = (typeof RESPONSE_FORMATS)[number];diff --git a/apps/papra-server/src/modules/documents/content-extraction/content-extraction-strategies/custom-http/custom-http.content-extraction-strategy.models.test.ts b/apps/papra-server/src/modules/documents/content-extraction/content-extraction-strategies/custom-http/custom-http.content-extraction-strategy.models.test.tsnew file mode 100644index 00000000..950b361b--- /dev/null+++ b/apps/papra-server/src/modules/documents/content-extraction/content-extraction-strategies/custom-http/custom-http.content-extraction-strategy.models.test.ts@@ -0,0 +1,140 @@+import { describe, expect, test } from 'vitest';+import {+ buildRequestBody,+ extractTextFromResponse,+} from './custom-http.content-extraction-strategy.models';++describe('custom-http.content-extraction-strategy.models', () => {+ describe('buildRequestBody', () => {+ test('the form-data format puts the file in the "file" field of a multipart body', async () => {+ const file = new File(['lorem ipsum'], 'file.txt', { type: 'text/plain' });++ const body = await buildRequestBody({ uploadFormat: 'form-data', file });++ expect(body).toBeInstanceOf(FormData);++ const uploadedFile = (body as FormData).get('file') as File;+ expect(uploadedFile.name).to.eql('file.txt');+ expect(uploadedFile.type).to.eql('text/plain');+ expect(await uploadedFile.text()).to.eql('lorem ipsum');+ });++ test('the json format embeds the base64-encoded file alongside its name and type', async () => {+ const file = new File(['lorem ipsum'], 'file.txt', { type: 'text/plain' });++ const body = await buildRequestBody({ uploadFormat: 'json', file });++ expect(body).to.eql({+ document: {+ filename: 'file.txt',+ type: 'text/plain',+ content: 'bG9yZW0gaXBzdW0=',+ },+ });+ });++ test('an unsupported upload format throws', async () => {+ const file = new File(['lorem ipsum'], 'file.txt', { type: 'text/plain' });++ await expect(buildRequestBody({ uploadFormat: 'xml' as never, file })).rejects.toThrow(+ 'Unsupported upload format.',+ );+ });+ });++ describe('extractTextFromResponse', () => {+ describe('when the response format is text', () => {+ test('the whole string response is used as the extracted text', () => {+ expect(+ extractTextFromResponse({+ response: 'extracted text',+ responseFormat: 'text',+ jsonResponseTextPath: [],+ }),+ ).to.eql({ text: 'extracted text' });+ });++ test('a non-string response throws', () => {+ expect(() =>+ extractTextFromResponse({+ response: { text: 'extracted text' },+ responseFormat: 'text',+ jsonResponseTextPath: [],+ }),+ ).to.throw('Expected response to be a string, but got object');+ });+ });++ describe('when the response format is json', () => {+ test('the text is read from the top-level key when the path has a single segment', () => {+ expect(+ extractTextFromResponse({+ response: { text: 'extracted text' },+ responseFormat: 'json',+ jsonResponseTextPath: ['text'],+ }),+ ).to.eql({ text: 'extracted text' });+ });++ test('the text is read by walking a nested path', () => {+ expect(+ extractTextFromResponse({+ response: { data: { result: { content: 'extracted text' } } },+ responseFormat: 'json',+ jsonResponseTextPath: ['data', 'result', 'content'],+ }),+ ).to.eql({ text: 'extracted text' });+ });++ test('a non-object response throws', () => {+ expect(() =>+ extractTextFromResponse({+ response: 'extracted text',+ responseFormat: 'json',+ jsonResponseTextPath: ['text'],+ }),+ ).to.throw('Expected response to be an object, but got string');+ });++ test('a null response throws', () => {+ expect(() =>+ extractTextFromResponse({+ response: null,+ responseFormat: 'json',+ jsonResponseTextPath: ['text'],+ }),+ ).to.throw('Expected response to be an object, but got object');+ });++ test('a missing key along the path throws, naming the missing key', () => {+ expect(() =>+ extractTextFromResponse({+ response: { data: { result: {} } },+ responseFormat: 'json',+ jsonResponseTextPath: ['data', 'result', 'content'],+ }),+ ).to.throw('Expected response to have key content.');+ });++ test('a non-string value at the end of the path throws', () => {+ expect(() =>+ extractTextFromResponse({+ response: { text: { nested: 'extracted text' } },+ responseFormat: 'json',+ jsonResponseTextPath: ['text'],+ }),+ ).to.throw('Expected response to be a string, but got object');+ });+ });++ test('an unsupported response format throws', () => {+ expect(() =>+ extractTextFromResponse({+ response: 'extracted text',+ responseFormat: 'xml' as never,+ jsonResponseTextPath: [],+ }),+ ).to.throw('Unsupported response format');+ });+ });+});diff --git a/apps/papra-server/src/modules/documents/content-extraction/content-extraction-strategies/custom-http/custom-http.content-extraction-strategy.models.ts b/apps/papra-server/src/modules/documents/content-extraction/content-extraction-strategies/custom-http/custom-http.content-extraction-strategy.models.tsnew file mode 100644index 00000000..fa7feb18--- /dev/null+++ b/apps/papra-server/src/modules/documents/content-extraction/content-extraction-strategies/custom-http/custom-http.content-extraction-strategy.models.ts@@ -0,0 +1,71 @@+import { fileToBase64 } from '../../../../shared/files/files';+import type {+ ResponseFormat,+ UploadFormat,+} from './custom-http.content-extraction-strategy.constants';++export async function buildRequestBody({+ uploadFormat,+ file,+}: {+ uploadFormat: UploadFormat;+ file: File;+}) {+ if (uploadFormat === 'form-data') {+ const formData = new FormData();+ formData.append('file', file);+ return formData;+ } else if (uploadFormat === 'json') {+ return {+ document: {+ filename: file.name,+ type: file.type,+ content: await fileToBase64(file),+ },+ };+ }++ throw new Error('Unsupported upload format.');+}++export function extractTextFromResponse({+ response,+ responseFormat,+ jsonResponseTextPath,+}: {+ response: unknown;+ responseFormat: ResponseFormat;+ jsonResponseTextPath: string[];+}): { text: string } {+ if (responseFormat === 'text') {+ if (typeof response !== 'string') {+ throw new Error(`Expected response to be a string, but got ${typeof response}`);+ }+ return { text: response };+ } else if (responseFormat === 'json') {+ if (typeof response !== 'object' || response === null) {+ throw new Error(`Expected response to be an object, but got ${typeof response}`);+ }++ let current: unknown = response;++ for (const key of jsonResponseTextPath) {+ if (+ typeof current !== 'object' ||+ current === null ||+ !Object.prototype.hasOwnProperty.call(current, key)+ ) {+ throw new Error(`Expected response to have key ${key}.`);+ }+ current = (current as Record<string, unknown>)[key];+ }++ if (typeof current !== 'string') {+ throw new Error(`Expected response to be a string, but got ${typeof current}`);+ }++ return { text: current };+ }++ throw new Error('Unsupported response format');+}diff --git a/apps/papra-server/src/modules/documents/content-extraction/content-extraction-strategies/custom-http/custom-http.content-extraction-strategy.ts b/apps/papra-server/src/modules/documents/content-extraction/content-extraction-strategies/custom-http/custom-http.content-extraction-strategy.tsnew file mode 100644index 00000000..754dcbd5--- /dev/null+++ b/apps/papra-server/src/modules/documents/content-extraction/content-extraction-strategies/custom-http/custom-http.content-extraction-strategy.ts@@ -0,0 +1,51 @@+import type { Config } from '../../../../config/config.types';+import type { ContentExtractionStrategy } from '../content-extraction-strategies.types';+import { isMimeTypeAllowed } from '../../../../shared/mime-types/mime-types.models';+import { extractTextFromDocumentWithCustomHttp } from './custom-http.content-extraction-strategy.usecases';+import { isNilOrEmptyString } from '../../../../shared/utils';++export function buildCustomHttpContentExtractionStrategy({+ config,+}: {+ config: Config;+}): ContentExtractionStrategy {+ const {+ url,+ headers,+ uploadFormat,+ responseFormat,+ jsonResponseTextPath,+ requestTimeout,+ mimeTypesAllowList,+ } = config.documentContentExtraction.strategy.customHttp;++ return {+ canExtractTextFromDocument: async ({ file }) => {+ return (+ !isNilOrEmptyString(url) &&+ isMimeTypeAllowed({+ mimeType: file.type,+ allowList: mimeTypesAllowList,+ })+ );+ },++ extractTextFromDocument: async ({ file }) => {+ if (!url) {+ throw new Error('Custom HTTP URL is not configured.');+ }++ const { text } = await extractTextFromDocumentWithCustomHttp({+ file,+ url,+ headers,+ uploadFormat,+ responseFormat,+ requestTimeout,+ jsonResponseTextPath,+ });++ return { text };+ },+ };+}diff --git a/apps/papra-server/src/modules/documents/content-extraction/content-extraction-strategies/custom-http/custom-http.content-extraction-strategy.usecases.ts b/apps/papra-server/src/modules/documents/content-extraction/content-extraction-strategies/custom-http/custom-http.content-extraction-strategy.usecases.tsnew file mode 100644index 00000000..34fd4f00--- /dev/null+++ b/apps/papra-server/src/modules/documents/content-extraction/content-extraction-strategies/custom-http/custom-http.content-extraction-strategy.usecases.ts@@ -0,0 +1,45 @@+import type {+ ResponseFormat,+ UploadFormat,+} from './custom-http.content-extraction-strategy.constants';+import { ofetch } from 'ofetch';+import {+ buildRequestBody,+ extractTextFromResponse,+} from './custom-http.content-extraction-strategy.models';++export async function extractTextFromDocumentWithCustomHttp({+ file,+ url,+ headers,+ uploadFormat,+ responseFormat,+ requestTimeout,+ jsonResponseTextPath,+}: {+ file: File;+ url: string;+ headers: Record<string, string>;+ uploadFormat: UploadFormat;+ requestTimeout: number;+ responseFormat: ResponseFormat;+ jsonResponseTextPath: string[];+}): Promise<{ text: string }> {+ const requestBody = await buildRequestBody({ uploadFormat, file });++ const response = await ofetch(url, {+ method: 'POST',+ headers,+ body: requestBody,+ responseType: responseFormat === 'json' ? 'json' : 'text',+ timeout: requestTimeout,+ });++ const { text } = extractTextFromResponse({+ response,+ responseFormat,+ jsonResponseTextPath,+ });++ return { text };+}diff --git a/apps/papra-server/src/modules/documents/content-extraction/content-extraction.config.ts b/apps/papra-server/src/modules/documents/content-extraction/content-extraction.config.tsindex 378e0137..c2cf57c5 100644--- a/apps/papra-server/src/modules/documents/content-extraction/content-extraction.config.ts+++ b/apps/papra-server/src/modules/documents/content-extraction/content-extraction.config.ts@@ -7,6 +7,7 @@ import {} from './content-extraction-strategies/content-extraction-strategies.constants';import { doclingConfig } from './content-extraction-strategies/docling/docling.content-extraction-strategy.config';import { azureDiConfig } from './content-extraction-strategies/azure-di/azure-di.content-extraction-strategy.config';+import { customHttpConfig } from './content-extraction-strategies/custom-http/custom-http.content-extraction-strategy.config';export const documentContentExtractionConfig = {extractionStrategies: {@@ -16,6 +17,7 @@ export const documentContentExtractionConfig = {`- \`${CONTENT_EXTRACTION_STRATEGIES.mistralOcr}\`: Uses the Mistral OCR API to extract text from documents. This strategy requires a valid Mistral API key.`,`- \`${CONTENT_EXTRACTION_STRATEGIES.docling}\`: Uses a Docling server to extract text from documents. This strategy requires a running [Docling server](https://github.com/docling-project/docling-serve).`,`- \`${CONTENT_EXTRACTION_STRATEGIES.azureDi}\`: Uses the Azure Document Intelligence service to extract text from documents. This strategy requires a valid Azure Document Intelligence endpoint and API key.`,+ `- \`${CONTENT_EXTRACTION_STRATEGIES.customHttp}\`: Uses a custom HTTP service to extract text from documents. This makes a POST request to the configured URL with the document file, see \`CONTENT_EXTRACTION_CUSTOM_HTTP_\` environment variables for configuration.`,].join('\n'),schema: v.pipe(v.string(),@@ -31,5 +33,6 @@ export const documentContentExtractionConfig = {mistralOcr: mistralOcrConfig,docling: doclingConfig,azureDi: azureDiConfig,+ customHttp: customHttpConfig,},} as const satisfies AppConfigDefinition;diff --git a/apps/papra-server/src/modules/shared/files/files.test.ts b/apps/papra-server/src/modules/shared/files/files.test.tsindex fe96771e..3307c496 100644--- a/apps/papra-server/src/modules/shared/files/files.test.ts+++ b/apps/papra-server/src/modules/shared/files/files.test.ts@@ -1,7 +1,15 @@import { describe, expect, test } from 'vitest';-import { fileToBase64DataUrl } from './files';+import { fileToBase64DataUrl, fileToBase64 } from './files';describe('files', () => {+ describe('fileToBase64', () => {+ test('builds a base64 string from a file', async () => {+ const file = new File(['lorem ipsum'], 'text-file.txt', { type: 'text/plain' });++ expect(await fileToBase64(file)).to.eql('bG9yZW0gaXBzdW0=');+ });+ });+describe('fileToBase64DataUrl', () => {test('builds a base64 data url from a file', async () => {const file = new File(['lorem ipsum'], 'text-file.txt', { type: 'text/plain' });diff --git a/apps/papra-server/src/modules/shared/files/files.ts b/apps/papra-server/src/modules/shared/files/files.tsindex 9cea6929..16a41bae 100644--- a/apps/papra-server/src/modules/shared/files/files.ts+++ b/apps/papra-server/src/modules/shared/files/files.ts@@ -1,5 +1,10 @@-export async function fileToBase64DataUrl(file: File): Promise<string> {+export async function fileToBase64(file: File): Promise<string> {const arrayBuffer = await file.arrayBuffer();const base64String = Buffer.from(arrayBuffer).toString('base64');+ return base64String;+}++export async function fileToBase64DataUrl(file: File): Promise<string> {+ const base64String = await fileToBase64(file);return `data:${file.type};base64,${base64String}`;}diff --git a/apps/papra-server/src/modules/shared/schemas/headers.schemas.test.ts b/apps/papra-server/src/modules/shared/schemas/headers.schemas.test.tsnew file mode 100644index 00000000..454bcdc6--- /dev/null+++ b/apps/papra-server/src/modules/shared/schemas/headers.schemas.test.ts@@ -0,0 +1,20 @@+import { describe, expect, test } from 'vitest';+import { headersDefinitionStringSchema } from './headers.schemas';+import * as v from 'valibot';++describe('headers.schemas', () => {+ describe('headersDefinitionStringSchema', () => {+ test('parses a valid JSON string into a record of string to string', () => {+ const input = '{"Content-Type": "application/json", "Authorization": "Bearer mytoken"}';+ const result = v.parse(headersDefinitionStringSchema, input);+ expect(result).toEqual({+ 'Content-Type': 'application/json',+ 'Authorization': 'Bearer mytoken',+ });+ });++ test('throws an error for an invalid JSON string', () => {+ expect(() => v.parse(headersDefinitionStringSchema, 'foo')).toThrow();+ });+ });+});diff --git a/apps/papra-server/src/modules/shared/schemas/headers.schemas.ts b/apps/papra-server/src/modules/shared/schemas/headers.schemas.tsnew file mode 100644index 00000000..b8971a62--- /dev/null+++ b/apps/papra-server/src/modules/shared/schemas/headers.schemas.ts@@ -0,0 +1,9 @@+import * as v from 'valibot';++export const headersDefinitionSchema = v.record(v.string(), v.string());++export const headersDefinitionStringSchema = v.pipe(+ v.string(),+ v.parseJson(),+ headersDefinitionSchema,+);