The minimalistic document archiving platform.
feat(ingestion-folders): make file stability polling configurable (#1173)
Corentin Thomassetc541b1f670ba
diff --git a/.changeset/salty-jokes-repair.md b/.changeset/salty-jokes-repair.mdnew file mode 100644index 00000000..02da39e6--- /dev/null+++ b/.changeset/salty-jokes-repair.md@@ -0,0 +1,8 @@+---+'@papra/app': patch+---++Added the possibility to configure the file size polling interval for ingestion folder watchers.++- `INGESTION_FOLDER_WATCHER_FILE_STABILITY_THRESHOLD_MS`: The amount of time in milliseconds for a file size to remain constant before being consumed. This helps to avoid processing files that are still being written to (e.g., scanners, cameras, network shares, etc.).+- `INGESTION_FOLDER_WATCHER_FILE_STABILITY_POLL_INTERVAL_MS`: The interval in milliseconds at which the file size is polled while waiting for write to finish.diff --git a/apps/papra-server/src/modules/ingestion-folders/ingestion-folders.config.ts b/apps/papra-server/src/modules/ingestion-folders/ingestion-folders.config.tsindex de2d649d..b1a5b19a 100644--- a/apps/papra-server/src/modules/ingestion-folders/ingestion-folders.config.ts+++ b/apps/papra-server/src/modules/ingestion-folders/ingestion-folders.config.ts@@ -31,6 +31,20 @@ export const ingestionFolderConfig = {default: 2 * IN_MS.SECOND,env: 'INGESTION_FOLDER_WATCHER_POLLING_INTERVAL_MS',},+ awaitWriteFinish: {+ stabilityThreshold: {+ doc: 'The amount of time in milliseconds for a file size to remain constant before being consumed. This helps to avoid processing files that are still being written to (e.g., scanners, cameras, network shares, etc.).',+ schema: coercedStrictlyPositiveIntegerSchema,+ default: 2 * IN_MS.SECOND,+ env: 'INGESTION_FOLDER_WATCHER_FILE_STABILITY_THRESHOLD_MS',+ },+ pollInterval: {+ doc: 'The interval in milliseconds at which the file size is polled while waiting for write to finish.',+ schema: coercedStrictlyPositiveIntegerSchema,+ default: 100,+ env: 'INGESTION_FOLDER_WATCHER_FILE_STABILITY_POLL_INTERVAL_MS',+ },+ },},processingConcurrency: {doc: 'The number of files that can be processed concurrently by the server. Increasing this can improve processing speed, but it will also increase CPU and memory usage.',diff --git a/apps/papra-server/src/modules/ingestion-folders/ingestion-folders.usecases.ts b/apps/papra-server/src/modules/ingestion-folders/ingestion-folders.usecases.tsindex 40143995..32f8abed 100644--- a/apps/papra-server/src/modules/ingestion-folders/ingestion-folders.usecases.ts+++ b/apps/papra-server/src/modules/ingestion-folders/ingestion-folders.usecases.ts@@ -49,7 +49,11 @@ export function createIngestionFolderWatcher({}) {const {folderRootPath,- watcher: { usePolling, pollingInterval },+ watcher: {+ usePolling,+ pollingInterval,+ awaitWriteFinish: { stabilityThreshold, pollInterval },+ },processingConcurrency,} = config.ingestionFolder;@@ -77,7 +81,10 @@ export function createIngestionFolderWatcher({.watch(folderRootPath, {persistent: true,followSymlinks: true,- awaitWriteFinish: true,+ awaitWriteFinish: {+ stabilityThreshold,+ pollInterval,+ },atomic: true,cwd,usePolling,@@ -114,6 +121,8 @@ export function createIngestionFolderWatcher({folderRootPath,usePolling,pollingInterval,+ stabilityThreshold,+ pollInterval,processingConcurrency,},'Ingestion folder watcher started',