Skip to content

armgit / papra

Public Imported from GitHub
0

The minimalistic document archiving platform.

feat(ingestion-folders): make file stability polling configurable (#1173)

Corentin Thomassetc541b1f670ba
diff --git a/.changeset/salty-jokes-repair.md b/.changeset/salty-jokes-repair.md
new file mode 100644
index 00000000..02da39e6
--- /dev/null
+++ b/.changeset/salty-jokes-repair.md
@@ -0,0 +1,8 @@
+---
+'@papra/app': patch
+---
+
+Added the possibility to configure the file size polling interval for ingestion folder watchers.
+
+- `INGESTION_FOLDER_WATCHER_FILE_STABILITY_THRESHOLD_MS`: The amount of time in milliseconds for a file size to remain constant before being consumed. This helps to avoid processing files that are still being written to (e.g., scanners, cameras, network shares, etc.).
+- `INGESTION_FOLDER_WATCHER_FILE_STABILITY_POLL_INTERVAL_MS`: The interval in milliseconds at which the file size is polled while waiting for write to finish.
diff --git a/apps/papra-server/src/modules/ingestion-folders/ingestion-folders.config.ts b/apps/papra-server/src/modules/ingestion-folders/ingestion-folders.config.ts
index de2d649d..b1a5b19a 100644
--- a/apps/papra-server/src/modules/ingestion-folders/ingestion-folders.config.ts
+++ b/apps/papra-server/src/modules/ingestion-folders/ingestion-folders.config.ts
@@ -31,6 +31,20 @@ export const ingestionFolderConfig = {
default: 2 * IN_MS.SECOND,
env: 'INGESTION_FOLDER_WATCHER_POLLING_INTERVAL_MS',
},
+ awaitWriteFinish: {
+ stabilityThreshold: {
+ doc: 'The amount of time in milliseconds for a file size to remain constant before being consumed. This helps to avoid processing files that are still being written to (e.g., scanners, cameras, network shares, etc.).',
+ schema: coercedStrictlyPositiveIntegerSchema,
+ default: 2 * IN_MS.SECOND,
+ env: 'INGESTION_FOLDER_WATCHER_FILE_STABILITY_THRESHOLD_MS',
+ },
+ pollInterval: {
+ doc: 'The interval in milliseconds at which the file size is polled while waiting for write to finish.',
+ schema: coercedStrictlyPositiveIntegerSchema,
+ default: 100,
+ env: 'INGESTION_FOLDER_WATCHER_FILE_STABILITY_POLL_INTERVAL_MS',
+ },
+ },
},
processingConcurrency: {
doc: 'The number of files that can be processed concurrently by the server. Increasing this can improve processing speed, but it will also increase CPU and memory usage.',
diff --git a/apps/papra-server/src/modules/ingestion-folders/ingestion-folders.usecases.ts b/apps/papra-server/src/modules/ingestion-folders/ingestion-folders.usecases.ts
index 40143995..32f8abed 100644
--- a/apps/papra-server/src/modules/ingestion-folders/ingestion-folders.usecases.ts
+++ b/apps/papra-server/src/modules/ingestion-folders/ingestion-folders.usecases.ts
@@ -49,7 +49,11 @@ export function createIngestionFolderWatcher({
}) {
const {
folderRootPath,
- watcher: { usePolling, pollingInterval },
+ watcher: {
+ usePolling,
+ pollingInterval,
+ awaitWriteFinish: { stabilityThreshold, pollInterval },
+ },
processingConcurrency,
} = config.ingestionFolder;
@@ -77,7 +81,10 @@ export function createIngestionFolderWatcher({
.watch(folderRootPath, {
persistent: true,
followSymlinks: true,
- awaitWriteFinish: true,
+ awaitWriteFinish: {
+ stabilityThreshold,
+ pollInterval,
+ },
atomic: true,
cwd,
usePolling,
@@ -114,6 +121,8 @@ export function createIngestionFolderWatcher({
folderRootPath,
usePolling,
pollingInterval,
+ stabilityThreshold,
+ pollInterval,
processingConcurrency,
},
'Ingestion folder watcher started',