Skip to content

OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.

Python

fix: Data ingestion fails due to OpenSearch "documents" index not found exception

Mike Pawlowskib877c384cce3
diff --git a/.env.example b/.env.example
index a1484525b..2e9737fcf 100644
--- a/.env.example
+++ b/.env.example
@@ -14,7 +14,7 @@ INGEST_SAMPLE_DATA=true
# LANGFLOW_TIMEOUT=2400
# LANGFLOW_CONNECT_TIMEOUT=30
-# Per-file processing timeout for document ingestion tasks (in seconds)
+# OPTIONAL: Per-file processing timeout for document ingestion tasks (in seconds)
# Should be >= LANGFLOW_TIMEOUT to allow long-running ingestion to complete
# Default: 3600 seconds (60 minutes) total timeout
# INGESTION_TIMEOUT=3600
diff --git a/docker-compose.yml b/docker-compose.yml
index 11870127c..7587e4350 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -81,6 +81,14 @@ services:
- AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID}
- AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY}
- OPENSEARCH_INDEX_NAME=${OPENSEARCH_INDEX_NAME:-documents}
+ - LOG_LEVEL=${LOG_LEVEL:-INFO}
+ - NO_COLOR=${NO_COLOR:-}
+ - ACCESS_LOG=${ACCESS_LOG:-true}
+ - LANGFLOW_TIMEOUT=${LANGFLOW_TIMEOUT:-2400}
+ - LANGFLOW_CONNECT_TIMEOUT=${LANGFLOW_CONNECT_TIMEOUT:-30}
+ - INGESTION_TIMEOUT=${INGESTION_TIMEOUT:-3600}
+ - UPLOAD_BATCH_SIZE=${UPLOAD_BATCH_SIZE:-25}
+ - MAX_WORKERS=${MAX_WORKERS:-4}
volumes:
- ${OPENRAG_DOCUMENTS_PATH:-./openrag-documents}:/app/openrag-documents:Z
- ${OPENRAG_KEYS_PATH:-./keys}:/app/keys:U,z
diff --git a/helm/openrag/templates/backend/deployment.yaml b/helm/openrag/templates/backend/deployment.yaml
index 0b71d3537..199b81014 100644
--- a/helm/openrag/templates/backend/deployment.yaml
+++ b/helm/openrag/templates/backend/deployment.yaml
@@ -121,6 +121,29 @@ spec:
value: {{ .Values.backend.features.disableIngestWithLangflow | quote }}
- name: INGEST_SAMPLE_DATA
value: {{ .Values.backend.features.ingestSampleData | quote }}
+ # Logging
+ - name: LOG_LEVEL
+ value: {{ .Values.backend.logLevel | quote }}
+ - name: ACCESS_LOG
+ value: {{ .Values.backend.accessLog | quote }}
+ {{- if .Values.backend.noColor }}
+ - name: NO_COLOR
+ value: "1"
+ {{- end }}
+ # Timeouts
+ - name: LANGFLOW_TIMEOUT
+ value: {{ .Values.backend.langflowTimeout | quote }}
+ - name: LANGFLOW_CONNECT_TIMEOUT
+ value: {{ .Values.backend.langflowConnectTimeout | quote }}
+ - name: INGESTION_TIMEOUT
+ value: {{ .Values.backend.ingestionTimeout | quote }}
+ # Concurrency
+ - name: UPLOAD_BATCH_SIZE
+ value: {{ .Values.backend.uploadBatchSize | quote }}
+ {{- if .Values.backend.maxWorkers }}
+ - name: MAX_WORKERS
+ value: {{ .Values.backend.maxWorkers | quote }}
+ {{- end }}
# LLM Provider keys
{{- if .Values.llmProviders.openai.enabled }}
- name: OPENAI_API_KEY
diff --git a/helm/openrag/values.yaml b/helm/openrag/values.yaml
index 3b2d35210..eb32e7607 100644
--- a/helm/openrag/values.yaml
+++ b/helm/openrag/values.yaml
@@ -165,6 +165,20 @@ backend:
size: 1Gi
mountPath: /app/config
+ # Logging
+ logLevel: "INFO" # DEBUG, INFO, WARNING, ERROR, CRITICAL
+ noColor: false # Set true to disable ANSI color codes in log output
+ accessLog: true # Set false to disable HTTP access log events
+
+ # Timeouts
+ langflowTimeout: 2400 # seconds; total timeout for Langflow HTTP calls (default: 40 min)
+ langflowConnectTimeout: 30 # seconds; connection timeout for Langflow HTTP calls
+ ingestionTimeout: 3600 # seconds; per-file processing timeout for ingestion tasks
+
+ # Concurrency
+ uploadBatchSize: 25 # Max files per batch when ingesting from a folder
+ maxWorkers: "" # Backend worker processes; empty = auto (min(4, CPU_COUNT // 2))
+
# Feature flags
features:
disableIngestWithLangflow: false # Set true to use traditional processor instead of Langflow