Skip to content

OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.

Python

fix: centralize backend paths and accept env var to define them, change deployments to centralize read only and writable volumes (#1353)

Lucas Oliveira0cefaacc64fa
diff --git a/.env.example b/.env.example
index 18d9e832d..ba5207354 100644
--- a/.env.example
+++ b/.env.example
@@ -203,8 +203,13 @@ NEXT_ALLOWED_DEV_ORIGINS=
# but can be pinned here to lock the deployment to a specific version when running OpenRAG with Make commands.
OPENRAG_VERSION=latest
-# The local documents path for ingestion.
-OPENRAG_DOCUMENTS_PATH=./openrag-documents
+# Path configuration (override defaults for custom directory layouts)
+# OPENRAG_DATA_PATH=./data
+# OPENRAG_KEYS_PATH=./keys
+# OPENRAG_CONFIG_PATH=./config
+# OPENRAG_FLOWS_PATH=./flows
+# OPENRAG_FLOWS_BACKUP_PATH= # defaults to <OPENRAG_FLOWS_PATH>/backup
+# OPENRAG_DOCUMENTS_PATH=./openrag-documents
# Langflow Configuration
# API Key for authenticating with Langflow.
diff --git a/docker-compose.yml b/docker-compose.yml
index fd024e88a..8a1d31ad7 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -105,6 +105,7 @@ services:
- ${OPENRAG_DOCUMENTS_PATH:-./openrag-documents}:/app/openrag-documents:U,z
- ${OPENRAG_KEYS_PATH:-./keys}:/app/keys:U,z
- ${OPENRAG_FLOWS_PATH:-./flows}:/app/flows:U,z
+ - ${OPENRAG_FLOWS_BACKUP_PATH:-./flows/backup}:/app/flows/backup:U,z
- ${OPENRAG_CONFIG_PATH:-./config}:/app/config:U,z
- ${OPENRAG_DATA_PATH:-./data}:/app/data:U,z
# Linux: Docker Engine does not auto-inject host.docker.internal; map it to the host gateway
diff --git a/kubernetes/helm/openrag/Chart.yaml b/kubernetes/helm/openrag/Chart.yaml
index ac9d0df3d..b8f71a686 100644
--- a/kubernetes/helm/openrag/Chart.yaml
+++ b/kubernetes/helm/openrag/Chart.yaml
@@ -2,7 +2,7 @@ apiVersion: v2
name: openrag
description: A Helm chart for deploying OpenRAG - an open-source agentic RAG platform
type: application
-version: 0.1.0
+version: 0.2.0
appVersion: "0.1.52"
keywords:
- openrag
diff --git a/kubernetes/helm/openrag/templates/NOTES.txt b/kubernetes/helm/openrag/templates/NOTES.txt
index 932754ba5..5d1a19d95 100644
--- a/kubernetes/helm/openrag/templates/NOTES.txt
+++ b/kubernetes/helm/openrag/templates/NOTES.txt
@@ -77,4 +77,15 @@ Check pod logs:
kubectl logs -n {{ include "openrag.namespace" . }} -l app.kubernetes.io/component=backend
kubectl logs -n {{ include "openrag.namespace" . }} -l app.kubernetes.io/component=frontend
+ === Breakthrough Notice / Breaking Changes ===
+
+CRITICAL: Chart version 0.2.0 introduces a major storage refactoring.
+Legacy PVCs (config, keys, documents) have been consolidated into 'data' and 'shared' PVCs.
+
+IF YOU ARE UPGRADING FROM A VERSION < 0.2.0:
+1. Manually migrate data from legacy PVCs (config, keys) to the new 'data' volume to avoid data loss.
+2. Old PVCs ({{ include "openrag.fullname" . }}-config, {{ include "openrag.fullname" . }}-keys) should be manually deleted after verification.
+5. The flag 'backend.persistence.documents.enabled' has been removed.
+ To disable default document loading, use 'backend.persistence.documents.loadDefaults: false' instead.
+
For more information, visit: https://github.com/langflow-ai/openrag
diff --git a/kubernetes/helm/openrag/templates/backend/backend-dotenv.yaml b/kubernetes/helm/openrag/templates/backend/backend-dotenv.yaml
index c6d96f741..7d901a0f2 100644
--- a/kubernetes/helm/openrag/templates/backend/backend-dotenv.yaml
+++ b/kubernetes/helm/openrag/templates/backend/backend-dotenv.yaml
@@ -164,9 +164,26 @@ stringData:
{{- if .Values.global.imageTag }}
OPENRAG_VERSION={{ .Values.global.imageTag | quote }}
{{- end }}
+ # Centralized Path Configuration
+ {{- if .Values.backend.persistence.data.dataPath }}
+ OPENRAG_DATA_PATH={{ .Values.backend.persistence.data.dataPath | quote }}
+ {{- end }}
+ {{- if .Values.backend.persistence.data.backupPath }}
+ OPENRAG_FLOWS_BACKUP_PATH={{ .Values.backend.persistence.data.backupPath | quote }}
+ {{- end }}
+ {{- if .Values.backend.persistence.data.configPath }}
+ OPENRAG_CONFIG_PATH={{ .Values.backend.persistence.data.configPath | quote }}
+ {{- end }}
+ {{- if .Values.backend.persistence.data.keysPath }}
+ OPENRAG_KEYS_PATH={{ .Values.backend.persistence.data.keysPath | quote }}
+ {{- end }}
+ {{- if .Values.backend.persistence.shared.flowsPath }}
+ OPENRAG_FLOWS_PATH={{ .Values.backend.persistence.shared.flowsPath | quote }}
+ {{- end }}
{{- if .Values.backend.persistence.documents.mountPath }}
OPENRAG_DOCUMENTS_PATH={{ .Values.backend.persistence.documents.mountPath | quote }}
{{- end }}
+
{{- if .Values.backend.encryption.key }}
OPENRAG_ENCRYPTION_KEY={{ .Values.backend.encryption.key | quote }}
{{- end }}
diff --git a/kubernetes/helm/openrag/templates/backend/deployment.yaml b/kubernetes/helm/openrag/templates/backend/deployment.yaml
index 7578590c5..c180b9a36 100644
--- a/kubernetes/helm/openrag/templates/backend/deployment.yaml
+++ b/kubernetes/helm/openrag/templates/backend/deployment.yaml
@@ -35,7 +35,7 @@ spec:
{{- toYaml . | nindent 8 }}
{{- end }}
initContainers:
- {{- if and .Values.backend.persistence.documents.enabled .Values.backend.persistence.documents.loadDefaults .Values.backend.persistence.documents.git.enabled }}
+ {{- if and .Values.backend.persistence.shared.enabled .Values.backend.persistence.documents.loadDefaults .Values.backend.persistence.documents.git.enabled }}
- name: load-default-documents
image: curlimages/curl:8.5.0
command: ["/bin/sh", "-c"]
@@ -103,8 +103,79 @@ spec:
echo "Documents already exist in $DOCS_DIR, skipping download."
fi
volumeMounts:
- - name: documents
- mountPath: {{ .Values.backend.persistence.documents.mountPath }}
+ - name: shared
+ mountPath: {{ .Values.backend.persistence.shared.mountPath | quote }}
+ {{- end }}
+ {{- if and .Values.backend.persistence.shared.enabled .Values.langflow.flows.loadDefaults .Values.langflow.flows.git.enabled }}
+ - name: load-default-flows
+ image: curlimages/curl:8.5.0
+ command: ["/bin/sh", "-c"]
+ args:
+ - |
+ set -eu
+
+ FLOWS_DIR="{{ .Values.backend.persistence.shared.flowsPath }}"
+
+ OWNER="{{ .Values.langflow.flows.git.owner }}"
+ REPO="{{ .Values.langflow.flows.git.repo }}"
+ BRANCH="{{ .Values.langflow.flows.git.branch }}"
+ {{- if .Values.langflow.flows.git.hash }}
+ GIT_REF="{{ .Values.langflow.flows.git.hash }}"
+ {{- else }}
+ GIT_REF="${BRANCH}"
+ {{- end }}
+ REPO_PATH="{{ .Values.langflow.flows.git.path }}"
+
+ # Raw base URL: https://raw.githubusercontent.com/<owner>/<repo>/<ref>/<path>
+ GITHUB_RAW_URL="https://raw.githubusercontent.com/${OWNER}/${REPO}/${GIT_REF}/${REPO_PATH}"
+
+ mkdir -p "$FLOWS_DIR"
+
+ if [ -z "$(ls -A "$FLOWS_DIR" 2>/dev/null)" ]; then
+ echo "Loading default flows from ${OWNER}/${REPO}@${GIT_REF}/${REPO_PATH} ..."
+
+ download_with_retry() {
+ file="$1"
+ url="${GITHUB_RAW_URL}/${file}"
+ dest="${FLOWS_DIR}/${file}"
+ max_retries=3
+ timeout=60
+ retry=0
+
+ while [ "$retry" -lt "$max_retries" ]; do
+ echo "Downloading ${file} (attempt $((retry + 1))/${max_retries})..."
+ if curl -f -L -m "$timeout" -o "$dest" "$url"; then
+ echo "OK: ${file}"
+ return 0
+ fi
+ retry=$((retry + 1))
+ if [ "$retry" -lt "$max_retries" ]; then
+ echo "Retrying ${file}..."
+ sleep 2
+ fi
+ done
+
+ echo "ERROR: Failed to download ${file} from ${url}"
+ return 1
+ }
+
+ failed=0
+{{- range .Values.langflow.flows.git.files }}
+ if ! download_with_retry "{{ . }}"; then failed=1; fi
+{{- end }}
+
+ if [ "$failed" -eq 1 ]; then
+ echo "ERROR: One or more flows failed to download"
+ exit 1
+ fi
+
+ echo "Flows loaded: $(ls "$FLOWS_DIR")"
+ else
+ echo "Flows already exist in $FLOWS_DIR, skipping download."
+ fi
+ volumeMounts:
+ - name: shared
+ mountPath: {{ .Values.backend.persistence.shared.mountPath | quote }}
{{- end }}
{{- if .Values.langflow.enabled }}
- name: wait-for-langflow
@@ -137,22 +208,13 @@ spec:
mountPath: /app/.env
subPath: .env
readOnly: true
- {{- if .Values.backend.persistence.documents.enabled }}
- - name: documents
- mountPath: {{ .Values.backend.persistence.documents.mountPath }}
- {{- end }}
- {{- if .Values.backend.persistence.keys.enabled }}
- - name: keys
- mountPath: {{ .Values.backend.persistence.keys.mountPath }}
- {{- end }}
- {{- if .Values.backend.persistence.config.enabled }}
- - name: config
- mountPath: {{ .Values.backend.persistence.config.mountPath }}
+ {{- if .Values.backend.persistence.data.enabled }}
+ - name: data
+ mountPath: {{ .Values.backend.persistence.data.mountPath | quote }}
{{- end }}
- {{- if .Values.langflow.persistence.enabled }}
- - name: flows
- mountPath: /app/flows
- subPath: {{ .Values.langflow.persistence.flowsSubPath }}
+ {{- if .Values.backend.persistence.shared.enabled }}
+ - name: shared
+ mountPath: {{ .Values.backend.persistence.shared.mountPath | quote }}
readOnly: true
{{- end }}
resources:
@@ -201,24 +263,15 @@ spec:
items:
- key: .env
path: .env
- {{- if .Values.backend.persistence.documents.enabled }}
- - name: documents
- emptyDir: {}
- {{- end }}
- {{- if .Values.backend.persistence.keys.enabled }}
- - name: keys
- persistentVolumeClaim:
- claimName: {{ include "openrag.fullname" . }}-keys
- {{- end }}
- {{- if .Values.backend.persistence.config.enabled }}
- - name: config
+ {{- if .Values.backend.persistence.data.enabled }}
+ - name: data
persistentVolumeClaim:
- claimName: {{ include "openrag.fullname" . }}-config
+ claimName: {{ include "openrag.fullname" . }}-data
{{- end }}
- {{- if .Values.langflow.persistence.enabled }}
- - name: flows
+ {{- if .Values.backend.persistence.shared.enabled }}
+ - name: shared
persistentVolumeClaim:
- claimName: {{ include "openrag.fullname" . }}-langflow
+ claimName: {{ include "openrag.fullname" . }}-shared
{{- end }}
{{- with .Values.nodeSelector }}
nodeSelector:
diff --git a/kubernetes/helm/openrag/templates/langflow/deployment.yaml b/kubernetes/helm/openrag/templates/langflow/deployment.yaml
index f0f32981c..a3bd63ec0 100644
--- a/kubernetes/helm/openrag/templates/langflow/deployment.yaml
+++ b/kubernetes/helm/openrag/templates/langflow/deployment.yaml
@@ -34,78 +34,6 @@ spec:
{{- toYaml . | nindent 8 }}
{{- end }}
- {{- if and .Values.langflow.flows.loadDefaults .Values.langflow.flows.git.enabled }}
- initContainers:
- - name: load-default-flows
- image: curlimages/curl:8.5.0
- command: ["/bin/sh", "-c"]
- args:
- - |
- set -eu
-
- FLOWS_DIR="{{ .Values.langflow.persistence.mountPath }}/{{ .Values.langflow.persistence.flowsSubPath }}"
-
- OWNER="{{ .Values.langflow.flows.git.owner }}"
- REPO="{{ .Values.langflow.flows.git.repo }}"
- BRANCH="{{ .Values.langflow.flows.git.branch }}"
- {{- if .Values.langflow.flows.git.hash }}
- GIT_REF="{{ .Values.langflow.flows.git.hash }}"
- {{- else }}
- GIT_REF="${BRANCH}"
- {{- end }}
- REPO_PATH="{{ .Values.langflow.flows.git.path }}"
-
- # Raw base URL: https://raw.githubusercontent.com/<owner>/<repo>/<ref>/<path>
- GITHUB_RAW_URL="https://raw.githubusercontent.com/${OWNER}/${REPO}/${GIT_REF}/${REPO_PATH}"
-
- mkdir -p "$FLOWS_DIR"
-
- if [ -z "$(ls -A "$FLOWS_DIR" 2>/dev/null)" ]; then
- echo "Loading default flows from ${OWNER}/${REPO}@${GIT_REF}/${REPO_PATH} ..."
-
- download_with_retry() {
- file="$1"
- url="${GITHUB_RAW_URL}/${file}"
- dest="${FLOWS_DIR}/${file}"
- max_retries=3
- timeout=60
- retry=0
-
- while [ "$retry" -lt "$max_retries" ]; do
- echo "Downloading ${file} (attempt $((retry + 1))/${max_retries})..."
- if curl -f -L -m "$timeout" -o "$dest" "$url"; then
- echo "OK: ${file}"
- return 0
- fi
- retry=$((retry + 1))
- if [ "$retry" -lt "$max_retries" ]; then
- echo "Retrying ${file}..."
- sleep 2
- fi
- done
-
- echo "ERROR: Failed to download ${file} from ${url}"
- return 1
- }
-
- failed=0
-{{- range .Values.langflow.flows.git.files }}
- if ! download_with_retry "{{ . }}"; then failed=1; fi
-{{- end }}
-
- if [ "$failed" -eq 1 ]; then
- echo "ERROR: One or more flows failed to download"
- exit 1
- fi
-
- echo "Flows loaded: $(ls "$FLOWS_DIR")"
- else
- echo "Flows already exist in $FLOWS_DIR, skipping download."
- fi
- volumeMounts:
- - name: langflow-data
- mountPath: {{ .Values.langflow.persistence.mountPath }}
- {{- end }}
containers:
- name: langflow
@@ -131,6 +59,9 @@ spec:
readOnly: true
- name: langflow-data
mountPath: {{ .Values.langflow.persistence.mountPath }}
+ - name: shared
+ mountPath: {{ .Values.backend.persistence.shared.mountPath | quote }}
+ readOnly: true
resources:
{{- toYaml .Values.langflow.resources | nindent 12 }}
@@ -172,6 +103,11 @@ spec:
{{- else }}
emptyDir: {}
{{- end }}
+ {{- if .Values.backend.persistence.shared.enabled }}
+ - name: shared
+ persistentVolumeClaim:
+ claimName: {{ include "openrag.fullname" . }}-shared
+ {{- end }}
{{- with .Values.nodeSelector }}
nodeSelector:
diff --git a/kubernetes/helm/openrag/templates/langflow/langflow-dotenv.yaml b/kubernetes/helm/openrag/templates/langflow/langflow-dotenv.yaml
index 7a2972b42..c7ffda957 100644
--- a/kubernetes/helm/openrag/templates/langflow/langflow-dotenv.yaml
+++ b/kubernetes/helm/openrag/templates/langflow/langflow-dotenv.yaml
@@ -37,7 +37,6 @@ stringData:
# Mounted as a file into the pod
# Langflow core settings
- LANGFLOW_LOAD_FLOWS_PATH={{ .Values.langflow.persistence.mountPath }}/{{ .Values.langflow.persistence.flowsSubPath }}
{{- if .Values.postgres.enabled }}
LANGFLOW_DATABASE_URL="postgresql://{{ .Values.postgres.username }}:{{ $postgresPassword }}@{{ include "openrag.fullname" . }}-postgres:{{ .Values.postgres.service.port }}/{{ .Values.postgres.database }}"
{{- else }}
diff --git a/kubernetes/helm/openrag/templates/storage/backend-sc.yaml b/kubernetes/helm/openrag/templates/storage/backend-sc.yaml
index 488e20082..2a7418bd6 100644
--- a/kubernetes/helm/openrag/templates/storage/backend-sc.yaml
+++ b/kubernetes/helm/openrag/templates/storage/backend-sc.yaml
@@ -1,4 +1,4 @@
-{{- if or (.Values.backend.persistence.config.enabled) (.Values.backend.persistence.keys.enabled) }}
+{{- if or (.Values.backend.persistence.data.enabled) (.Values.backend.persistence.shared.enabled) }}
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
@@ -18,5 +18,5 @@ parameters:
{{- end }}
reclaimPolicy: {{ .Values.backend.storageClassReclaimPolicy | default "Delete" }}
volumeBindingMode: {{ .Values.backend.storageClassVolumeBindingMode | default "WaitForFirstConsumer" }}
-allowVolumeExpansion: {{ .Values.backend.storageClassAllowVolumeExpansion | default true }}
+allowVolumeExpansion: {{ default true .Values.backend.storageClassAllowVolumeExpansion }}
{{- end }}
\ No newline at end of file
diff --git a/kubernetes/helm/openrag/templates/storage/config-pvc.yaml b/kubernetes/helm/openrag/templates/storage/config-pvc.yaml
deleted file mode 100644
index d5ced07fc..000000000
--- a/kubernetes/helm/openrag/templates/storage/config-pvc.yaml
+++ /dev/null
@@ -1,18 +0,0 @@
-{{- if and .Values.backend.enabled .Values.backend.persistence.config.enabled }}
-apiVersion: v1
-kind: PersistentVolumeClaim
-metadata:
- name: {{ include "openrag.fullname" . }}-config
- namespace: {{ include "openrag.namespace" . }}
- labels:
- {{- include "openrag.backend.labels" . | nindent 4 }}
-spec:
- accessModes:
- - {{ .Values.backend.persistence.config.accessMode }}
- {{- if .Values.backend.persistence.config.storageClass }}
- storageClassName: {{ .Values.backend.persistence.config.storageClass | quote }}
- {{- end }}
- resources:
- requests:
- storage: {{ .Values.backend.persistence.config.size }}
-{{- end }}
diff --git a/kubernetes/helm/openrag/templates/storage/data-pvc.yaml b/kubernetes/helm/openrag/templates/storage/data-pvc.yaml
new file mode 100644
index 000000000..a30cd09dd
--- /dev/null
+++ b/kubernetes/helm/openrag/templates/storage/data-pvc.yaml
@@ -0,0 +1,18 @@
+{{- if and .Values.backend.enabled .Values.backend.persistence.data.enabled }}
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+ name: {{ include "openrag.fullname" . }}-data
+ namespace: {{ include "openrag.namespace" . }}
+ labels:
+ {{- include "openrag.backend.labels" . | nindent 4 }}
+spec:
+ accessModes:
+ - {{ .Values.backend.persistence.data.accessMode }}
+ {{- if .Values.backend.persistence.data.storageClass }}
+ storageClassName: {{ .Values.backend.persistence.data.storageClass | quote }}
+ {{- end }}
+ resources:
+ requests:
+ storage: {{ .Values.backend.persistence.data.size }}
+{{- end }}
diff --git a/kubernetes/helm/openrag/templates/storage/keys-pvc.yaml b/kubernetes/helm/openrag/templates/storage/keys-pvc.yaml
deleted file mode 100644
index 6d2acd4c7..000000000
--- a/kubernetes/helm/openrag/templates/storage/keys-pvc.yaml
+++ /dev/null
@@ -1,18 +0,0 @@
-{{- if and .Values.backend.enabled .Values.backend.persistence.keys.enabled }}
-apiVersion: v1
-kind: PersistentVolumeClaim
-metadata:
- name: {{ include "openrag.fullname" . }}-keys
- namespace: {{ include "openrag.namespace" . }}
- labels:
- {{- include "openrag.backend.labels" . | nindent 4 }}
-spec:
- accessModes:
- - {{ .Values.backend.persistence.keys.accessMode }}
- {{- if .Values.backend.persistence.keys.storageClass }}
- storageClassName: {{ .Values.backend.persistence.keys.storageClass | quote }}
- {{- end }}
- resources:
- requests:
- storage: {{ .Values.backend.persistence.keys.size }}
-{{- end }}
diff --git a/kubernetes/helm/openrag/templates/storage/shared-pvc.yaml b/kubernetes/helm/openrag/templates/storage/shared-pvc.yaml
new file mode 100644
index 000000000..8d3ad4820
--- /dev/null
+++ b/kubernetes/helm/openrag/templates/storage/shared-pvc.yaml
@@ -0,0 +1,18 @@
+{{- if and .Values.backend.persistence.shared.enabled (or .Values.backend.enabled .Values.langflow.enabled) }}
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+ name: {{ include "openrag.fullname" . }}-shared
+ namespace: {{ include "openrag.namespace" . }}
+ labels:
+ {{- include "openrag.backend.labels" . | nindent 4 }}
+spec:
+ accessModes:
+ - {{ .Values.backend.persistence.shared.accessMode }}
+ {{- if .Values.backend.persistence.shared.storageClass }}
+ storageClassName: {{ .Values.backend.persistence.shared.storageClass | quote }}
+ {{- end }}
+ resources:
+ requests:
+ storage: {{ .Values.backend.persistence.shared.size }}
+{{- end }}
diff --git a/kubernetes/helm/openrag/values.yaml b/kubernetes/helm/openrag/values.yaml
index cdf0ace24..a1fc3b221 100644
--- a/kubernetes/helm/openrag/values.yaml
+++ b/kubernetes/helm/openrag/values.yaml
@@ -238,41 +238,42 @@ backend:
logLevel: "INFO" # DEBUG, INFO, WARNING, ERROR
serviceName: "openrag"
-
- # Persistence for documents, keys, and config
+ # Persistence consolidated into two main volumes:
+ # 1. /data: internal application state (data, backups, config, keys)
+ # 2. /shared: shared content (documents, flows)
persistence:
- documents:
+ data:
enabled: true
storageClass: ""
accessMode: ReadWriteOnce
+ size: 10Gi
+ mountPath: /data
+ dataPath: "/data/data"
+ backupPath: "/data/backup"
+ configPath: "/data/config"
+ keysPath: "/data/keys"
+ shared:
+ enabled: true
+ storageClass: ""
+ accessMode: ReadWriteMany # Often beneficial for shared content between backend/langflow
size: 50Gi
- mountPath: /app/openrag-documents
- loadDefaults: true # Load default documents on first deployment
+ mountPath: /shared
+ flowsPath: "/shared/flows"
+ documents:
+ mountPath: "/shared/openrag-documents"
+ loadDefaults: true
git:
enabled: true
owner: langflow-ai
repo: openrag
branch: main
- # Optional: specific git commit hash (overrides branch if set)
- #hash: ""
path: openrag-documents
files:
- docling.pdf
- ibm_anthropic.pdf
- openrag-documentation.pdf
- warmup_ocr.pdf
- keys:
- enabled: true
- storageClass: ""
- accessMode: ReadWriteOnce
- size: 1Gi
- mountPath: /app/keys
- config:
- enabled: true
- storageClass: ""
- accessMode: ReadWriteOnce
- size: 1Gi
- mountPath: /app/config
+
# Logging
logLevel: "INFO" # DEBUG, INFO, WARNING, ERROR, CRITICAL
diff --git a/src/config/config_manager.py b/src/config/config_manager.py
index fea87ba43..2d04d8dd8 100644
--- a/src/config/config_manager.py
+++ b/src/config/config_manager.py
@@ -165,7 +165,11 @@ class ConfigManager:
Args:
config_file: Path to configuration file. Defaults to 'config.yaml' in project root.
"""
- self.config_file = Path(config_file) if config_file else Path("config/config.yaml")
+ if config_file:
+ self.config_file = Path(config_file)
+ else:
+ from config.paths import get_config_file_path
+ self.config_file = Path(get_config_file_path())
self._config: Optional[OpenRAGConfig] = None
diff --git a/src/config/paths.py b/src/config/paths.py
new file mode 100644
index 000000000..41eed34fd
--- /dev/null
+++ b/src/config/paths.py
@@ -0,0 +1,93 @@
+"""Centralized path helpers for OpenRAG.
+
+This module is the single source of truth for the OPENRAG_*_PATH environment variables.
+Variables are read directly via ``os.getenv`` here to avoid circular imports
+(settings → config_manager → paths → settings).
+"""
+
+import os
+
+
+# ---------------------------------------------------------------------------
+# Documents directory
+# ---------------------------------------------------------------------------
+def get_documents_path() -> str:
+ """Return the path to the documents directory.
+
+ Environment variable: OPENRAG_DOCUMENTS_PATH
+ Default: ``openrag-documents`` (relative to the working directory)
+ """
+ return os.getenv("OPENRAG_DOCUMENTS_PATH") or "openrag-documents"
+
+
+# ---------------------------------------------------------------------------
+# JWT keys directory
+# ---------------------------------------------------------------------------
+def get_keys_path() -> str:
+ """Return the path to the JWT keys directory.
+
+ Environment variable: OPENRAG_KEYS_PATH
+ Default: ``keys`` (relative to the working directory)
+ """
+ return os.getenv("OPENRAG_KEYS_PATH") or "keys"
+
+
+# ---------------------------------------------------------------------------
+# Flows directory
+# ---------------------------------------------------------------------------
+def get_flows_path() -> str:
+ """Return the path to the flows directory.
+
+ Environment variable: OPENRAG_FLOWS_PATH
+ Default: ``flows`` (relative to the working directory)
+ """
+ return os.getenv("OPENRAG_FLOWS_PATH") or "flows"
+
+
+def get_flows_backup_path() -> str:
+ """Return the path to the flows backup directory.
+
+ Environment variable: OPENRAG_FLOWS_BACKUP_PATH
+ Default: ``<flows_path>/backup``
+ """
+ return os.getenv("OPENRAG_FLOWS_BACKUP_PATH") or os.path.join(get_flows_path(), "backup")
+
+
+# ---------------------------------------------------------------------------
+# Config directory (holds config.yaml)
+# ---------------------------------------------------------------------------
+def get_config_path() -> str:
+ """Return the path to the configuration directory.
+
+ Environment variable: OPENRAG_CONFIG_PATH
+ Default: ``config`` (relative to the working directory)
+ """
+ return os.getenv("OPENRAG_CONFIG_PATH") or "config"
+
+
+def get_config_file_path() -> str:
+ """Return the full path to the config.yaml file."""
+ return os.path.join(get_config_path(), "config.yaml")
+
+
+# ---------------------------------------------------------------------------
+# Data directory (conversations, tokens, connections, etc.)
+# ---------------------------------------------------------------------------
+def get_data_path() -> str:
+ """Return the path to the data directory.
+
+ Environment variable: OPENRAG_DATA_PATH
+ Default: ``data`` (relative to the working directory)
+ """
+ return os.getenv("OPENRAG_DATA_PATH") or "data"
+
+
+def get_data_file(filename: str) -> str:
+ """Return a full path for a file inside the data directory.
+
+ Example::
+
+ get_data_file("conversations.json")
+ # → "data/conversations.json" (or $OPENRAG_DATA_PATH/conversations.json)
+ """
+ return os.path.join(get_data_path(), filename)
diff --git a/src/config/settings.py b/src/config/settings.py
index 3a8af891b..b992d2dbc 100644
--- a/src/config/settings.py
+++ b/src/config/settings.py
@@ -1,3 +1,4 @@
+from config.paths import get_flows_path
import asyncio
import os
from utils.env_utils import get_env_int, get_env_float
@@ -101,7 +102,6 @@ LANGFLOW_CONNECT_TIMEOUT = get_env_float("LANGFLOW_CONNECT_TIMEOUT", 30.0) # 30
# Default: 3600 seconds (60 minutes)
INGESTION_TIMEOUT = get_env_int("INGESTION_TIMEOUT", 3600)
-
def is_no_auth_mode():
"""Check if we're running in no-auth mode (OAuth credentials missing)"""
if IBM_AUTH_ENABLED:
@@ -830,24 +830,32 @@ class AppClients:
)
-# Component template paths
-WATSONX_LLM_COMPONENT_PATH = os.getenv(
- "WATSONX_LLM_COMPONENT_PATH", "flows/components/watsonx_llm.json"
+# Component template paths — derived from the centralized flows directory
+def _component_path(env_var: str, filename: str) -> str:
+ """Return a component path, using the env var override or the centralized flows dir."""
+ env_val = os.getenv(env_var)
+ if env_val:
+ return env_val
+ flows_dir = get_flows_path()
+ return os.path.join(flows_dir, "components", filename)
+
+WATSONX_LLM_COMPONENT_PATH = _component_path(
+ "WATSONX_LLM_COMPONENT_PATH", "watsonx_llm.json"
)
-WATSONX_LLM_TEXT_COMPONENT_PATH = os.getenv(
- "WATSONX_LLM_TEXT_COMPONENT_PATH", "flows/components/watsonx_llm_text.json"
+WATSONX_LLM_TEXT_COMPONENT_PATH = _component_path(
+ "WATSONX_LLM_TEXT_COMPONENT_PATH", "watsonx_llm_text.json"
)
-WATSONX_EMBEDDING_COMPONENT_PATH = os.getenv(
- "WATSONX_EMBEDDING_COMPONENT_PATH", "flows/components/watsonx_embedding.json"
+WATSONX_EMBEDDING_COMPONENT_PATH = _component_path(
+ "WATSONX_EMBEDDING_COMPONENT_PATH", "watsonx_embedding.json"
)
-OLLAMA_LLM_COMPONENT_PATH = os.getenv(
- "OLLAMA_LLM_COMPONENT_PATH", "flows/components/ollama_llm.json"
+OLLAMA_LLM_COMPONENT_PATH = _component_path(
+ "OLLAMA_LLM_COMPONENT_PATH", "ollama_llm.json"
)
-OLLAMA_LLM_TEXT_COMPONENT_PATH = os.getenv(
- "OLLAMA_LLM_TEXT_COMPONENT_PATH", "flows/components/ollama_llm_text.json"
+OLLAMA_LLM_TEXT_COMPONENT_PATH = _component_path(
+ "OLLAMA_LLM_TEXT_COMPONENT_PATH", "ollama_llm_text.json"
)
-OLLAMA_EMBEDDING_COMPONENT_PATH = os.getenv(
- "OLLAMA_EMBEDDING_COMPONENT_PATH", "flows/components/ollama_embedding.json"
+OLLAMA_EMBEDDING_COMPONENT_PATH = _component_path(
+ "OLLAMA_EMBEDDING_COMPONENT_PATH", "ollama_embedding.json"
)
# Component IDs in flows
diff --git a/src/connectors/connection_manager.py b/src/connectors/connection_manager.py
index 8081e0b7f..8646d72aa 100644
--- a/src/connectors/connection_manager.py
+++ b/src/connectors/connection_manager.py
@@ -39,8 +39,9 @@ class ConnectionConfig:
class ConnectionManager:
"""Manages multiple connector connections with persistence"""
- def __init__(self, connections_file: str = "data/connections.json"):
- self.connections_file = Path(connections_file)
+ def __init__(self, connections_file: str = None):
+ from config.paths import get_data_file
+ self.connections_file = Path(connections_file or get_data_file("connections.json"))
# Ensure data directory exists
self.connections_file.parent.mkdir(parents=True, exist_ok=True)
self.connections: Dict[str, ConnectionConfig] = {}
diff --git a/src/connectors/google_drive/connector.py b/src/connectors/google_drive/connector.py
index e55f62619..53b3e8eea 100644
--- a/src/connectors/google_drive/connector.py
+++ b/src/connectors/google_drive/connector.py
@@ -97,8 +97,9 @@ class GoogleDriveConnector(BaseConnector):
client_id = config.get("client_id") or env_client_id
client_secret = config.get("client_secret") or env_client_secret
- # Token file default - use data/ directory for persistence
- token_file = config.get("token_file") or "data/google_drive_token.json"
+ # Token file default - use data directory for persistence
+ from config.paths import get_data_file
+ token_file = config.get("token_file") or get_data_file("google_drive_token.json")
Path(token_file).parent.mkdir(parents=True, exist_ok=True)
if not isinstance(client_id, str) or not client_id.strip():
diff --git a/src/connectors/onedrive/connector.py b/src/connectors/onedrive/connector.py
index e6348db77..5a9c5eeda 100644
--- a/src/connectors/onedrive/connector.py
+++ b/src/connectors/onedrive/connector.py
@@ -61,8 +61,9 @@ class OneDriveConnector(BaseConnector):
except Exception as e:
logger.debug(f"Failed to get client_secret: {e}")
- # Token file setup - use data/ directory for persistence
- token_file = config.get("token_file") or "data/onedrive_token.json"
+ # Token file setup - use data directory for persistence
+ from config.paths import get_data_file
+ token_file = config.get("token_file") or get_data_file("onedrive_token.json")
Path(token_file).parent.mkdir(parents=True, exist_ok=True)
# Only initialize OAuth if we have credentials
@@ -74,7 +75,7 @@ class OneDriveConnector(BaseConnector):
oauth_token_file = config["token_file"]
else:
# Use a per-connection cache file to avoid collisions with other connectors
- oauth_token_file = f"data/onedrive_token_{connection_id}.json"
+ oauth_token_file = get_data_file(f"onedrive_token_{connection_id}.json")
# MSA & org both work via /common for OneDrive personal testing
authority = "https://login.microsoftonline.com/common"
diff --git a/src/connectors/sharepoint/connector.py b/src/connectors/sharepoint/connector.py
index 6900b64c4..c8f8fafa4 100644
--- a/src/connectors/sharepoint/connector.py
+++ b/src/connectors/sharepoint/connector.py
@@ -68,8 +68,9 @@ class SharePointConnector(BaseConnector):
logger.debug(f"Failed to get client_secret: {e}")
pass # Credentials not available, that's OK for listing
- # Token file setup - use data/ directory for persistence
- token_file = config.get("token_file") or "data/sharepoint_token.json"
+ # Token file setup - use data directory for persistence
+ from config.paths import get_data_file
+ token_file = config.get("token_file") or get_data_file("sharepoint_token.json")
Path(token_file).parent.mkdir(parents=True, exist_ok=True)
# Only initialize OAuth if we have credentials
@@ -80,7 +81,7 @@ class SharePointConnector(BaseConnector):
if config.get("token_file"):
oauth_token_file = config["token_file"]
else:
- oauth_token_file = f"data/sharepoint_token_{connection_id}.json"
+ oauth_token_file = get_data_file(f"sharepoint_token_{connection_id}.json")
authority = f"https://login.microsoftonline.com/{self.tenant_id}" if self.tenant_id != "common" else "https://login.microsoftonline.com/common"
diff --git a/src/main.py b/src/main.py
index a2ade69c9..1b89a38a9 100644
--- a/src/main.py
+++ b/src/main.py
@@ -14,7 +14,6 @@ from html.parser import HTMLParser
from connectors.langflow_connector_service import LangflowConnectorService
from connectors.service import ConnectorService
from services.flows_service import FlowsService
-from utils.container_utils import detect_container_environment
from utils.embeddings import create_dynamic_index_body
from utils.logging_config import configure_from_env, get_logger
from utils.encryption import enforce_startup_prerequisites
@@ -360,7 +359,8 @@ async def init_index_when_ready(opensearch_client=None):
def generate_jwt_keys():
"""Generate RSA keys for JWT signing if they don't exist"""
- keys_dir = "keys"
+ from config.paths import get_keys_path
+ keys_dir = get_keys_path()
private_key_path = os.path.join(keys_dir, "private_key.pem")
public_key_path = os.path.join(keys_dir, "public_key.pem")
@@ -417,17 +417,10 @@ def generate_jwt_keys():
def _get_documents_dir():
"""Get the documents directory path, handling both Docker and local environments."""
- # In Docker, the volume is mounted at /app/openrag-documents
- # Locally, we use openrag-documents
- container_env = detect_container_environment()
- if container_env:
- path = os.path.abspath("/app/openrag-documents")
- logger.debug(f"Running in {container_env}, using container path: {path}")
- return path
- else:
- path = os.path.abspath(os.path.join(os.getcwd(), "openrag-documents"))
- logger.debug(f"Running locally, using local path: {path}")
- return path
+ from config.paths import get_documents_path
+ path = get_documents_path()
+ logger.debug(f"Using documents path: {path}")
+ return path
def _should_use_url_default_docs_ingest() -> bool:
diff --git a/src/services/auth_service.py b/src/services/auth_service.py
index f5c701efb..3c04fa26d 100644
--- a/src/services/auth_service.py
+++ b/src/services/auth_service.py
@@ -86,8 +86,9 @@ class AuthService:
# We'll validate client credentials when creating the connector
- # Create connection configuration - use data/ directory for persistence
- token_file = f"data/{connector_type}_{purpose}_{uuid.uuid4().hex[:8]}.json"
+ # Create connection configuration - use data directory for persistence
+ from config.paths import get_data_file
+ token_file = get_data_file(f"{connector_type}_{purpose}_{uuid.uuid4().hex[:8]}.json")
effective_redirect_uri = OAUTH_BROKER_URL or redirect_uri
config = {
"token_file": token_file,
diff --git a/src/services/conversation_persistence_service.py b/src/services/conversation_persistence_service.py
index 8af36efff..d7049fd31 100644
--- a/src/services/conversation_persistence_service.py
+++ b/src/services/conversation_persistence_service.py
@@ -16,8 +16,9 @@ logger = get_logger(__name__)
class ConversationPersistenceService:
"""Simple service to persist conversations to disk"""
- def __init__(self, storage_file: str = "data/conversations.json"):
- self.storage_file = storage_file
+ def __init__(self, storage_file: str = None):
+ from config.paths import get_data_file
+ self.storage_file = storage_file or get_data_file("conversations.json")
# Ensure data directory exists
os.makedirs(os.path.dirname(self.storage_file), exist_ok=True)
self.lock = threading.Lock()
diff --git a/src/services/flows_service.py b/src/services/flows_service.py
index f06d7b948..ba9957bc1 100644
--- a/src/services/flows_service.py
+++ b/src/services/flows_service.py
@@ -80,15 +80,13 @@ class FlowsService:
def _get_flows_directory(self):
"""Get the flows directory path"""
- current_file_dir = os.path.dirname(os.path.abspath(__file__)) # src/services/
- src_dir = os.path.dirname(current_file_dir) # src/
- project_root = os.path.dirname(src_dir) # project root
- return os.path.join(project_root, "flows")
+ from config.paths import get_flows_path
+ return get_flows_path()
def _get_backup_directory(self):
"""Get the backup directory path"""
- flows_dir = self._get_flows_directory()
- backup_dir = os.path.join(flows_dir, "backup")
+ from config.paths import get_flows_backup_path
+ backup_dir = get_flows_backup_path()
os.makedirs(backup_dir, exist_ok=True)
return backup_dir
diff --git a/src/services/langflow_file_service.py b/src/services/langflow_file_service.py
index 21e946d2d..bceef3f5a 100644
--- a/src/services/langflow_file_service.py
+++ b/src/services/langflow_file_service.py
@@ -353,7 +353,8 @@ class LangflowFileService:
max_attempts = 2
last_error: Exception | None = None
- flow_file = Path(__file__).resolve().parents[2] / "flows" / "openrag_url_mcp.json"
+ from config.paths import get_flows_path
+ flow_file = Path(get_flows_path()) / "openrag_url_mcp.json"
if not flow_file.exists():
raise ValueError(
"LANGFLOW_URL_INGEST_FLOW_ID is invalid and "
diff --git a/src/services/session_ownership_service.py b/src/services/session_ownership_service.py
index 8fcd83084..2025d867c 100644
--- a/src/services/session_ownership_service.py
+++ b/src/services/session_ownership_service.py
@@ -15,7 +15,8 @@ class SessionOwnershipService:
"""Simple service to track which user owns which session"""
def __init__(self):
- self.ownership_file = "data/session_ownership.json"
+ from config.paths import get_data_file
+ self.ownership_file = get_data_file("session_ownership.json")
# Ensure data directory exists
os.makedirs(os.path.dirname(self.ownership_file), exist_ok=True)
self.ownership_data = self._load_ownership_data()
diff --git a/src/session_manager.py b/src/session_manager.py
index ff40950a5..8185c2146 100644
--- a/src/session_manager.py
+++ b/src/session_manager.py
@@ -50,17 +50,19 @@ class SessionManager:
def __init__(
self,
secret_key: str = None,
- private_key_path: str = "keys/private_key.pem",
- public_key_path: str = "keys/public_key.pem",
+ private_key_path: str = None,
+ public_key_path: str = None,
):
+ from config.paths import get_keys_path
+ keys_dir = get_keys_path()
self.secret_key = secret_key # Keep for backward compatibility
self.users: Dict[str, User] = {} # user_id -> User
self.user_opensearch_clients: Dict[
str, Any
] = {} # user_id -> OpenSearch client
- self.private_key_path = private_key_path
- self.public_key_path = public_key_path
+ self.private_key_path = private_key_path or os.path.join(keys_dir, "private_key.pem")
+ self.public_key_path = public_key_path or os.path.join(keys_dir, "public_key.pem")
# Configure JWT signing (checks env var first, falls back to key files)
self._configure_jwt_signing()