OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.
Python
fix: centralize backend paths and accept env var to define them, change deployments to centralize read only and writable volumes (#1353)
Lucas Oliveira0cefaacc64fa
diff --git a/.env.example b/.env.exampleindex 18d9e832d..ba5207354 100644--- a/.env.example+++ b/.env.example@@ -203,8 +203,13 @@ NEXT_ALLOWED_DEV_ORIGINS=# but can be pinned here to lock the deployment to a specific version when running OpenRAG with Make commands.OPENRAG_VERSION=latest-# The local documents path for ingestion.-OPENRAG_DOCUMENTS_PATH=./openrag-documents+# Path configuration (override defaults for custom directory layouts)+# OPENRAG_DATA_PATH=./data+# OPENRAG_KEYS_PATH=./keys+# OPENRAG_CONFIG_PATH=./config+# OPENRAG_FLOWS_PATH=./flows+# OPENRAG_FLOWS_BACKUP_PATH= # defaults to <OPENRAG_FLOWS_PATH>/backup+# OPENRAG_DOCUMENTS_PATH=./openrag-documents# Langflow Configuration# API Key for authenticating with Langflow.diff --git a/docker-compose.yml b/docker-compose.ymlindex fd024e88a..8a1d31ad7 100644--- a/docker-compose.yml+++ b/docker-compose.yml@@ -105,6 +105,7 @@ services:- ${OPENRAG_DOCUMENTS_PATH:-./openrag-documents}:/app/openrag-documents:U,z- ${OPENRAG_KEYS_PATH:-./keys}:/app/keys:U,z- ${OPENRAG_FLOWS_PATH:-./flows}:/app/flows:U,z+ - ${OPENRAG_FLOWS_BACKUP_PATH:-./flows/backup}:/app/flows/backup:U,z- ${OPENRAG_CONFIG_PATH:-./config}:/app/config:U,z- ${OPENRAG_DATA_PATH:-./data}:/app/data:U,z# Linux: Docker Engine does not auto-inject host.docker.internal; map it to the host gatewaydiff --git a/kubernetes/helm/openrag/Chart.yaml b/kubernetes/helm/openrag/Chart.yamlindex ac9d0df3d..b8f71a686 100644--- a/kubernetes/helm/openrag/Chart.yaml+++ b/kubernetes/helm/openrag/Chart.yaml@@ -2,7 +2,7 @@ apiVersion: v2name: openragdescription: A Helm chart for deploying OpenRAG - an open-source agentic RAG platformtype: application-version: 0.1.0+version: 0.2.0appVersion: "0.1.52"keywords:- openragdiff --git a/kubernetes/helm/openrag/templates/NOTES.txt b/kubernetes/helm/openrag/templates/NOTES.txtindex 932754ba5..5d1a19d95 100644--- a/kubernetes/helm/openrag/templates/NOTES.txt+++ b/kubernetes/helm/openrag/templates/NOTES.txt@@ -77,4 +77,15 @@ Check pod logs:kubectl logs -n {{ include "openrag.namespace" . }} -l app.kubernetes.io/component=backendkubectl logs -n {{ include "openrag.namespace" . }} -l app.kubernetes.io/component=frontend+ === Breakthrough Notice / Breaking Changes ===++CRITICAL: Chart version 0.2.0 introduces a major storage refactoring.+Legacy PVCs (config, keys, documents) have been consolidated into 'data' and 'shared' PVCs.++IF YOU ARE UPGRADING FROM A VERSION < 0.2.0:+1. Manually migrate data from legacy PVCs (config, keys) to the new 'data' volume to avoid data loss.+2. Old PVCs ({{ include "openrag.fullname" . }}-config, {{ include "openrag.fullname" . }}-keys) should be manually deleted after verification.+5. The flag 'backend.persistence.documents.enabled' has been removed.+ To disable default document loading, use 'backend.persistence.documents.loadDefaults: false' instead.+For more information, visit: https://github.com/langflow-ai/openragdiff --git a/kubernetes/helm/openrag/templates/backend/backend-dotenv.yaml b/kubernetes/helm/openrag/templates/backend/backend-dotenv.yamlindex c6d96f741..7d901a0f2 100644--- a/kubernetes/helm/openrag/templates/backend/backend-dotenv.yaml+++ b/kubernetes/helm/openrag/templates/backend/backend-dotenv.yaml@@ -164,9 +164,26 @@ stringData:{{- if .Values.global.imageTag }}OPENRAG_VERSION={{ .Values.global.imageTag | quote }}{{- end }}+ # Centralized Path Configuration+ {{- if .Values.backend.persistence.data.dataPath }}+ OPENRAG_DATA_PATH={{ .Values.backend.persistence.data.dataPath | quote }}+ {{- end }}+ {{- if .Values.backend.persistence.data.backupPath }}+ OPENRAG_FLOWS_BACKUP_PATH={{ .Values.backend.persistence.data.backupPath | quote }}+ {{- end }}+ {{- if .Values.backend.persistence.data.configPath }}+ OPENRAG_CONFIG_PATH={{ .Values.backend.persistence.data.configPath | quote }}+ {{- end }}+ {{- if .Values.backend.persistence.data.keysPath }}+ OPENRAG_KEYS_PATH={{ .Values.backend.persistence.data.keysPath | quote }}+ {{- end }}+ {{- if .Values.backend.persistence.shared.flowsPath }}+ OPENRAG_FLOWS_PATH={{ .Values.backend.persistence.shared.flowsPath | quote }}+ {{- end }}{{- if .Values.backend.persistence.documents.mountPath }}OPENRAG_DOCUMENTS_PATH={{ .Values.backend.persistence.documents.mountPath | quote }}{{- end }}+{{- if .Values.backend.encryption.key }}OPENRAG_ENCRYPTION_KEY={{ .Values.backend.encryption.key | quote }}{{- end }}diff --git a/kubernetes/helm/openrag/templates/backend/deployment.yaml b/kubernetes/helm/openrag/templates/backend/deployment.yamlindex 7578590c5..c180b9a36 100644--- a/kubernetes/helm/openrag/templates/backend/deployment.yaml+++ b/kubernetes/helm/openrag/templates/backend/deployment.yaml@@ -35,7 +35,7 @@ spec:{{- toYaml . | nindent 8 }}{{- end }}initContainers:- {{- if and .Values.backend.persistence.documents.enabled .Values.backend.persistence.documents.loadDefaults .Values.backend.persistence.documents.git.enabled }}+ {{- if and .Values.backend.persistence.shared.enabled .Values.backend.persistence.documents.loadDefaults .Values.backend.persistence.documents.git.enabled }}- name: load-default-documentsimage: curlimages/curl:8.5.0command: ["/bin/sh", "-c"]@@ -103,8 +103,79 @@ spec:echo "Documents already exist in $DOCS_DIR, skipping download."fivolumeMounts:- - name: documents- mountPath: {{ .Values.backend.persistence.documents.mountPath }}+ - name: shared+ mountPath: {{ .Values.backend.persistence.shared.mountPath | quote }}+ {{- end }}+ {{- if and .Values.backend.persistence.shared.enabled .Values.langflow.flows.loadDefaults .Values.langflow.flows.git.enabled }}+ - name: load-default-flows+ image: curlimages/curl:8.5.0+ command: ["/bin/sh", "-c"]+ args:+ - |+ set -eu++ FLOWS_DIR="{{ .Values.backend.persistence.shared.flowsPath }}"++ OWNER="{{ .Values.langflow.flows.git.owner }}"+ REPO="{{ .Values.langflow.flows.git.repo }}"+ BRANCH="{{ .Values.langflow.flows.git.branch }}"+ {{- if .Values.langflow.flows.git.hash }}+ GIT_REF="{{ .Values.langflow.flows.git.hash }}"+ {{- else }}+ GIT_REF="${BRANCH}"+ {{- end }}+ REPO_PATH="{{ .Values.langflow.flows.git.path }}"++ # Raw base URL: https://raw.githubusercontent.com/<owner>/<repo>/<ref>/<path>+ GITHUB_RAW_URL="https://raw.githubusercontent.com/${OWNER}/${REPO}/${GIT_REF}/${REPO_PATH}"++ mkdir -p "$FLOWS_DIR"++ if [ -z "$(ls -A "$FLOWS_DIR" 2>/dev/null)" ]; then+ echo "Loading default flows from ${OWNER}/${REPO}@${GIT_REF}/${REPO_PATH} ..."++ download_with_retry() {+ file="$1"+ url="${GITHUB_RAW_URL}/${file}"+ dest="${FLOWS_DIR}/${file}"+ max_retries=3+ timeout=60+ retry=0++ while [ "$retry" -lt "$max_retries" ]; do+ echo "Downloading ${file} (attempt $((retry + 1))/${max_retries})..."+ if curl -f -L -m "$timeout" -o "$dest" "$url"; then+ echo "OK: ${file}"+ return 0+ fi+ retry=$((retry + 1))+ if [ "$retry" -lt "$max_retries" ]; then+ echo "Retrying ${file}..."+ sleep 2+ fi+ done++ echo "ERROR: Failed to download ${file} from ${url}"+ return 1+ }++ failed=0+{{- range .Values.langflow.flows.git.files }}+ if ! download_with_retry "{{ . }}"; then failed=1; fi+{{- end }}++ if [ "$failed" -eq 1 ]; then+ echo "ERROR: One or more flows failed to download"+ exit 1+ fi++ echo "Flows loaded: $(ls "$FLOWS_DIR")"+ else+ echo "Flows already exist in $FLOWS_DIR, skipping download."+ fi+ volumeMounts:+ - name: shared+ mountPath: {{ .Values.backend.persistence.shared.mountPath | quote }}{{- end }}{{- if .Values.langflow.enabled }}- name: wait-for-langflow@@ -137,22 +208,13 @@ spec:mountPath: /app/.envsubPath: .envreadOnly: true- {{- if .Values.backend.persistence.documents.enabled }}- - name: documents- mountPath: {{ .Values.backend.persistence.documents.mountPath }}- {{- end }}- {{- if .Values.backend.persistence.keys.enabled }}- - name: keys- mountPath: {{ .Values.backend.persistence.keys.mountPath }}- {{- end }}- {{- if .Values.backend.persistence.config.enabled }}- - name: config- mountPath: {{ .Values.backend.persistence.config.mountPath }}+ {{- if .Values.backend.persistence.data.enabled }}+ - name: data+ mountPath: {{ .Values.backend.persistence.data.mountPath | quote }}{{- end }}- {{- if .Values.langflow.persistence.enabled }}- - name: flows- mountPath: /app/flows- subPath: {{ .Values.langflow.persistence.flowsSubPath }}+ {{- if .Values.backend.persistence.shared.enabled }}+ - name: shared+ mountPath: {{ .Values.backend.persistence.shared.mountPath | quote }}readOnly: true{{- end }}resources:@@ -201,24 +263,15 @@ spec:items:- key: .envpath: .env- {{- if .Values.backend.persistence.documents.enabled }}- - name: documents- emptyDir: {}- {{- end }}- {{- if .Values.backend.persistence.keys.enabled }}- - name: keys- persistentVolumeClaim:- claimName: {{ include "openrag.fullname" . }}-keys- {{- end }}- {{- if .Values.backend.persistence.config.enabled }}- - name: config+ {{- if .Values.backend.persistence.data.enabled }}+ - name: datapersistentVolumeClaim:- claimName: {{ include "openrag.fullname" . }}-config+ claimName: {{ include "openrag.fullname" . }}-data{{- end }}- {{- if .Values.langflow.persistence.enabled }}- - name: flows+ {{- if .Values.backend.persistence.shared.enabled }}+ - name: sharedpersistentVolumeClaim:- claimName: {{ include "openrag.fullname" . }}-langflow+ claimName: {{ include "openrag.fullname" . }}-shared{{- end }}{{- with .Values.nodeSelector }}nodeSelector:diff --git a/kubernetes/helm/openrag/templates/langflow/deployment.yaml b/kubernetes/helm/openrag/templates/langflow/deployment.yamlindex f0f32981c..a3bd63ec0 100644--- a/kubernetes/helm/openrag/templates/langflow/deployment.yaml+++ b/kubernetes/helm/openrag/templates/langflow/deployment.yaml@@ -34,78 +34,6 @@ spec:{{- toYaml . | nindent 8 }}{{- end }}- {{- if and .Values.langflow.flows.loadDefaults .Values.langflow.flows.git.enabled }}- initContainers:- - name: load-default-flows- image: curlimages/curl:8.5.0- command: ["/bin/sh", "-c"]- args:- - |- set -eu-- FLOWS_DIR="{{ .Values.langflow.persistence.mountPath }}/{{ .Values.langflow.persistence.flowsSubPath }}"-- OWNER="{{ .Values.langflow.flows.git.owner }}"- REPO="{{ .Values.langflow.flows.git.repo }}"- BRANCH="{{ .Values.langflow.flows.git.branch }}"- {{- if .Values.langflow.flows.git.hash }}- GIT_REF="{{ .Values.langflow.flows.git.hash }}"- {{- else }}- GIT_REF="${BRANCH}"- {{- end }}- REPO_PATH="{{ .Values.langflow.flows.git.path }}"-- # Raw base URL: https://raw.githubusercontent.com/<owner>/<repo>/<ref>/<path>- GITHUB_RAW_URL="https://raw.githubusercontent.com/${OWNER}/${REPO}/${GIT_REF}/${REPO_PATH}"-- mkdir -p "$FLOWS_DIR"-- if [ -z "$(ls -A "$FLOWS_DIR" 2>/dev/null)" ]; then- echo "Loading default flows from ${OWNER}/${REPO}@${GIT_REF}/${REPO_PATH} ..."-- download_with_retry() {- file="$1"- url="${GITHUB_RAW_URL}/${file}"- dest="${FLOWS_DIR}/${file}"- max_retries=3- timeout=60- retry=0-- while [ "$retry" -lt "$max_retries" ]; do- echo "Downloading ${file} (attempt $((retry + 1))/${max_retries})..."- if curl -f -L -m "$timeout" -o "$dest" "$url"; then- echo "OK: ${file}"- return 0- fi- retry=$((retry + 1))- if [ "$retry" -lt "$max_retries" ]; then- echo "Retrying ${file}..."- sleep 2- fi- done-- echo "ERROR: Failed to download ${file} from ${url}"- return 1- }-- failed=0-{{- range .Values.langflow.flows.git.files }}- if ! download_with_retry "{{ . }}"; then failed=1; fi-{{- end }}-- if [ "$failed" -eq 1 ]; then- echo "ERROR: One or more flows failed to download"- exit 1- fi-- echo "Flows loaded: $(ls "$FLOWS_DIR")"- else- echo "Flows already exist in $FLOWS_DIR, skipping download."- fi- volumeMounts:- - name: langflow-data- mountPath: {{ .Values.langflow.persistence.mountPath }}- {{- end }}containers:- name: langflow@@ -131,6 +59,9 @@ spec:readOnly: true- name: langflow-datamountPath: {{ .Values.langflow.persistence.mountPath }}+ - name: shared+ mountPath: {{ .Values.backend.persistence.shared.mountPath | quote }}+ readOnly: trueresources:{{- toYaml .Values.langflow.resources | nindent 12 }}@@ -172,6 +103,11 @@ spec:{{- else }}emptyDir: {}{{- end }}+ {{- if .Values.backend.persistence.shared.enabled }}+ - name: shared+ persistentVolumeClaim:+ claimName: {{ include "openrag.fullname" . }}-shared+ {{- end }}{{- with .Values.nodeSelector }}nodeSelector:diff --git a/kubernetes/helm/openrag/templates/langflow/langflow-dotenv.yaml b/kubernetes/helm/openrag/templates/langflow/langflow-dotenv.yamlindex 7a2972b42..c7ffda957 100644--- a/kubernetes/helm/openrag/templates/langflow/langflow-dotenv.yaml+++ b/kubernetes/helm/openrag/templates/langflow/langflow-dotenv.yaml@@ -37,7 +37,6 @@ stringData:# Mounted as a file into the pod# Langflow core settings- LANGFLOW_LOAD_FLOWS_PATH={{ .Values.langflow.persistence.mountPath }}/{{ .Values.langflow.persistence.flowsSubPath }}{{- if .Values.postgres.enabled }}LANGFLOW_DATABASE_URL="postgresql://{{ .Values.postgres.username }}:{{ $postgresPassword }}@{{ include "openrag.fullname" . }}-postgres:{{ .Values.postgres.service.port }}/{{ .Values.postgres.database }}"{{- else }}diff --git a/kubernetes/helm/openrag/templates/storage/backend-sc.yaml b/kubernetes/helm/openrag/templates/storage/backend-sc.yamlindex 488e20082..2a7418bd6 100644--- a/kubernetes/helm/openrag/templates/storage/backend-sc.yaml+++ b/kubernetes/helm/openrag/templates/storage/backend-sc.yaml@@ -1,4 +1,4 @@-{{- if or (.Values.backend.persistence.config.enabled) (.Values.backend.persistence.keys.enabled) }}+{{- if or (.Values.backend.persistence.data.enabled) (.Values.backend.persistence.shared.enabled) }}apiVersion: storage.k8s.io/v1kind: StorageClassmetadata:@@ -18,5 +18,5 @@ parameters:{{- end }}reclaimPolicy: {{ .Values.backend.storageClassReclaimPolicy | default "Delete" }}volumeBindingMode: {{ .Values.backend.storageClassVolumeBindingMode | default "WaitForFirstConsumer" }}-allowVolumeExpansion: {{ .Values.backend.storageClassAllowVolumeExpansion | default true }}+allowVolumeExpansion: {{ default true .Values.backend.storageClassAllowVolumeExpansion }}{{- end }}\ No newline at end of filediff --git a/kubernetes/helm/openrag/templates/storage/config-pvc.yaml b/kubernetes/helm/openrag/templates/storage/config-pvc.yamldeleted file mode 100644index d5ced07fc..000000000--- a/kubernetes/helm/openrag/templates/storage/config-pvc.yaml+++ /dev/null@@ -1,18 +0,0 @@-{{- if and .Values.backend.enabled .Values.backend.persistence.config.enabled }}-apiVersion: v1-kind: PersistentVolumeClaim-metadata:- name: {{ include "openrag.fullname" . }}-config- namespace: {{ include "openrag.namespace" . }}- labels:- {{- include "openrag.backend.labels" . | nindent 4 }}-spec:- accessModes:- - {{ .Values.backend.persistence.config.accessMode }}- {{- if .Values.backend.persistence.config.storageClass }}- storageClassName: {{ .Values.backend.persistence.config.storageClass | quote }}- {{- end }}- resources:- requests:- storage: {{ .Values.backend.persistence.config.size }}-{{- end }}diff --git a/kubernetes/helm/openrag/templates/storage/data-pvc.yaml b/kubernetes/helm/openrag/templates/storage/data-pvc.yamlnew file mode 100644index 000000000..a30cd09dd--- /dev/null+++ b/kubernetes/helm/openrag/templates/storage/data-pvc.yaml@@ -0,0 +1,18 @@+{{- if and .Values.backend.enabled .Values.backend.persistence.data.enabled }}+apiVersion: v1+kind: PersistentVolumeClaim+metadata:+ name: {{ include "openrag.fullname" . }}-data+ namespace: {{ include "openrag.namespace" . }}+ labels:+ {{- include "openrag.backend.labels" . | nindent 4 }}+spec:+ accessModes:+ - {{ .Values.backend.persistence.data.accessMode }}+ {{- if .Values.backend.persistence.data.storageClass }}+ storageClassName: {{ .Values.backend.persistence.data.storageClass | quote }}+ {{- end }}+ resources:+ requests:+ storage: {{ .Values.backend.persistence.data.size }}+{{- end }}diff --git a/kubernetes/helm/openrag/templates/storage/keys-pvc.yaml b/kubernetes/helm/openrag/templates/storage/keys-pvc.yamldeleted file mode 100644index 6d2acd4c7..000000000--- a/kubernetes/helm/openrag/templates/storage/keys-pvc.yaml+++ /dev/null@@ -1,18 +0,0 @@-{{- if and .Values.backend.enabled .Values.backend.persistence.keys.enabled }}-apiVersion: v1-kind: PersistentVolumeClaim-metadata:- name: {{ include "openrag.fullname" . }}-keys- namespace: {{ include "openrag.namespace" . }}- labels:- {{- include "openrag.backend.labels" . | nindent 4 }}-spec:- accessModes:- - {{ .Values.backend.persistence.keys.accessMode }}- {{- if .Values.backend.persistence.keys.storageClass }}- storageClassName: {{ .Values.backend.persistence.keys.storageClass | quote }}- {{- end }}- resources:- requests:- storage: {{ .Values.backend.persistence.keys.size }}-{{- end }}diff --git a/kubernetes/helm/openrag/templates/storage/shared-pvc.yaml b/kubernetes/helm/openrag/templates/storage/shared-pvc.yamlnew file mode 100644index 000000000..8d3ad4820--- /dev/null+++ b/kubernetes/helm/openrag/templates/storage/shared-pvc.yaml@@ -0,0 +1,18 @@+{{- if and .Values.backend.persistence.shared.enabled (or .Values.backend.enabled .Values.langflow.enabled) }}+apiVersion: v1+kind: PersistentVolumeClaim+metadata:+ name: {{ include "openrag.fullname" . }}-shared+ namespace: {{ include "openrag.namespace" . }}+ labels:+ {{- include "openrag.backend.labels" . | nindent 4 }}+spec:+ accessModes:+ - {{ .Values.backend.persistence.shared.accessMode }}+ {{- if .Values.backend.persistence.shared.storageClass }}+ storageClassName: {{ .Values.backend.persistence.shared.storageClass | quote }}+ {{- end }}+ resources:+ requests:+ storage: {{ .Values.backend.persistence.shared.size }}+{{- end }}diff --git a/kubernetes/helm/openrag/values.yaml b/kubernetes/helm/openrag/values.yamlindex cdf0ace24..a1fc3b221 100644--- a/kubernetes/helm/openrag/values.yaml+++ b/kubernetes/helm/openrag/values.yaml@@ -238,41 +238,42 @@ backend:logLevel: "INFO" # DEBUG, INFO, WARNING, ERRORserviceName: "openrag"-- # Persistence for documents, keys, and config+ # Persistence consolidated into two main volumes:+ # 1. /data: internal application state (data, backups, config, keys)+ # 2. /shared: shared content (documents, flows)persistence:- documents:+ data:enabled: truestorageClass: ""accessMode: ReadWriteOnce+ size: 10Gi+ mountPath: /data+ dataPath: "/data/data"+ backupPath: "/data/backup"+ configPath: "/data/config"+ keysPath: "/data/keys"+ shared:+ enabled: true+ storageClass: ""+ accessMode: ReadWriteMany # Often beneficial for shared content between backend/langflowsize: 50Gi- mountPath: /app/openrag-documents- loadDefaults: true # Load default documents on first deployment+ mountPath: /shared+ flowsPath: "/shared/flows"+ documents:+ mountPath: "/shared/openrag-documents"+ loadDefaults: truegit:enabled: trueowner: langflow-airepo: openragbranch: main- # Optional: specific git commit hash (overrides branch if set)- #hash: ""path: openrag-documentsfiles:- docling.pdf- ibm_anthropic.pdf- openrag-documentation.pdf- warmup_ocr.pdf- keys:- enabled: true- storageClass: ""- accessMode: ReadWriteOnce- size: 1Gi- mountPath: /app/keys- config:- enabled: true- storageClass: ""- accessMode: ReadWriteOnce- size: 1Gi- mountPath: /app/config+# LogginglogLevel: "INFO" # DEBUG, INFO, WARNING, ERROR, CRITICALdiff --git a/src/config/config_manager.py b/src/config/config_manager.pyindex fea87ba43..2d04d8dd8 100644--- a/src/config/config_manager.py+++ b/src/config/config_manager.py@@ -165,7 +165,11 @@ class ConfigManager:Args:config_file: Path to configuration file. Defaults to 'config.yaml' in project root."""- self.config_file = Path(config_file) if config_file else Path("config/config.yaml")+ if config_file:+ self.config_file = Path(config_file)+ else:+ from config.paths import get_config_file_path+ self.config_file = Path(get_config_file_path())self._config: Optional[OpenRAGConfig] = Nonediff --git a/src/config/paths.py b/src/config/paths.pynew file mode 100644index 000000000..41eed34fd--- /dev/null+++ b/src/config/paths.py@@ -0,0 +1,93 @@+"""Centralized path helpers for OpenRAG.++This module is the single source of truth for the OPENRAG_*_PATH environment variables.+Variables are read directly via ``os.getenv`` here to avoid circular imports+(settings → config_manager → paths → settings).+"""++import os+++# ---------------------------------------------------------------------------+# Documents directory+# ---------------------------------------------------------------------------+def get_documents_path() -> str:+ """Return the path to the documents directory.++ Environment variable: OPENRAG_DOCUMENTS_PATH+ Default: ``openrag-documents`` (relative to the working directory)+ """+ return os.getenv("OPENRAG_DOCUMENTS_PATH") or "openrag-documents"+++# ---------------------------------------------------------------------------+# JWT keys directory+# ---------------------------------------------------------------------------+def get_keys_path() -> str:+ """Return the path to the JWT keys directory.++ Environment variable: OPENRAG_KEYS_PATH+ Default: ``keys`` (relative to the working directory)+ """+ return os.getenv("OPENRAG_KEYS_PATH") or "keys"+++# ---------------------------------------------------------------------------+# Flows directory+# ---------------------------------------------------------------------------+def get_flows_path() -> str:+ """Return the path to the flows directory.++ Environment variable: OPENRAG_FLOWS_PATH+ Default: ``flows`` (relative to the working directory)+ """+ return os.getenv("OPENRAG_FLOWS_PATH") or "flows"+++def get_flows_backup_path() -> str:+ """Return the path to the flows backup directory.++ Environment variable: OPENRAG_FLOWS_BACKUP_PATH+ Default: ``<flows_path>/backup``+ """+ return os.getenv("OPENRAG_FLOWS_BACKUP_PATH") or os.path.join(get_flows_path(), "backup")+++# ---------------------------------------------------------------------------+# Config directory (holds config.yaml)+# ---------------------------------------------------------------------------+def get_config_path() -> str:+ """Return the path to the configuration directory.++ Environment variable: OPENRAG_CONFIG_PATH+ Default: ``config`` (relative to the working directory)+ """+ return os.getenv("OPENRAG_CONFIG_PATH") or "config"+++def get_config_file_path() -> str:+ """Return the full path to the config.yaml file."""+ return os.path.join(get_config_path(), "config.yaml")+++# ---------------------------------------------------------------------------+# Data directory (conversations, tokens, connections, etc.)+# ---------------------------------------------------------------------------+def get_data_path() -> str:+ """Return the path to the data directory.++ Environment variable: OPENRAG_DATA_PATH+ Default: ``data`` (relative to the working directory)+ """+ return os.getenv("OPENRAG_DATA_PATH") or "data"+++def get_data_file(filename: str) -> str:+ """Return a full path for a file inside the data directory.++ Example::++ get_data_file("conversations.json")+ # → "data/conversations.json" (or $OPENRAG_DATA_PATH/conversations.json)+ """+ return os.path.join(get_data_path(), filename)diff --git a/src/config/settings.py b/src/config/settings.pyindex 3a8af891b..b992d2dbc 100644--- a/src/config/settings.py+++ b/src/config/settings.py@@ -1,3 +1,4 @@+from config.paths import get_flows_pathimport asyncioimport osfrom utils.env_utils import get_env_int, get_env_float@@ -101,7 +102,6 @@ LANGFLOW_CONNECT_TIMEOUT = get_env_float("LANGFLOW_CONNECT_TIMEOUT", 30.0) # 30# Default: 3600 seconds (60 minutes)INGESTION_TIMEOUT = get_env_int("INGESTION_TIMEOUT", 3600)-def is_no_auth_mode():"""Check if we're running in no-auth mode (OAuth credentials missing)"""if IBM_AUTH_ENABLED:@@ -830,24 +830,32 @@ class AppClients:)-# Component template paths-WATSONX_LLM_COMPONENT_PATH = os.getenv(- "WATSONX_LLM_COMPONENT_PATH", "flows/components/watsonx_llm.json"+# Component template paths — derived from the centralized flows directory+def _component_path(env_var: str, filename: str) -> str:+ """Return a component path, using the env var override or the centralized flows dir."""+ env_val = os.getenv(env_var)+ if env_val:+ return env_val+ flows_dir = get_flows_path()+ return os.path.join(flows_dir, "components", filename)++WATSONX_LLM_COMPONENT_PATH = _component_path(+ "WATSONX_LLM_COMPONENT_PATH", "watsonx_llm.json")-WATSONX_LLM_TEXT_COMPONENT_PATH = os.getenv(- "WATSONX_LLM_TEXT_COMPONENT_PATH", "flows/components/watsonx_llm_text.json"+WATSONX_LLM_TEXT_COMPONENT_PATH = _component_path(+ "WATSONX_LLM_TEXT_COMPONENT_PATH", "watsonx_llm_text.json")-WATSONX_EMBEDDING_COMPONENT_PATH = os.getenv(- "WATSONX_EMBEDDING_COMPONENT_PATH", "flows/components/watsonx_embedding.json"+WATSONX_EMBEDDING_COMPONENT_PATH = _component_path(+ "WATSONX_EMBEDDING_COMPONENT_PATH", "watsonx_embedding.json")-OLLAMA_LLM_COMPONENT_PATH = os.getenv(- "OLLAMA_LLM_COMPONENT_PATH", "flows/components/ollama_llm.json"+OLLAMA_LLM_COMPONENT_PATH = _component_path(+ "OLLAMA_LLM_COMPONENT_PATH", "ollama_llm.json")-OLLAMA_LLM_TEXT_COMPONENT_PATH = os.getenv(- "OLLAMA_LLM_TEXT_COMPONENT_PATH", "flows/components/ollama_llm_text.json"+OLLAMA_LLM_TEXT_COMPONENT_PATH = _component_path(+ "OLLAMA_LLM_TEXT_COMPONENT_PATH", "ollama_llm_text.json")-OLLAMA_EMBEDDING_COMPONENT_PATH = os.getenv(- "OLLAMA_EMBEDDING_COMPONENT_PATH", "flows/components/ollama_embedding.json"+OLLAMA_EMBEDDING_COMPONENT_PATH = _component_path(+ "OLLAMA_EMBEDDING_COMPONENT_PATH", "ollama_embedding.json")# Component IDs in flowsdiff --git a/src/connectors/connection_manager.py b/src/connectors/connection_manager.pyindex 8081e0b7f..8646d72aa 100644--- a/src/connectors/connection_manager.py+++ b/src/connectors/connection_manager.py@@ -39,8 +39,9 @@ class ConnectionConfig:class ConnectionManager:"""Manages multiple connector connections with persistence"""- def __init__(self, connections_file: str = "data/connections.json"):- self.connections_file = Path(connections_file)+ def __init__(self, connections_file: str = None):+ from config.paths import get_data_file+ self.connections_file = Path(connections_file or get_data_file("connections.json"))# Ensure data directory existsself.connections_file.parent.mkdir(parents=True, exist_ok=True)self.connections: Dict[str, ConnectionConfig] = {}diff --git a/src/connectors/google_drive/connector.py b/src/connectors/google_drive/connector.pyindex e55f62619..53b3e8eea 100644--- a/src/connectors/google_drive/connector.py+++ b/src/connectors/google_drive/connector.py@@ -97,8 +97,9 @@ class GoogleDriveConnector(BaseConnector):client_id = config.get("client_id") or env_client_idclient_secret = config.get("client_secret") or env_client_secret- # Token file default - use data/ directory for persistence- token_file = config.get("token_file") or "data/google_drive_token.json"+ # Token file default - use data directory for persistence+ from config.paths import get_data_file+ token_file = config.get("token_file") or get_data_file("google_drive_token.json")Path(token_file).parent.mkdir(parents=True, exist_ok=True)if not isinstance(client_id, str) or not client_id.strip():diff --git a/src/connectors/onedrive/connector.py b/src/connectors/onedrive/connector.pyindex e6348db77..5a9c5eeda 100644--- a/src/connectors/onedrive/connector.py+++ b/src/connectors/onedrive/connector.py@@ -61,8 +61,9 @@ class OneDriveConnector(BaseConnector):except Exception as e:logger.debug(f"Failed to get client_secret: {e}")- # Token file setup - use data/ directory for persistence- token_file = config.get("token_file") or "data/onedrive_token.json"+ # Token file setup - use data directory for persistence+ from config.paths import get_data_file+ token_file = config.get("token_file") or get_data_file("onedrive_token.json")Path(token_file).parent.mkdir(parents=True, exist_ok=True)# Only initialize OAuth if we have credentials@@ -74,7 +75,7 @@ class OneDriveConnector(BaseConnector):oauth_token_file = config["token_file"]else:# Use a per-connection cache file to avoid collisions with other connectors- oauth_token_file = f"data/onedrive_token_{connection_id}.json"+ oauth_token_file = get_data_file(f"onedrive_token_{connection_id}.json")# MSA & org both work via /common for OneDrive personal testingauthority = "https://login.microsoftonline.com/common"diff --git a/src/connectors/sharepoint/connector.py b/src/connectors/sharepoint/connector.pyindex 6900b64c4..c8f8fafa4 100644--- a/src/connectors/sharepoint/connector.py+++ b/src/connectors/sharepoint/connector.py@@ -68,8 +68,9 @@ class SharePointConnector(BaseConnector):logger.debug(f"Failed to get client_secret: {e}")pass # Credentials not available, that's OK for listing- # Token file setup - use data/ directory for persistence- token_file = config.get("token_file") or "data/sharepoint_token.json"+ # Token file setup - use data directory for persistence+ from config.paths import get_data_file+ token_file = config.get("token_file") or get_data_file("sharepoint_token.json")Path(token_file).parent.mkdir(parents=True, exist_ok=True)# Only initialize OAuth if we have credentials@@ -80,7 +81,7 @@ class SharePointConnector(BaseConnector):if config.get("token_file"):oauth_token_file = config["token_file"]else:- oauth_token_file = f"data/sharepoint_token_{connection_id}.json"+ oauth_token_file = get_data_file(f"sharepoint_token_{connection_id}.json")authority = f"https://login.microsoftonline.com/{self.tenant_id}" if self.tenant_id != "common" else "https://login.microsoftonline.com/common"diff --git a/src/main.py b/src/main.pyindex a2ade69c9..1b89a38a9 100644--- a/src/main.py+++ b/src/main.py@@ -14,7 +14,6 @@ from html.parser import HTMLParserfrom connectors.langflow_connector_service import LangflowConnectorServicefrom connectors.service import ConnectorServicefrom services.flows_service import FlowsService-from utils.container_utils import detect_container_environmentfrom utils.embeddings import create_dynamic_index_bodyfrom utils.logging_config import configure_from_env, get_loggerfrom utils.encryption import enforce_startup_prerequisites@@ -360,7 +359,8 @@ async def init_index_when_ready(opensearch_client=None):def generate_jwt_keys():"""Generate RSA keys for JWT signing if they don't exist"""- keys_dir = "keys"+ from config.paths import get_keys_path+ keys_dir = get_keys_path()private_key_path = os.path.join(keys_dir, "private_key.pem")public_key_path = os.path.join(keys_dir, "public_key.pem")@@ -417,17 +417,10 @@ def generate_jwt_keys():def _get_documents_dir():"""Get the documents directory path, handling both Docker and local environments."""- # In Docker, the volume is mounted at /app/openrag-documents- # Locally, we use openrag-documents- container_env = detect_container_environment()- if container_env:- path = os.path.abspath("/app/openrag-documents")- logger.debug(f"Running in {container_env}, using container path: {path}")- return path- else:- path = os.path.abspath(os.path.join(os.getcwd(), "openrag-documents"))- logger.debug(f"Running locally, using local path: {path}")- return path+ from config.paths import get_documents_path+ path = get_documents_path()+ logger.debug(f"Using documents path: {path}")+ return pathdef _should_use_url_default_docs_ingest() -> bool:diff --git a/src/services/auth_service.py b/src/services/auth_service.pyindex f5c701efb..3c04fa26d 100644--- a/src/services/auth_service.py+++ b/src/services/auth_service.py@@ -86,8 +86,9 @@ class AuthService:# We'll validate client credentials when creating the connector- # Create connection configuration - use data/ directory for persistence- token_file = f"data/{connector_type}_{purpose}_{uuid.uuid4().hex[:8]}.json"+ # Create connection configuration - use data directory for persistence+ from config.paths import get_data_file+ token_file = get_data_file(f"{connector_type}_{purpose}_{uuid.uuid4().hex[:8]}.json")effective_redirect_uri = OAUTH_BROKER_URL or redirect_uriconfig = {"token_file": token_file,diff --git a/src/services/conversation_persistence_service.py b/src/services/conversation_persistence_service.pyindex 8af36efff..d7049fd31 100644--- a/src/services/conversation_persistence_service.py+++ b/src/services/conversation_persistence_service.py@@ -16,8 +16,9 @@ logger = get_logger(__name__)class ConversationPersistenceService:"""Simple service to persist conversations to disk"""- def __init__(self, storage_file: str = "data/conversations.json"):- self.storage_file = storage_file+ def __init__(self, storage_file: str = None):+ from config.paths import get_data_file+ self.storage_file = storage_file or get_data_file("conversations.json")# Ensure data directory existsos.makedirs(os.path.dirname(self.storage_file), exist_ok=True)self.lock = threading.Lock()diff --git a/src/services/flows_service.py b/src/services/flows_service.pyindex f06d7b948..ba9957bc1 100644--- a/src/services/flows_service.py+++ b/src/services/flows_service.py@@ -80,15 +80,13 @@ class FlowsService:def _get_flows_directory(self):"""Get the flows directory path"""- current_file_dir = os.path.dirname(os.path.abspath(__file__)) # src/services/- src_dir = os.path.dirname(current_file_dir) # src/- project_root = os.path.dirname(src_dir) # project root- return os.path.join(project_root, "flows")+ from config.paths import get_flows_path+ return get_flows_path()def _get_backup_directory(self):"""Get the backup directory path"""- flows_dir = self._get_flows_directory()- backup_dir = os.path.join(flows_dir, "backup")+ from config.paths import get_flows_backup_path+ backup_dir = get_flows_backup_path()os.makedirs(backup_dir, exist_ok=True)return backup_dirdiff --git a/src/services/langflow_file_service.py b/src/services/langflow_file_service.pyindex 21e946d2d..bceef3f5a 100644--- a/src/services/langflow_file_service.py+++ b/src/services/langflow_file_service.py@@ -353,7 +353,8 @@ class LangflowFileService:max_attempts = 2last_error: Exception | None = None- flow_file = Path(__file__).resolve().parents[2] / "flows" / "openrag_url_mcp.json"+ from config.paths import get_flows_path+ flow_file = Path(get_flows_path()) / "openrag_url_mcp.json"if not flow_file.exists():raise ValueError("LANGFLOW_URL_INGEST_FLOW_ID is invalid and "diff --git a/src/services/session_ownership_service.py b/src/services/session_ownership_service.pyindex 8fcd83084..2025d867c 100644--- a/src/services/session_ownership_service.py+++ b/src/services/session_ownership_service.py@@ -15,7 +15,8 @@ class SessionOwnershipService:"""Simple service to track which user owns which session"""def __init__(self):- self.ownership_file = "data/session_ownership.json"+ from config.paths import get_data_file+ self.ownership_file = get_data_file("session_ownership.json")# Ensure data directory existsos.makedirs(os.path.dirname(self.ownership_file), exist_ok=True)self.ownership_data = self._load_ownership_data()diff --git a/src/session_manager.py b/src/session_manager.pyindex ff40950a5..8185c2146 100644--- a/src/session_manager.py+++ b/src/session_manager.py@@ -50,17 +50,19 @@ class SessionManager:def __init__(self,secret_key: str = None,- private_key_path: str = "keys/private_key.pem",- public_key_path: str = "keys/public_key.pem",+ private_key_path: str = None,+ public_key_path: str = None,):+ from config.paths import get_keys_path+ keys_dir = get_keys_path()self.secret_key = secret_key # Keep for backward compatibilityself.users: Dict[str, User] = {} # user_id -> Userself.user_opensearch_clients: Dict[str, Any] = {} # user_id -> OpenSearch client- self.private_key_path = private_key_path- self.public_key_path = public_key_path+ self.private_key_path = private_key_path or os.path.join(keys_dir, "private_key.pem")+ self.public_key_path = public_key_path or os.path.join(keys_dir, "public_key.pem")# Configure JWT signing (checks env var first, falls back to key files)self._configure_jwt_signing()