OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.
Python
fix: persist Langflow database across container restarts
Mike Pawlowski2b1456afc827
diff --git a/.gitignore b/.gitignoreindex 7da9d1140..02d024e00 100644--- a/.gitignore+++ b/.gitignore@@ -32,6 +32,9 @@ wheels/# OpenSearch data directoryopensearch-data/+# Langflow data directory (ignore contents, keep directory via .gitkeep)+/langflow-data/*+!/langflow-data/.gitkeepnode_modulesdiff --git a/Dockerfile.langflow b/Dockerfile.langflowindex 2f8286cd0..73793121b 100644--- a/Dockerfile.langflow+++ b/Dockerfile.langflow@@ -1,7 +1,13 @@FROM langflowai/langflow:1.8.0-RUN pip install uv+# (+) Install uv+# (+) Pre-create the Langflow data directory with correct ownership.+# - This ensures named Docker volumes are initialised with uid=1000 so+# the non-root container user can write to the mounted path.+RUN set -ex \+ && pip install uv \+ && mkdir -p /app/langflow-dataEXPOSE 7860-CMD ["langflow", "run", "--host", "0.0.0.0", "--port", "7860"]\ No newline at end of file+CMD ["langflow", "run", "--host", "0.0.0.0", "--port", "7860"]diff --git a/Makefile b/Makefileindex f5d391e8f..5fb3d948c 100644--- a/Makefile+++ b/Makefile@@ -502,6 +502,7 @@ factory-reset: ## Complete reset (stop, remove volumes, clear data, remove imageecho " - Stop all containers"; \echo " - Remove all volumes"; \echo " - Delete opensearch-data directory"; \+ echo " - Delete langflow-data directory"; \echo " - Delete config directory"; \echo " - Delete JWT keys (private_key.pem, public_key.pem)"; \echo " - Remove OpenRAG images"; \@@ -525,6 +526,11 @@ factory-reset: ## Complete reset (stop, remove volumes, clear data, remove imagerm -rf opensearch-data/* 2>/dev/null || true; \echo "$(PURPLE)opensearch-data removed$(NC)"; \fi; \+ if [ -d "langflow-data" ]; then \+ echo "Removing langflow-data..."; \+ rm -rf langflow-data/* 2>/dev/null || true; \+ echo "$(PURPLE)langflow-data removed$(NC)"; \+ fi; \if [ -d "config" ]; then \echo "Removing config..."; \rm -rf config; \diff --git a/docker-compose.yml b/docker-compose.ymlindex c4a225178..6a2a07f16 100644--- a/docker-compose.yml+++ b/docker-compose.yml@@ -127,7 +127,7 @@ services:langflow:volumes:- ${OPENRAG_FLOWS_PATH:-./flows}:/app/flows:U,z- - ${LANGFLOW_DATA_PATH:-./langflow-data}:/root/.langflow:U,z+ - ${LANGFLOW_DATA_PATH:-./langflow-data}:/app/langflow-data:U,zimage: langflowai/openrag-langflow:${OPENRAG_VERSION:-latest}build:context: .@@ -146,8 +146,8 @@ services:- WATSONX_URL=${WATSONX_URL:-${WATSONX_ENDPOINT}}- WATSONX_PROJECT_ID=${WATSONX_PROJECT_ID}- OLLAMA_BASE_URL=${OLLAMA_ENDPOINT}- - LANGFLOW_LOAD_FLOWS_PATH=/app/flows- - LANGFLOW_DATABASE_URL=sqlite:////root/.langflow/langflow.db+ - LANGFLOW_CONFIG_DIR=/app/langflow-data+ - LANGFLOW_DATABASE_URL=sqlite:////app/langflow-data/langflow.db- LANGFLOW_SECRET_KEY=${LANGFLOW_SECRET_KEY}- JWT=None- OWNER=Nonediff --git a/langflow-data/.gitkeep b/langflow-data/.gitkeepnew file mode 100644index 000000000..e69de29bbdiff --git a/src/main.py b/src/main.pyindex 041c573cc..8eae9b293 100644--- a/src/main.py+++ b/src/main.py@@ -1200,6 +1200,15 @@ async def startup_tasks(services):# Update MCP servers with provider credentials (especially important for no-auth mode)await _update_mcp_servers_with_provider_credentials(services)+ # Ensure all configured flows exist in Langflow (create-only, never overwrites).+ # This replaces LANGFLOW_LOAD_FLOWS_PATH, which performed a blind upsert on+ # every container start and discarded any user edits made in the Langflow UI.+ try:+ flows_service = services["flows_service"]+ await flows_service.ensure_flows_exist()+ except Exception as e:+ logger.warning("Failed to ensure Langflow flows exist at startup", error=str(e))+# Check if flows were reset and reapply settings if config is editedtry:config = get_openrag_config()diff --git a/src/services/flows_service.py b/src/services/flows_service.pyindex 00b381fb4..137af1080 100644--- a/src/services/flows_service.py+++ b/src/services/flows_service.py@@ -47,7 +47,7 @@ class FlowsService:resolved_url = Nonefor cand in candidates:test_url = replace_localhost_patterns(endpoint, cand)-+logger.debug(f"Probing Ollama candidate via Langflow: {test_url}")try:response = await clients.langflow_request(@@ -61,7 +61,7 @@ class FlowsService:except Exception as e:logger.debug(f"Probe failed for {test_url}: {e}")continue-+if not resolved_url:# Fallback to simple transformation if probing failsresolved_url = transform_localhost_url(endpoint)@@ -95,23 +95,23 @@ class FlowsService:def _get_latest_backup_path(self, flow_id: str, flow_type: str):"""Get the path to the latest backup file for a flow.-+Args:flow_id: The flow IDflow_type: The flow type name-+Returns:str: Path to latest backup file, or None if no backup exists"""backup_dir = self._get_backup_directory()-+if not os.path.exists(backup_dir):return None-+# Find all backup files for this flowbackup_files = []prefix = f"{flow_type}_"-+try:for filename in os.listdir(backup_dir):if filename.startswith(prefix) and filename.endswith(".json"):@@ -122,10 +122,10 @@ class FlowsService:except Exception as e:logger.warning(f"Error reading backup directory: {str(e)}")return None-+if not backup_files:return None-+# Return the most recent backup (highest mtime)backup_files.sort(key=lambda x: x[0], reverse=True)return backup_files[0][1]@@ -134,17 +134,17 @@ class FlowsService:"""Compare two flow structures to see if they're different.Normalizes both flows before comparison.-+Args:flow1: First flow dataflow2: Second flow data-+Returns:bool: True if flows are different, False if they're the same"""normalized1 = self._normalize_flow_structure(flow1)normalized2 = self._normalize_flow_structure(flow2)-+# Compare normalized structuresreturn normalized1 != normalized2@@ -152,10 +152,10 @@ class FlowsService:"""Backup all flows from Langflow to the backup folder.Only backs up flows that have changed since the last backup.-+Args:only_if_changed: If True, only backup flows that differ from latest backup-+Returns:dict: Summary of backup operations with success/failure status"""@@ -200,7 +200,7 @@ class FlowsService:flow_locked = current_flow.get("locked", False)latest_backup_path = self._get_latest_backup_path(flow_id, flow_type)has_backups = latest_backup_path is not None-+# If flow is locked and no backups exist, skip backupif flow_locked and not has_backups:logger.debug(@@ -212,13 +212,13 @@ class FlowsService:"reason": "locked_without_backups",})continue-+# Check if we need to backup (only if changed)if only_if_changed and has_backups:try:with open(latest_backup_path, "r") as f:latest_backup = json.load(f)-+# Compare flowsif not self._compare_flows(current_flow, latest_backup):logger.debug(@@ -280,12 +280,12 @@ class FlowsService:async def _backup_flow(self, flow_id: str, flow_type: str, flow_data: dict = None):"""Backup a single flow to the backup folder.-+Args:flow_id: The flow ID to backupflow_type: The flow type name (nudges, retrieval, ingest, url_ingest)flow_data: The flow data to backup (if None, fetches from API)-+Returns:str: Path to the backup file, or None if backup failed"""@@ -717,7 +717,7 @@ class FlowsService:for node in nodes:node_data = node.get("data", {})node_template = node_data.get("node", {})-+normalized_node = {"id": node.get("id"), # Keep ID for edge matching"type": node.get("type"),@@ -775,20 +775,20 @@ class FlowsService:# Compare entire normalized structures exactly# Sort nodes and edges for consistent comparisonnormalized_langflow["data"]["nodes"] = sorted(- normalized_langflow["data"]["nodes"],+ normalized_langflow["data"]["nodes"],key=lambda x: (x.get("id", ""), x.get("type", "")))normalized_file["data"]["nodes"] = sorted(- normalized_file["data"]["nodes"],+ normalized_file["data"]["nodes"],key=lambda x: (x.get("id", ""), x.get("type", "")))normalized_langflow["data"]["edges"] = sorted(- normalized_langflow["data"]["edges"],+ normalized_langflow["data"]["edges"],key=lambda x: (x.get("source", ""), x.get("target", ""), x.get("sourceHandle", ""), x.get("targetHandle", "")))normalized_file["data"]["edges"] = sorted(- normalized_file["data"]["edges"],+ normalized_file["data"]["edges"],key=lambda x: (x.get("source", ""), x.get("target", ""), x.get("sourceHandle", ""), x.get("targetHandle", "")))@@ -799,6 +799,67 @@ class FlowsService:logger.error(f"Error comparing flow {flow_id} with file: {str(e)}")return False+ async def ensure_flows_exist(self):+ """+ Ensure all configured flows exist in Langflow.++ Creates flows from their JSON files if they are not already present in+ the Langflow database. This is intentionally create-only: it never+ patches or overwrites an existing flow, preserving any edits the user+ has made in the Langflow UI.++ This replaces the LANGFLOW_LOAD_FLOWS_PATH mechanism, which performed a+ blind upsert on every container start and discarded user edits.+ """+ flow_configs = [+ ("nudges", NUDGES_FLOW_ID),+ ("retrieval", LANGFLOW_CHAT_FLOW_ID),+ ("ingest", LANGFLOW_INGEST_FLOW_ID),+ ("url_ingest", LANGFLOW_URL_INGEST_FLOW_ID),+ ]++ for flow_type, flow_id in flow_configs:+ if not flow_id:+ continue++ try:+ response = await clients.langflow_request(+ "GET", f"/api/v1/flows/{flow_id}"+ )+ if response.status_code == 200:+ logger.info(+ f"Flow {flow_type} (ID: {flow_id}) already exists, skipping creation"+ )+ continue++ flow_path = self._find_flow_file_by_id(flow_id)+ if not flow_path:+ logger.warning(+ f"No flow file found for {flow_type} (ID: {flow_id}), cannot create"+ )+ continue++ with open(flow_path, "r") as f:+ flow_data = json.load(f)++ response = await clients.langflow_request(+ "PUT", f"/api/v1/flows/{flow_id}", json=flow_data+ )+ if response.status_code in (200, 201):+ logger.info(+ f"Created {flow_type} flow (ID: {flow_id}) from {os.path.basename(flow_path)}"+ )+ else:+ logger.warning(+ f"Failed to create {flow_type} flow (ID: {flow_id}): "+ f"HTTP {response.status_code} — {response.text}"+ )++ except Exception as e:+ logger.error(+ f"Error ensuring {flow_type} flow (ID: {flow_id}) exists: {e}"+ )+async def check_flows_reset(self):"""Check if any flows have been reset by comparing with JSON files.@@ -819,7 +880,7 @@ class FlowsService:logger.info(f"Checking if {flow_type} flow (ID: {flow_id}) was reset")is_reset = await self._compare_flow_with_file(flow_id)-+if is_reset:logger.info(f"Flow {flow_type} (ID: {flow_id}) appears to have been reset")reset_flows.append(flow_type)@@ -827,7 +888,7 @@ class FlowsService:logger.info(f"Flow {flow_type} (ID: {flow_id}) does not match reset state")return reset_flows-+async def change_langflow_model_value(self,provider: str,@@ -917,23 +978,23 @@ class FlowsService:# Get all embedding nodes in the flowembedding_nodes = self._find_nodes_in_flow(flow_data, display_name=OPENAI_EMBEDDING_COMPONENT_DISPLAY_NAME)logger.info(f"Found {len(embedding_nodes)} embedding nodes in flow {flow_name} with display name '{OPENAI_EMBEDDING_COMPONENT_DISPLAY_NAME}'")-+# Count configured embedding-enabled providersconfig_obj = get_openrag_config()configured_providers = []if config_obj.providers.openai.configured: configured_providers.append("openai")if config_obj.providers.watsonx.configured: configured_providers.append("watsonx")if config_obj.providers.ollama.configured: configured_providers.append("ollama")-+# Ensure current provider is in the list for counting purposes if it's being configuredif provider in ["openai", "watsonx", "ollama"] and provider not in configured_providers:configured_providers.append(provider)-+all_possible = ["openai", "watsonx", "ollama"]configured_providers = [p for p in all_possible if p in configured_providers]provider_count = len(configured_providers)logger.info(f"Configured embedding providers: {configured_providers} (count: {provider_count})")-+# Determine slot mapping contextif provider_count == 1:logger.info("Configuration mode: all 3 slots belong to the single active provider")@@ -948,7 +1009,7 @@ class FlowsService:for node, idx in embedding_nodes:if self._get_node_provider(node) == provider_display:matched_nodes.append((node, idx))-+if matched_nodes:logger.info(f"Found {len(matched_nodes)} nodes already configured for provider '{provider}'")for node, idx in matched_nodes:@@ -1035,7 +1096,7 @@ class FlowsService:# Only call if code field exists (custom components should have code)if "code" in template and "value" in template["code"]:code_value = template["code"]["value"]-+try:update_payload = {"code": code_value,@@ -1044,11 +1105,11 @@ class FlowsService:"field_value": model,"tool_mode": False,}-+response = await clients.langflow_request("POST", "/api/v1/custom_component/update", json=update_payload)-+if response.status_code == 200:response_data = response.json()# Update template with the new template from response.data@@ -1161,11 +1222,11 @@ class FlowsService:"model_id": model_value,"enabled": True}]-+response = await clients.langflow_request("POST", "/api/v1/models/enabled_models", json=enable_payload)-+if response.status_code == 200:logger.info(f"Successfully enabled model {model_value} for provider {provider_name}")else: