Skip to content

OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.

Python
71.1 KB
Raw Blame
import asyncio
import json
import os
from datetime import datetime
from typing import Any

from cachetools import LRUCache

from config.settings import (
    AGENT_COMPONENT_DISPLAY_NAME,
    LANGFLOW_CHAT_FLOW_ID,
    LANGFLOW_INGEST_FLOW_ID,
    LANGFLOW_URL,
    LANGFLOW_URL_INGEST_FLOW_ID,
    NUDGES_FLOW_ID,
    OPENAI_EMBEDDING_COMPONENT_DISPLAY_NAME,
    OPENAI_LLM_COMPONENT_DISPLAY_NAME,
    clients,
    get_openrag_config,
)
from utils.langflow_utils import enable_mcp_none_for_project
from utils.logging_config import get_logger
from utils.telemetry import Category, MessageId, TelemetryClient

logger = get_logger(__name__)

_UNSET = object()
_FLOW_FILE_MTIME_TAG_PREFIX = "openrag:file-mtime:"


class FlowsService:
    def __init__(self) -> None:
        # Ephemeral dict of dismissed flow update notifications per user_id
        self._dismissed_updates: dict[str, set[str]] = {}
        # Cache for flow file mappings to avoid repeated filesystem scans
        self._flow_file_cache: dict[str, str] = {}
        # Semaphores to prevent race conditions when updating the same flow concurrently
        self._flow_locks = LRUCache(maxsize=128)
        self._lock_creation_lock = asyncio.Lock()
        # Cache for enabled models to avoid redundant API calls
        self._enabled_models_cache: dict[str, Any] | None = None
        self._enabled_models_cache_time: datetime | None = None
        self._enabled_models_cache_ttl = 60  # seconds
        self._enabled_models_lock = asyncio.Lock()
        # Cache for available custom flow updates
        self._custom_updates_cache: list[dict[str, Any]] | None = None

    def dismiss_flows_updates(
        self, flow_types: list[str] | None = None, user_id: str | None = None
    ) -> None:
        """Dismiss flow update notifications ephemerally in backend memory per user."""
        if not hasattr(self, "_dismissed_updates") or not isinstance(self._dismissed_updates, dict):
            self._dismissed_updates = {}
        target_user = user_id or "default"
        if target_user not in self._dismissed_updates:
            self._dismissed_updates[target_user] = set()

        if flow_types:
            self._dismissed_updates[target_user].update(flow_types)
        else:
            self._dismissed_updates[target_user].update(
                ["nudges", "retrieval", "ingest", "url_ingest"]
            )

    def clear_dismissed_update(self, flow_type: str, user_id: str | None = None):
        if hasattr(self, "_dismissed_updates") and isinstance(self._dismissed_updates, dict):
            if user_id:
                if user_id in self._dismissed_updates:
                    self._dismissed_updates[user_id].discard(flow_type)
            else:
                for user_set in self._dismissed_updates.values():
                    user_set.discard(flow_type)

    async def resolve_ollama_url(self, endpoint: str, force_refresh: bool = False) -> str:
        """Find the correct Ollama URL by probing candidates via Langflow's validate-provider API."""
        from config.config_manager import config_manager

        config = config_manager.get_config()

        from utils.container_utils import (
            get_container_host,
            is_localhost_url,
            replace_localhost_patterns,
            transform_localhost_url,
        )

        # If not forcing, check if we already have a resolved endpoint for this original endpoint
        if not force_refresh and config.providers.ollama.resolved_endpoint:
            # We only use the cached one if the original endpoint is still a localhost one
            if is_localhost_url(endpoint):
                logger.debug(
                    f"Using cached resolved Ollama URL: {config.providers.ollama.resolved_endpoint}"
                )
                return config.providers.ollama.resolved_endpoint

        if not is_localhost_url(endpoint):
            return endpoint

        # Candidates to probe
        candidates = ["host.containers.internal", "host.docker.internal"]
        detected_host = get_container_host()
        if detected_host and detected_host not in candidates:
            candidates.insert(0, detected_host)

        resolved_url = None
        for cand in candidates:
            test_url = replace_localhost_patterns(endpoint, cand)

            logger.debug(f"Probing Ollama candidate via Langflow: {test_url}")
            try:
                response = await clients.langflow_request(
                    "POST",
                    "/api/v1/models/validate-provider",
                    json={"provider": "Ollama", "variables": {"OLLAMA_BASE_URL": test_url}},
                )
                if response.status_code in (200, 201) and response.json().get("valid"):
                    logger.info(f"Resolved Ollama URL via Langflow: {test_url}")
                    resolved_url = test_url
                    break
            except Exception as e:
                logger.debug(f"Probe failed for {test_url}: {e}")
                continue

        if not resolved_url:
            # Fallback to simple transformation if probing fails
            resolved_url = transform_localhost_url(endpoint)

        # Cache the result if it changed
        if resolved_url and resolved_url != config.providers.ollama.resolved_endpoint:
            config.providers.ollama.resolved_endpoint = resolved_url
            config_manager.save_config_file(config)
            logger.debug(f"Saved resolved Ollama URL to config: {resolved_url}")

        return resolved_url

    async def _get_flow_lock(self, flow_id: str) -> asyncio.Lock:
        """Get or create an asyncio.Lock for a specific flow ID"""
        async with self._lock_creation_lock:
            if flow_id not in self._flow_locks:
                self._flow_locks[flow_id] = asyncio.Lock()
            return self._flow_locks[flow_id]

    def _get_flows_directory(self):
        """Get the flows directory path"""
        from config.paths import get_flows_path

        return get_flows_path()

    def _get_backup_directory(self):
        """Get the backup directory path"""
        from config.paths import get_flows_backup_path

        backup_dir = get_flows_backup_path()
        os.makedirs(backup_dir, exist_ok=True)
        return backup_dir

    def _get_latest_backup_path(self, flow_id: str, flow_type: str):
        """
        Get the path to the latest backup file for a flow.

        Args:
            flow_id: The flow ID
            flow_type: The flow type name

        Returns:
            str: Path to latest backup file, or None if no backup exists
        """
        backup_dir = self._get_backup_directory()

        if not os.path.exists(backup_dir):
            return None

        # Find all backup files for this flow
        backup_files = []
        prefix = f"{flow_type}_"

        try:
            for filename in os.listdir(backup_dir):
                if filename.startswith(prefix) and filename.endswith(".json"):
                    file_path = os.path.join(backup_dir, filename)
                    # Get modification time for sorting
                    mtime = os.path.getmtime(file_path)
                    backup_files.append((mtime, file_path))
        except Exception as e:
            logger.warning(f"Error reading backup directory: {str(e)}")
            return None

        if not backup_files:
            return None

        # Return the most recent backup (highest mtime)
        backup_files.sort(key=lambda x: x[0], reverse=True)
        return backup_files[0][1]

    def _compare_flows(self, flow1: dict, flow2: dict):
        """
        Compare two flow structures to see if they're different.
        Normalizes both flows before comparison.

        Args:
            flow1: First flow data
            flow2: Second flow data

        Returns:
            bool: True if flows are different, False if they're the same
        """
        normalized1 = self._normalize_flow_structure(flow1)
        normalized2 = self._normalize_flow_structure(flow2)

        # Compare normalized structures
        return normalized1 != normalized2

    async def backup_all_flows(self, only_if_changed=True):
        """
        Backup all flows from Langflow to the backup folder.
        Only backs up flows that have changed since the last backup.

        Args:
            only_if_changed: If True, only backup flows that differ from latest backup

        Returns:
            dict: Summary of backup operations with success/failure status
        """
        backup_results = {
            "success": True,
            "backed_up": [],
            "skipped": [],
            "failed": [],
        }

        flow_configs = [
            ("nudges", NUDGES_FLOW_ID),
            ("retrieval", LANGFLOW_CHAT_FLOW_ID),
            ("ingest", LANGFLOW_INGEST_FLOW_ID),
            ("url_ingest", LANGFLOW_URL_INGEST_FLOW_ID),
        ]

        logger.info("Starting periodic backup of Langflow flows")

        async def backup_single_flow(flow_type, flow_id):
            if not flow_id:
                return None

            try:
                # Get current flow from Langflow
                response = await clients.langflow_request("GET", f"/api/v1/flows/{flow_id}")
                if response.status_code != 200:
                    logger.warning(
                        f"Failed to get flow {flow_id} for backup: HTTP {response.status_code}"
                    )
                    return {
                        "type": "failed",
                        "data": {
                            "flow_type": flow_type,
                            "flow_id": flow_id,
                            "error": f"HTTP {response.status_code}",
                        },
                    }

                current_flow = response.json()

                # Check if flow is locked and if we should skip backup
                flow_locked = current_flow.get("locked", False)
                latest_backup_path = self._get_latest_backup_path(flow_id, flow_type)
                has_backups = latest_backup_path is not None

                # If flow is locked and no backups exist, skip backup
                if flow_locked and not has_backups:
                    logger.debug(
                        f"Flow {flow_type} (ID: {flow_id}) is locked and has no backups, skipping backup"
                    )
                    return {
                        "type": "skipped",
                        "data": {
                            "flow_type": flow_type,
                            "flow_id": flow_id,
                            "reason": "locked_without_backups",
                        },
                    }

                # Check if we need to backup (only if changed)
                if only_if_changed and has_backups:
                    try:
                        with open(latest_backup_path) as f:
                            latest_backup = json.load(f)

                        # Compare flows
                        if not self._compare_flows(current_flow, latest_backup):
                            logger.debug(
                                f"Flow {flow_type} (ID: {flow_id}) unchanged, skipping backup"
                            )
                            return {
                                "type": "skipped",
                                "data": {
                                    "flow_type": flow_type,
                                    "flow_id": flow_id,
                                    "reason": "unchanged",
                                },
                            }
                    except Exception as e:
                        logger.warning(
                            f"Failed to read latest backup for {flow_type} (ID: {flow_id}): {str(e)}"
                        )

                # Backup the flow
                backup_path = await self._backup_flow(flow_id, flow_type, current_flow)
                if backup_path:
                    return {
                        "type": "backed_up",
                        "data": {
                            "flow_type": flow_type,
                            "flow_id": flow_id,
                            "backup_path": backup_path,
                        },
                    }
                else:
                    return {
                        "type": "failed",
                        "data": {
                            "flow_type": flow_type,
                            "flow_id": flow_id,
                            "error": "Backup returned None",
                        },
                    }
            except Exception as e:
                logger.error(f"Failed to backup {flow_type} flow (ID: {flow_id}): {str(e)}")
                return {
                    "type": "failed",
                    "data": {
                        "flow_type": flow_type,
                        "flow_id": flow_id,
                        "error": str(e),
                    },
                }

        tasks = [backup_single_flow(ft, fi) for ft, fi in flow_configs]
        results = await asyncio.gather(*tasks)

        for result in results:
            if not result:
                continue
            res_type = result.get("type")
            if res_type in ("backed_up", "skipped", "failed"):
                target_list = backup_results[res_type]
                if isinstance(target_list, list):
                    target_list.append(result.get("data"))
            if res_type == "failed":
                backup_results["success"] = False

        logger.info(
            "Completed periodic backup of flows",
            backed_up_count=len(backup_results["backed_up"]),
            skipped_count=len(backup_results["skipped"]),
            failed_count=len(backup_results["failed"]),
        )

        # Send telemetry event
        if backup_results["failed"]:
            await TelemetryClient.send_event(
                Category.FLOW_OPERATIONS, MessageId.ORB_FLOW_BACKUP_FAILED
            )
        else:
            await TelemetryClient.send_event(
                Category.FLOW_OPERATIONS, MessageId.ORB_FLOW_BACKUP_COMPLETE
            )

        return backup_results

    async def _backup_flow(self, flow_id: str, flow_type: str, flow_data: dict | None = None):
        """
        Backup a single flow to the backup folder.

        Args:
            flow_id: The flow ID to backup
            flow_type: The flow type name (nudges, retrieval, ingest, url_ingest)
            flow_data: The flow data to backup (if None, fetches from API)

        Returns:
            str: Path to the backup file, or None if backup failed
        """
        try:
            # Get flow data if not provided
            if flow_data is None:
                response = await clients.langflow_request("GET", f"/api/v1/flows/{flow_id}")
                if response.status_code != 200:
                    logger.warning(
                        f"Failed to get flow {flow_id} for backup: HTTP {response.status_code}"
                    )
                    return None
                flow_data = response.json()

            # Create backup directory if it doesn't exist
            backup_dir = self._get_backup_directory()

            # Generate backup filename with timestamp
            timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
            backup_filename = f"{flow_type}_{timestamp}.json"
            backup_path = os.path.join(backup_dir, backup_filename)

            # Save flow to backup file
            with open(backup_path, "w") as f:
                json.dump(flow_data, f, indent=2, ensure_ascii=False)

            logger.info(
                f"Backed up {flow_type} flow (ID: {flow_id}) to {backup_filename}",
                backup_path=backup_path,
            )

            return backup_path

        except Exception as e:
            logger.error(
                f"Failed to backup flow {flow_id} ({flow_type}): {str(e)}",
                error=str(e),
            )
            return None

    def _find_flow_file_by_id(self, flow_id: str):
        """
        Scan the flows directory and find the JSON file that contains the specified flow ID.

        Args:
            flow_id: The flow ID to search for

        Returns:
            str: The path to the flow file, or None if not found
        """
        if not flow_id:
            raise ValueError("flow_id is required")

        # Check cache first
        if flow_id in self._flow_file_cache:
            cached_path = self._flow_file_cache[flow_id]
            if os.path.exists(cached_path):
                return cached_path
            else:
                # Remove stale cache entry
                del self._flow_file_cache[flow_id]

        flows_dir = self._get_flows_directory()

        if not os.path.exists(flows_dir):
            logger.warning(f"Flows directory not found: {flows_dir}")
            return None

        # Scan all JSON files in the flows directory
        try:
            for filename in os.listdir(flows_dir):
                if not filename.endswith(".json"):
                    continue

                file_path = os.path.join(flows_dir, filename)

                try:
                    with open(file_path) as f:
                        flow_data = json.load(f)

                    # Check if this file contains the flow we're looking for
                    if flow_data.get("id") == flow_id:
                        # Cache the result
                        self._flow_file_cache[flow_id] = file_path
                        logger.info(f"Found flow {flow_id} in file: {filename}")
                        return file_path

                except (json.JSONDecodeError, FileNotFoundError) as e:
                    logger.warning(f"Error reading flow file {filename}: {e}")
                    continue

        except Exception as e:
            logger.error(f"Error scanning flows directory: {e}")
            return None

        logger.warning(f"Flow with ID {flow_id} not found in flows directory")
        return None

    def _get_flow_id_by_type(self, flow_type: str) -> str:
        if flow_type == "nudges":
            flow_id = NUDGES_FLOW_ID
        elif flow_type == "retrieval":
            flow_id = LANGFLOW_CHAT_FLOW_ID
        elif flow_type == "ingest":
            flow_id = LANGFLOW_INGEST_FLOW_ID
        elif flow_type == "url_ingest":
            flow_id = LANGFLOW_URL_INGEST_FLOW_ID
        else:
            raise ValueError(
                "flow_type must be either 'nudges', 'retrieval', 'ingest', or 'url_ingest'"
            )
        if not flow_id:
            raise ValueError(f"Flow ID not configured for flow_type '{flow_type}'")
        return flow_id

    async def reset_langflow_flow(self, flow_type: str):
        """Reset a Langflow flow by uploading the corresponding JSON file

        Args:
            flow_type: Either 'nudges', 'retrieval', 'ingest', or 'url_ingest'

        Returns:
            dict: Success/error response
        """
        flow_id = self._get_flow_id_by_type(flow_type)
        flow_lock = await self._get_flow_lock(flow_id)
        async with flow_lock:
            result = await self._reset_langflow_flow_locked(flow_type, flow_id)
        if result.get("success"):
            self._custom_updates_cache = None
            try:
                config = get_openrag_config()
                if config.edited:
                    from api.settings import reapply_all_settings

                    await reapply_all_settings()
            except Exception as e:
                logger.error(f"Error reapplying settings after flow reset: {e}")
        return result

    async def _unlock_flow(self, flow_id: str):
        """Unlock a flow before making changes."""
        try:
            response = await clients.langflow_request(
                "PATCH",
                f"/api/v1/flows/{flow_id}",
                json={"locked": False},
                idempotent=True,
            )
            if response.status_code not in (200, 404):
                logger.warning(
                    f"Failed to unlock flow {flow_id}: HTTP {response.status_code} - {response.text}"
                )
        except Exception as e:
            logger.warning(f"Error unlocking flow {flow_id}: {e}")

    async def _lock_flow(self, flow_id: str):
        """Lock a flow after making changes."""
        try:
            response = await clients.langflow_request(
                "PATCH",
                f"/api/v1/flows/{flow_id}",
                json={"locked": True},
                idempotent=True,
            )
            if response.status_code not in (200, 404):
                logger.warning(
                    f"Failed to lock flow {flow_id}: HTTP {response.status_code} - {response.text}"
                )
        except Exception as e:
            logger.warning(f"Error locking flow {flow_id}: {e}")

    async def _reset_langflow_flow_locked(self, flow_type: str, flow_id: str):
        if not LANGFLOW_URL:
            raise ValueError("LANGFLOW_URL environment variable is required")

        # Dynamically find the flow file by ID
        flow_path = self._find_flow_file_by_id(flow_id)
        if not flow_path:
            raise FileNotFoundError(f"Flow file not found for flow ID: {flow_id}")

        # Load flow JSON file
        try:
            with open(flow_path) as f:
                flow_data = json.load(f)
            self._stamp_flow_with_file_mtime(flow_data, flow_path)
            logger.info(
                f"Successfully loaded flow data for {flow_type} from {os.path.basename(flow_path)}"
            )
        except json.JSONDecodeError as e:
            raise ValueError(f"Invalid JSON in flow file {flow_path}: {e}") from e
        except FileNotFoundError as e:
            raise ValueError(f"Flow file not found: {flow_path}") from e

        await self._unlock_flow(flow_id)

        # Replace the complete flow so the stamped source timestamp is persisted.
        try:
            response = await clients.langflow_request(
                "PUT",
                f"/api/v1/flows/{flow_id}",
                json=flow_data,
                idempotent=True,
            )

            if response.status_code == 200:
                await self._lock_flow(flow_id)
                try:
                    res_json = response.json()
                    project_id = res_json.get("folder_id")
                    if project_id and (
                        flow_type == "url_ingest" or flow_id == LANGFLOW_URL_INGEST_FLOW_ID
                    ):
                        await enable_mcp_none_for_project(project_id)
                except Exception as err:
                    logger.warning(f"Failed to enable unauthenticated MCP for {flow_type}: {err}")
                logger.info(
                    f"Successfully reset {flow_type} flow",
                    flow_id=flow_id,
                    flow_file=os.path.basename(flow_path),
                )
                return {
                    "success": True,
                    "message": f"Successfully reset {flow_type} flow",
                    "flow_id": flow_id,
                    "flow_type": flow_type,
                }
            else:
                error_text = response.text
                logger.error(
                    f"Failed to reset {flow_type} flow",
                    status_code=response.status_code,
                    error=error_text,
                )
                return {
                    "success": False,
                    "error": f"Failed to reset flow: HTTP {response.status_code} - {error_text}",
                }
        except Exception as e:
            logger.error(f"Error while resetting {flow_type} flow", error=str(e))
            return {"success": False, "error": f"Error: {str(e)}"}

    def _find_node_in_flow(self, flow_data, node_id=None, display_name=None):
        """
        Helper function to find a node in flow data by ID or display name.
        Returns tuple of (node, node_index) or (None, None) if not found.
        """
        nodes = flow_data.get("data", {}).get("nodes", [])

        for i, node in enumerate(nodes):
            node_data = node.get("data", {})
            node_template = node_data.get("node", {})

            # Check by ID if provided
            if node_id and node_data.get("id") == node_id:
                return node, i

            # Check by display_name if provided
            if display_name and node_template.get("display_name") == display_name:
                return node, i

        return None, None

    def _find_nodes_in_flow(self, flow_data, display_name=None):
        """Find all nodes in flow data by display name."""
        nodes = flow_data.get("data", {}).get("nodes", [])
        found = []
        for i, node in enumerate(nodes):
            node_data = node.get("data", {})
            node_template = node_data.get("node", {})
            if display_name and node_template.get("display_name") == display_name:
                found.append((node, i))
        return found

    def _get_node_provider(self, node):
        """Get the provider name currently set in a node."""
        template = node.get("data", {}).get("node", {}).get("template", {})
        model_val = template.get("model", {}).get("value")
        if isinstance(model_val, list) and len(model_val) > 0:
            return model_val[0].get("provider")
        return None

    def _get_provider_name_display(self, provider: str) -> str:
        """Map internal provider key to Langflow display name."""
        if provider == "watsonx":
            return "IBM WatsonX"
        if provider == "ollama":
            return "Ollama"
        if provider == "anthropic":
            return "Anthropic"
        return "OpenAI"

    async def _update_flow_field(
        self,
        flow_id: str,
        field_name: str,
        field_value: Any,
        node_display_name: str | None = None,
        expected_value: Any = _UNSET,
    ):
        """
        Generic helper function to update any field in any Langflow component.

        Args:
            flow_id: The ID of the flow to update
            field_name: The name of the field to update (e.g., 'model_name', 'system_message', 'docling_serve_opts')
            field_value: The new value to set
            node_display_name: The display name to search for (optional)
            expected_value: Optional expected current value to revalidate before writing
        """
        if not flow_id:
            raise ValueError("flow_id is required")

        # Get the current flow data from Langflow
        response = await clients.langflow_request("GET", f"/api/v1/flows/{flow_id}")

        if response.status_code != 200:
            raise Exception(f"Failed to get flow: HTTP {response.status_code} - {response.text}")

        flow_data = response.json()

        # Find the target component by display name first, then by ID as fallback
        target_node, target_node_index = None, None
        if node_display_name:
            target_node, target_node_index = self._find_node_in_flow(
                flow_data, display_name=node_display_name
            )

        if target_node is None:
            identifier = node_display_name
            raise Exception(f"Component '{identifier}' not found in flow {flow_id}")

        # Update the field value directly in the existing node
        template = target_node.get("data", {}).get("node", {}).get("template", {})
        if template.get(field_name):
            if expected_value is not _UNSET:
                current_val = template[field_name].get("value")
                if current_val != expected_value:
                    raise Exception(
                        f"Field '{field_name}' in '{node_display_name}' component has changed from expected value"
                    )
            flow_data["data"]["nodes"][target_node_index]["data"]["node"]["template"][field_name][
                "value"
            ] = field_value
            if (
                "options"
                in flow_data["data"]["nodes"][target_node_index]["data"]["node"]["template"][
                    field_name
                ]
                and field_value
                not in flow_data["data"]["nodes"][target_node_index]["data"]["node"]["template"][
                    field_name
                ]["options"]
            ):
                flow_data["data"]["nodes"][target_node_index]["data"]["node"]["template"][
                    field_name
                ]["options"].append(field_value)
        else:
            identifier = node_display_name
            raise Exception(f"{field_name} field not found in {identifier} component")

        await self._unlock_flow(flow_id)

        # Update the flow via PATCH request
        patch_response = await clients.langflow_request(
            "PATCH",
            f"/api/v1/flows/{flow_id}",
            json=flow_data,
            idempotent=True,
        )

        if patch_response.status_code != 200:
            raise Exception(
                f"Failed to update flow: HTTP {patch_response.status_code} - {patch_response.text}"
            )

        await self._lock_flow(flow_id)

    async def get_chat_flow_system_prompt(self) -> str | None:
        """Helper function to get the current system prompt from the chat flow"""
        if not LANGFLOW_CHAT_FLOW_ID:
            raise ValueError("LANGFLOW_CHAT_FLOW_ID is not configured")

        response = await clients.langflow_request("GET", f"/api/v1/flows/{LANGFLOW_CHAT_FLOW_ID}")
        if response.status_code != 200:
            raise Exception(
                f"Failed to get flow {LANGFLOW_CHAT_FLOW_ID}: HTTP {response.status_code} - {response.text}"
            )

        flow_data = response.json()
        target_node, _ = self._find_node_in_flow(
            flow_data, display_name=AGENT_COMPONENT_DISPLAY_NAME
        )
        if target_node is None:
            raise Exception(
                f"Component '{AGENT_COMPONENT_DISPLAY_NAME}' not found in flow {LANGFLOW_CHAT_FLOW_ID}"
            )

        return (
            target_node.get("data", {})
            .get("node", {})
            .get("template", {})
            .get("system_prompt", {})
            .get("value")
        )

    async def update_chat_flow_system_prompt(
        self, system_prompt: str, expected_prompt: Any = _UNSET
    ):
        """Helper function to update the system prompt in the chat flow"""
        if not LANGFLOW_CHAT_FLOW_ID:
            raise ValueError("LANGFLOW_CHAT_FLOW_ID is not configured")

        await self._update_flow_field(
            LANGFLOW_CHAT_FLOW_ID,
            "system_prompt",
            system_prompt,
            node_display_name=AGENT_COMPONENT_DISPLAY_NAME,
            expected_value=expected_prompt,
        )

    async def update_flow_docling_preset(self, preset: str, preset_config: dict):
        """Helper function to update docling preset in the ingest flow"""
        if not LANGFLOW_INGEST_FLOW_ID:
            raise ValueError("LANGFLOW_INGEST_FLOW_ID is not configured")

        from config.settings import DOCLING_COMPONENT_DISPLAY_NAME

        await self._update_flow_field(
            LANGFLOW_INGEST_FLOW_ID,
            "docling_serve_opts",
            preset_config,
            node_display_name=DOCLING_COMPONENT_DISPLAY_NAME,
        )

    async def update_ingest_flow_chunk_size(self, chunk_size: int):
        """Helper function to update chunk size in the ingest flow"""
        if not LANGFLOW_INGEST_FLOW_ID:
            raise ValueError("LANGFLOW_INGEST_FLOW_ID is not configured")
        await self._update_flow_field(
            LANGFLOW_INGEST_FLOW_ID,
            "chunk_size",
            chunk_size,
            node_display_name="Split Text",
        )

    async def update_ingest_flow_chunk_overlap(self, chunk_overlap: int):
        """Helper function to update chunk overlap in the ingest flow"""
        if not LANGFLOW_INGEST_FLOW_ID:
            raise ValueError("LANGFLOW_INGEST_FLOW_ID is not configured")
        await self._update_flow_field(
            LANGFLOW_INGEST_FLOW_ID,
            "chunk_overlap",
            chunk_overlap,
            node_display_name="Split Text",
        )

    def _normalize_flow_structure(self, flow_data: dict[str, Any]) -> dict[str, Any]:
        """
        Normalize flow structure for comparison by removing dynamic fields.
        Keeps structural elements: nodes (types, display names, template definitions), edges (connections).
        Removes: IDs, timestamps, positions, dynamic runtime field values, and dynamic options lists.
        """
        normalized: dict[str, Any] = {"data": {"nodes": [], "edges": []}}

        # Normalize nodes - keep structural info including template definitions
        nodes = flow_data.get("data", {}).get("nodes", [])
        for node in nodes:
            node_data = node.get("data", {})
            node_template = node_data.get("node", {})

            template = node_template.get("template", {})
            normalized_template: dict[str, Any] = {}
            if isinstance(template, dict):
                for field_name, field_def in template.items():
                    if isinstance(field_def, dict):
                        normalized_template[field_name] = {
                            "type": field_def.get("type"),
                            "display_name": field_def.get("display_name"),
                            "required": field_def.get("required"),
                            "input_types": field_def.get("input_types"),
                        }
                    else:
                        normalized_template[field_name] = {"value": str(field_def)}

            normalized_node = {
                "id": node.get("id"),  # Keep ID for edge matching
                "type": node.get("type"),
                "data": {
                    "node": {
                        "display_name": node_template.get("display_name"),
                        "name": node_template.get("name"),
                        "base_classes": node_template.get("base_classes", []),
                        "template": normalized_template,
                    }
                },
            }
            normalized["data"]["nodes"].append(normalized_node)

        # Normalize edges - keep only connections
        edges = flow_data.get("data", {}).get("edges", [])
        for edge in edges:
            normalized_edge = {
                "source": edge.get("source"),
                "target": edge.get("target"),
                "sourceHandle": edge.get("sourceHandle"),
                "targetHandle": edge.get("targetHandle"),
            }
            normalized["data"]["edges"].append(normalized_edge)

        return normalized

    async def _check_flow_update(
        self, flow_type: str, flow_id: str, flow_data: dict[str, Any] | None = None
    ) -> dict[str, Any] | None:
        """
        Check whether the source flow file is newer than the version stored in Langflow.

        The backend stamps each uploaded flow with an
        ``openrag:file-mtime:<nanoseconds>`` tag. Comparing the current file
        timestamp to that tag avoids relying on Langflow's unrelated
        ``updated_at`` value.

        Returns flow update info dict if an update is available, None otherwise.
        """
        if not flow_id:
            return None
        flow_path = self._find_flow_file_by_id(flow_id)
        if not flow_path or not os.path.exists(flow_path):
            return None

        try:
            source_mtime = os.stat(flow_path).st_mtime_ns
        except OSError as error:
            logger.warning(f"Could not stat bundled flow {flow_path} for update check: {error}")
            return None

        if flow_data is None:
            try:
                response = await clients.langflow_request("GET", f"/api/v1/flows/{flow_id}")
                if response.status_code != 200:
                    return None
                flow_data = response.json()
            except Exception as e:
                logger.error(f"Failed to fetch flow {flow_id} for update check: {e}")
                return None

        stored_mtime = self._get_flow_file_mtime_tag(flow_data)
        if stored_mtime is None or source_mtime > stored_mtime:
            is_locked = flow_data.get("locked", False)
            return {
                "flow_type": flow_type,
                "flow_id": flow_id,
                "is_custom": not is_locked,
            }
        return None

    @staticmethod
    def _stamp_flow_with_file_mtime(flow_data: dict[str, Any], flow_path: str) -> None:
        """Embed the source flow's modification timestamp into its Langflow tags."""
        tags = flow_data.get("tags")
        if tags is None:
            tags = []
        if not isinstance(tags, list) or not all(isinstance(tag, str) for tag in tags):
            raise ValueError(f"Flow {flow_data.get('id', '<unknown>')} has invalid tags")
        flow_data["tags"] = [
            *[tag for tag in tags if not tag.startswith(_FLOW_FILE_MTIME_TAG_PREFIX)],
            f"{_FLOW_FILE_MTIME_TAG_PREFIX}{os.stat(flow_path).st_mtime_ns}",
        ]

    @staticmethod
    def _get_flow_file_mtime_tag(flow_data: dict[str, Any]) -> int | None:
        """Return the source-file modification timestamp recorded on a Langflow flow."""
        tags = flow_data.get("tags")
        if not isinstance(tags, list):
            return None
        for tag in tags:
            if not isinstance(tag, str) or not tag.startswith(_FLOW_FILE_MTIME_TAG_PREFIX):
                continue
            try:
                return int(tag.removeprefix(_FLOW_FILE_MTIME_TAG_PREFIX))
            except ValueError:
                continue
        return None

    async def ensure_flows_exist(self) -> set[str]:
        """
        Ensure all configured flows exist in Langflow.

        Creates flows from their JSON files if they are not already present in
        the Langflow database.  This is intentionally create-only: it never
        patches or overwrites an existing flow, preserving any edits the user
        has made in the Langflow UI.

        Returns the set of flow type names that were actually created.
        """
        flow_configs = [
            ("nudges", NUDGES_FLOW_ID),
            ("retrieval", LANGFLOW_CHAT_FLOW_ID),
            ("ingest", LANGFLOW_INGEST_FLOW_ID),
            ("url_ingest", LANGFLOW_URL_INGEST_FLOW_ID),
        ]
        created_flow_types: set[str] = set()

        async def ensure_single_flow_exists(flow_type, flow_id):
            if not flow_id:
                return None
            try:
                response = await clients.langflow_request("GET", f"/api/v1/flows/{flow_id}")
                if response.status_code == 200:
                    logger.info(
                        f"Flow {flow_type} (ID: {flow_id}) already exists, skipping creation"
                    )
                    try:
                        project_id = response.json().get("folder_id")
                        if project_id and flow_type == "url_ingest":
                            await enable_mcp_none_for_project(project_id)
                    except Exception as err:
                        logger.warning(f"Failed to check MCP project auth for {flow_type}: {err}")
                    return None

                if response.status_code != 404:
                    logger.warning(
                        f"Unexpected status checking {flow_type} flow (ID: {flow_id}): "
                        f"HTTP {response.status_code} — skipping creation to avoid overwriting existing data"
                    )
                    return None

                flow_path = self._find_flow_file_by_id(flow_id)
                if not flow_path:
                    logger.warning(
                        f"No flow file found for {flow_type} (ID: {flow_id}), cannot create"
                    )
                    return None

                with open(flow_path) as f:
                    flow_data = json.load(f)
                self._stamp_flow_with_file_mtime(flow_data, flow_path)

                response = await clients.langflow_request(
                    "PUT", f"/api/v1/flows/{flow_id}", json=flow_data
                )
                if response.status_code in (200, 201):
                    logger.info(
                        f"Created {flow_type} flow (ID: {flow_id}) from {os.path.basename(flow_path)}"
                    )
                    try:
                        project_id = response.json().get("folder_id")
                        if project_id and flow_type == "url_ingest":
                            await enable_mcp_none_for_project(project_id)
                    except Exception as err:
                        logger.warning(f"Failed to check MCP project auth for {flow_type}: {err}")
                    return flow_type
                else:
                    logger.warning(
                        f"Failed to create {flow_type} flow (ID: {flow_id}): "
                        f"HTTP {response.status_code} — {response.text}"
                    )
                    return None
            except Exception as e:
                logger.error(f"Error ensuring {flow_type} flow (ID: {flow_id}) exists: {e}")
                return None

        tasks = [ensure_single_flow_exists(ft, fi) for ft, fi in flow_configs]
        results = await asyncio.gather(*tasks)
        created_flow_types = {r for r in results if r is not None}

        if created_flow_types:
            try:
                config = get_openrag_config()
                if config.edited:
                    logger.info(
                        f"Newly created flows detected ({', '.join(created_flow_types)}), reapplying settings."
                    )
                    from api.settings import reapply_all_settings

                    await reapply_all_settings()
                    logger.info("Successfully reapplied settings for newly created flows")
            except Exception as e:
                logger.error(f"Failed to reapply settings for newly created flows: {e}")

        return created_flow_types

    async def enable_mcp_none_for_url_ingest_flow(self) -> bool:
        """Configure auth_type='none' for MCP on the URL ingest flow project."""
        flow_id = LANGFLOW_URL_INGEST_FLOW_ID
        if not flow_id:
            return False
        try:
            resp = await clients.langflow_request("GET", f"/api/v1/flows/{flow_id}")
            if resp.status_code == 200:
                project_id = resp.json().get("folder_id")
                if project_id:
                    return await enable_mcp_none_for_project(project_id)
        except Exception as e:
            logger.warning(
                f"Error enabling MCP auth_type='none' for URL ingest flow {flow_id}: {e}"
            )
        return False

    async def change_langflow_model_value(
        self,
        provider: str,
        embedding_model: str | None = None,
        llm_model: str | None = None,
        force_embedding_update: bool = False,
        force_llm_update: bool = False,
        flow_configs: list | None = None,
    ):
        """
        Change dropdown values for provider-specific components across flows

        Args:
            provider: Any LiteLLM provider. Non-legacy providers use Langflow's
                OpenAI-compatible component, which points at the OpenRAG proxy.
            embedding_model: The embedding model name to set
            llm_model: The LLM model name to set
            force_embedding_update: If True, update embeddings even if model is None
            force_llm_update: If True, update LLM even if model is None
            flow_configs: Optional list of flow configs to update
        """
        from services.model_catalog import is_known_provider

        if not is_known_provider(provider):
            raise ValueError(f"Unknown LiteLLM provider: {provider}")
        flow_provider = (
            provider if provider in {"watsonx", "ollama", "openai", "anthropic"} else "openai"
        )

        try:
            # Use provided flow_configs or default to all flows
            if flow_configs is None:
                flow_configs = [
                    {"name": "nudges", "flow_id": NUDGES_FLOW_ID},
                    {"name": "retrieval", "flow_id": LANGFLOW_CHAT_FLOW_ID},
                    {"name": "ingest", "flow_id": LANGFLOW_INGEST_FLOW_ID},
                    {"name": "url_ingest", "flow_id": LANGFLOW_URL_INGEST_FLOW_ID},
                ]

            tasks = []
            for config in flow_configs:
                tasks.append(
                    self._update_provider_components(
                        config,
                        flow_provider,
                        embedding_model=embedding_model,
                        llm_model=llm_model,
                        force_embedding_update=force_embedding_update,
                        force_llm_update=force_llm_update,
                    )
                )

            # Process all flows simultaneously
            raw_results = await asyncio.gather(*tasks, return_exceptions=True)

            results: list[Any] = []
            for i, result in enumerate(raw_results):
                if isinstance(result, Exception):
                    flow_name = flow_configs[i]["name"]
                    logger.error(f"Error updating {flow_name} flow: {str(result)}")
                    results.append({"flow": flow_name, "success": False, "error": str(result)})
                else:
                    results.append(result)

            return {
                "success": all(r.get("success", False) for r in results),
                "results": results,
            }
        except Exception as e:
            logger.error(f"Error in change_langflow_model_value: {str(e)}")
            return {"success": False, "error": str(e)}

    async def _update_provider_components(
        self,
        config: dict,
        provider_name: str,
        embedding_model: str | None = None,
        llm_model: str | None = None,
        force_embedding_update: bool = False,
        force_llm_update: bool = False,
    ):
        """Update provider components and their dropdown values in a flow"""
        flow_id = config["flow_id"]
        flow_lock = await self._get_flow_lock(flow_id)
        async with flow_lock:
            return await self._update_provider_components_locked(
                config,
                provider_name,
                embedding_model=embedding_model,
                llm_model=llm_model,
                force_embedding_update=force_embedding_update,
                force_llm_update=force_llm_update,
            )

    async def _update_provider_components_locked(
        self,
        config: dict,
        provider_name: str,
        embedding_model: str | None = None,
        llm_model: str | None = None,
        force_embedding_update: bool = False,
        force_llm_update: bool = False,
    ):
        """Internal helper to update provider components when flow_lock is already held"""
        flow_name = config["name"]
        flow_id = config["flow_id"]
        # Get flow data from Langflow API instead of file
        response = await clients.langflow_request("GET", f"/api/v1/flows/{flow_id}")

        if response.status_code != 200:
            raise Exception(
                f"Failed to get flow from Langflow: HTTP {response.status_code} - {response.text}"
            )

        flow_data = response.json()
        provider = provider_name
        updates_made = []
        node_tasks: list[Any] = []

        async def wrap_node_update(node, p, m, label):
            if await self._update_component_fields(node, p, m):
                return label
            return None

        # Get all embedding nodes in the flow to check if flow is new (no 3 embeddings)
        embedding_nodes = self._find_nodes_in_flow(
            flow_data, display_name=OPENAI_EMBEDDING_COMPONENT_DISPLAY_NAME
        )
        logger.info(
            f"Found {len(embedding_nodes)} embedding nodes in flow {flow_name} with display name '{OPENAI_EMBEDDING_COMPONENT_DISPLAY_NAME}'"
        )
        is_new_flow = len(embedding_nodes) < 3

        provider_name = self._get_provider_name_display(provider)

        # Update embedding component
        if not get_openrag_config().knowledge.disable_ingest_with_langflow and (
            embedding_model or force_embedding_update
        ):
            if is_new_flow:
                if embedding_model:
                    await self._enable_model_in_langflow(provider_name, embedding_model)
                logger.info(
                    f"Flow {flow_name} has {len(embedding_nodes)} embedding nodes (< 3). "
                    f"Flow is updated to generic embedding component; enabled model in Langflow, skipping legacy component patching."
                )
            else:
                # Count configured embedding-enabled providers
                config_obj = get_openrag_config()
                configured_providers = []
                if config_obj.providers.openai.configured:
                    configured_providers.append("openai")
                if config_obj.providers.watsonx.configured:
                    configured_providers.append("watsonx")
                if config_obj.providers.ollama.configured:
                    configured_providers.append("ollama")

                # Ensure current provider is in the list for counting purposes if it's being configured
                if (
                    provider in ["openai", "watsonx", "ollama"]
                    and provider not in configured_providers
                ):
                    configured_providers.append(provider)

                all_possible = ["openai", "watsonx", "ollama"]
                configured_providers = [p for p in all_possible if p in configured_providers]
                provider_count = len(configured_providers)
                logger.info(
                    f"Configured embedding providers: {configured_providers} (count: {provider_count})"
                )

                # 1. Check if any node is already this provider - always update those first
                matched_nodes = []
                provider_display = provider_name
                for node, idx in embedding_nodes:
                    if self._get_node_provider(node) == provider_display:
                        matched_nodes.append((node, idx))

                if matched_nodes:
                    logger.info(
                        f"Found {len(matched_nodes)} nodes already configured for provider '{provider}'"
                    )
                    for node, _idx in matched_nodes:
                        node_tasks.append(
                            wrap_node_update(
                                node,
                                provider,
                                embedding_model,
                                f"embedding model: {embedding_model} (updated existing {provider} node)",
                            )
                        )
                else:
                    # 2. No existing node matched, use slot-based logic
                    try:
                        p_index = configured_providers.index(provider)
                        logger.info(
                            f"Using slot-based logic for provider '{provider}' (p_index: {p_index}, total configured: {provider_count})"
                        )

                        if provider_count == 1:
                            # Single provider mode: update all available nodes (up to 3)
                            logger.info(
                                f"Single provider mode: updating all available embedding nodes (available: {len(embedding_nodes)})"
                            )
                            for i in range(min(3, len(embedding_nodes))):
                                node, idx = embedding_nodes[i]
                                node_tasks.append(
                                    wrap_node_update(
                                        node,
                                        provider,
                                        embedding_model,
                                        f"embedding model: {embedding_model} (set node {i + 1})",
                                    )
                                )
                        else:
                            # Multiple providers: each gets one slot based on its list index
                            if p_index < len(embedding_nodes):
                                node, idx = embedding_nodes[p_index]
                                logger.info(
                                    f"Multiple provider mode: assigning provider '{provider}' to node slot {p_index} (node {p_index + 1})"
                                )
                                node_tasks.append(
                                    wrap_node_update(
                                        node,
                                        provider,
                                        embedding_model,
                                        f"embedding model: {embedding_model} (set node {p_index + 1})",
                                    )
                                )
                            else:
                                logger.info(
                                    f"Provider index {p_index} exceeds available embedding nodes ({len(embedding_nodes)}) - skipping automatic assignment"
                                )
                    except ValueError:
                        logger.warning(
                            f"Current provider '{provider}' not found in configured providers list: {configured_providers}"
                        )

        # Update LLM component (if exists in this flow)
        if llm_model or force_llm_update:
            if is_new_flow:
                if llm_model:
                    await self._enable_model_in_langflow(provider_name, llm_model)
                logger.info(
                    f"Flow {flow_name} has {len(embedding_nodes)} embedding nodes (< 3). "
                    f"Flow is updated to generic components; enabled model in Langflow, skipping legacy LLM component patching."
                )
            else:
                llm_node, _ = self._find_node_in_flow(
                    flow_data, display_name=OPENAI_LLM_COMPONENT_DISPLAY_NAME
                )
                if llm_node:
                    node_tasks.append(
                        wrap_node_update(llm_node, provider, llm_model, f"llm model: {llm_model}")
                    )

                agent_node, _ = self._find_node_in_flow(
                    flow_data, display_name=AGENT_COMPONENT_DISPLAY_NAME
                )
                if agent_node:
                    node_tasks.append(
                        wrap_node_update(
                            agent_node, provider, llm_model, f"agent model: {llm_model}"
                        )
                    )

        # Execute all node updates simultaneously
        if node_tasks:
            node_results = await asyncio.gather(*node_tasks)
            updates_made.extend([r for r in node_results if r])

        # If no updates were made, return skip message
        if not updates_made:
            return {
                "flow": flow_name,
                "success": True,
                "message": f"No compatible components found in {flow_name} flow (skipped)",
                "flow_id": flow_id,
            }

        logger.info(f"Updated {', '.join(updates_made)} in {flow_name} flow")

        await self._unlock_flow(flow_id)

        # PATCH the updated flow
        response = await clients.langflow_request(
            "PATCH",
            f"/api/v1/flows/{flow_id}",
            json=flow_data,
            idempotent=True,
        )

        if response.status_code != 200:
            raise Exception(f"Failed to update flow: HTTP {response.status_code} - {response.text}")

        await self._lock_flow(flow_id)

        return {
            "flow": flow_name,
            "success": True,
            "message": f"Successfully updated {', '.join(updates_made)}",
            "flow_id": flow_id,
        }

    async def _update_component_langflow(self, template, model: Any):
        # Call custom_component/update endpoint to get updated template
        # Only call if code field exists (custom components should have code)
        if "code" in template and "value" in template["code"]:
            code_value = template["code"]["value"]

            try:
                update_payload = {
                    "code": code_value,
                    "template": template,
                    "field": "model",
                    "field_value": model,
                    "tool_mode": False,
                }

                response = await clients.langflow_request(
                    "POST", "/api/v1/custom_component/update", json=update_payload
                )

                if response.status_code == 200:
                    response_data = response.json()
                    # Update template with the new template from response.data
                    if "template" in response_data:
                        # Update the template in component_node
                        return response_data["template"]
                    else:
                        logger.warning("Response from custom_component/update missing 'data' field")
                else:
                    logger.warning(
                        f"Failed to call custom_component/update: HTTP {response.status_code} - {response.text}"
                    )
            except Exception as e:
                logger.error(f"Error calling custom_component/update: {str(e)}")
                # Continue with manual updates even if API call fails

    async def _update_component_fields(
        self,
        component_node,
        provider: str,
        model_value: str,
    ):
        """Update fields in a component node based on provider and component type"""
        template = component_node.get("data", {}).get("node", {}).get("template", {})
        if not template:
            return False

        updated = False
        provider_name = self._get_provider_name_display(provider)

        # Update model field and call custom_component/update endpoint
        if "model" in template:
            if "options" not in template["model"]:
                logger.warning(f"Model field not found in template for provider {provider_name}")
                return False

            # Enable the model in Langflow first
            if model_value:
                await self._enable_model_in_langflow(provider_name, model_value)
            # Update template via Langflow API to get latest options
            template = (
                await self._update_component_langflow(template, template["model"]["value"])
                or template
            )
            component_node["data"]["node"]["template"] = template

            # Find the specific model option for the provider
            if model_value:
                model_options = [
                    item
                    for item in template["model"].get("options", [])
                    if item.get("provider") == provider_name and item.get("name") == model_value
                ]
            else:
                # If no specific model provided, pick the first available one for this provider
                model_options = [
                    item
                    for item in template["model"].get("options", [])
                    if item.get("provider") == provider_name
                ][:1]
                if model_options:
                    logger.info(
                        f"Using first available model '{model_options[0].get('name')}' for provider {provider_name}"
                    )

            if not model_options:
                logger.warning(
                    f"Model {model_value or 'ANY'} not found for provider {provider_name}"
                )
                return False

            template["model"]["value"] = model_options

            template = await self._update_component_langflow(template, model_options) or template
            component_node["data"]["node"]["template"] = template

            updated = True

        # Point every model component at the OpenRAG OpenAI-compatible proxy.
        # Real provider secrets never leave OpenRAG; Langflow sends the caller JWT.
        proxy_fields = {
            "api_key": "OPENRAG_LLM_TOKEN",
            "openai_api_key": "OPENRAG_LLM_TOKEN",
            "api_base": "OPENRAG_LLM_BASE_URL",
            "openai_api_base": "OPENRAG_LLM_BASE_URL",
        }
        for field, global_name in proxy_fields.items():
            if field in template:
                template[field]["value"] = global_name
                template[field]["load_from_db"] = True
                updated = True

        return updated

    async def _enable_model_in_langflow(self, provider_name: str, model_value: str):
        """Ensure the specified model is enabled in Langflow."""
        try:
            enabled_data = None
            async with self._enabled_models_lock:
                # Check if cache is still valid
                if (
                    self._enabled_models_cache
                    and self._enabled_models_cache_time
                    and (datetime.now() - self._enabled_models_cache_time).total_seconds()
                    < self._enabled_models_cache_ttl
                ):
                    enabled_data = self._enabled_models_cache
                else:
                    # Cache miss or stale, fetch from Langflow
                    get_response = await clients.langflow_request(
                        "GET", "/api/v1/models/enabled_models"
                    )
                    if get_response.status_code == 200:
                        enabled_data = get_response.json().get("enabled_models", {})
                        self._enabled_models_cache = enabled_data
                        self._enabled_models_cache_time = datetime.now()
                        logger.debug("Refreshed enabled models cache from Langflow")

            if enabled_data:
                provider_models = enabled_data.get(provider_name, {})
                if provider_models.get(model_value) is True:
                    logger.info(
                        f"Model {model_value} for provider {provider_name} is already enabled. Skipping."
                    )
                    return False

            enable_payload = [{"provider": provider_name, "model_id": model_value, "enabled": True}]

            response = await clients.langflow_request(
                "POST", "/api/v1/models/enabled_models", json=enable_payload
            )

            if response.status_code == 200:
                logger.info(
                    f"Successfully enabled model {model_value} for provider {provider_name}"
                )
                async with self._enabled_models_lock:
                    self._enabled_models_cache = None
                return True
            else:
                logger.warning(
                    f"Failed to enable model: HTTP {response.status_code} - {response.text}"
                )
                async with self._enabled_models_lock:
                    self._enabled_models_cache = None
        except Exception as e:
            logger.error(f"Error enabling model in Langflow: {str(e)}")
        return False

    def _get_provider_component_ids(self, provider: str):
        """Helper to get component display names for various providers."""
        from config.settings import (
            OLLAMA_EMBEDDING_COMPONENT_DISPLAY_NAME,
            OLLAMA_LLM_COMPONENT_DISPLAY_NAME,
            OPENAI_EMBEDDING_COMPONENT_DISPLAY_NAME,
            OPENAI_LLM_COMPONENT_DISPLAY_NAME,
            WATSONX_EMBEDDING_COMPONENT_DISPLAY_NAME,
            WATSONX_LLM_COMPONENT_DISPLAY_NAME,
        )

        if provider == "openai":
            return (OPENAI_EMBEDDING_COMPONENT_DISPLAY_NAME, OPENAI_LLM_COMPONENT_DISPLAY_NAME)
        elif provider == "watsonx":
            return (WATSONX_EMBEDDING_COMPONENT_DISPLAY_NAME, WATSONX_LLM_COMPONENT_DISPLAY_NAME)
        elif provider == "ollama":
            return (OLLAMA_EMBEDDING_COMPONENT_DISPLAY_NAME, OLLAMA_LLM_COMPONENT_DISPLAY_NAME)
        elif provider == "anthropic":
            return (None, "Anthropic")
        return (None, None)

    async def get_flows_updates_available(self, user_id: str | None = None) -> list[dict[str, Any]]:
        """
        Check for available flow updates.
        Uses cached update results from startup/previous checks if available.
        Surfaces all available updates regardless of locked status.
        """
        target_user = user_id or "default"
        if not hasattr(self, "_custom_updates_cache") or self._custom_updates_cache is None:
            flow_configs = [
                ("nudges", NUDGES_FLOW_ID),
                ("retrieval", LANGFLOW_CHAT_FLOW_ID),
                ("ingest", LANGFLOW_INGEST_FLOW_ID),
                ("url_ingest", LANGFLOW_URL_INGEST_FLOW_ID),
            ]
            cache = []
            for flow_type, flow_id in flow_configs:
                update_info = await self._check_flow_update(flow_type, flow_id)
                if update_info:
                    cache.append(update_info)
            self._custom_updates_cache = cache

        updates = []
        for item in self._custom_updates_cache:
            is_dismissed = False
            if hasattr(self, "_dismissed_updates") and isinstance(self._dismissed_updates, dict):
                is_dismissed = item["flow_type"] in self._dismissed_updates.get(target_user, set())
            updates.append(
                {
                    **item,
                    "dismissed": is_dismissed,
                }
            )

        return updates

    async def bulk_update_flows(self, flow_types: list[str], backup_custom: bool = True):
        """
        Bulk updates multiple flows.
        If backup_custom is True, flows are backed up in Langflow before resetting.
        Local file backups are always saved via _backup_flow prior to resetting.
        Returns the backup flow details if any.
        """
        results: list[dict[str, Any]] = []
        flow_configs_dict = {
            "nudges": NUDGES_FLOW_ID,
            "retrieval": LANGFLOW_CHAT_FLOW_ID,
            "ingest": LANGFLOW_INGEST_FLOW_ID,
            "url_ingest": LANGFLOW_URL_INGEST_FLOW_ID,
        }

        for flow_type in flow_types:
            flow_id = flow_configs_dict.get(flow_type)
            if not flow_id:
                results.append(
                    {"flow_type": flow_type, "success": False, "error": "Invalid flow_type"}
                )
                continue

            flow_lock = await self._get_flow_lock(flow_id)
            async with flow_lock:
                backup_path = None
                backup_flow_id = None
                flow_check_ok = False

                try:
                    response = await clients.langflow_request("GET", f"/api/v1/flows/{flow_id}")
                    if response.status_code == 200:
                        flow_check_ok = True
                        flow_data = response.json()

                        # Always attempt local file backup for safety
                        backup_path = await self._backup_flow(flow_id, flow_type, flow_data)

                        if backup_custom:
                            logger.info(f"Backing up flow {flow_type} in Langflow before update.")
                            # Create a backup flow directly in Langflow
                            backup_payload = dict(flow_data)
                            backup_payload.pop("id", None)
                            flow_name = flow_data.get("name", flow_type.capitalize())
                            timestamp = datetime.now().strftime("%Y-%m-%d %H:%M")
                            backup_payload["name"] = f"Backup - {flow_name} ({timestamp})"
                            backup_payload["description"] = (
                                f"Backup of flow '{flow_name}' created before flow update on {timestamp}. "
                                "Use this flow to reference or redo your modifications."
                            )
                            backup_payload["locked"] = False

                            lf_create_res = await clients.langflow_request(
                                "POST", "/api/v1/flows/", json=backup_payload
                            )
                            if lf_create_res.status_code in (200, 201):
                                created_flow = lf_create_res.json()
                                backup_flow_id = created_flow.get("id")
                                logger.info(
                                    f"Created backup flow in Langflow for {flow_type}: '{backup_payload['name']}' (ID: {backup_flow_id})"
                                )
                            else:
                                logger.warning(
                                    f"Failed to create backup flow in Langflow for {flow_type}: HTTP {lf_create_res.status_code}"
                                )
                except Exception as e:
                    logger.error(f"Failed to check/backup flow {flow_id} before update: {e}")

                if not flow_check_ok:
                    logger.error(
                        f"Pre-flight check failed for flow {flow_type}; aborting update to protect flow modifications."
                    )
                    results.append(
                        {
                            "flow_type": flow_type,
                            "success": False,
                            "error": "Pre-flight check failed; update aborted to avoid losing changes",
                            "backup_path": None,
                            "backup_flow_id": None,
                        }
                    )
                    continue

                # If backup_custom is enabled, abort update if BOTH backups failed
                if backup_custom and backup_path is None and backup_flow_id is None:
                    logger.error(
                        f"Backup failed for flow {flow_type}; aborting update to protect flow modifications."
                    )
                    results.append(
                        {
                            "flow_type": flow_type,
                            "success": False,
                            "error": "Backup failed; update aborted to avoid losing changes",
                            "backup_path": None,
                            "backup_flow_id": None,
                        }
                    )
                    continue

                try:
                    reset_res = await self._reset_langflow_flow_locked(flow_type, flow_id)
                    if reset_res.get("success"):
                        self.clear_dismissed_update(flow_type)
                    results.append(
                        {
                            "flow_type": flow_type,
                            "success": reset_res.get("success", False),
                            "error": reset_res.get("error"),
                            "backup_path": backup_path,
                            "backup_flow_id": backup_flow_id,
                        }
                    )
                except Exception as e:
                    results.append(
                        {
                            "flow_type": flow_type,
                            "success": False,
                            "error": str(e),
                            "backup_path": backup_path,
                            "backup_flow_id": backup_flow_id,
                        }
                    )

        if results and any(r.get("success") for r in results):
            self._custom_updates_cache = None
            try:
                config = get_openrag_config()
                if config.edited:
                    from api.settings import reapply_all_settings

                    await reapply_all_settings()
            except Exception as e:
                logger.error(f"Error reapplying settings after bulk update: {e}")

        return results