OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.
Python
fix: remove default fields on startup, simplify startup logic for updating global variables in LF (#2267)
Lucas Oliveiraa63eee967768
diff --git a/src/api/settings/langflow_sync.py b/src/api/settings/langflow_sync.pyindex 956fb4858..d53437440 100644--- a/src/api/settings/langflow_sync.py+++ b/src/api/settings/langflow_sync.py@@ -122,91 +122,172 @@ def _required_generic_global_values(config) -> dict[str, str]:async def ensure_required_langflow_global_variables(config=None):- """Ensure load_from_db plain-string globals are Generic for backwards compatibility."""+ """Ensure plain-string globals are Generic and remove any Apply To (default_fields) fields."""config = config or get_openrag_config()required_values = _required_generic_global_values(config)+ existing_by_name = {}+ try:+ response = await clients.langflow_request("GET", "/api/v1/variables/")+ if response.status_code == 200:+ existing_variables = response.json()+ existing_by_name = {v.get("name"): v for v in existing_variables if v.get("name")}+ else:+ logger.warning(+ "Could not fetch Langflow variables at startup",+ status_code=response.status_code,+ )+ return+ except Exception as e:+ logger.warning("Could not fetch Langflow variables at startup", error=str(e))+ return++ # Update or fix existing variables in a single operation per variable+ for name, var in existing_by_name.items():+ try:+ var_id = var.get("id")+ if not var_id:+ continue++ is_generic = name in LANGFLOW_GENERIC_GLOBAL_VARIABLES+ target_type = "Generic" if is_generic else var.get("type", "Credential")++ curr_type = var.get("type")+ curr_val = var.get("value", "")+ has_default_fields = bool(var.get("default_fields"))++ target_val = _string_value(required_values.get(name)) if is_generic else curr_val++ if curr_type != target_type:+ logger.info(+ "Migrating Langflow global variable type",+ variable_name=name,+ old_type=curr_type,+ new_type=target_type,+ )+ del_resp = await clients.langflow_request("DELETE", f"/api/v1/variables/{var_id}")+ if not (200 <= del_resp.status_code < 300):+ raise RuntimeError(+ f"Failed to delete Langflow global variable {name!r}: status_code={del_resp.status_code}"+ )+ recreate_payload = {+ "name": name,+ "value": target_val,+ "default_fields": [],+ "type": target_type,+ }+ recreate_resp = await clients.langflow_request(+ "POST", "/api/v1/variables/", json=recreate_payload+ )+ if not (200 <= recreate_resp.status_code < 300):+ raise RuntimeError(+ f"Failed to recreate Langflow global variable {name!r}: status_code={recreate_resp.status_code}"+ )+ elif has_default_fields or (is_generic and curr_val != target_val):+ logger.info(+ "Updating Langflow global variable", variable_name=name, variable_id=var_id+ )+ patch_payload = {+ "id": var_id,+ "name": name,+ "default_fields": [],+ "type": target_type,+ }+ if is_generic:+ patch_payload["value"] = target_val+ patch_resp = await clients.langflow_request(+ "PATCH", f"/api/v1/variables/{var_id}", json=patch_payload+ )+ if not (200 <= patch_resp.status_code < 300):+ raise RuntimeError(+ f"Failed to patch Langflow global variable {name!r}: status_code={patch_resp.status_code}"+ )+ except Exception as e:+ logger.warning(+ "Failed to update Langflow global variable", variable_name=name, error=str(e)+ )++ # Create missing generic variablesfor name in sorted(LANGFLOW_GENERIC_GLOBAL_VARIABLES):- await _upsert_langflow_global_variable(name, _string_value(required_values.get(name, "")))+ if name not in existing_by_name:+ try:+ target_val = _string_value(required_values.get(name, ""))+ await _upsert_langflow_global_variable(name, target_val)+ except Exception as e:+ logger.warning(+ "Failed to create Langflow global variable", variable_name=name, error=str(e)+ )async def _update_langflow_global_variables(config, flows_service=None):"""Update Langflow global variables for all configured providers"""- try:- # WatsonX global variables- if config.providers.watsonx.api_key:- await _upsert_langflow_global_variable(- "WATSONX_APIKEY", config.providers.watsonx.api_key- )- logger.info("Set WATSONX_APIKEY global variable in Langflow")+ errors: list[str] = []- if config.providers.watsonx.project_id:- await _upsert_langflow_global_variable(- "WATSONX_PROJECT_ID", config.providers.watsonx.project_id+ async def _safe_upsert(name: str, value: str):+ try:+ await _upsert_langflow_global_variable(name, value)+ logger.info("Set global variable in Langflow", variable_name=name)+ except Exception as e:+ logger.warning(+ "Failed to set global variable in Langflow", variable_name=name, error=str(e))- logger.info("Set WATSONX_PROJECT_ID global variable in Langflow")+ errors.append(f"{name}: {e}")- if config.providers.watsonx.endpoint:- await _upsert_langflow_global_variable("WATSONX_URL", config.providers.watsonx.endpoint)- logger.info("Set WATSONX_URL global variable in Langflow")+ providers = getattr(config, "providers", None)- # OpenAI global variables- if config.providers.openai.api_key:- await _upsert_langflow_global_variable(- "OPENAI_API_KEY", config.providers.openai.api_key- )- logger.info("Set OPENAI_API_KEY global variable in Langflow")+ # WatsonX global variables+ watsonx = getattr(providers, "watsonx", None)+ if watsonx:+ if getattr(watsonx, "api_key", None):+ await _safe_upsert("WATSONX_APIKEY", watsonx.api_key)- # Anthropic global variables- if config.providers.anthropic.api_key:- await _upsert_langflow_global_variable(- "ANTHROPIC_API_KEY", config.providers.anthropic.api_key- )- logger.info("Set ANTHROPIC_API_KEY global variable in Langflow")+ if getattr(watsonx, "project_id", None):+ await _safe_upsert("WATSONX_PROJECT_ID", watsonx.project_id)++ if getattr(watsonx, "endpoint", None):+ await _safe_upsert("WATSONX_URL", watsonx.endpoint)++ # OpenAI global variables+ openai = getattr(providers, "openai", None)+ if openai and getattr(openai, "api_key", None):+ await _safe_upsert("OPENAI_API_KEY", openai.api_key)++ # Anthropic global variables+ anthropic = getattr(providers, "anthropic", None)+ if anthropic and getattr(anthropic, "api_key", None):+ await _safe_upsert("ANTHROPIC_API_KEY", anthropic.api_key)- # Ollama global variables- if config.providers.ollama.endpoint:+ # Ollama global variables+ ollama = getattr(providers, "ollama", None)+ if ollama and getattr(ollama, "endpoint", None):+ try:if not flows_service:flows_service = _get_flows_service()- endpoint = await flows_service.resolve_ollama_url(- config.providers.ollama.endpoint, force_refresh=True- )- await _upsert_langflow_global_variable("OLLAMA_BASE_URL", endpoint)- logger.info("Set OLLAMA_BASE_URL global variable in Langflow")+ endpoint = await flows_service.resolve_ollama_url(ollama.endpoint, force_refresh=True)+ await _safe_upsert("OLLAMA_BASE_URL", endpoint)+ except Exception as e:+ logger.warning("Failed to resolve OLLAMA_BASE_URL", error=str(e))+ errors.append(f"OLLAMA_BASE_URL resolution: {e}")- if config.knowledge.embedding_model:- await _upsert_langflow_global_variable(- "SELECTED_EMBEDDING_MODEL", config.knowledge.embedding_model- )- logger.info(- f"Set SELECTED_EMBEDDING_MODEL global variable to {config.knowledge.embedding_model}"- )- if config.knowledge.embedding_provider:- mapped_provider = map_provider(config.knowledge.embedding_provider)- await _upsert_langflow_global_variable(- "SELECTED_EMBEDDING_MODEL_PROVIDER", mapped_provider- )- logger.info(- f"Set SELECTED_EMBEDDING_MODEL_PROVIDER global variable to {mapped_provider}"- )- if config.agent.llm_model:- await _upsert_langflow_global_variable(- "SELECTED_LANGUAGE_MODEL", config.agent.llm_model- )- logger.info(f"Set SELECTED_LANGUAGE_MODEL global variable to {config.agent.llm_model}")- if config.agent.llm_provider:- mapped_llm_provider = map_provider(config.agent.llm_provider)- await _upsert_langflow_global_variable(- "SELECTED_LANGUAGE_MODEL_PROVIDER", mapped_llm_provider- )- logger.info(- f"Set SELECTED_LANGUAGE_MODEL_PROVIDER global variable to {mapped_llm_provider}"- )+ knowledge = getattr(config, "knowledge", None)+ if getattr(knowledge, "embedding_model", None):+ await _safe_upsert("SELECTED_EMBEDDING_MODEL", config.knowledge.embedding_model)- except Exception as e:- logger.error(f"Failed to update Langflow global variables: {str(e)}")- raise+ if getattr(knowledge, "embedding_provider", None):+ mapped_provider = map_provider(config.knowledge.embedding_provider)+ await _safe_upsert("SELECTED_EMBEDDING_MODEL_PROVIDER", mapped_provider)++ agent = getattr(config, "agent", None)+ if getattr(agent, "llm_model", None):+ await _safe_upsert("SELECTED_LANGUAGE_MODEL", config.agent.llm_model)++ if getattr(agent, "llm_provider", None):+ mapped_llm_provider = map_provider(config.agent.llm_provider)+ await _safe_upsert("SELECTED_LANGUAGE_MODEL_PROVIDER", mapped_llm_provider)++ if errors:+ raise RuntimeError(f"Failed to update Langflow global variable(s): {', '.join(errors)}")async def _run_async_post_save_langflow_updates(diff --git a/src/config/settings.py b/src/config/settings.pyindex 6b1d15e96..25911386c 100644--- a/src/config/settings.py+++ b/src/config/settings.py@@ -1550,28 +1550,26 @@ class AppClients:name: str,value: str,variable_type: str = "Credential",+ target_variable: dict | None = None,):"""Update an existing global variable in Langflow via API"""try:- # First, get all variables to find the one with the matching name- get_response = await self.langflow_request("GET", "/api/v1/variables/")-- if get_response.status_code != 200:- logger.error(- "Failed to retrieve variables for update",- variable_name=name,- status_code=get_response.status_code,- )- return+ if not target_variable:+ get_response = await self.langflow_request("GET", "/api/v1/variables/")- variables = get_response.json()- target_variable = None+ if get_response.status_code != 200:+ logger.error(+ "Failed to retrieve variables for update",+ variable_name=name,+ status_code=get_response.status_code,+ )+ return- # Find the variable with matching name- for variable in variables:- if variable.get("name") == name:- target_variable = variable- break+ variables = get_response.json()+ for variable in variables:+ if variable.get("name") == name:+ target_variable = variable+ breakif not target_variable:logger.error("Variable not found for update", variable_name=name)@@ -1602,7 +1600,7 @@ class AppClients:recreate_payload = {"name": name,"value": value,- "default_fields": target_variable.get("default_fields", []),+ "default_fields": [],"type": variable_type,}recreate_response = await self.langflow_request(@@ -1628,7 +1626,8 @@ class AppClients:returncurrent_value = target_variable.get("value")- if current_value == value:+ current_default_fields = target_variable.get("default_fields", [])+ if current_value == value and not current_default_fields:logger.debug("Langflow global variable already up to date, skipping update",variable_name=name,@@ -1641,7 +1640,7 @@ class AppClients:"id": variable_id,"name": name,"value": value,- "default_fields": target_variable.get("default_fields", []),+ "default_fields": [],"type": variable_type,}diff --git a/tests/unit/test_langflow_global_variables.py b/tests/unit/test_langflow_global_variables.pyindex c11a4c6cc..60ef90bb9 100644--- a/tests/unit/test_langflow_global_variables.py+++ b/tests/unit/test_langflow_global_variables.py@@ -88,7 +88,7 @@ async def test_update_langflow_global_variable_recreates_when_type_changes():"json": {"name": "OPENSEARCH_INDEX_NAME","value": "documents-v2",- "default_fields": ["OpenRAG", "Index"],+ "default_fields": [],"type": "Generic",}},@@ -133,7 +133,7 @@ async def test_update_langflow_global_variable_patches_when_type_matches():"id": "var-1","name": "OPENAI_API_KEY","value": "new-secret",- "default_fields": ["OpenAI", "OpenAI API Key"],+ "default_fields": [],"type": "Credential",}},@@ -141,6 +141,76 @@ async def test_update_langflow_global_variable_patches_when_type_matches():]+async def test_ensure_required_langflow_global_variables_removes_apply_to_fields(monkeypatch):+ langflow_calls = []++ async def mock_langflow_request(method, endpoint, **kwargs):+ langflow_calls.append((method, endpoint, kwargs))+ if method == "GET":+ return _Response(+ json_data=[+ {+ "id": "var-1",+ "name": "OPENAI_API_KEY",+ "value": "secret",+ "type": "Credential",+ "default_fields": ["OpenAI", "api_key"],+ },+ {+ "id": "var-2",+ "name": "DOCLING_SERVE_URL",+ "value": langflow_sync.settings.get_langflow_docling_url(),+ "type": "Generic",+ "default_fields": [],+ },+ ]+ )+ return _Response(status_code=200)++ monkeypatch.setattr(+ langflow_sync.clients,+ "langflow_request",+ mock_langflow_request,+ raising=True,+ )++ create_calls = []++ async def create_variable(name, value, modify=False, variable_type="Credential"):+ create_calls.append((name, value, modify, variable_type))++ monkeypatch.setattr(+ langflow_sync.clients,+ "_create_langflow_global_variable",+ create_variable,+ raising=True,+ )++ config = SimpleNamespace(+ providers=SimpleNamespace(),+ knowledge=SimpleNamespace(),+ )++ await langflow_sync.ensure_required_langflow_global_variables(config)++ # Verify PATCH call was made to remove default_fields from var-1+ patch_calls = [c for c in langflow_calls if c[0] == "PATCH"]+ assert len(patch_calls) == 1+ assert patch_calls[0] == (+ "PATCH",+ "/api/v1/variables/var-1",+ {+ "json": {+ "id": "var-1",+ "name": "OPENAI_API_KEY",+ "default_fields": [],+ "type": "Credential",+ }+ },+ )++@pytest.mark.asyncioasync def test_update_langflow_global_variables_marks_non_secret_provider_fields_generic(monkeypatch,@@ -198,6 +268,18 @@ async def test_update_langflow_global_variables_marks_non_secret_provider_fieldsasync def test_ensure_required_langflow_global_variables_creates_all_generics(monkeypatch):calls = []+ async def mock_langflow_request(method, endpoint, **kwargs):+ if method == "GET":+ return _Response(status_code=200, json_data=[])+ return _Response(status_code=200)++ monkeypatch.setattr(+ langflow_sync.clients,+ "langflow_request",+ mock_langflow_request,+ raising=True,+ )+async def create_variable(name, value, modify=False, variable_type="Credential"):calls.append((name, value, modify, variable_type))@@ -226,3 +308,150 @@ async def test_ensure_required_langflow_global_variables_creates_all_generics(moassert all(variable_type == "Generic" for *_, variable_type in calls)assert ("OPENSEARCH_INDEX_NAME", "documents-v2", True, "Generic") in callsassert ("SELECTED_EMBEDDING_MODEL", "text-embedding-3-large", True, "Generic") in calls+++async def test_update_langflow_global_variables_continues_when_one_fails(monkeypatch):+ calls = []+ attempted_names = []++ async def create_variable(name, value, modify=False, variable_type="Credential"):+ attempted_names.append(name)+ if name == "WATSONX_APIKEY":+ raise RuntimeError("Simulated network failure on WATSONX_APIKEY")+ calls.append((name, value, modify, variable_type))++ monkeypatch.setattr(+ langflow_sync.clients,+ "_create_langflow_global_variable",+ create_variable,+ raising=True,+ )++ config = SimpleNamespace(+ providers=SimpleNamespace(+ watsonx=SimpleNamespace(api_key="watson-key", project_id="watson-project"),+ openai=SimpleNamespace(api_key="openai-key"),+ ),+ knowledge=SimpleNamespace(embedding_model="embedding-model", embedding_provider=None),+ agent=SimpleNamespace(llm_model=None, llm_provider=None),+ )++ # Calling sync should raise RuntimeError at the end because WATSONX_APIKEY failed,+ # but it should have attempted all other variables first.+ with pytest.raises(RuntimeError) as exc_info:+ await langflow_sync._update_langflow_global_variables(config)++ assert "WATSONX_APIKEY" in str(exc_info.value)+ assert "WATSONX_APIKEY" in attempted_names+ names = {name for name, *_ in calls}+ assert "WATSONX_APIKEY" not in names+ assert "WATSONX_PROJECT_ID" in names+ assert "OPENAI_API_KEY" in names+ assert "SELECTED_EMBEDDING_MODEL" in names+++async def test_ensure_required_langflow_global_variables_get_500_sends_no_post_requests(+ monkeypatch,+):+ langflow_calls = []++ async def mock_langflow_request(method, endpoint, **kwargs):+ langflow_calls.append((method, endpoint, kwargs))+ if method == "GET":+ return _Response(status_code=500, text="Internal Server Error")+ return _Response(status_code=200)++ monkeypatch.setattr(+ langflow_sync.clients,+ "langflow_request",+ mock_langflow_request,+ raising=True,+ )++ create_calls = []++ async def create_variable(name, value, modify=False, variable_type="Credential"):+ create_calls.append((name, value, modify, variable_type))++ monkeypatch.setattr(+ langflow_sync.clients,+ "_create_langflow_global_variable",+ create_variable,+ raising=True,+ )++ config = SimpleNamespace(+ providers=SimpleNamespace(),+ knowledge=SimpleNamespace(),+ )++ await langflow_sync.ensure_required_langflow_global_variables(config)++ assert len(create_calls) == 0+ assert len(langflow_calls) == 1+ assert langflow_calls[0][0] == "GET"+++async def test_ensure_required_langflow_global_variables_handles_failed_delete_post_patch(+ monkeypatch,+):+ langflow_calls = []++ async def mock_langflow_request(method, endpoint, **kwargs):+ langflow_calls.append((method, endpoint, kwargs))+ if method == "GET":+ return _Response(+ json_data=[+ {+ "id": "var-del-fail",+ "name": "OPENSEARCH_INDEX_NAME",+ "value": "documents",+ "type": "Credential",+ },+ {+ "id": "var-post-fail",+ "name": "WATSONX_URL",+ "value": "https://watson.example",+ "type": "Credential",+ },+ {+ "id": "var-patch-fail",+ "name": "OPENAI_API_KEY",+ "value": "secret",+ "type": "Credential",+ "default_fields": ["OpenAI", "api_key"],+ },+ ]+ )+ if method == "DELETE" and "var-del-fail" in endpoint:+ return _Response(status_code=500)+ if method == "DELETE" and "var-post-fail" in endpoint:+ return _Response(status_code=204)+ if method == "POST" and endpoint == "/api/v1/variables/":+ return _Response(status_code=500)+ if method == "PATCH" and "var-patch-fail" in endpoint:+ return _Response(status_code=500)+ return _Response(status_code=200)++ monkeypatch.setattr(+ langflow_sync.clients,+ "langflow_request",+ mock_langflow_request,+ raising=True,+ )++ config = SimpleNamespace(+ providers=SimpleNamespace(),+ knowledge=SimpleNamespace(),+ )++ # Should complete without raising exception even though individual DELETE, POST, PATCH failed+ await langflow_sync.ensure_required_langflow_global_variables(config)++ methods = [c[0] for c in langflow_calls]+ assert "DELETE" in methods+ assert "POST" in methods+ assert "PATCH" in methods