OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.
Python
Service-ize default URL docs ingestion flow.
Olfa Maslah3455d93b011d
diff --git a/src/main.py b/src/main.pyindex b014e977b..890834fdf 100644--- a/src/main.py+++ b/src/main.py@@ -3,7 +3,6 @@ import asyncioimport atexitimport hashlibimport httpx-import jsonimport osimport subprocess@@ -395,6 +394,7 @@ async def ingest_default_documents_when_ready()if use_url_ingest:await _ingest_default_documents_url(+ langflow_file_service=langflow_file_service,session_manager=session_manager,docs_url=DEFAULT_DOCS_URL,crawl_depth=DEFAULT_DOCS_CRAWL_DEPTH,@@ -508,17 +508,13 @@ async def _ingest_default_documents_langflow(async def _ingest_default_documents_url(- session_manager, docs_url: str, crawl_depth: int+ langflow_file_service, session_manager, docs_url: str, crawl_depth: int):"""Ingest default docs by crawling a URL with the Langflow URL ingest flow."""- if not LANGFLOW_URL_INGEST_FLOW_ID:- raise ValueError("LANGFLOW_URL_INGEST_FLOW_ID is not configured")- if not docs_url:- raise ValueError("DEFAULT_DOCS_URL is not configured")+ if langflow_file_service is None:+ raise ValueError("LangflowFileService is not available")from session_manager import AnonymousUser- from utils.langflow_headers import add_provider_credentials_to_headers- from config.settings import get_openrag_configanonymous_user = AnonymousUser()effective_jwt = None@@ -529,48 +525,21 @@ async def _ingest_default_documents_url(if hasattr(session_manager, "_anonymous_jwt"):effective_jwt = session_manager._anonymous_jwt- payload = {- "input_value": docs_url,- "input_type": "chat",- "output_type": "text",- }- config = get_openrag_config()- headers = {- "X-Langflow-Global-Var-JWT": str(effective_jwt) if effective_jwt else "",- "X-Langflow-Global-Var-OWNER": "",- "X-Langflow-Global-Var-OWNER_NAME": str(anonymous_user.name),- "X-Langflow-Global-Var-OWNER_EMAIL": str(anonymous_user.email),- "X-Langflow-Global-Var-CONNECTOR_TYPE": "system_default",- "X-Langflow-Global-Var-SELECTED_EMBEDDING_MODEL": str(- config.knowledge.embedding_model- ),- # Prefer global variable passthrough over component tweaks so the- # flow stays stable even if component IDs change.- "X-Langflow-Global-Var-DOCS_URL": str(docs_url),- "X-Langflow-Global-Var-DOCS_URLS": json.dumps([docs_url]),- "X-Langflow-Global-Var-DOCS_CRAWL_DEPTH": str(crawl_depth),- "X-Langflow-Global-Var-DOCS_PREVENT_OUTSIDE": "true",- }- add_provider_credentials_to_headers(headers, config)-logger.info("Running default URL docs ingestion flow",docs_url=docs_url,crawl_depth=crawl_depth,)- resp = await clients.langflow_request(- "POST",- f"/api/v1/run/{LANGFLOW_URL_INGEST_FLOW_ID}",- json=payload,- headers=headers,+ await langflow_file_service.run_url_ingestion_flow(+ docs_url=docs_url,+ crawl_depth=crawl_depth,+ jwt_token=effective_jwt,+ owner=None,+ owner_name=anonymous_user.name,+ owner_email=anonymous_user.email,+ connector_type="system_default",+ prevent_outside=True,)- if resp.status_code >= 400:- logger.error(- "Default URL docs ingestion failed",- status_code=resp.status_code,- body=resp.text[:1000],- )- resp.raise_for_status()async def _delete_existing_default_docs(session_manager):diff --git a/src/services/langflow_file_service.py b/src/services/langflow_file_service.pyindex ac02be607..9361a75ce 100644--- a/src/services/langflow_file_service.py+++ b/src/services/langflow_file_service.py@@ -1,7 +1,7 @@from typing import Any, Dict, List, Optionalimport json-from config.settings import LANGFLOW_INGEST_FLOW_ID, clients+from config.settings import LANGFLOW_INGEST_FLOW_ID, LANGFLOW_URL_INGEST_FLOW_ID, clientsfrom utils.logging_config import get_loggerlogger = get_logger(__name__)@@ -10,6 +10,7 @@ logger = get_logger(__name__)class LangflowFileService:def __init__(self):self.flow_id_ingest = LANGFLOW_INGEST_FLOW_ID+ self.flow_id_url_ingest = LANGFLOW_URL_INGEST_FLOW_IDasync def upload_user_file(self, file_tuple, jwt_token: Optional[str] = None@@ -226,6 +227,86 @@ class LangflowFileService:raisereturn resp_json+ async def run_url_ingestion_flow(+ self,+ docs_url: str,+ crawl_depth: int,+ jwt_token: Optional[str] = None,+ owner: Optional[str] = None,+ owner_name: Optional[str] = None,+ owner_email: Optional[str] = None,+ connector_type: str = "system_default",+ prevent_outside: bool = True,+ ) -> Dict[str, Any]:+ """Run URL-based docs ingestion flow using Langflow global variable passthrough."""+ if not self.flow_id_url_ingest:+ logger.error("[LF] LANGFLOW_URL_INGEST_FLOW_ID is not configured")+ raise ValueError("LANGFLOW_URL_INGEST_FLOW_ID is not configured")+ if not docs_url:+ raise ValueError("DEFAULT_DOCS_URL is not configured")++ payload: Dict[str, Any] = {+ "input_value": docs_url,+ "input_type": "chat",+ "output_type": "text",+ }++ from config.settings import get_openrag_config+ from utils.langflow_headers import add_provider_credentials_to_headers++ config = get_openrag_config()+ embedding_model = config.knowledge.embedding_model+ headers = {+ "X-Langflow-Global-Var-JWT": str(jwt_token) if jwt_token else "",+ "X-Langflow-Global-Var-OWNER": str(owner) if owner else "",+ "X-Langflow-Global-Var-OWNER_NAME": str(owner_name) if owner_name else "",+ "X-Langflow-Global-Var-OWNER_EMAIL": str(owner_email) if owner_email else "",+ "X-Langflow-Global-Var-CONNECTOR_TYPE": str(connector_type),+ "X-Langflow-Global-Var-SELECTED_EMBEDDING_MODEL": str(embedding_model),+ "X-Langflow-Global-Var-DOCS_URL": str(docs_url),+ "X-Langflow-Global-Var-DOCS_URLS": json.dumps([docs_url]),+ "X-Langflow-Global-Var-DOCS_CRAWL_DEPTH": str(crawl_depth),+ "X-Langflow-Global-Var-DOCS_PREVENT_OUTSIDE": "true"+ if prevent_outside+ else "false",+ }+ add_provider_credentials_to_headers(headers, config)++ logger.info(+ "[LF] Running URL ingestion flow",+ docs_url=docs_url,+ crawl_depth=crawl_depth,+ )+ resp = await clients.langflow_request(+ "POST",+ f"/api/v1/run/{self.flow_id_url_ingest}",+ json=payload,+ headers=headers,+ )+ if resp.status_code >= 400:+ logger.error(+ "[LF] URL ingestion flow failed",+ status_code=resp.status_code,+ reason=resp.reason_phrase,+ body=resp.text[:1000],+ )+ resp.raise_for_status()++ content_type = resp.headers.get("content-type", "")+ if "application/json" not in content_type:+ logger.error(+ "[LF] Unexpected URL ingestion response content type",+ content_type=content_type,+ status_code=resp.status_code,+ body=resp.text[:1000],+ )+ raise ValueError(+ f"Langflow returned {content_type} instead of JSON for URL ingestion. "+ f"Response preview: {resp.text[:500]}"+ )++ return resp.json()+async def upload_and_ingest_file(self,file_tuple,