OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.
Python
fix onboarding docs ingestion to use URL source safely
Olfa Maslah236409a313e6
diff --git a/.env.example b/.env.exampleindex f3bdc0f1f..dc3278889 100644--- a/.env.example+++ b/.env.example@@ -3,10 +3,21 @@# If unset or false, Langflow pipeline will be used (default: upload -> ingest -> delete)DISABLE_INGEST_WITH_LANGFLOW=false-# Set to false to skip ingesting sample data (pdfs) on startup+# Set to false to skip ingesting OpenRAG sample docs during onboarding# Default: trueINGEST_SAMPLE_DATA=true+# Default OpenRAG docs source for sample ingestion during onboarding.+# - url: crawl DEFAULT_DOCS_URL using URL ingestion flow+# - files: ingest files from OPENRAG_DOCUMENTS_PATH+DEFAULT_DOCS_INGEST_SOURCE=url++# URL used when DEFAULT_DOCS_INGEST_SOURCE=url+DEFAULT_DOCS_URL=https://docs.openr.ag++# Crawl depth used when DEFAULT_DOCS_INGEST_SOURCE=url+DEFAULT_DOCS_CRAWL_DEPTH=2+# Langflow HTTP timeout configuration (in seconds)# For large documents (300+ pages), ingestion can take 30+ minutes# Increase these values if you experience timeouts with very large PDFsdiff --git a/frontend/app/chat/_types/types.ts b/frontend/app/chat/_types/types.tsindex c605da7c3..c00ce97d6 100644--- a/frontend/app/chat/_types/types.ts+++ b/frontend/app/chat/_types/types.ts@@ -45,6 +45,7 @@ export interface SelectedFilters {data_sources: string[];document_types: string[];owners: string[];+ connector_types: string[];}export interface KnowledgeFilterData {diff --git a/frontend/app/chat/page.tsx b/frontend/app/chat/page.tsxindex 44d8c70ac..71aeee4d8 100644--- a/frontend/app/chat/page.tsx+++ b/frontend/app/chat/page.tsx@@ -667,6 +667,7 @@ function ChatPage() {data_sources: [],document_types: [],owners: [],+ connector_types: [],};processed.data_sources = filters.data_sources.includes("*")? []@@ -675,11 +676,16 @@ function ChatPage() {? []: filters.document_types;processed.owners = filters.owners.includes("*") ? [] : filters.owners;+ processed.connector_types =+ filters.connector_types?.includes("*")+ ? []+ : (filters.connector_types ?? []);const hasFilters =processed.data_sources.length > 0 ||processed.document_types.length > 0 ||- processed.owners.length > 0;+ processed.owners.length > 0 ||+ processed.connector_types.length > 0;return hasFilters ? processed : undefined;})(): undefined;@@ -708,6 +714,7 @@ function ChatPage() {data_sources: [],document_types: [],owners: [],+ connector_types: [],};processed.data_sources = filters.data_sources.includes("*")? []@@ -716,11 +723,16 @@ function ChatPage() {? []: filters.document_types;processed.owners = filters.owners.includes("*") ? [] : filters.owners;+ processed.connector_types =+ filters.connector_types?.includes("*")+ ? []+ : (filters.connector_types ?? []);const hasFilters =processed.data_sources.length > 0 ||processed.document_types.length > 0 ||- processed.owners.length > 0;+ processed.owners.length > 0 ||+ processed.connector_types.length > 0;return hasFilters ? processed : undefined;})(): undefined;@@ -791,6 +803,7 @@ function ChatPage() {data_sources: [],document_types: [],owners: [],+ connector_types: [],};// Only copy non-wildcard arraysprocessed.data_sources = filters.data_sources.includes("*")@@ -802,12 +815,17 @@ function ChatPage() {processed.owners = filters.owners.includes("*")? []: filters.owners;+ processed.connector_types =+ filters.connector_types?.includes("*")+ ? []+ : (filters.connector_types ?? []);// Only include filters if any array has valuesconst hasFilters =processed.data_sources.length > 0 ||processed.document_types.length > 0 ||- processed.owners.length > 0;+ processed.owners.length > 0 ||+ processed.connector_types.length > 0;return hasFilters ? { filters: processed } : {};})()),limit: parsedFilterData?.limit ?? 10,diff --git a/frontend/app/onboarding/_components/onboarding-content.tsx b/frontend/app/onboarding/_components/onboarding-content.tsxindex e98dc2294..de7900713 100644--- a/frontend/app/onboarding/_components/onboarding-content.tsx+++ b/frontend/app/onboarding/_components/onboarding-content.tsx@@ -18,9 +18,10 @@ import OnboardingUpload from "./onboarding-upload";// Filters for OpenRAG documentationconst OPENRAG_DOCS_FILTERS: SelectedFilters = {- data_sources: ["openrag-documentation.pdf"],- document_types: [],- owners: [],+ data_sources: ["*"],+ document_types: ["*"],+ owners: ["*"],+ connector_types: ["system_default"],};export function OnboardingContent({diff --git a/src/api/settings.py b/src/api/settings.pyindex 562dac8e8..4058805d7 100644--- a/src/api/settings.py+++ b/src/api/settings.py@@ -1135,10 +1135,12 @@ async def _create_openrag_docs_filter(query_data = json.dumps({"query": "","filters": {- "data_sources": ["openrag-documentation.pdf"],+ # URL-based docs ingestion produces many source URLs.+ # Filter by connector type to target the OpenRAG system docs set.+ "data_sources": ["*"],"document_types": ["*"],"owners": ["*"],- "connector_types": ["*"],+ "connector_types": ["system_default"],},"limit": 10,"scoreThreshold": 0,diff --git a/src/config/settings.py b/src/config/settings.pyindex b4d1e9e28..c5578218a 100644--- a/src/config/settings.py+++ b/src/config/settings.py@@ -61,6 +61,13 @@ INGEST_SAMPLE_DATA = os.getenv("INGEST_SAMPLE_DATA", "true").lower() in ("true", "1", "yes")+# Default OpenRAG docs sample ingestion source+# - "url": crawl DEFAULT_DOCS_URL with URL ingestion flow+# - "files": ingest files from the openrag-documents directory+DEFAULT_DOCS_INGEST_SOURCE = os.getenv("DEFAULT_DOCS_INGEST_SOURCE", "url").lower()+DEFAULT_DOCS_URL = os.getenv("DEFAULT_DOCS_URL", "https://docs.openr.ag")+DEFAULT_DOCS_CRAWL_DEPTH = get_env_int("DEFAULT_DOCS_CRAWL_DEPTH", 2)+# Maximum number of files to upload / ingest (in batch) per task when adding knowledge via folderUPLOAD_BATCH_SIZE = get_env_int("UPLOAD_BATCH_SIZE", 25)diff --git a/src/main.py b/src/main.pyindex 598263f78..fe752f659 100644--- a/src/main.py+++ b/src/main.py@@ -44,11 +44,15 @@ from api.v1 import chat as v1_chat, search as v1_search, documents as v1_documen# Configuration and setupfrom config.settings import (+ DEFAULT_DOCS_CRAWL_DEPTH,+ DEFAULT_DOCS_INGEST_SOURCE,+ DEFAULT_DOCS_URL,API_KEYS_INDEX_BODY,API_KEYS_INDEX_NAME,DISABLE_INGEST_WITH_LANGFLOW,INGESTION_TIMEOUT,INDEX_BODY,+ LANGFLOW_URL_INGEST_FLOW_ID,SESSION_SECRET,clients,get_embedding_model,@@ -323,13 +327,37 @@ def _get_documents_dir():async def ingest_default_documents_when_ready(document_service, task_service, langflow_file_service, session_manager):- """Scan the local documents folder and ingest files like a non-auth upload."""+ """Ingest default OpenRAG docs during onboarding."""try:logger.info("Ingesting default documents when ready",disable_langflow_ingest=DISABLE_INGEST_WITH_LANGFLOW,+ ingest_source=DEFAULT_DOCS_INGEST_SOURCE,)await TelemetryClient.send_event(Category.DOCUMENT_INGESTION, MessageId.ORB_DOC_DEFAULT_START)+ use_url_ingest = (+ DEFAULT_DOCS_INGEST_SOURCE == "url"+ and not DISABLE_INGEST_WITH_LANGFLOW+ and bool(LANGFLOW_URL_INGEST_FLOW_ID)+ )+ if (+ DEFAULT_DOCS_INGEST_SOURCE == "url"+ and not DISABLE_INGEST_WITH_LANGFLOW+ and not LANGFLOW_URL_INGEST_FLOW_ID+ ):+ logger.warning(+ "URL docs ingestion flow id is missing; falling back to file-based default ingestion",+ ingest_source=DEFAULT_DOCS_INGEST_SOURCE,+ )+ if use_url_ingest:+ await _ingest_default_documents_url(+ session_manager=session_manager,+ docs_url=DEFAULT_DOCS_URL,+ crawl_depth=DEFAULT_DOCS_CRAWL_DEPTH,+ )+ await TelemetryClient.send_event(Category.DOCUMENT_INGESTION, MessageId.ORB_DOC_DEFAULT_COMPLETE)+ return+base_dir = _get_documents_dir()if not os.path.isdir(base_dir):raise FileNotFoundError(f"Default documents directory not found: {base_dir}")@@ -426,6 +454,67 @@ async def _ingest_default_documents_langflow(file_count=len(file_paths),)++async def _ingest_default_documents_url(session_manager, docs_url: str, crawl_depth: int):+ """Ingest default docs by crawling a URL with the Langflow URL ingest flow."""+ if not LANGFLOW_URL_INGEST_FLOW_ID:+ raise ValueError("LANGFLOW_URL_INGEST_FLOW_ID is not configured")+ if not docs_url:+ raise ValueError("DEFAULT_DOCS_URL is not configured")++ from session_manager import AnonymousUser+ from utils.langflow_headers import add_provider_credentials_to_headers+ from config.settings import get_openrag_config++ anonymous_user = AnonymousUser()+ effective_jwt = None+ if session_manager:+ session_manager.get_user_opensearch_client(anonymous_user.user_id, effective_jwt)+ if hasattr(session_manager, "_anonymous_jwt"):+ effective_jwt = session_manager._anonymous_jwt++ payload = {+ "input_value": docs_url,+ "input_type": "chat",+ "output_type": "text",+ "tweaks": {+ "URLComponent-lnA0q": {+ "urls": [docs_url],+ "max_depth": crawl_depth,+ "prevent_outside": True,+ }+ },+ }+ config = get_openrag_config()+ headers = {+ "X-Langflow-Global-Var-JWT": str(effective_jwt) if effective_jwt else "",+ "X-Langflow-Global-Var-OWNER": "",+ "X-Langflow-Global-Var-OWNER_NAME": str(anonymous_user.name),+ "X-Langflow-Global-Var-OWNER_EMAIL": str(anonymous_user.email),+ "X-Langflow-Global-Var-CONNECTOR_TYPE": "system_default",+ "X-Langflow-Global-Var-SELECTED_EMBEDDING_MODEL": str(config.knowledge.embedding_model),+ }+ add_provider_credentials_to_headers(headers, config)++ logger.info(+ "Running default URL docs ingestion flow",+ docs_url=docs_url,+ crawl_depth=crawl_depth,+ )+ resp = await clients.langflow_request(+ "POST",+ f"/api/v1/run/{LANGFLOW_URL_INGEST_FLOW_ID}",+ json=payload,+ headers=headers,+ )+ if resp.status_code >= 400:+ logger.error(+ "Default URL docs ingestion failed",+ status_code=resp.status_code,+ body=resp.text[:1000],+ )+ resp.raise_for_status()+async def health_check(request: Request):"""Simple liveness probe: Indicates that the OpenRAG Backend service is online and running."""return JSONResponse({"status": "ok"}, status_code=200)