OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.
Python
fix: stabilize default OpenRAG URL docs ingestion and metadata handling
Olfa Maslahd282828b8e38
diff --git a/.env.example b/.env.exampleindex 847e757b7..a5a34d88c 100644--- a/.env.example+++ b/.env.example@@ -13,7 +13,7 @@ INGEST_SAMPLE_DATA=trueDEFAULT_DOCS_INGEST_SOURCE=url# URL used when DEFAULT_DOCS_INGEST_SOURCE=url-DEFAULT_DOCS_URL=https://docs.openr.ag+DEFAULT_DOCS_URL=https://www.openr.ag/# Crawl depth used when DEFAULT_DOCS_INGEST_SOURCE=urlDEFAULT_DOCS_CRAWL_DEPTH=2diff --git a/frontend/app/knowledge/page.tsx b/frontend/app/knowledge/page.tsxindex ba197026d..b027bad17 100644--- a/frontend/app/knowledge/page.tsx+++ b/frontend/app/knowledge/page.tsx@@ -107,7 +107,7 @@ function SearchPage() {return file?.connector_type === "system_default";}, []);- const hasOpenragRefreshCue = tasks.some((task) => {+ const hasOpenragRefreshCueFromTasks = tasks.some((task) => {const isTaskActive =task.status === "pending" ||task.status === "running" ||@@ -119,10 +119,13 @@ function SearchPage() {return Object.entries(task.files).some(([fileKey, fileInfo]) => {const filename = (fileInfo as { filename?: string })?.filename ?? "";return (- filename === "OpenRAG docs refresh" || fileKey.includes("docs.openr.ag")+ filename === "OpenRAG docs refresh" ||+ fileKey.includes("openr.ag"));});});+ const hasOpenragRefreshCue =+ refreshOpenragDocsMutation.isPending || hasOpenragRefreshCueFromTasks;// Show toast notification for search errorsuseEffect(() => {@@ -180,7 +183,7 @@ function SearchPage() {// The table should only show indexed docs, not orchestration task labels.if (taskFile.filename === "OpenRAG docs refresh" ||- taskFile.source_url.includes("docs.openr.ag")+ taskFile.source_url.includes("openr.ag")) {return false;}@@ -198,6 +201,28 @@ function SearchPage() {});// Combine task files first, then backend filesconst fileResults = [...backendFiles, ...filteredTaskFiles];+ const hasOpenragDocsEntry = fileResults.some(+ (file) => file.connector_type === "system_default",+ );+ const isDefaultKnowledgeView =+ !queryOverride || queryOverride.trim().length === 0 || queryOverride === "*";+ const shouldShowDefaultOpenragPlaceholder =+ isDefaultKnowledgeView &&+ !isFetching &&+ !hasOpenragDocsEntry &&+ fileResults.length === 0;++ const placeholderOpenragDoc: File = {+ filename: "What is OpenRAG docs",+ mimetype: "text/markdown",+ source_url: "https://www.openr.ag/",+ size: 0,+ connector_type: "system_default",+ status: "unavailable",+ };+ const gridRows = shouldShowDefaultOpenragPlaceholder+ ? [placeholderOpenragDoc]+ : fileResults;const gridRef = useRef<AgGridReact>(null);const columnDefs: ColDef<File>[] = [@@ -567,7 +592,7 @@ function SearchPage() {loading={isFetching}ref={gridRef}theme={themeQuartz.withParams({ browserColorScheme: "inherit" })}- rowData={fileResults}+ rowData={gridRows}rowSelection="multiple"rowMultiSelectWithClick={false}suppressRowClickSelection={true}diff --git a/src/config/settings.py b/src/config/settings.pyindex 4db7433b2..0c13a8e9e 100644--- a/src/config/settings.py+++ b/src/config/settings.py@@ -65,7 +65,7 @@ INGEST_SAMPLE_DATA = os.getenv(# - "url": crawl DEFAULT_DOCS_URL with URL ingestion flow# - "files": ingest files from the openrag-documents directoryDEFAULT_DOCS_INGEST_SOURCE = os.getenv("DEFAULT_DOCS_INGEST_SOURCE", "url").lower()-DEFAULT_DOCS_URL = os.getenv("DEFAULT_DOCS_URL", "https://docs.openr.ag")+DEFAULT_DOCS_URL = os.getenv("DEFAULT_DOCS_URL", "https://www.openr.ag/")DEFAULT_DOCS_CRAWL_DEPTH = get_env_int("DEFAULT_DOCS_CRAWL_DEPTH", 2)FETCH_OPENRAG_DOCS_AT_STARTUP = os.getenv("FETCH_OPENRAG_DOCS_AT_STARTUP", "false"diff --git a/src/main.py b/src/main.pyindex d3f17924f..46b3348a8 100644--- a/src/main.py+++ b/src/main.py@@ -2,9 +2,12 @@ from utils.version_utils import OPENRAG_VERSIONimport asyncioimport atexitimport hashlib+import htmlimport httpximport os+import reimport subprocess+import tempfile# Configure structured logging earlyfrom connectors.langflow_connector_service import LangflowConnectorService@@ -361,12 +364,7 @@ def _get_documents_dir():def _should_use_url_default_docs_ingest() -> bool:"""Return whether default docs ingestion should use URL crawling."""- return (- DEFAULT_DOCS_INGEST_SOURCE == "url"- and not DISABLE_INGEST_WITH_LANGFLOW- and bool(LANGFLOW_URL_INGEST_FLOW_ID)- and bool(DEFAULT_DOCS_URL)- )+ return DEFAULT_DOCS_INGEST_SOURCE == "url" and bool(DEFAULT_DOCS_URL)async def ingest_default_documents_when_ready(@@ -383,18 +381,9 @@ async def ingest_default_documents_when_ready(Category.DOCUMENT_INGESTION, MessageId.ORB_DOC_DEFAULT_START)use_url_ingest = _should_use_url_default_docs_ingest()- if (- DEFAULT_DOCS_INGEST_SOURCE == "url"- and not DISABLE_INGEST_WITH_LANGFLOW- and not LANGFLOW_URL_INGEST_FLOW_ID- ):- logger.warning(- "URL docs ingestion flow id is missing; falling back to file-based default ingestion",- ingest_source=DEFAULT_DOCS_INGEST_SOURCE,- )if use_url_ingest:await _ingest_default_documents_url(- session_manager=session_manager,+ document_service=document_service,docs_url=DEFAULT_DOCS_URL,crawl_depth=DEFAULT_DOCS_CRAWL_DEPTH,)@@ -453,7 +442,6 @@ async def _ingest_default_documents_langflow(# Use AnonymousUser details for default documentsfrom session_manager import AnonymousUser-anonymous_user = AnonymousUser()# Get JWT token using same logic as DocumentFileProcessor@@ -507,70 +495,108 @@ async def _ingest_default_documents_langflow(async def _ingest_default_documents_url(- session_manager, docs_url: str, crawl_depth: int+ document_service,+ docs_url: str,+ crawl_depth: int,):- """Ingest default docs by crawling a URL with the Langflow URL ingest flow."""- if not LANGFLOW_URL_INGEST_FLOW_ID:- raise ValueError("LANGFLOW_URL_INGEST_FLOW_ID is not configured")+ """Ingest default docs from URL using OpenRAG ingestion logic (no Langflow)."""if not docs_url:raise ValueError("DEFAULT_DOCS_URL is not configured")- from session_manager import AnonymousUser- from utils.langflow_headers import add_provider_credentials_to_headers- from config.settings import get_openrag_config-- anonymous_user = AnonymousUser()- effective_jwt = None- if session_manager:- session_manager.get_user_opensearch_client(- anonymous_user.user_id, effective_jwt- )- if hasattr(session_manager, "_anonymous_jwt"):- effective_jwt = session_manager._anonymous_jwt-- payload = {- "input_value": docs_url,- "input_type": "chat",- "output_type": "text",- "tweaks": {- "URLComponent-lnA0q": {- "urls": [docs_url],- "max_depth": crawl_depth,- "prevent_outside": True,- }- },- }- config = get_openrag_config()- headers = {- "X-Langflow-Global-Var-JWT": str(effective_jwt) if effective_jwt else "",- "X-Langflow-Global-Var-OWNER": "",- "X-Langflow-Global-Var-OWNER_NAME": str(anonymous_user.name),- "X-Langflow-Global-Var-OWNER_EMAIL": str(anonymous_user.email),- "X-Langflow-Global-Var-CONNECTOR_TYPE": "system_default",- "X-Langflow-Global-Var-SELECTED_EMBEDDING_MODEL": str(- config.knowledge.embedding_model- ),- }- add_provider_credentials_to_headers(headers, config)-logger.info(- "Running default URL docs ingestion flow",+ "Running default URL docs ingestion with OpenRAG processor",docs_url=docs_url,crawl_depth=crawl_depth,)- resp = await clients.langflow_request(- "POST",- f"/api/v1/run/{LANGFLOW_URL_INGEST_FLOW_ID}",- json=payload,- headers=headers,+ temp_file_path = await _materialize_default_docs_url_as_text_file(+ docs_url=docs_url,+ crawl_depth=crawl_depth,)- if resp.status_code >= 400:- logger.error(- "Default URL docs ingestion failed",- status_code=resp.status_code,- body=resp.text[:1000],+ try:+ from models.processors import DocumentFileProcessor+ from utils.hash_utils import hash_id++ processor = DocumentFileProcessor(+ document_service,+ owner_user_id=None,+ jwt_token=None,+ owner_name=None,+ owner_email=None,+ is_sample_data=True,+ connector_type="system_default",)- resp.raise_for_status()+ await processor.process_document_standard(+ file_path=temp_file_path,+ file_hash=hash_id(temp_file_path),+ owner_user_id=None,+ original_filename="openrag-url-default.txt",+ jwt_token=None,+ owner_name=None,+ owner_email=None,+ file_size=os.path.getsize(temp_file_path),+ connector_type="system_default",+ is_sample_data=True,+ )+ finally:+ try:+ os.unlink(temp_file_path)+ except FileNotFoundError:+ pass+ except Exception as e:+ logger.warning(+ "Failed to clean temporary default URL docs file",+ path=temp_file_path,+ error=str(e),+ )+++async def _materialize_default_docs_url_as_text_file(+ docs_url: str,+ crawl_depth: int,+) -> str:+ """Fetch URL content and write a temporary text file for OpenRAG ingestion."""+ async with httpx.AsyncClient(timeout=30.0, follow_redirects=True) as client:+ response = await client.get(docs_url)+ response.raise_for_status()+ raw_html = response.text++ title_match = re.search(+ r"<title[^>]*>(.*?)</title>", raw_html, flags=re.IGNORECASE | re.DOTALL+ )+ title = html.unescape(title_match.group(1).strip()) if title_match else "OpenRAG"++ no_scripts = re.sub(+ r"<script[^>]*>.*?</script>",+ " ",+ raw_html,+ flags=re.IGNORECASE | re.DOTALL,+ )+ no_styles = re.sub(+ r"<style[^>]*>.*?</style>",+ " ",+ no_scripts,+ flags=re.IGNORECASE | re.DOTALL,+ )+ text_only = re.sub(r"<[^>]+>", " ", no_styles)+ normalized_text = re.sub(r"\s+", " ", html.unescape(text_only)).strip()++ content = (+ f"{title}\n\n"+ f"Source URL: {docs_url}\n"+ f"Crawl depth: {crawl_depth}\n\n"+ f"{normalized_text}\n"+ )++ temp_file = tempfile.NamedTemporaryFile(+ mode="w",+ suffix=".txt",+ prefix="openrag-url-default-",+ delete=False,+ encoding="utf-8",+ )+ with temp_file:+ temp_file.write(content)+ return temp_file.nameasync def _delete_existing_default_docs(session_manager):@@ -882,7 +908,9 @@ async def opensearch_health_ready(request):)-async def _ingest_default_documents_openrag(document_service, task_service, file_paths):+async def _ingest_default_documents_openrag(+ document_service, task_service, file_paths, connector_type: str = "local"+):"""Ingest default documents using traditional OpenRAG processor."""logger.info("Using traditional OpenRAG ingestion for default documents",@@ -899,6 +927,7 @@ async def _ingest_default_documents_openrag(document_service, task_service, fileowner_name=None,owner_email=None,is_sample_data=True, # Mark as sample data+ connector_type=connector_type,)task_id = await task_service.create_custom_task("anonymous", file_paths, processor)diff --git a/src/models/processors.py b/src/models/processors.pyindex f3b71ed07..2392ed712 100644--- a/src/models/processors.py+++ b/src/models/processors.py@@ -171,13 +171,25 @@ class TaskProcessor:acl: DocumentACL instance with access control information"""import datetime- from config.settings import clients, get_embedding_model, get_index_name+ from config.settings import (+ clients,+ get_embedding_model,+ get_index_name,+ get_openrag_config,+ )from services.document_service import chunk_texts_for_embeddingsfrom utils.document_processing import extract_relevantfrom utils.embedding_fields import get_embedding_field_name, ensure_embedding_field_exists- # Use provided embedding model or fall back to default- embedding_model = embedding_model or get_embedding_model()+ # Use provided embedding model or configured model.+ # get_embedding_model() returns empty string when Langflow ingest is enabled,+ # but OpenRAG processors still need a concrete embedding model.+ configured_embedding_model = get_openrag_config().knowledge.embedding_model+ embedding_model = (+ embedding_model+ or configured_embedding_model+ or get_embedding_model()+ )# Get user's OpenSearch client with JWT for OIDC authopensearch_client = self.document_service.session_manager.get_user_opensearch_client(@@ -323,6 +335,7 @@ class DocumentFileProcessor(TaskProcessor):owner_name: str = None,owner_email: str = None,is_sample_data: bool = False,+ connector_type: str = "local",):super().__init__(document_service)self.owner_user_id = owner_user_id@@ -330,6 +343,7 @@ class DocumentFileProcessor(TaskProcessor):self.owner_name = owner_nameself.owner_email = owner_emailself.is_sample_data = is_sample_data+ self.connector_type = connector_typeasync def process_item(self, upload_task: UploadTask, item: str, file_task: FileTask@@ -363,7 +377,7 @@ class DocumentFileProcessor(TaskProcessor):owner_name=self.owner_name,owner_email=self.owner_email,file_size=file_size,- connector_type="local",+ connector_type=self.connector_type,is_sample_data=self.is_sample_data,)