Skip to content

OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.

Python

fix: stabilize default OpenRAG URL docs ingestion and metadata handling

Olfa Maslahd282828b8e38
diff --git a/.env.example b/.env.example
index 847e757b7..a5a34d88c 100644
--- a/.env.example
+++ b/.env.example
@@ -13,7 +13,7 @@ INGEST_SAMPLE_DATA=true
DEFAULT_DOCS_INGEST_SOURCE=url
# URL used when DEFAULT_DOCS_INGEST_SOURCE=url
-DEFAULT_DOCS_URL=https://docs.openr.ag
+DEFAULT_DOCS_URL=https://www.openr.ag/
# Crawl depth used when DEFAULT_DOCS_INGEST_SOURCE=url
DEFAULT_DOCS_CRAWL_DEPTH=2
diff --git a/frontend/app/knowledge/page.tsx b/frontend/app/knowledge/page.tsx
index ba197026d..b027bad17 100644
--- a/frontend/app/knowledge/page.tsx
+++ b/frontend/app/knowledge/page.tsx
@@ -107,7 +107,7 @@ function SearchPage() {
return file?.connector_type === "system_default";
}, []);
- const hasOpenragRefreshCue = tasks.some((task) => {
+ const hasOpenragRefreshCueFromTasks = tasks.some((task) => {
const isTaskActive =
task.status === "pending" ||
task.status === "running" ||
@@ -119,10 +119,13 @@ function SearchPage() {
return Object.entries(task.files).some(([fileKey, fileInfo]) => {
const filename = (fileInfo as { filename?: string })?.filename ?? "";
return (
- filename === "OpenRAG docs refresh" || fileKey.includes("docs.openr.ag")
+ filename === "OpenRAG docs refresh" ||
+ fileKey.includes("openr.ag")
);
});
});
+ const hasOpenragRefreshCue =
+ refreshOpenragDocsMutation.isPending || hasOpenragRefreshCueFromTasks;
// Show toast notification for search errors
useEffect(() => {
@@ -180,7 +183,7 @@ function SearchPage() {
// The table should only show indexed docs, not orchestration task labels.
if (
taskFile.filename === "OpenRAG docs refresh" ||
- taskFile.source_url.includes("docs.openr.ag")
+ taskFile.source_url.includes("openr.ag")
) {
return false;
}
@@ -198,6 +201,28 @@ function SearchPage() {
});
// Combine task files first, then backend files
const fileResults = [...backendFiles, ...filteredTaskFiles];
+ const hasOpenragDocsEntry = fileResults.some(
+ (file) => file.connector_type === "system_default",
+ );
+ const isDefaultKnowledgeView =
+ !queryOverride || queryOverride.trim().length === 0 || queryOverride === "*";
+ const shouldShowDefaultOpenragPlaceholder =
+ isDefaultKnowledgeView &&
+ !isFetching &&
+ !hasOpenragDocsEntry &&
+ fileResults.length === 0;
+
+ const placeholderOpenragDoc: File = {
+ filename: "What is OpenRAG docs",
+ mimetype: "text/markdown",
+ source_url: "https://www.openr.ag/",
+ size: 0,
+ connector_type: "system_default",
+ status: "unavailable",
+ };
+ const gridRows = shouldShowDefaultOpenragPlaceholder
+ ? [placeholderOpenragDoc]
+ : fileResults;
const gridRef = useRef<AgGridReact>(null);
const columnDefs: ColDef<File>[] = [
@@ -567,7 +592,7 @@ function SearchPage() {
loading={isFetching}
ref={gridRef}
theme={themeQuartz.withParams({ browserColorScheme: "inherit" })}
- rowData={fileResults}
+ rowData={gridRows}
rowSelection="multiple"
rowMultiSelectWithClick={false}
suppressRowClickSelection={true}
diff --git a/src/config/settings.py b/src/config/settings.py
index 4db7433b2..0c13a8e9e 100644
--- a/src/config/settings.py
+++ b/src/config/settings.py
@@ -65,7 +65,7 @@ INGEST_SAMPLE_DATA = os.getenv(
# - "url": crawl DEFAULT_DOCS_URL with URL ingestion flow
# - "files": ingest files from the openrag-documents directory
DEFAULT_DOCS_INGEST_SOURCE = os.getenv("DEFAULT_DOCS_INGEST_SOURCE", "url").lower()
-DEFAULT_DOCS_URL = os.getenv("DEFAULT_DOCS_URL", "https://docs.openr.ag")
+DEFAULT_DOCS_URL = os.getenv("DEFAULT_DOCS_URL", "https://www.openr.ag/")
DEFAULT_DOCS_CRAWL_DEPTH = get_env_int("DEFAULT_DOCS_CRAWL_DEPTH", 2)
FETCH_OPENRAG_DOCS_AT_STARTUP = os.getenv(
"FETCH_OPENRAG_DOCS_AT_STARTUP", "false"
diff --git a/src/main.py b/src/main.py
index d3f17924f..46b3348a8 100644
--- a/src/main.py
+++ b/src/main.py
@@ -2,9 +2,12 @@ from utils.version_utils import OPENRAG_VERSION
import asyncio
import atexit
import hashlib
+import html
import httpx
import os
+import re
import subprocess
+import tempfile
# Configure structured logging early
from connectors.langflow_connector_service import LangflowConnectorService
@@ -361,12 +364,7 @@ def _get_documents_dir():
def _should_use_url_default_docs_ingest() -> bool:
"""Return whether default docs ingestion should use URL crawling."""
- return (
- DEFAULT_DOCS_INGEST_SOURCE == "url"
- and not DISABLE_INGEST_WITH_LANGFLOW
- and bool(LANGFLOW_URL_INGEST_FLOW_ID)
- and bool(DEFAULT_DOCS_URL)
- )
+ return DEFAULT_DOCS_INGEST_SOURCE == "url" and bool(DEFAULT_DOCS_URL)
async def ingest_default_documents_when_ready(
@@ -383,18 +381,9 @@ async def ingest_default_documents_when_ready(
Category.DOCUMENT_INGESTION, MessageId.ORB_DOC_DEFAULT_START
)
use_url_ingest = _should_use_url_default_docs_ingest()
- if (
- DEFAULT_DOCS_INGEST_SOURCE == "url"
- and not DISABLE_INGEST_WITH_LANGFLOW
- and not LANGFLOW_URL_INGEST_FLOW_ID
- ):
- logger.warning(
- "URL docs ingestion flow id is missing; falling back to file-based default ingestion",
- ingest_source=DEFAULT_DOCS_INGEST_SOURCE,
- )
if use_url_ingest:
await _ingest_default_documents_url(
- session_manager=session_manager,
+ document_service=document_service,
docs_url=DEFAULT_DOCS_URL,
crawl_depth=DEFAULT_DOCS_CRAWL_DEPTH,
)
@@ -453,7 +442,6 @@ async def _ingest_default_documents_langflow(
# Use AnonymousUser details for default documents
from session_manager import AnonymousUser
-
anonymous_user = AnonymousUser()
# Get JWT token using same logic as DocumentFileProcessor
@@ -507,70 +495,108 @@ async def _ingest_default_documents_langflow(
async def _ingest_default_documents_url(
- session_manager, docs_url: str, crawl_depth: int
+ document_service,
+ docs_url: str,
+ crawl_depth: int,
):
- """Ingest default docs by crawling a URL with the Langflow URL ingest flow."""
- if not LANGFLOW_URL_INGEST_FLOW_ID:
- raise ValueError("LANGFLOW_URL_INGEST_FLOW_ID is not configured")
+ """Ingest default docs from URL using OpenRAG ingestion logic (no Langflow)."""
if not docs_url:
raise ValueError("DEFAULT_DOCS_URL is not configured")
- from session_manager import AnonymousUser
- from utils.langflow_headers import add_provider_credentials_to_headers
- from config.settings import get_openrag_config
-
- anonymous_user = AnonymousUser()
- effective_jwt = None
- if session_manager:
- session_manager.get_user_opensearch_client(
- anonymous_user.user_id, effective_jwt
- )
- if hasattr(session_manager, "_anonymous_jwt"):
- effective_jwt = session_manager._anonymous_jwt
-
- payload = {
- "input_value": docs_url,
- "input_type": "chat",
- "output_type": "text",
- "tweaks": {
- "URLComponent-lnA0q": {
- "urls": [docs_url],
- "max_depth": crawl_depth,
- "prevent_outside": True,
- }
- },
- }
- config = get_openrag_config()
- headers = {
- "X-Langflow-Global-Var-JWT": str(effective_jwt) if effective_jwt else "",
- "X-Langflow-Global-Var-OWNER": "",
- "X-Langflow-Global-Var-OWNER_NAME": str(anonymous_user.name),
- "X-Langflow-Global-Var-OWNER_EMAIL": str(anonymous_user.email),
- "X-Langflow-Global-Var-CONNECTOR_TYPE": "system_default",
- "X-Langflow-Global-Var-SELECTED_EMBEDDING_MODEL": str(
- config.knowledge.embedding_model
- ),
- }
- add_provider_credentials_to_headers(headers, config)
-
logger.info(
- "Running default URL docs ingestion flow",
+ "Running default URL docs ingestion with OpenRAG processor",
docs_url=docs_url,
crawl_depth=crawl_depth,
)
- resp = await clients.langflow_request(
- "POST",
- f"/api/v1/run/{LANGFLOW_URL_INGEST_FLOW_ID}",
- json=payload,
- headers=headers,
+ temp_file_path = await _materialize_default_docs_url_as_text_file(
+ docs_url=docs_url,
+ crawl_depth=crawl_depth,
)
- if resp.status_code >= 400:
- logger.error(
- "Default URL docs ingestion failed",
- status_code=resp.status_code,
- body=resp.text[:1000],
+ try:
+ from models.processors import DocumentFileProcessor
+ from utils.hash_utils import hash_id
+
+ processor = DocumentFileProcessor(
+ document_service,
+ owner_user_id=None,
+ jwt_token=None,
+ owner_name=None,
+ owner_email=None,
+ is_sample_data=True,
+ connector_type="system_default",
)
- resp.raise_for_status()
+ await processor.process_document_standard(
+ file_path=temp_file_path,
+ file_hash=hash_id(temp_file_path),
+ owner_user_id=None,
+ original_filename="openrag-url-default.txt",
+ jwt_token=None,
+ owner_name=None,
+ owner_email=None,
+ file_size=os.path.getsize(temp_file_path),
+ connector_type="system_default",
+ is_sample_data=True,
+ )
+ finally:
+ try:
+ os.unlink(temp_file_path)
+ except FileNotFoundError:
+ pass
+ except Exception as e:
+ logger.warning(
+ "Failed to clean temporary default URL docs file",
+ path=temp_file_path,
+ error=str(e),
+ )
+
+
+async def _materialize_default_docs_url_as_text_file(
+ docs_url: str,
+ crawl_depth: int,
+) -> str:
+ """Fetch URL content and write a temporary text file for OpenRAG ingestion."""
+ async with httpx.AsyncClient(timeout=30.0, follow_redirects=True) as client:
+ response = await client.get(docs_url)
+ response.raise_for_status()
+ raw_html = response.text
+
+ title_match = re.search(
+ r"<title[^>]*>(.*?)</title>", raw_html, flags=re.IGNORECASE | re.DOTALL
+ )
+ title = html.unescape(title_match.group(1).strip()) if title_match else "OpenRAG"
+
+ no_scripts = re.sub(
+ r"<script[^>]*>.*?</script>",
+ " ",
+ raw_html,
+ flags=re.IGNORECASE | re.DOTALL,
+ )
+ no_styles = re.sub(
+ r"<style[^>]*>.*?</style>",
+ " ",
+ no_scripts,
+ flags=re.IGNORECASE | re.DOTALL,
+ )
+ text_only = re.sub(r"<[^>]+>", " ", no_styles)
+ normalized_text = re.sub(r"\s+", " ", html.unescape(text_only)).strip()
+
+ content = (
+ f"{title}\n\n"
+ f"Source URL: {docs_url}\n"
+ f"Crawl depth: {crawl_depth}\n\n"
+ f"{normalized_text}\n"
+ )
+
+ temp_file = tempfile.NamedTemporaryFile(
+ mode="w",
+ suffix=".txt",
+ prefix="openrag-url-default-",
+ delete=False,
+ encoding="utf-8",
+ )
+ with temp_file:
+ temp_file.write(content)
+ return temp_file.name
async def _delete_existing_default_docs(session_manager):
@@ -882,7 +908,9 @@ async def opensearch_health_ready(request):
)
-async def _ingest_default_documents_openrag(document_service, task_service, file_paths):
+async def _ingest_default_documents_openrag(
+ document_service, task_service, file_paths, connector_type: str = "local"
+):
"""Ingest default documents using traditional OpenRAG processor."""
logger.info(
"Using traditional OpenRAG ingestion for default documents",
@@ -899,6 +927,7 @@ async def _ingest_default_documents_openrag(document_service, task_service, file
owner_name=None,
owner_email=None,
is_sample_data=True, # Mark as sample data
+ connector_type=connector_type,
)
task_id = await task_service.create_custom_task("anonymous", file_paths, processor)
diff --git a/src/models/processors.py b/src/models/processors.py
index f3b71ed07..2392ed712 100644
--- a/src/models/processors.py
+++ b/src/models/processors.py
@@ -171,13 +171,25 @@ class TaskProcessor:
acl: DocumentACL instance with access control information
"""
import datetime
- from config.settings import clients, get_embedding_model, get_index_name
+ from config.settings import (
+ clients,
+ get_embedding_model,
+ get_index_name,
+ get_openrag_config,
+ )
from services.document_service import chunk_texts_for_embeddings
from utils.document_processing import extract_relevant
from utils.embedding_fields import get_embedding_field_name, ensure_embedding_field_exists
- # Use provided embedding model or fall back to default
- embedding_model = embedding_model or get_embedding_model()
+ # Use provided embedding model or configured model.
+ # get_embedding_model() returns empty string when Langflow ingest is enabled,
+ # but OpenRAG processors still need a concrete embedding model.
+ configured_embedding_model = get_openrag_config().knowledge.embedding_model
+ embedding_model = (
+ embedding_model
+ or configured_embedding_model
+ or get_embedding_model()
+ )
# Get user's OpenSearch client with JWT for OIDC auth
opensearch_client = self.document_service.session_manager.get_user_opensearch_client(
@@ -323,6 +335,7 @@ class DocumentFileProcessor(TaskProcessor):
owner_name: str = None,
owner_email: str = None,
is_sample_data: bool = False,
+ connector_type: str = "local",
):
super().__init__(document_service)
self.owner_user_id = owner_user_id
@@ -330,6 +343,7 @@ class DocumentFileProcessor(TaskProcessor):
self.owner_name = owner_name
self.owner_email = owner_email
self.is_sample_data = is_sample_data
+ self.connector_type = connector_type
async def process_item(
self, upload_task: UploadTask, item: str, file_task: FileTask
@@ -363,7 +377,7 @@ class DocumentFileProcessor(TaskProcessor):
owner_name=self.owner_name,
owner_email=self.owner_email,
file_size=file_size,
- connector_type="local",
+ connector_type=self.connector_type,
is_sample_data=self.is_sample_data,
)