Skip to content

OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.

Python

Support URL ingest for default OpenRAG docs

Edwin Josed6ef157fb8a1
diff --git a/.env.example b/.env.example
index a5a34d88c..dce12c465 100644
--- a/.env.example
+++ b/.env.example
@@ -15,8 +15,6 @@ DEFAULT_DOCS_INGEST_SOURCE=url
# URL used when DEFAULT_DOCS_INGEST_SOURCE=url
DEFAULT_DOCS_URL=https://www.openr.ag/
-# Crawl depth used when DEFAULT_DOCS_INGEST_SOURCE=url
-DEFAULT_DOCS_CRAWL_DEPTH=2
# Check for newer OpenRAG docs and update knowledge on every startup.
# Default: false (manual fetch remains available in UI)
diff --git a/src/config/settings.py b/src/config/settings.py
index 0c13a8e9e..b1df97074 100644
--- a/src/config/settings.py
+++ b/src/config/settings.py
@@ -64,9 +64,13 @@ INGEST_SAMPLE_DATA = os.getenv(
# Default OpenRAG docs sample ingestion source
# - "url": crawl DEFAULT_DOCS_URL with URL ingestion flow
# - "files": ingest files from the openrag-documents directory
+
DEFAULT_DOCS_INGEST_SOURCE = os.getenv("DEFAULT_DOCS_INGEST_SOURCE", "url").lower()
-DEFAULT_DOCS_URL = os.getenv("DEFAULT_DOCS_URL", "https://www.openr.ag/")
+DEFAULT_DOCS_URL = os.getenv("DEFAULT_DOCS_URL", "https://docs.openr.ag/")
+#TODO: Enable this when the flow is updated to use the new variables
+
DEFAULT_DOCS_CRAWL_DEPTH = get_env_int("DEFAULT_DOCS_CRAWL_DEPTH", 2)
+
FETCH_OPENRAG_DOCS_AT_STARTUP = os.getenv(
"FETCH_OPENRAG_DOCS_AT_STARTUP", "false"
).lower() in ("true", "1", "yes")
diff --git a/src/main.py b/src/main.py
index 0630ec5fd..994d464db 100644
--- a/src/main.py
+++ b/src/main.py
@@ -97,6 +97,7 @@ logger = get_logger(__name__)
# Files to exclude from startup ingestion
EXCLUDED_INGESTION_FILES = {"warmup_ocr.pdf"}
+URL_INGEST_EXCLUDED_INGESTION_FILES = {"openrag-documentation.pdf"}
class _VisibleTextHTMLParser(HTMLParser):
@@ -392,6 +393,40 @@ def _should_use_url_default_docs_ingest() -> bool:
return DEFAULT_DOCS_INGEST_SOURCE == "url" and bool(DEFAULT_DOCS_URL)
+async def ingest_openrag_docs_when_ready(
+ document_service, task_service, langflow_file_service, session_manager
+):
+ """Ingest OpenRAG docs during onboarding."""
+ use_url_ingest = _should_use_url_default_docs_ingest()
+ if use_url_ingest:
+ try:
+ await TelemetryClient.send_event(
+ Category.DOCUMENT_INGESTION, MessageId.ORB_DOC_DEFAULT_URL_START
+ )
+ if DISABLE_INGEST_WITH_LANGFLOW:
+ await _ingest_default_documents_url(
+ document_service=document_service,
+ docs_url=DEFAULT_DOCS_URL,
+ crawl_depth=DEFAULT_DOCS_CRAWL_DEPTH,
+ )
+ else:
+ await _ingest_default_documents_url_langflow(
+ langflow_file_service=langflow_file_service,
+ session_manager=session_manager,
+ task_service=task_service,
+ docs_url=DEFAULT_DOCS_URL,
+ crawl_depth=DEFAULT_DOCS_CRAWL_DEPTH,
+ )
+ await TelemetryClient.send_event(
+ Category.DOCUMENT_INGESTION, MessageId.ORB_DOC_DEFAULT_URL_COMPLETE
+ )
+ except Exception as e:
+ logger.error("Default URL documents ingestion failed", error=str(e))
+ await TelemetryClient.send_event(
+ Category.DOCUMENT_INGESTION, MessageId.ORB_DOC_DEFAULT_URL_FAILED
+ )
+
+
async def ingest_default_documents_when_ready(
document_service, task_service, langflow_file_service, session_manager
):
@@ -405,17 +440,9 @@ async def ingest_default_documents_when_ready(
await TelemetryClient.send_event(
Category.DOCUMENT_INGESTION, MessageId.ORB_DOC_DEFAULT_START
)
- use_url_ingest = _should_use_url_default_docs_ingest()
- if use_url_ingest:
- await _ingest_default_documents_url(
- document_service=document_service,
- docs_url=DEFAULT_DOCS_URL,
- crawl_depth=DEFAULT_DOCS_CRAWL_DEPTH,
- )
- await TelemetryClient.send_event(
- Category.DOCUMENT_INGESTION, MessageId.ORB_DOC_DEFAULT_COMPLETE
- )
- return
+ await ingest_openrag_docs_when_ready(
+ document_service, task_service, langflow_file_service, session_manager
+ )
base_dir = _get_documents_dir()
if not os.path.isdir(base_dir):
@@ -423,12 +450,16 @@ async def ingest_default_documents_when_ready(
f"Default documents directory not found: {base_dir}"
)
- # Collect files recursively, excluding warmup files
+ excluded_files = set(EXCLUDED_INGESTION_FILES)
+ if _should_use_url_default_docs_ingest():
+ excluded_files.update(URL_INGEST_EXCLUDED_INGESTION_FILES)
+
+ # Collect files recursively, excluding warmup files and URL-ingested docs
file_paths = [
os.path.join(root, fn)
for root, _, files in os.walk(base_dir)
for fn in files
- if fn not in EXCLUDED_INGESTION_FILES
+ if fn not in excluded_files
]
if not file_paths:
@@ -467,6 +498,7 @@ async def _ingest_default_documents_langflow(
# Use AnonymousUser details for default documents
from session_manager import AnonymousUser
+
anonymous_user = AnonymousUser()
# Get JWT token using same logic as DocumentFileProcessor
@@ -519,6 +551,56 @@ async def _ingest_default_documents_langflow(
)
+async def _ingest_default_documents_url_langflow(
+ langflow_file_service,
+ session_manager,
+ task_service,
+ docs_url: str,
+ crawl_depth: int,
+):
+ """Ingest default URL docs using the Langflow URL ingestion pipeline."""
+ if not docs_url:
+ raise ValueError("DEFAULT_DOCS_URL is not configured")
+
+ logger.info(
+ "Using Langflow URL ingestion pipeline for default documents",
+ docs_url=docs_url,
+ crawl_depth=crawl_depth,
+ )
+
+ from models.url import LangflowUrlProcessor
+ from session_manager import AnonymousUser
+
+ anonymous_user = AnonymousUser()
+ effective_jwt = None
+
+ if session_manager:
+ session_manager.get_user_opensearch_client(
+ anonymous_user.user_id, effective_jwt
+ )
+ if hasattr(session_manager, "_anonymous_jwt"):
+ effective_jwt = session_manager._anonymous_jwt
+
+ processor = LangflowUrlProcessor(
+ langflow_file_service=langflow_file_service,
+ session_manager=session_manager,
+ docs_url=docs_url,
+ crawl_depth=crawl_depth,
+ owner_user_id=None,
+ jwt_token=effective_jwt,
+ owner_name=anonymous_user.name,
+ owner_email=anonymous_user.email,
+ connector_type="openrag_docs",
+ )
+ task_id = await task_service.create_custom_task(None, [docs_url], processor)
+
+ logger.info(
+ "Started Langflow URL ingestion task for default documents",
+ task_id=task_id,
+ docs_url=docs_url,
+ )
+
+
async def _ingest_default_documents_url(
document_service,
docs_url: str,
@@ -616,7 +698,7 @@ async def _materialize_default_docs_url_as_text_file(
return temp_file.name
-async def _delete_existing_default_docs(session_manager):
+async def _delete_existing_default_docs(session_manager, connector_type: str):
"""Delete previously ingested default OpenRAG docs before reingestion."""
from session_manager import AnonymousUser
@@ -647,7 +729,7 @@ async def _delete_existing_default_docs(session_manager):
{
"bool": {
"must": [
- {"term": {"connector_type": "system_default"}},
+ {"term": {"connector_type": connector_type}},
{"term": {"owner_email": anonymous_user.email}},
]
}
@@ -705,8 +787,8 @@ async def _reingest_default_docs_on_upgrade_if_needed(
previous_version=previous_version,
current_version=current_version,
)
- await _delete_existing_default_docs(session_manager)
- await ingest_default_documents_when_ready(
+ await _delete_existing_default_docs(session_manager, connector_type="openrag_docs")
+ await ingest_openrag_docs_when_ready(
document_service,
task_service,
langflow_file_service,
@@ -863,8 +945,8 @@ async def refresh_default_openrag_docs(
previous_signature=previous_signature,
new_signature=signature,
)
- await _delete_existing_default_docs(session_manager)
- await ingest_default_documents_when_ready(
+ await _delete_existing_default_docs(session_manager, connector_type="openrag_docs")
+ await ingest_openrag_docs_when_ready(
document_service,
task_service,
langflow_file_service,
diff --git a/src/services/langflow_file_service.py b/src/services/langflow_file_service.py
index 6aabc3bc3..c48378b39 100644
--- a/src/services/langflow_file_service.py
+++ b/src/services/langflow_file_service.py
@@ -239,7 +239,7 @@ class LangflowFileService:
owner: Optional[str] = None,
owner_name: Optional[str] = None,
owner_email: Optional[str] = None,
- connector_type: str = "system_default",
+ connector_type: str = "url",
prevent_outside: bool = True,
) -> Dict[str, Any]:
"""Run URL-based docs ingestion flow using Langflow global variable passthrough."""
@@ -267,12 +267,12 @@ class LangflowFileService:
"X-Langflow-Global-Var-OWNER_EMAIL": str(owner_email) if owner_email else "",
"X-Langflow-Global-Var-CONNECTOR_TYPE": str(connector_type),
"X-Langflow-Global-Var-SELECTED_EMBEDDING_MODEL": str(embedding_model),
- "X-Langflow-Global-Var-DOCS_URL": str(docs_url),
- "X-Langflow-Global-Var-DOCS_URLS": json.dumps([docs_url]),
- "X-Langflow-Global-Var-DOCS_CRAWL_DEPTH": str(crawl_depth),
- "X-Langflow-Global-Var-DOCS_PREVENT_OUTSIDE": "true"
- if prevent_outside
- else "false",
+ #TODO: Uncomment these when the flow is updated to use the new variables
+ # "X-Langflow-Global-Var-DOCS_URL": str(docs_url),
+ # "X-Langflow-Global-Var-DOCS_CRAWL_DEPTH": str(crawl_depth),
+ # "X-Langflow-Global-Var-DOCS_PREVENT_OUTSIDE": "true"
+ # if prevent_outside
+ # else "false",
}
add_provider_credentials_to_headers(headers, config)
diff --git a/src/utils/telemetry/message_id.py b/src/utils/telemetry/message_id.py
index 0a3ac16b9..7b0f17d45 100644
--- a/src/utils/telemetry/message_id.py
+++ b/src/utils/telemetry/message_id.py
@@ -66,6 +66,12 @@ class MessageId:
ORB_DOC_INGEST_FAILED = "ORB_DOC_INGEST_FAILED"
# Message: Default documents ingestion started
ORB_DOC_DEFAULT_START = "ORB_DOC_DEFAULT_START"
+ # Message: Default URL documents ingestion started
+ ORB_DOC_DEFAULT_URL_START = "ORB_DOC_DEFAULT_URL_START"
+ # Message: Default URL documents ingestion completed
+ ORB_DOC_DEFAULT_URL_COMPLETE = "ORB_DOC_DEFAULT_URL_COMPLETE"
+ # Message: Default URL documents ingestion failed
+ ORB_DOC_DEFAULT_URL_FAILED = "ORB_DOC_DEFAULT_URL_FAILED"
# Message: Default documents ingestion completed
ORB_DOC_DEFAULT_COMPLETE = "ORB_DOC_DEFAULT_COMPLETE"
# Message: Default documents ingestion failed