Skip to content

OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.

Python

fix onboarding docs ingestion to use URL source safely

Olfa Maslah236409a313e6
diff --git a/.env.example b/.env.example
index f3bdc0f1f..dc3278889 100644
--- a/.env.example
+++ b/.env.example
@@ -3,10 +3,21 @@
# If unset or false, Langflow pipeline will be used (default: upload -> ingest -> delete)
DISABLE_INGEST_WITH_LANGFLOW=false
-# Set to false to skip ingesting sample data (pdfs) on startup
+# Set to false to skip ingesting OpenRAG sample docs during onboarding
# Default: true
INGEST_SAMPLE_DATA=true
+# Default OpenRAG docs source for sample ingestion during onboarding.
+# - url: crawl DEFAULT_DOCS_URL using URL ingestion flow
+# - files: ingest files from OPENRAG_DOCUMENTS_PATH
+DEFAULT_DOCS_INGEST_SOURCE=url
+
+# URL used when DEFAULT_DOCS_INGEST_SOURCE=url
+DEFAULT_DOCS_URL=https://docs.openr.ag
+
+# Crawl depth used when DEFAULT_DOCS_INGEST_SOURCE=url
+DEFAULT_DOCS_CRAWL_DEPTH=2
+
# Langflow HTTP timeout configuration (in seconds)
# For large documents (300+ pages), ingestion can take 30+ minutes
# Increase these values if you experience timeouts with very large PDFs
diff --git a/frontend/app/chat/_types/types.ts b/frontend/app/chat/_types/types.ts
index c605da7c3..c00ce97d6 100644
--- a/frontend/app/chat/_types/types.ts
+++ b/frontend/app/chat/_types/types.ts
@@ -45,6 +45,7 @@ export interface SelectedFilters {
data_sources: string[];
document_types: string[];
owners: string[];
+ connector_types: string[];
}
export interface KnowledgeFilterData {
diff --git a/frontend/app/chat/page.tsx b/frontend/app/chat/page.tsx
index 44d8c70ac..71aeee4d8 100644
--- a/frontend/app/chat/page.tsx
+++ b/frontend/app/chat/page.tsx
@@ -667,6 +667,7 @@ function ChatPage() {
data_sources: [],
document_types: [],
owners: [],
+ connector_types: [],
};
processed.data_sources = filters.data_sources.includes("*")
? []
@@ -675,11 +676,16 @@ function ChatPage() {
? []
: filters.document_types;
processed.owners = filters.owners.includes("*") ? [] : filters.owners;
+ processed.connector_types =
+ filters.connector_types?.includes("*")
+ ? []
+ : (filters.connector_types ?? []);
const hasFilters =
processed.data_sources.length > 0 ||
processed.document_types.length > 0 ||
- processed.owners.length > 0;
+ processed.owners.length > 0 ||
+ processed.connector_types.length > 0;
return hasFilters ? processed : undefined;
})()
: undefined;
@@ -708,6 +714,7 @@ function ChatPage() {
data_sources: [],
document_types: [],
owners: [],
+ connector_types: [],
};
processed.data_sources = filters.data_sources.includes("*")
? []
@@ -716,11 +723,16 @@ function ChatPage() {
? []
: filters.document_types;
processed.owners = filters.owners.includes("*") ? [] : filters.owners;
+ processed.connector_types =
+ filters.connector_types?.includes("*")
+ ? []
+ : (filters.connector_types ?? []);
const hasFilters =
processed.data_sources.length > 0 ||
processed.document_types.length > 0 ||
- processed.owners.length > 0;
+ processed.owners.length > 0 ||
+ processed.connector_types.length > 0;
return hasFilters ? processed : undefined;
})()
: undefined;
@@ -791,6 +803,7 @@ function ChatPage() {
data_sources: [],
document_types: [],
owners: [],
+ connector_types: [],
};
// Only copy non-wildcard arrays
processed.data_sources = filters.data_sources.includes("*")
@@ -802,12 +815,17 @@ function ChatPage() {
processed.owners = filters.owners.includes("*")
? []
: filters.owners;
+ processed.connector_types =
+ filters.connector_types?.includes("*")
+ ? []
+ : (filters.connector_types ?? []);
// Only include filters if any array has values
const hasFilters =
processed.data_sources.length > 0 ||
processed.document_types.length > 0 ||
- processed.owners.length > 0;
+ processed.owners.length > 0 ||
+ processed.connector_types.length > 0;
return hasFilters ? { filters: processed } : {};
})()),
limit: parsedFilterData?.limit ?? 10,
diff --git a/frontend/app/onboarding/_components/onboarding-content.tsx b/frontend/app/onboarding/_components/onboarding-content.tsx
index e98dc2294..de7900713 100644
--- a/frontend/app/onboarding/_components/onboarding-content.tsx
+++ b/frontend/app/onboarding/_components/onboarding-content.tsx
@@ -18,9 +18,10 @@ import OnboardingUpload from "./onboarding-upload";
// Filters for OpenRAG documentation
const OPENRAG_DOCS_FILTERS: SelectedFilters = {
- data_sources: ["openrag-documentation.pdf"],
- document_types: [],
- owners: [],
+ data_sources: ["*"],
+ document_types: ["*"],
+ owners: ["*"],
+ connector_types: ["system_default"],
};
export function OnboardingContent({
diff --git a/src/api/settings.py b/src/api/settings.py
index 562dac8e8..4058805d7 100644
--- a/src/api/settings.py
+++ b/src/api/settings.py
@@ -1135,10 +1135,12 @@ async def _create_openrag_docs_filter(
query_data = json.dumps({
"query": "",
"filters": {
- "data_sources": ["openrag-documentation.pdf"],
+ # URL-based docs ingestion produces many source URLs.
+ # Filter by connector type to target the OpenRAG system docs set.
+ "data_sources": ["*"],
"document_types": ["*"],
"owners": ["*"],
- "connector_types": ["*"],
+ "connector_types": ["system_default"],
},
"limit": 10,
"scoreThreshold": 0,
diff --git a/src/config/settings.py b/src/config/settings.py
index b4d1e9e28..c5578218a 100644
--- a/src/config/settings.py
+++ b/src/config/settings.py
@@ -61,6 +61,13 @@ INGEST_SAMPLE_DATA = os.getenv(
"INGEST_SAMPLE_DATA", "true"
).lower() in ("true", "1", "yes")
+# Default OpenRAG docs sample ingestion source
+# - "url": crawl DEFAULT_DOCS_URL with URL ingestion flow
+# - "files": ingest files from the openrag-documents directory
+DEFAULT_DOCS_INGEST_SOURCE = os.getenv("DEFAULT_DOCS_INGEST_SOURCE", "url").lower()
+DEFAULT_DOCS_URL = os.getenv("DEFAULT_DOCS_URL", "https://docs.openr.ag")
+DEFAULT_DOCS_CRAWL_DEPTH = get_env_int("DEFAULT_DOCS_CRAWL_DEPTH", 2)
+
# Maximum number of files to upload / ingest (in batch) per task when adding knowledge via folder
UPLOAD_BATCH_SIZE = get_env_int("UPLOAD_BATCH_SIZE", 25)
diff --git a/src/main.py b/src/main.py
index 598263f78..fe752f659 100644
--- a/src/main.py
+++ b/src/main.py
@@ -44,11 +44,15 @@ from api.v1 import chat as v1_chat, search as v1_search, documents as v1_documen
# Configuration and setup
from config.settings import (
+ DEFAULT_DOCS_CRAWL_DEPTH,
+ DEFAULT_DOCS_INGEST_SOURCE,
+ DEFAULT_DOCS_URL,
API_KEYS_INDEX_BODY,
API_KEYS_INDEX_NAME,
DISABLE_INGEST_WITH_LANGFLOW,
INGESTION_TIMEOUT,
INDEX_BODY,
+ LANGFLOW_URL_INGEST_FLOW_ID,
SESSION_SECRET,
clients,
get_embedding_model,
@@ -323,13 +327,37 @@ def _get_documents_dir():
async def ingest_default_documents_when_ready(
document_service, task_service, langflow_file_service, session_manager
):
- """Scan the local documents folder and ingest files like a non-auth upload."""
+ """Ingest default OpenRAG docs during onboarding."""
try:
logger.info(
"Ingesting default documents when ready",
disable_langflow_ingest=DISABLE_INGEST_WITH_LANGFLOW,
+ ingest_source=DEFAULT_DOCS_INGEST_SOURCE,
)
await TelemetryClient.send_event(Category.DOCUMENT_INGESTION, MessageId.ORB_DOC_DEFAULT_START)
+ use_url_ingest = (
+ DEFAULT_DOCS_INGEST_SOURCE == "url"
+ and not DISABLE_INGEST_WITH_LANGFLOW
+ and bool(LANGFLOW_URL_INGEST_FLOW_ID)
+ )
+ if (
+ DEFAULT_DOCS_INGEST_SOURCE == "url"
+ and not DISABLE_INGEST_WITH_LANGFLOW
+ and not LANGFLOW_URL_INGEST_FLOW_ID
+ ):
+ logger.warning(
+ "URL docs ingestion flow id is missing; falling back to file-based default ingestion",
+ ingest_source=DEFAULT_DOCS_INGEST_SOURCE,
+ )
+ if use_url_ingest:
+ await _ingest_default_documents_url(
+ session_manager=session_manager,
+ docs_url=DEFAULT_DOCS_URL,
+ crawl_depth=DEFAULT_DOCS_CRAWL_DEPTH,
+ )
+ await TelemetryClient.send_event(Category.DOCUMENT_INGESTION, MessageId.ORB_DOC_DEFAULT_COMPLETE)
+ return
+
base_dir = _get_documents_dir()
if not os.path.isdir(base_dir):
raise FileNotFoundError(f"Default documents directory not found: {base_dir}")
@@ -426,6 +454,67 @@ async def _ingest_default_documents_langflow(
file_count=len(file_paths),
)
+
+async def _ingest_default_documents_url(session_manager, docs_url: str, crawl_depth: int):
+ """Ingest default docs by crawling a URL with the Langflow URL ingest flow."""
+ if not LANGFLOW_URL_INGEST_FLOW_ID:
+ raise ValueError("LANGFLOW_URL_INGEST_FLOW_ID is not configured")
+ if not docs_url:
+ raise ValueError("DEFAULT_DOCS_URL is not configured")
+
+ from session_manager import AnonymousUser
+ from utils.langflow_headers import add_provider_credentials_to_headers
+ from config.settings import get_openrag_config
+
+ anonymous_user = AnonymousUser()
+ effective_jwt = None
+ if session_manager:
+ session_manager.get_user_opensearch_client(anonymous_user.user_id, effective_jwt)
+ if hasattr(session_manager, "_anonymous_jwt"):
+ effective_jwt = session_manager._anonymous_jwt
+
+ payload = {
+ "input_value": docs_url,
+ "input_type": "chat",
+ "output_type": "text",
+ "tweaks": {
+ "URLComponent-lnA0q": {
+ "urls": [docs_url],
+ "max_depth": crawl_depth,
+ "prevent_outside": True,
+ }
+ },
+ }
+ config = get_openrag_config()
+ headers = {
+ "X-Langflow-Global-Var-JWT": str(effective_jwt) if effective_jwt else "",
+ "X-Langflow-Global-Var-OWNER": "",
+ "X-Langflow-Global-Var-OWNER_NAME": str(anonymous_user.name),
+ "X-Langflow-Global-Var-OWNER_EMAIL": str(anonymous_user.email),
+ "X-Langflow-Global-Var-CONNECTOR_TYPE": "system_default",
+ "X-Langflow-Global-Var-SELECTED_EMBEDDING_MODEL": str(config.knowledge.embedding_model),
+ }
+ add_provider_credentials_to_headers(headers, config)
+
+ logger.info(
+ "Running default URL docs ingestion flow",
+ docs_url=docs_url,
+ crawl_depth=crawl_depth,
+ )
+ resp = await clients.langflow_request(
+ "POST",
+ f"/api/v1/run/{LANGFLOW_URL_INGEST_FLOW_ID}",
+ json=payload,
+ headers=headers,
+ )
+ if resp.status_code >= 400:
+ logger.error(
+ "Default URL docs ingestion failed",
+ status_code=resp.status_code,
+ body=resp.text[:1000],
+ )
+ resp.raise_for_status()
+
async def health_check(request: Request):
"""Simple liveness probe: Indicates that the OpenRAG Backend service is online and running."""
return JSONResponse({"status": "ok"}, status_code=200)