Skip to content

OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.

Python

Merge branch 'main' into pdf-doc-build

Olfa Maslahd7a238e95fdb
diff --cc src/api/settings.py
index d3b83546d,6b6c31607..ba7b20f69
--- a/src/api/settings.py
+++ b/src/api/settings.py
@@@ -174,12 -168,8 +174,13 @@@ class OnboardingResponse(BaseModel)
edited: bool
sample_data_ingested: bool
openrag_docs_filter_id: Optional[str] = None
+ task_id: Optional[str] = None
+
+class RefreshOpenRAGDocsResponse(BaseModel):
+ message: str
+ refreshed: bool
+
class DoclingConfig(BaseModel):
do_ocr: bool
ocr_engine: str
diff --cc src/main.py
index eccee56a3,76fbf0cf8..13a7662e0
--- a/src/main.py
+++ b/src/main.py
@@@ -431,15 -354,13 +431,17 @@@ async def ingest_default_documents_when
langflow_file_service, session_manager, task_service, file_paths
)
- await TelemetryClient.send_event(Category.DOCUMENT_INGESTION, MessageId.ORB_DOC_DEFAULT_COMPLETE)
+ await TelemetryClient.send_event(
+ Category.DOCUMENT_INGESTION, MessageId.ORB_DOC_DEFAULT_COMPLETE
+ )
+ return task_id
+
except Exception as e:
logger.error("Default documents ingestion failed", error=str(e))
- await TelemetryClient.send_event(Category.DOCUMENT_INGESTION, MessageId.ORB_DOC_DEFAULT_FAILED)
+ await TelemetryClient.send_event(
+ Category.DOCUMENT_INGESTION, MessageId.ORB_DOC_DEFAULT_FAILED
+ )
raise
@@@ -507,357 -415,8 +496,359 @@@ async def _ingest_default_documents_lan
file_count=len(file_paths),
)
+ return task_id
+async def _ingest_default_documents_url(
+ langflow_file_service, session_manager, task_service, docs_url: str, crawl_depth: int
+):
+ """Ingest default docs by crawling a URL with tracked task progress."""
+ if langflow_file_service is None:
+ raise ValueError("LangflowFileService is not available")
+ if task_service is None:
+ raise ValueError("TaskService is not available")
+
+ from session_manager import AnonymousUser
+ from models.processors import LangflowUrlProcessor
+
+ anonymous_user = AnonymousUser()
+ effective_jwt = None
+ if session_manager:
+ session_manager.get_user_opensearch_client(
+ anonymous_user.user_id, effective_jwt
+ )
+ if hasattr(session_manager, "_anonymous_jwt"):
+ effective_jwt = session_manager._anonymous_jwt
+
+ logger.info(
+ "Running default URL docs ingestion flow",
+ docs_url=docs_url,
+ crawl_depth=crawl_depth,
+ )
+ processor = LangflowUrlProcessor(
+ langflow_file_service=langflow_file_service,
+ session_manager=session_manager,
+ docs_url=docs_url,
+ crawl_depth=crawl_depth,
+ jwt_token=effective_jwt,
+ owner_user_id=anonymous_user.user_id,
+ owner_name=anonymous_user.name,
+ owner_email=anonymous_user.email,
+ connector_type="system_default",
+ prevent_outside=True,
+ )
+ task_id = await task_service.create_custom_task(
+ user_id=None, # Anonymous shared task
+ items=[docs_url],
+ processor=processor,
+ original_filenames={docs_url: "OpenRAG docs refresh"},
+ )
+ logger.info(
+ "Started URL ingestion task for default documents",
+ task_id=task_id,
+ docs_url=docs_url,
+ )
+
+
+async def _delete_existing_default_docs(
+ session_manager, include_system_default: bool = True
+):
+ """Delete previously ingested default OpenRAG docs before reingestion."""
+ from session_manager import AnonymousUser
+
+ if session_manager is None:
+ logger.warning(
+ "Session manager unavailable; skipping default docs cleanup before reingestion"
+ )
+ return
+
+ anonymous_user = AnonymousUser()
+ effective_jwt = None
+ if session_manager:
+ session_manager.get_user_opensearch_client(
+ anonymous_user.user_id, effective_jwt
+ )
+ if hasattr(session_manager, "_anonymous_jwt"):
+ effective_jwt = session_manager._anonymous_jwt
+
+ opensearch_client = session_manager.get_user_opensearch_client(
+ anonymous_user.user_id, effective_jwt
+ )
+ should_clauses = []
+ if include_system_default:
+ # URL-based default docs are ingested as system_default and owned
+ # by the anonymous onboarding user.
+ should_clauses.append(
+ {
+ "bool": {
+ "must": [
+ {"term": {"connector_type": "system_default"}},
+ {"term": {"owner_email": anonymous_user.email}},
+ ]
+ }
+ }
+ )
+
+ # Legacy file-based default docs were ingested as local and
+ # marked with is_sample_data=true.
+ should_clauses.append(
+ {
+ "bool": {
+ "must": [
+ {"term": {"connector_type": "local"}},
+ {"term": {"is_sample_data": "true"}},
+ ]
+ }
+ }
+ )
+
+ delete_query = {
+ "query": {
+ "bool": {
+ "should": should_clauses,
+ "minimum_should_match": 1,
+ }
+ }
+ }
+ result = await opensearch_client.delete_by_query(
+ index=get_index_name(),
+ body=delete_query,
+ conflicts="proceed",
+ )
+ logger.info(
+ "Deleted existing default OpenRAG docs before reingestion",
+ deleted_chunks=result.get("deleted", 0),
+ )
+
+
+async def _reingest_default_docs_on_upgrade_if_needed(
+ document_service,
+ task_service,
+ langflow_file_service,
+ session_manager,
+):
+ """Reingest default OpenRAG docs once when app version changes."""
+ config = get_openrag_config()
+
+ previous_version = config.onboarding.openrag_docs_ingested_version
+ current_version = OPENRAG_VERSION
+ should_reingest = bool(previous_version) and previous_version != current_version
+
+ # Legacy installs may not have a stored docs ingestion version.
+ # Use the presence of the OpenRAG docs filter as the signal that docs were
+ # already onboarded, independent of whether config.edited is set.
+ if not previous_version and config.onboarding.openrag_docs_filter_id:
+ should_reingest = True
+
+ if not should_reingest:
+ return False
+
+ logger.info(
+ "Detected OpenRAG upgrade; reingesting default docs",
+ previous_version=previous_version,
+ current_version=current_version,
+ )
+ await _delete_existing_default_docs(session_manager)
+ await ingest_default_documents_when_ready(
+ document_service,
+ task_service,
+ langflow_file_service,
+ session_manager,
+ )
+ config.onboarding.openrag_docs_ingested_version = current_version
+ if _should_use_url_default_docs_ingest():
+ # Refresh signature metadata after upgrade reingestion so startup
+ # signature checks don't trigger an immediate duplicate ingest.
+ config.onboarding.openrag_docs_remote_signature = (
+ await _get_remote_docs_signature(DEFAULT_DOCS_URL)
+ )
+ else:
+ config.onboarding.openrag_docs_remote_signature = None
+ if not config_manager.save_config_file(config):
+ logger.warning(
+ "Default docs were reingested but failed to persist metadata",
+ current_version=current_version,
+ signature=config.onboarding.openrag_docs_remote_signature,
+ )
+ return True
+
+
+async def _get_remote_docs_signature(docs_url: str):
+ """Get a signature for remote docs to detect content updates."""
+ try:
+ async with httpx.AsyncClient(timeout=10.0, follow_redirects=True) as client:
+ head_response = await client.head(docs_url)
+ if head_response.status_code >= 400:
+ get_response = await client.get(docs_url)
+ if get_response.status_code >= 400:
+ logger.warning(
+ "Failed to fetch remote docs signature",
+ docs_url=docs_url,
+ status_code=get_response.status_code,
+ )
+ return None
+ return hashlib.sha256(get_response.text.encode("utf-8")).hexdigest()
+
+ etag = (head_response.headers.get("etag") or "").strip()
+ last_modified = (head_response.headers.get("last-modified") or "").strip()
+ if etag:
+ # Prefer ETag when available: it is typically the strongest
+ # cache validator and stays stable if extra cache headers
+ # appear/disappear without content changes.
+ return f"etag={etag}"
+ if last_modified:
+ return f"last_modified={last_modified}"
+
+ # HEAD has no body. If cache headers are missing, fetch the page body.
+ get_response = await client.get(docs_url)
+ if get_response.status_code >= 400:
+ logger.warning(
+ "Failed to fetch remote docs signature body fallback",
+ docs_url=docs_url,
+ status_code=get_response.status_code,
+ )
+ return None
+ return hashlib.sha256(get_response.text.encode("utf-8")).hexdigest()
+ except Exception as e:
+ logger.warning(
+ "Unable to retrieve remote docs signature",
+ docs_url=docs_url,
+ error=str(e),
+ )
+ return None
+
+
+async def refresh_default_openrag_docs(
+ document_service,
+ task_service,
+ langflow_file_service,
+ session_manager,
+ force: bool = False,
+ reason: str = "startup",
+):
+ """Refresh OpenRAG docs if remote content changed or when forced."""
+ await TelemetryClient.send_event(
+ Category.DOCUMENT_INGESTION,
+ MessageId.ORB_DOC_REFRESH_START,
+ metadata={"reason": reason, "force": force},
+ )
+ try:
+ if not _should_use_url_default_docs_ingest():
+ logger.info(
+ "Skipping OpenRAG docs refresh: URL ingestion is not active",
+ ingest_source=DEFAULT_DOCS_INGEST_SOURCE,
+ disable_langflow_ingest=DISABLE_INGEST_WITH_LANGFLOW,
+ has_url_ingest_flow_id=bool(LANGFLOW_URL_INGEST_FLOW_ID),
+ has_docs_url=bool(DEFAULT_DOCS_URL),
+ )
+ await TelemetryClient.send_event(
+ Category.DOCUMENT_INGESTION,
+ MessageId.ORB_DOC_REFRESH_SKIPPED,
+ metadata={
+ "reason": reason,
+ "force": force,
+ "skip_reason": "url_ingestion_inactive",
+ },
+ )
+ return False
+
+ config = get_openrag_config()
+ if not config.edited:
+ logger.info("Skipping OpenRAG docs refresh: onboarding not completed")
+ await TelemetryClient.send_event(
+ Category.DOCUMENT_INGESTION,
+ MessageId.ORB_DOC_REFRESH_SKIPPED,
+ metadata={
+ "reason": reason,
+ "force": force,
+ "skip_reason": "onboarding_not_completed",
+ },
+ )
+ return False
+
+ signature = await _get_remote_docs_signature(DEFAULT_DOCS_URL)
+ if not signature and not force:
+ await TelemetryClient.send_event(
+ Category.DOCUMENT_INGESTION,
+ MessageId.ORB_DOC_REFRESH_SKIPPED,
+ metadata={
+ "reason": reason,
+ "force": force,
+ "skip_reason": "signature_unavailable",
+ },
+ )
+ return False
+
+ previous_signature = config.onboarding.openrag_docs_remote_signature
+ should_refresh = force or (
+ signature is not None and signature != previous_signature
+ )
+ if not should_refresh:
+ logger.info(
+ "OpenRAG docs refresh skipped: remote signature unchanged",
+ signature=signature,
+ )
+ await TelemetryClient.send_event(
+ Category.DOCUMENT_INGESTION,
+ MessageId.ORB_DOC_REFRESH_SKIPPED,
+ metadata={
+ "reason": reason,
+ "force": force,
+ "skip_reason": "signature_unchanged",
+ },
+ )
+ return False
+
+ logger.info(
+ "Refreshing default OpenRAG docs",
+ reason=reason,
+ force=force,
+ previous_signature=previous_signature,
+ new_signature=signature,
+ )
+ # Keep existing system_default docs in place for manual refresh.
+ # URL refresh ingestion runs asynchronously, so pre-deleting could
+ # make docs disappear if the downstream flow returns success without
+ # actually indexing documents.
+ await _delete_existing_default_docs(
+ session_manager, include_system_default=False
+ )
+ await ingest_default_documents_when_ready(
+ document_service,
+ task_service,
+ langflow_file_service,
+ session_manager,
+ )
+ config.onboarding.openrag_docs_ingested_version = OPENRAG_VERSION
+ # Keep docs version/signature metadata consistent after a refresh.
+ # If signature retrieval failed, persist None explicitly instead of
+ # leaving a stale previous signature value.
+ config.onboarding.openrag_docs_remote_signature = signature
+ if not config_manager.save_config_file(config):
+ logger.warning(
+ "OpenRAG docs refreshed but failed to persist metadata",
+ version=config.onboarding.openrag_docs_ingested_version,
+ signature=config.onboarding.openrag_docs_remote_signature,
+ )
+ await TelemetryClient.send_event(
+ Category.DOCUMENT_INGESTION,
+ MessageId.ORB_DOC_REFRESH_COMPLETE,
+ metadata={"reason": reason, "force": force},
+ )
+ return True
+ except Exception as e:
+ await TelemetryClient.send_event(
+ Category.DOCUMENT_INGESTION,
+ MessageId.ORB_DOC_REFRESH_FAILED,
+ metadata={
+ "reason": reason,
+ "force": force,
+ "error_type": type(e).__name__,
+ },
+ )
+ raise
+
+
++
async def health_check(request: Request):
"""Simple liveness probe: Indicates that the OpenRAG Backend service is online and running."""
return JSONResponse({"status": "ok"}, status_code=200)