OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.
Python
Merge branch 'main' into pdf-doc-build
Olfa Maslahd7a238e95fdb
diff --cc src/api/settings.pyindex d3b83546d,6b6c31607..ba7b20f69--- a/src/api/settings.py+++ b/src/api/settings.py@@@ -174,12 -168,8 +174,13 @@@ class OnboardingResponse(BaseModel)edited: boolsample_data_ingested: boolopenrag_docs_filter_id: Optional[str] = None+ task_id: Optional[str] = None++class RefreshOpenRAGDocsResponse(BaseModel):+ message: str+ refreshed: bool+class DoclingConfig(BaseModel):do_ocr: boolocr_engine: strdiff --cc src/main.pyindex eccee56a3,76fbf0cf8..13a7662e0--- a/src/main.py+++ b/src/main.py@@@ -431,15 -354,13 +431,17 @@@ async def ingest_default_documents_whenlangflow_file_service, session_manager, task_service, file_paths)- await TelemetryClient.send_event(Category.DOCUMENT_INGESTION, MessageId.ORB_DOC_DEFAULT_COMPLETE)+ await TelemetryClient.send_event(+ Category.DOCUMENT_INGESTION, MessageId.ORB_DOC_DEFAULT_COMPLETE+ )+ return task_id+except Exception as e:logger.error("Default documents ingestion failed", error=str(e))- await TelemetryClient.send_event(Category.DOCUMENT_INGESTION, MessageId.ORB_DOC_DEFAULT_FAILED)+ await TelemetryClient.send_event(+ Category.DOCUMENT_INGESTION, MessageId.ORB_DOC_DEFAULT_FAILED+ )raise@@@ -507,357 -415,8 +496,359 @@@ async def _ingest_default_documents_lanfile_count=len(file_paths),)+ return task_id+async def _ingest_default_documents_url(+ langflow_file_service, session_manager, task_service, docs_url: str, crawl_depth: int+):+ """Ingest default docs by crawling a URL with tracked task progress."""+ if langflow_file_service is None:+ raise ValueError("LangflowFileService is not available")+ if task_service is None:+ raise ValueError("TaskService is not available")++ from session_manager import AnonymousUser+ from models.processors import LangflowUrlProcessor++ anonymous_user = AnonymousUser()+ effective_jwt = None+ if session_manager:+ session_manager.get_user_opensearch_client(+ anonymous_user.user_id, effective_jwt+ )+ if hasattr(session_manager, "_anonymous_jwt"):+ effective_jwt = session_manager._anonymous_jwt++ logger.info(+ "Running default URL docs ingestion flow",+ docs_url=docs_url,+ crawl_depth=crawl_depth,+ )+ processor = LangflowUrlProcessor(+ langflow_file_service=langflow_file_service,+ session_manager=session_manager,+ docs_url=docs_url,+ crawl_depth=crawl_depth,+ jwt_token=effective_jwt,+ owner_user_id=anonymous_user.user_id,+ owner_name=anonymous_user.name,+ owner_email=anonymous_user.email,+ connector_type="system_default",+ prevent_outside=True,+ )+ task_id = await task_service.create_custom_task(+ user_id=None, # Anonymous shared task+ items=[docs_url],+ processor=processor,+ original_filenames={docs_url: "OpenRAG docs refresh"},+ )+ logger.info(+ "Started URL ingestion task for default documents",+ task_id=task_id,+ docs_url=docs_url,+ )+++async def _delete_existing_default_docs(+ session_manager, include_system_default: bool = True+):+ """Delete previously ingested default OpenRAG docs before reingestion."""+ from session_manager import AnonymousUser++ if session_manager is None:+ logger.warning(+ "Session manager unavailable; skipping default docs cleanup before reingestion"+ )+ return++ anonymous_user = AnonymousUser()+ effective_jwt = None+ if session_manager:+ session_manager.get_user_opensearch_client(+ anonymous_user.user_id, effective_jwt+ )+ if hasattr(session_manager, "_anonymous_jwt"):+ effective_jwt = session_manager._anonymous_jwt++ opensearch_client = session_manager.get_user_opensearch_client(+ anonymous_user.user_id, effective_jwt+ )+ should_clauses = []+ if include_system_default:+ # URL-based default docs are ingested as system_default and owned+ # by the anonymous onboarding user.+ should_clauses.append(+ {+ "bool": {+ "must": [+ {"term": {"connector_type": "system_default"}},+ {"term": {"owner_email": anonymous_user.email}},+ ]+ }+ }+ )++ # Legacy file-based default docs were ingested as local and+ # marked with is_sample_data=true.+ should_clauses.append(+ {+ "bool": {+ "must": [+ {"term": {"connector_type": "local"}},+ {"term": {"is_sample_data": "true"}},+ ]+ }+ }+ )++ delete_query = {+ "query": {+ "bool": {+ "should": should_clauses,+ "minimum_should_match": 1,+ }+ }+ }+ result = await opensearch_client.delete_by_query(+ index=get_index_name(),+ body=delete_query,+ conflicts="proceed",+ )+ logger.info(+ "Deleted existing default OpenRAG docs before reingestion",+ deleted_chunks=result.get("deleted", 0),+ )+++async def _reingest_default_docs_on_upgrade_if_needed(+ document_service,+ task_service,+ langflow_file_service,+ session_manager,+):+ """Reingest default OpenRAG docs once when app version changes."""+ config = get_openrag_config()++ previous_version = config.onboarding.openrag_docs_ingested_version+ current_version = OPENRAG_VERSION+ should_reingest = bool(previous_version) and previous_version != current_version++ # Legacy installs may not have a stored docs ingestion version.+ # Use the presence of the OpenRAG docs filter as the signal that docs were+ # already onboarded, independent of whether config.edited is set.+ if not previous_version and config.onboarding.openrag_docs_filter_id:+ should_reingest = True++ if not should_reingest:+ return False++ logger.info(+ "Detected OpenRAG upgrade; reingesting default docs",+ previous_version=previous_version,+ current_version=current_version,+ )+ await _delete_existing_default_docs(session_manager)+ await ingest_default_documents_when_ready(+ document_service,+ task_service,+ langflow_file_service,+ session_manager,+ )+ config.onboarding.openrag_docs_ingested_version = current_version+ if _should_use_url_default_docs_ingest():+ # Refresh signature metadata after upgrade reingestion so startup+ # signature checks don't trigger an immediate duplicate ingest.+ config.onboarding.openrag_docs_remote_signature = (+ await _get_remote_docs_signature(DEFAULT_DOCS_URL)+ )+ else:+ config.onboarding.openrag_docs_remote_signature = None+ if not config_manager.save_config_file(config):+ logger.warning(+ "Default docs were reingested but failed to persist metadata",+ current_version=current_version,+ signature=config.onboarding.openrag_docs_remote_signature,+ )+ return True+++async def _get_remote_docs_signature(docs_url: str):+ """Get a signature for remote docs to detect content updates."""+ try:+ async with httpx.AsyncClient(timeout=10.0, follow_redirects=True) as client:+ head_response = await client.head(docs_url)+ if head_response.status_code >= 400:+ get_response = await client.get(docs_url)+ if get_response.status_code >= 400:+ logger.warning(+ "Failed to fetch remote docs signature",+ docs_url=docs_url,+ status_code=get_response.status_code,+ )+ return None+ return hashlib.sha256(get_response.text.encode("utf-8")).hexdigest()++ etag = (head_response.headers.get("etag") or "").strip()+ last_modified = (head_response.headers.get("last-modified") or "").strip()+ if etag:+ # Prefer ETag when available: it is typically the strongest+ # cache validator and stays stable if extra cache headers+ # appear/disappear without content changes.+ return f"etag={etag}"+ if last_modified:+ return f"last_modified={last_modified}"++ # HEAD has no body. If cache headers are missing, fetch the page body.+ get_response = await client.get(docs_url)+ if get_response.status_code >= 400:+ logger.warning(+ "Failed to fetch remote docs signature body fallback",+ docs_url=docs_url,+ status_code=get_response.status_code,+ )+ return None+ return hashlib.sha256(get_response.text.encode("utf-8")).hexdigest()+ except Exception as e:+ logger.warning(+ "Unable to retrieve remote docs signature",+ docs_url=docs_url,+ error=str(e),+ )+ return None+++async def refresh_default_openrag_docs(+ document_service,+ task_service,+ langflow_file_service,+ session_manager,+ force: bool = False,+ reason: str = "startup",+):+ """Refresh OpenRAG docs if remote content changed or when forced."""+ await TelemetryClient.send_event(+ Category.DOCUMENT_INGESTION,+ MessageId.ORB_DOC_REFRESH_START,+ metadata={"reason": reason, "force": force},+ )+ try:+ if not _should_use_url_default_docs_ingest():+ logger.info(+ "Skipping OpenRAG docs refresh: URL ingestion is not active",+ ingest_source=DEFAULT_DOCS_INGEST_SOURCE,+ disable_langflow_ingest=DISABLE_INGEST_WITH_LANGFLOW,+ has_url_ingest_flow_id=bool(LANGFLOW_URL_INGEST_FLOW_ID),+ has_docs_url=bool(DEFAULT_DOCS_URL),+ )+ await TelemetryClient.send_event(+ Category.DOCUMENT_INGESTION,+ MessageId.ORB_DOC_REFRESH_SKIPPED,+ metadata={+ "reason": reason,+ "force": force,+ "skip_reason": "url_ingestion_inactive",+ },+ )+ return False++ config = get_openrag_config()+ if not config.edited:+ logger.info("Skipping OpenRAG docs refresh: onboarding not completed")+ await TelemetryClient.send_event(+ Category.DOCUMENT_INGESTION,+ MessageId.ORB_DOC_REFRESH_SKIPPED,+ metadata={+ "reason": reason,+ "force": force,+ "skip_reason": "onboarding_not_completed",+ },+ )+ return False++ signature = await _get_remote_docs_signature(DEFAULT_DOCS_URL)+ if not signature and not force:+ await TelemetryClient.send_event(+ Category.DOCUMENT_INGESTION,+ MessageId.ORB_DOC_REFRESH_SKIPPED,+ metadata={+ "reason": reason,+ "force": force,+ "skip_reason": "signature_unavailable",+ },+ )+ return False++ previous_signature = config.onboarding.openrag_docs_remote_signature+ should_refresh = force or (+ signature is not None and signature != previous_signature+ )+ if not should_refresh:+ logger.info(+ "OpenRAG docs refresh skipped: remote signature unchanged",+ signature=signature,+ )+ await TelemetryClient.send_event(+ Category.DOCUMENT_INGESTION,+ MessageId.ORB_DOC_REFRESH_SKIPPED,+ metadata={+ "reason": reason,+ "force": force,+ "skip_reason": "signature_unchanged",+ },+ )+ return False++ logger.info(+ "Refreshing default OpenRAG docs",+ reason=reason,+ force=force,+ previous_signature=previous_signature,+ new_signature=signature,+ )+ # Keep existing system_default docs in place for manual refresh.+ # URL refresh ingestion runs asynchronously, so pre-deleting could+ # make docs disappear if the downstream flow returns success without+ # actually indexing documents.+ await _delete_existing_default_docs(+ session_manager, include_system_default=False+ )+ await ingest_default_documents_when_ready(+ document_service,+ task_service,+ langflow_file_service,+ session_manager,+ )+ config.onboarding.openrag_docs_ingested_version = OPENRAG_VERSION+ # Keep docs version/signature metadata consistent after a refresh.+ # If signature retrieval failed, persist None explicitly instead of+ # leaving a stale previous signature value.+ config.onboarding.openrag_docs_remote_signature = signature+ if not config_manager.save_config_file(config):+ logger.warning(+ "OpenRAG docs refreshed but failed to persist metadata",+ version=config.onboarding.openrag_docs_ingested_version,+ signature=config.onboarding.openrag_docs_remote_signature,+ )+ await TelemetryClient.send_event(+ Category.DOCUMENT_INGESTION,+ MessageId.ORB_DOC_REFRESH_COMPLETE,+ metadata={"reason": reason, "force": force},+ )+ return True+ except Exception as e:+ await TelemetryClient.send_event(+ Category.DOCUMENT_INGESTION,+ MessageId.ORB_DOC_REFRESH_FAILED,+ metadata={+ "reason": reason,+ "force": force,+ "error_type": type(e).__name__,+ },+ )+ raise++++async def health_check(request: Request):"""Simple liveness probe: Indicates that the OpenRAG Backend service is online and running."""return JSONResponse({"status": "ok"}, status_code=200)