OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.
Python
Improve default docs refresh flow to be simpler and avoid duplicate startup reingestion.
Olfa Maslah4daeb1b51aab
diff --git a/.env.example b/.env.exampleindex dc3278889..847e757b7 100644--- a/.env.example+++ b/.env.example@@ -18,6 +18,10 @@ DEFAULT_DOCS_URL=https://docs.openr.ag# Crawl depth used when DEFAULT_DOCS_INGEST_SOURCE=urlDEFAULT_DOCS_CRAWL_DEPTH=2+# Check for newer OpenRAG docs and update knowledge on every startup.+# Default: false (manual fetch remains available in UI)+FETCH_OPENRAG_DOCS_AT_STARTUP=false+# Langflow HTTP timeout configuration (in seconds)# For large documents (300+ pages), ingestion can take 30+ minutes# Increase these values if you experience timeouts with very large PDFsdiff --git a/frontend/app/api/mutations/useRefreshOpenragDocs.ts b/frontend/app/api/mutations/useRefreshOpenragDocs.tsnew file mode 100644index 000000000..14b6289a2--- /dev/null+++ b/frontend/app/api/mutations/useRefreshOpenragDocs.ts@@ -0,0 +1,37 @@+"use client";++import { useMutation, useQueryClient } from "@tanstack/react-query";++interface RefreshOpenRAGDocsResponse {+ message: string;+ refreshed: boolean;+}++const refreshOpenragDocs = async (): Promise<RefreshOpenRAGDocsResponse> => {+ const response = await fetch("/api/openrag-docs/refresh", {+ method: "POST",+ headers: {+ "Content-Type": "application/json",+ },+ });++ if (!response.ok) {+ const error = await response.json();+ throw new Error(error.detail || error.error || "Failed to refresh OpenRAG docs");+ }++ return response.json();+};++export const useRefreshOpenragDocs = () => {+ const queryClient = useQueryClient();++ return useMutation({+ mutationFn: refreshOpenragDocs,+ onSettled: () => {+ queryClient.invalidateQueries({ queryKey: ["tasks"], exact: false });+ queryClient.invalidateQueries({ queryKey: ["search"], exact: false });+ queryClient.invalidateQueries({ queryKey: ["settings"], exact: false });+ },+ });+};diff --git a/frontend/app/knowledge/page.tsx b/frontend/app/knowledge/page.tsxindex abca9792a..98e30fe2a 100644--- a/frontend/app/knowledge/page.tsx+++ b/frontend/app/knowledge/page.tsx@@ -43,6 +43,7 @@ import GoogleDriveIcon from "../../components/icons/google-drive-logo";import OneDriveIcon from "../../components/icons/one-drive-logo";import SharePointIcon from "../../components/icons/share-point-logo";import { useDeleteDocument } from "../api/mutations/useDeleteDocument";+import { useRefreshOpenragDocs } from "../api/mutations/useRefreshOpenragDocs";import { useSyncAllConnectors } from "../api/mutations/useSyncConnector";// Function to get the appropriate icon for a connector type@@ -86,6 +87,7 @@ function SearchPage() {const deleteDocumentMutation = useDeleteDocument();const syncAllConnectorsMutation = useSyncAllConnectors();+ const refreshOpenragDocsMutation = useRefreshOpenragDocs();useEffect(() => {refreshTasks();@@ -379,6 +381,37 @@ function SearchPage() {{/* Search Input Area */}<div className="flex-1 flex items-center flex-shrink-0 flex-wrap-reverse gap-3 mb-6"><KnowledgeSearchInput />+ <Button+ type="button"+ variant="outline"+ className="rounded-lg flex-shrink-0"+ disabled={refreshOpenragDocsMutation.isPending}+ onClick={async () => {+ try {+ toast.info("Refreshing OpenRAG docs...");+ const result = await refreshOpenragDocsMutation.mutateAsync();+ toast.success(result.message);+ } catch (error) {+ toast.error(+ error instanceof Error+ ? error.message+ : "Failed to refresh OpenRAG docs",+ );+ }+ }}+ >+ {refreshOpenragDocsMutation.isPending ? (+ <>+ <RefreshCw className="h-4 w-4 mr-2 animate-spin" />+ Refreshing docs...+ </>+ ) : (+ <>+ <RefreshCw className="h-4 w-4 mr-2" />+ Fetch latest docs+ </>+ )}+ </Button><Buttontype="button"variant="outline"diff --git a/src/api/settings.py b/src/api/settings.pyindex 5e6329887..d3b83546d 100644--- a/src/api/settings.py+++ b/src/api/settings.py@@ -8,6 +8,7 @@ from utils.logging_config import get_loggerfrom utils.telemetry import TelemetryClient, Category, MessageIdfrom utils.version_utils import OPENRAG_VERSIONfrom config.settings import (+ DEFAULT_DOCS_URL,DISABLE_INGEST_WITH_LANGFLOW,INGEST_SAMPLE_DATA,LANGFLOW_URL,@@ -86,6 +87,7 @@ class OnboardingStateBody(BaseModel):openrag_docs_filter_id: Optional[str] = Noneuser_doc_filter_id: Optional[str] = Noneopenrag_docs_ingested_version: Optional[str] = None+ openrag_docs_remote_signature: Optional[str] = Noneclass DoclingPresetBody(BaseModel):@@ -104,6 +106,7 @@ class OnboardingStateConfig(BaseModel):openrag_docs_filter_id: Optional[str]user_doc_filter_id: Optional[str]openrag_docs_ingested_version: Optional[str]+ openrag_docs_remote_signature: Optional[str]class OpenAIProviderConfig(BaseModel):has_api_key: bool@@ -172,6 +175,11 @@ class OnboardingResponse(BaseModel):sample_data_ingested: boolopenrag_docs_filter_id: Optional[str] = None++class RefreshOpenRAGDocsResponse(BaseModel):+ message: str+ refreshed: bool+class DoclingConfig(BaseModel):do_ocr: boolocr_engine: str@@ -315,6 +323,7 @@ async def get_settings(openrag_docs_filter_id=openrag_config.onboarding.openrag_docs_filter_id,user_doc_filter_id=openrag_config.onboarding.user_doc_filter_id,openrag_docs_ingested_version=openrag_config.onboarding.openrag_docs_ingested_version,+ openrag_docs_remote_signature=openrag_config.onboarding.openrag_docs_remote_signature,),providers=ProvidersConfig(openai=OpenAIProviderConfig(@@ -1015,6 +1024,17 @@ async def onboarding(session_manager,)current_config.onboarding.openrag_docs_ingested_version = OPENRAG_VERSION+ from main import (+ _get_remote_docs_signature,+ _should_use_url_default_docs_ingest,+ )++ if _should_use_url_default_docs_ingest():+ current_config.onboarding.openrag_docs_remote_signature = (+ await _get_remote_docs_signature(DEFAULT_DOCS_URL)+ )+ else:+ current_config.onboarding.openrag_docs_remote_signature = Nonelogger.info("Sample data ingestion completed successfully")except Exception as e:@@ -1522,6 +1542,7 @@ async def rollback_onboarding(current_config.knowledge.embedding_provider = "openai" # Reset to defaultcurrent_config.knowledge.embedding_model = ""current_config.onboarding.openrag_docs_ingested_version = None+ current_config.onboarding.openrag_docs_remote_signature = None# Mark config as not edited so user can go through onboarding againcurrent_config.edited = False@@ -1638,3 +1659,38 @@ async def update_docling_preset(except Exception as e:logger.error("Failed to update docling settings", error=str(e))raise HTTPException(status_code=500, detail=f"Failed to update docling settings: {str(e)}")+++async def refresh_openrag_docs(+ document_service=Depends(get_document_service),+ task_service=Depends(get_task_service),+ langflow_file_service=Depends(get_langflow_file_service),+ session_manager=Depends(get_session_manager),+ user: User = Depends(get_current_user),+) -> RefreshOpenRAGDocsResponse:+ """Manually refresh OpenRAG docs ingestion on demand."""+ try:+ from main import refresh_default_openrag_docs++ refreshed = await refresh_default_openrag_docs(+ document_service=document_service,+ task_service=task_service,+ langflow_file_service=langflow_file_service,+ session_manager=session_manager,+ force=True,+ reason="manual",+ )+ return RefreshOpenRAGDocsResponse(+ message=(+ "OpenRAG docs were refreshed."+ if refreshed+ else "OpenRAG docs refresh was skipped by current configuration."+ ),+ refreshed=refreshed,+ )+ except Exception as e:+ logger.error("Failed to refresh OpenRAG docs on demand", error=str(e))+ raise HTTPException(+ status_code=500,+ detail=f"Failed to refresh OpenRAG docs: {str(e)}",+ )diff --git a/src/config/config_manager.py b/src/config/config_manager.pyindex 67cbd58d2..78f39f853 100644--- a/src/config/config_manager.py+++ b/src/config/config_manager.py@@ -98,6 +98,7 @@ class OnboardingState:openrag_docs_filter_id: Optional[str] = field(default=None)user_doc_filter_id: Optional[str] = field(default=None)openrag_docs_ingested_version: Optional[str] = field(default=None)+ openrag_docs_remote_signature: Optional[str] = field(default=None)@dataclassdiff --git a/src/config/settings.py b/src/config/settings.pyindex c5578218a..6fd6eddb5 100644--- a/src/config/settings.py+++ b/src/config/settings.py@@ -67,6 +67,9 @@ INGEST_SAMPLE_DATA = os.getenv(DEFAULT_DOCS_INGEST_SOURCE = os.getenv("DEFAULT_DOCS_INGEST_SOURCE", "url").lower()DEFAULT_DOCS_URL = os.getenv("DEFAULT_DOCS_URL", "https://docs.openr.ag")DEFAULT_DOCS_CRAWL_DEPTH = get_env_int("DEFAULT_DOCS_CRAWL_DEPTH", 2)+FETCH_OPENRAG_DOCS_AT_STARTUP = os.getenv(+ "FETCH_OPENRAG_DOCS_AT_STARTUP", "false"+).lower() in ("true", "1", "yes")# Maximum number of files to upload / ingest (in batch) per task when adding knowledge via folderUPLOAD_BATCH_SIZE = get_env_int("UPLOAD_BATCH_SIZE", 25)diff --git a/src/main.py b/src/main.pyindex 1c67fdaf8..f07dfbee9 100644--- a/src/main.py+++ b/src/main.py@@ -1,6 +1,8 @@from utils.version_utils import OPENRAG_VERSIONimport asyncioimport atexit+import hashlib+import httpximport osimport subprocess@@ -50,6 +52,7 @@ from config.settings import (API_KEYS_INDEX_BODY,API_KEYS_INDEX_NAME,DISABLE_INGEST_WITH_LANGFLOW,+ FETCH_OPENRAG_DOCS_AT_STARTUP,INGESTION_TIMEOUT,INDEX_BODY,LANGFLOW_URL_INGEST_FLOW_ID,@@ -325,6 +328,16 @@ def _get_documents_dir():return path+def _should_use_url_default_docs_ingest() -> bool:+ """Return whether default docs ingestion should use URL crawling."""+ return (+ DEFAULT_DOCS_INGEST_SOURCE == "url"+ and not DISABLE_INGEST_WITH_LANGFLOW+ and bool(LANGFLOW_URL_INGEST_FLOW_ID)+ and bool(DEFAULT_DOCS_URL)+ )++async def ingest_default_documents_when_ready(document_service, task_service, langflow_file_service, session_manager):@@ -336,11 +349,7 @@ async def ingest_default_documents_when_ready(ingest_source=DEFAULT_DOCS_INGEST_SOURCE,)await TelemetryClient.send_event(Category.DOCUMENT_INGESTION, MessageId.ORB_DOC_DEFAULT_START)- use_url_ingest = (- DEFAULT_DOCS_INGEST_SOURCE == "url"- and not DISABLE_INGEST_WITH_LANGFLOW- and bool(LANGFLOW_URL_INGEST_FLOW_ID)- )+ use_url_ingest = _should_use_url_default_docs_ingest()if (DEFAULT_DOCS_INGEST_SOURCE == "url"and not DISABLE_INGEST_WITH_LANGFLOW@@ -578,7 +587,7 @@ async def _reingest_default_docs_on_upgrade_if_needed(should_reingest = Trueif not should_reingest:- return+ return Falselogger.info("Detected OpenRAG upgrade; reingesting default docs",@@ -593,11 +602,124 @@ async def _reingest_default_docs_on_upgrade_if_needed(session_manager,)config.onboarding.openrag_docs_ingested_version = current_version+ if _should_use_url_default_docs_ingest():+ # Refresh signature metadata after upgrade reingestion so startup+ # signature checks don't trigger an immediate duplicate ingest.+ config.onboarding.openrag_docs_remote_signature = await _get_remote_docs_signature(+ DEFAULT_DOCS_URL+ )+ else:+ config.onboarding.openrag_docs_remote_signature = Noneif not config_manager.save_config_file(config):logger.warning(- "Default docs were reingested but failed to persist ingested version",+ "Default docs were reingested but failed to persist metadata",current_version=current_version,+ signature=config.onboarding.openrag_docs_remote_signature,+ )+ return True+++async def _get_remote_docs_signature(docs_url: str):+ """Get a signature for remote docs to detect content updates."""+ try:+ async with httpx.AsyncClient(timeout=10.0, follow_redirects=True) as client:+ head_response = await client.head(docs_url)+ if head_response.status_code >= 400:+ get_response = await client.get(docs_url)+ if get_response.status_code >= 400:+ logger.warning(+ "Failed to fetch remote docs signature",+ docs_url=docs_url,+ status_code=get_response.status_code,+ )+ return None+ return hashlib.sha256(get_response.text.encode("utf-8")).hexdigest()++ etag = (head_response.headers.get("etag") or "").strip()+ last_modified = (head_response.headers.get("last-modified") or "").strip()+ if etag or last_modified:+ return f"etag={etag}|last_modified={last_modified}"++ # HEAD has no body. If cache headers are missing, fetch the page body.+ get_response = await client.get(docs_url)+ if get_response.status_code >= 400:+ logger.warning(+ "Failed to fetch remote docs signature body fallback",+ docs_url=docs_url,+ status_code=get_response.status_code,+ )+ return None+ return hashlib.sha256(get_response.text.encode("utf-8")).hexdigest()+ except Exception as e:+ logger.warning(+ "Unable to retrieve remote docs signature",+ docs_url=docs_url,+ error=str(e),+ )+ return None+++async def refresh_default_openrag_docs(+ document_service,+ task_service,+ langflow_file_service,+ session_manager,+ force: bool = False,+ reason: str = "startup",+):+ """Refresh OpenRAG docs if remote content changed or when forced."""+ if not _should_use_url_default_docs_ingest():+ logger.info(+ "Skipping OpenRAG docs refresh: URL ingestion is not active",+ ingest_source=DEFAULT_DOCS_INGEST_SOURCE,+ disable_langflow_ingest=DISABLE_INGEST_WITH_LANGFLOW,+ has_url_ingest_flow_id=bool(LANGFLOW_URL_INGEST_FLOW_ID),+ has_docs_url=bool(DEFAULT_DOCS_URL),+ )+ return False++ config = get_openrag_config()+ if not config.edited:+ logger.info("Skipping OpenRAG docs refresh: onboarding not completed")+ return False++ signature = await _get_remote_docs_signature(DEFAULT_DOCS_URL)+ if not signature and not force:+ return False++ previous_signature = config.onboarding.openrag_docs_remote_signature+ should_refresh = force or (signature is not None and signature != previous_signature)+ if not should_refresh:+ logger.info(+ "OpenRAG docs refresh skipped: remote signature unchanged",+ signature=signature,+ )+ return False++ logger.info(+ "Refreshing default OpenRAG docs",+ reason=reason,+ force=force,+ previous_signature=previous_signature,+ new_signature=signature,+ )+ await _delete_existing_default_docs(session_manager)+ await ingest_default_documents_when_ready(+ document_service,+ task_service,+ langflow_file_service,+ session_manager,+ )+ config.onboarding.openrag_docs_ingested_version = OPENRAG_VERSION+ if signature:+ config.onboarding.openrag_docs_remote_signature = signature+ if not config_manager.save_config_file(config):+ logger.warning(+ "OpenRAG docs refreshed but failed to persist metadata",+ version=config.onboarding.openrag_docs_ingested_version,+ signature=config.onboarding.openrag_docs_remote_signature,)+ return Trueasync def health_check(request: Request):"""Simple liveness probe: Indicates that the OpenRAG Backend service is online and running."""@@ -745,8 +867,9 @@ async def startup_tasks(services):await configure_alerting_security()# Reingest bundled OpenRAG docs once after application upgrade.+ upgrade_reingested = Falsetry:- await _reingest_default_docs_on_upgrade_if_needed(+ upgrade_reingested = await _reingest_default_docs_on_upgrade_if_needed(services["document_service"],services["task_service"],services["langflow_file_service"],@@ -755,6 +878,19 @@ async def startup_tasks(services):except Exception as e:logger.warning("Default docs reingestion on upgrade failed", error=str(e))+ if FETCH_OPENRAG_DOCS_AT_STARTUP and not upgrade_reingested:+ try:+ await refresh_default_openrag_docs(+ services["document_service"],+ services["task_service"],+ services["langflow_file_service"],+ services["session_manager"],+ force=False,+ reason="startup",+ )+ except Exception as e:+ logger.warning("OpenRAG docs startup refresh failed", error=str(e))+# Update MCP servers with provider credentials (especially important for no-auth mode)await _update_mcp_servers_with_provider_credentials(services)@@ -966,6 +1102,7 @@ async def create_app():app.add_api_route("/settings", settings.get_settings, methods=["GET"], tags=["internal"])app.add_api_route("/settings", settings.update_settings, methods=["POST"], tags=["internal"])app.add_api_route("/onboarding/state", settings.update_onboarding_state, methods=["POST"], tags=["internal"])+ app.add_api_route("/openrag-docs/refresh", settings.refresh_openrag_docs, methods=["POST"], tags=["internal"])# Provider health check endpointapp.add_api_route("/provider/health", provider_health.check_provider_health, methods=["GET"], tags=["internal"])diff --git a/tests/unit/test_main_docs_signature.py b/tests/unit/test_main_docs_signature.pynew file mode 100644index 000000000..c2c45a697--- /dev/null+++ b/tests/unit/test_main_docs_signature.py@@ -0,0 +1,77 @@+import hashlib++import pytest+++class _FakeResponse:+ def __init__(self, status_code: int, headers: dict | None = None, text: str = ""):+ self.status_code = status_code+ self.headers = headers or {}+ self.text = text+++async def test_signature_fallback_uses_get_body_when_head_has_no_cache_headers(monkeypatch):+ """When HEAD lacks ETag/Last-Modified, fallback must hash GET body content."""+ from main import _get_remote_docs_signature++ calls: list[str] = []++ class FakeAsyncClient:+ def __init__(self, *args, **kwargs):+ pass++ async def __aenter__(self):+ return self++ async def __aexit__(self, exc_type, exc, tb):+ return False++ async def head(self, _url):+ calls.append("head")+ return _FakeResponse(200, headers={}, text="")++ async def get(self, _url):+ calls.append("get")+ return _FakeResponse(200, headers={}, text="docs-content-v1")++ monkeypatch.setattr("main.httpx.AsyncClient", FakeAsyncClient)++ signature = await _get_remote_docs_signature("https://docs.example")+ expected = hashlib.sha256("docs-content-v1".encode("utf-8")).hexdigest()++ assert signature == expected+ assert calls == ["head", "get"]+++async def test_signature_changes_when_get_body_changes_without_cache_headers(monkeypatch):+ """Without cache headers, different GET content should produce different signatures."""+ from main import _get_remote_docs_signature++ get_bodies = ["docs-content-v1", "docs-content-v2"]++ class FakeAsyncClient:+ def __init__(self, *args, **kwargs):+ pass++ async def __aenter__(self):+ return self++ async def __aexit__(self, exc_type, exc, tb):+ return False++ async def head(self, _url):+ return _FakeResponse(200, headers={}, text="")++ async def get(self, _url):+ return _FakeResponse(200, headers={}, text=get_bodies.pop(0))++ monkeypatch.setattr("main.httpx.AsyncClient", FakeAsyncClient)++ signature_1 = await _get_remote_docs_signature("https://docs.example")+ signature_2 = await _get_remote_docs_signature("https://docs.example")++ assert signature_1 is not None+ assert signature_2 is not None+ assert signature_1 != signature_2