Skip to content

OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.

Python

Improve default docs refresh flow to be simpler and avoid duplicate startup reingestion.

Olfa Maslah4daeb1b51aab
diff --git a/.env.example b/.env.example
index dc3278889..847e757b7 100644
--- a/.env.example
+++ b/.env.example
@@ -18,6 +18,10 @@ DEFAULT_DOCS_URL=https://docs.openr.ag
# Crawl depth used when DEFAULT_DOCS_INGEST_SOURCE=url
DEFAULT_DOCS_CRAWL_DEPTH=2
+# Check for newer OpenRAG docs and update knowledge on every startup.
+# Default: false (manual fetch remains available in UI)
+FETCH_OPENRAG_DOCS_AT_STARTUP=false
+
# Langflow HTTP timeout configuration (in seconds)
# For large documents (300+ pages), ingestion can take 30+ minutes
# Increase these values if you experience timeouts with very large PDFs
diff --git a/frontend/app/api/mutations/useRefreshOpenragDocs.ts b/frontend/app/api/mutations/useRefreshOpenragDocs.ts
new file mode 100644
index 000000000..14b6289a2
--- /dev/null
+++ b/frontend/app/api/mutations/useRefreshOpenragDocs.ts
@@ -0,0 +1,37 @@
+"use client";
+
+import { useMutation, useQueryClient } from "@tanstack/react-query";
+
+interface RefreshOpenRAGDocsResponse {
+ message: string;
+ refreshed: boolean;
+}
+
+const refreshOpenragDocs = async (): Promise<RefreshOpenRAGDocsResponse> => {
+ const response = await fetch("/api/openrag-docs/refresh", {
+ method: "POST",
+ headers: {
+ "Content-Type": "application/json",
+ },
+ });
+
+ if (!response.ok) {
+ const error = await response.json();
+ throw new Error(error.detail || error.error || "Failed to refresh OpenRAG docs");
+ }
+
+ return response.json();
+};
+
+export const useRefreshOpenragDocs = () => {
+ const queryClient = useQueryClient();
+
+ return useMutation({
+ mutationFn: refreshOpenragDocs,
+ onSettled: () => {
+ queryClient.invalidateQueries({ queryKey: ["tasks"], exact: false });
+ queryClient.invalidateQueries({ queryKey: ["search"], exact: false });
+ queryClient.invalidateQueries({ queryKey: ["settings"], exact: false });
+ },
+ });
+};
diff --git a/frontend/app/knowledge/page.tsx b/frontend/app/knowledge/page.tsx
index abca9792a..98e30fe2a 100644
--- a/frontend/app/knowledge/page.tsx
+++ b/frontend/app/knowledge/page.tsx
@@ -43,6 +43,7 @@ import GoogleDriveIcon from "../../components/icons/google-drive-logo";
import OneDriveIcon from "../../components/icons/one-drive-logo";
import SharePointIcon from "../../components/icons/share-point-logo";
import { useDeleteDocument } from "../api/mutations/useDeleteDocument";
+import { useRefreshOpenragDocs } from "../api/mutations/useRefreshOpenragDocs";
import { useSyncAllConnectors } from "../api/mutations/useSyncConnector";
// Function to get the appropriate icon for a connector type
@@ -86,6 +87,7 @@ function SearchPage() {
const deleteDocumentMutation = useDeleteDocument();
const syncAllConnectorsMutation = useSyncAllConnectors();
+ const refreshOpenragDocsMutation = useRefreshOpenragDocs();
useEffect(() => {
refreshTasks();
@@ -379,6 +381,37 @@ function SearchPage() {
{/* Search Input Area */}
<div className="flex-1 flex items-center flex-shrink-0 flex-wrap-reverse gap-3 mb-6">
<KnowledgeSearchInput />
+ <Button
+ type="button"
+ variant="outline"
+ className="rounded-lg flex-shrink-0"
+ disabled={refreshOpenragDocsMutation.isPending}
+ onClick={async () => {
+ try {
+ toast.info("Refreshing OpenRAG docs...");
+ const result = await refreshOpenragDocsMutation.mutateAsync();
+ toast.success(result.message);
+ } catch (error) {
+ toast.error(
+ error instanceof Error
+ ? error.message
+ : "Failed to refresh OpenRAG docs",
+ );
+ }
+ }}
+ >
+ {refreshOpenragDocsMutation.isPending ? (
+ <>
+ <RefreshCw className="h-4 w-4 mr-2 animate-spin" />
+ Refreshing docs...
+ </>
+ ) : (
+ <>
+ <RefreshCw className="h-4 w-4 mr-2" />
+ Fetch latest docs
+ </>
+ )}
+ </Button>
<Button
type="button"
variant="outline"
diff --git a/src/api/settings.py b/src/api/settings.py
index 5e6329887..d3b83546d 100644
--- a/src/api/settings.py
+++ b/src/api/settings.py
@@ -8,6 +8,7 @@ from utils.logging_config import get_logger
from utils.telemetry import TelemetryClient, Category, MessageId
from utils.version_utils import OPENRAG_VERSION
from config.settings import (
+ DEFAULT_DOCS_URL,
DISABLE_INGEST_WITH_LANGFLOW,
INGEST_SAMPLE_DATA,
LANGFLOW_URL,
@@ -86,6 +87,7 @@ class OnboardingStateBody(BaseModel):
openrag_docs_filter_id: Optional[str] = None
user_doc_filter_id: Optional[str] = None
openrag_docs_ingested_version: Optional[str] = None
+ openrag_docs_remote_signature: Optional[str] = None
class DoclingPresetBody(BaseModel):
@@ -104,6 +106,7 @@ class OnboardingStateConfig(BaseModel):
openrag_docs_filter_id: Optional[str]
user_doc_filter_id: Optional[str]
openrag_docs_ingested_version: Optional[str]
+ openrag_docs_remote_signature: Optional[str]
class OpenAIProviderConfig(BaseModel):
has_api_key: bool
@@ -172,6 +175,11 @@ class OnboardingResponse(BaseModel):
sample_data_ingested: bool
openrag_docs_filter_id: Optional[str] = None
+
+class RefreshOpenRAGDocsResponse(BaseModel):
+ message: str
+ refreshed: bool
+
class DoclingConfig(BaseModel):
do_ocr: bool
ocr_engine: str
@@ -315,6 +323,7 @@ async def get_settings(
openrag_docs_filter_id=openrag_config.onboarding.openrag_docs_filter_id,
user_doc_filter_id=openrag_config.onboarding.user_doc_filter_id,
openrag_docs_ingested_version=openrag_config.onboarding.openrag_docs_ingested_version,
+ openrag_docs_remote_signature=openrag_config.onboarding.openrag_docs_remote_signature,
),
providers=ProvidersConfig(
openai=OpenAIProviderConfig(
@@ -1015,6 +1024,17 @@ async def onboarding(
session_manager,
)
current_config.onboarding.openrag_docs_ingested_version = OPENRAG_VERSION
+ from main import (
+ _get_remote_docs_signature,
+ _should_use_url_default_docs_ingest,
+ )
+
+ if _should_use_url_default_docs_ingest():
+ current_config.onboarding.openrag_docs_remote_signature = (
+ await _get_remote_docs_signature(DEFAULT_DOCS_URL)
+ )
+ else:
+ current_config.onboarding.openrag_docs_remote_signature = None
logger.info("Sample data ingestion completed successfully")
except Exception as e:
@@ -1522,6 +1542,7 @@ async def rollback_onboarding(
current_config.knowledge.embedding_provider = "openai" # Reset to default
current_config.knowledge.embedding_model = ""
current_config.onboarding.openrag_docs_ingested_version = None
+ current_config.onboarding.openrag_docs_remote_signature = None
# Mark config as not edited so user can go through onboarding again
current_config.edited = False
@@ -1638,3 +1659,38 @@ async def update_docling_preset(
except Exception as e:
logger.error("Failed to update docling settings", error=str(e))
raise HTTPException(status_code=500, detail=f"Failed to update docling settings: {str(e)}")
+
+
+async def refresh_openrag_docs(
+ document_service=Depends(get_document_service),
+ task_service=Depends(get_task_service),
+ langflow_file_service=Depends(get_langflow_file_service),
+ session_manager=Depends(get_session_manager),
+ user: User = Depends(get_current_user),
+) -> RefreshOpenRAGDocsResponse:
+ """Manually refresh OpenRAG docs ingestion on demand."""
+ try:
+ from main import refresh_default_openrag_docs
+
+ refreshed = await refresh_default_openrag_docs(
+ document_service=document_service,
+ task_service=task_service,
+ langflow_file_service=langflow_file_service,
+ session_manager=session_manager,
+ force=True,
+ reason="manual",
+ )
+ return RefreshOpenRAGDocsResponse(
+ message=(
+ "OpenRAG docs were refreshed."
+ if refreshed
+ else "OpenRAG docs refresh was skipped by current configuration."
+ ),
+ refreshed=refreshed,
+ )
+ except Exception as e:
+ logger.error("Failed to refresh OpenRAG docs on demand", error=str(e))
+ raise HTTPException(
+ status_code=500,
+ detail=f"Failed to refresh OpenRAG docs: {str(e)}",
+ )
diff --git a/src/config/config_manager.py b/src/config/config_manager.py
index 67cbd58d2..78f39f853 100644
--- a/src/config/config_manager.py
+++ b/src/config/config_manager.py
@@ -98,6 +98,7 @@ class OnboardingState:
openrag_docs_filter_id: Optional[str] = field(default=None)
user_doc_filter_id: Optional[str] = field(default=None)
openrag_docs_ingested_version: Optional[str] = field(default=None)
+ openrag_docs_remote_signature: Optional[str] = field(default=None)
@dataclass
diff --git a/src/config/settings.py b/src/config/settings.py
index c5578218a..6fd6eddb5 100644
--- a/src/config/settings.py
+++ b/src/config/settings.py
@@ -67,6 +67,9 @@ INGEST_SAMPLE_DATA = os.getenv(
DEFAULT_DOCS_INGEST_SOURCE = os.getenv("DEFAULT_DOCS_INGEST_SOURCE", "url").lower()
DEFAULT_DOCS_URL = os.getenv("DEFAULT_DOCS_URL", "https://docs.openr.ag")
DEFAULT_DOCS_CRAWL_DEPTH = get_env_int("DEFAULT_DOCS_CRAWL_DEPTH", 2)
+FETCH_OPENRAG_DOCS_AT_STARTUP = os.getenv(
+ "FETCH_OPENRAG_DOCS_AT_STARTUP", "false"
+).lower() in ("true", "1", "yes")
# Maximum number of files to upload / ingest (in batch) per task when adding knowledge via folder
UPLOAD_BATCH_SIZE = get_env_int("UPLOAD_BATCH_SIZE", 25)
diff --git a/src/main.py b/src/main.py
index 1c67fdaf8..f07dfbee9 100644
--- a/src/main.py
+++ b/src/main.py
@@ -1,6 +1,8 @@
from utils.version_utils import OPENRAG_VERSION
import asyncio
import atexit
+import hashlib
+import httpx
import os
import subprocess
@@ -50,6 +52,7 @@ from config.settings import (
API_KEYS_INDEX_BODY,
API_KEYS_INDEX_NAME,
DISABLE_INGEST_WITH_LANGFLOW,
+ FETCH_OPENRAG_DOCS_AT_STARTUP,
INGESTION_TIMEOUT,
INDEX_BODY,
LANGFLOW_URL_INGEST_FLOW_ID,
@@ -325,6 +328,16 @@ def _get_documents_dir():
return path
+def _should_use_url_default_docs_ingest() -> bool:
+ """Return whether default docs ingestion should use URL crawling."""
+ return (
+ DEFAULT_DOCS_INGEST_SOURCE == "url"
+ and not DISABLE_INGEST_WITH_LANGFLOW
+ and bool(LANGFLOW_URL_INGEST_FLOW_ID)
+ and bool(DEFAULT_DOCS_URL)
+ )
+
+
async def ingest_default_documents_when_ready(
document_service, task_service, langflow_file_service, session_manager
):
@@ -336,11 +349,7 @@ async def ingest_default_documents_when_ready(
ingest_source=DEFAULT_DOCS_INGEST_SOURCE,
)
await TelemetryClient.send_event(Category.DOCUMENT_INGESTION, MessageId.ORB_DOC_DEFAULT_START)
- use_url_ingest = (
- DEFAULT_DOCS_INGEST_SOURCE == "url"
- and not DISABLE_INGEST_WITH_LANGFLOW
- and bool(LANGFLOW_URL_INGEST_FLOW_ID)
- )
+ use_url_ingest = _should_use_url_default_docs_ingest()
if (
DEFAULT_DOCS_INGEST_SOURCE == "url"
and not DISABLE_INGEST_WITH_LANGFLOW
@@ -578,7 +587,7 @@ async def _reingest_default_docs_on_upgrade_if_needed(
should_reingest = True
if not should_reingest:
- return
+ return False
logger.info(
"Detected OpenRAG upgrade; reingesting default docs",
@@ -593,11 +602,124 @@ async def _reingest_default_docs_on_upgrade_if_needed(
session_manager,
)
config.onboarding.openrag_docs_ingested_version = current_version
+ if _should_use_url_default_docs_ingest():
+ # Refresh signature metadata after upgrade reingestion so startup
+ # signature checks don't trigger an immediate duplicate ingest.
+ config.onboarding.openrag_docs_remote_signature = await _get_remote_docs_signature(
+ DEFAULT_DOCS_URL
+ )
+ else:
+ config.onboarding.openrag_docs_remote_signature = None
if not config_manager.save_config_file(config):
logger.warning(
- "Default docs were reingested but failed to persist ingested version",
+ "Default docs were reingested but failed to persist metadata",
current_version=current_version,
+ signature=config.onboarding.openrag_docs_remote_signature,
+ )
+ return True
+
+
+async def _get_remote_docs_signature(docs_url: str):
+ """Get a signature for remote docs to detect content updates."""
+ try:
+ async with httpx.AsyncClient(timeout=10.0, follow_redirects=True) as client:
+ head_response = await client.head(docs_url)
+ if head_response.status_code >= 400:
+ get_response = await client.get(docs_url)
+ if get_response.status_code >= 400:
+ logger.warning(
+ "Failed to fetch remote docs signature",
+ docs_url=docs_url,
+ status_code=get_response.status_code,
+ )
+ return None
+ return hashlib.sha256(get_response.text.encode("utf-8")).hexdigest()
+
+ etag = (head_response.headers.get("etag") or "").strip()
+ last_modified = (head_response.headers.get("last-modified") or "").strip()
+ if etag or last_modified:
+ return f"etag={etag}|last_modified={last_modified}"
+
+ # HEAD has no body. If cache headers are missing, fetch the page body.
+ get_response = await client.get(docs_url)
+ if get_response.status_code >= 400:
+ logger.warning(
+ "Failed to fetch remote docs signature body fallback",
+ docs_url=docs_url,
+ status_code=get_response.status_code,
+ )
+ return None
+ return hashlib.sha256(get_response.text.encode("utf-8")).hexdigest()
+ except Exception as e:
+ logger.warning(
+ "Unable to retrieve remote docs signature",
+ docs_url=docs_url,
+ error=str(e),
+ )
+ return None
+
+
+async def refresh_default_openrag_docs(
+ document_service,
+ task_service,
+ langflow_file_service,
+ session_manager,
+ force: bool = False,
+ reason: str = "startup",
+):
+ """Refresh OpenRAG docs if remote content changed or when forced."""
+ if not _should_use_url_default_docs_ingest():
+ logger.info(
+ "Skipping OpenRAG docs refresh: URL ingestion is not active",
+ ingest_source=DEFAULT_DOCS_INGEST_SOURCE,
+ disable_langflow_ingest=DISABLE_INGEST_WITH_LANGFLOW,
+ has_url_ingest_flow_id=bool(LANGFLOW_URL_INGEST_FLOW_ID),
+ has_docs_url=bool(DEFAULT_DOCS_URL),
+ )
+ return False
+
+ config = get_openrag_config()
+ if not config.edited:
+ logger.info("Skipping OpenRAG docs refresh: onboarding not completed")
+ return False
+
+ signature = await _get_remote_docs_signature(DEFAULT_DOCS_URL)
+ if not signature and not force:
+ return False
+
+ previous_signature = config.onboarding.openrag_docs_remote_signature
+ should_refresh = force or (signature is not None and signature != previous_signature)
+ if not should_refresh:
+ logger.info(
+ "OpenRAG docs refresh skipped: remote signature unchanged",
+ signature=signature,
+ )
+ return False
+
+ logger.info(
+ "Refreshing default OpenRAG docs",
+ reason=reason,
+ force=force,
+ previous_signature=previous_signature,
+ new_signature=signature,
+ )
+ await _delete_existing_default_docs(session_manager)
+ await ingest_default_documents_when_ready(
+ document_service,
+ task_service,
+ langflow_file_service,
+ session_manager,
+ )
+ config.onboarding.openrag_docs_ingested_version = OPENRAG_VERSION
+ if signature:
+ config.onboarding.openrag_docs_remote_signature = signature
+ if not config_manager.save_config_file(config):
+ logger.warning(
+ "OpenRAG docs refreshed but failed to persist metadata",
+ version=config.onboarding.openrag_docs_ingested_version,
+ signature=config.onboarding.openrag_docs_remote_signature,
)
+ return True
async def health_check(request: Request):
"""Simple liveness probe: Indicates that the OpenRAG Backend service is online and running."""
@@ -745,8 +867,9 @@ async def startup_tasks(services):
await configure_alerting_security()
# Reingest bundled OpenRAG docs once after application upgrade.
+ upgrade_reingested = False
try:
- await _reingest_default_docs_on_upgrade_if_needed(
+ upgrade_reingested = await _reingest_default_docs_on_upgrade_if_needed(
services["document_service"],
services["task_service"],
services["langflow_file_service"],
@@ -755,6 +878,19 @@ async def startup_tasks(services):
except Exception as e:
logger.warning("Default docs reingestion on upgrade failed", error=str(e))
+ if FETCH_OPENRAG_DOCS_AT_STARTUP and not upgrade_reingested:
+ try:
+ await refresh_default_openrag_docs(
+ services["document_service"],
+ services["task_service"],
+ services["langflow_file_service"],
+ services["session_manager"],
+ force=False,
+ reason="startup",
+ )
+ except Exception as e:
+ logger.warning("OpenRAG docs startup refresh failed", error=str(e))
+
# Update MCP servers with provider credentials (especially important for no-auth mode)
await _update_mcp_servers_with_provider_credentials(services)
@@ -966,6 +1102,7 @@ async def create_app():
app.add_api_route("/settings", settings.get_settings, methods=["GET"], tags=["internal"])
app.add_api_route("/settings", settings.update_settings, methods=["POST"], tags=["internal"])
app.add_api_route("/onboarding/state", settings.update_onboarding_state, methods=["POST"], tags=["internal"])
+ app.add_api_route("/openrag-docs/refresh", settings.refresh_openrag_docs, methods=["POST"], tags=["internal"])
# Provider health check endpoint
app.add_api_route("/provider/health", provider_health.check_provider_health, methods=["GET"], tags=["internal"])
diff --git a/tests/unit/test_main_docs_signature.py b/tests/unit/test_main_docs_signature.py
new file mode 100644
index 000000000..c2c45a697
--- /dev/null
+++ b/tests/unit/test_main_docs_signature.py
@@ -0,0 +1,77 @@
+import hashlib
+
+import pytest
+
+
+class _FakeResponse:
+ def __init__(self, status_code: int, headers: dict | None = None, text: str = ""):
+ self.status_code = status_code
+ self.headers = headers or {}
+ self.text = text
+
+
+async def test_signature_fallback_uses_get_body_when_head_has_no_cache_headers(monkeypatch):
+ """When HEAD lacks ETag/Last-Modified, fallback must hash GET body content."""
+ from main import _get_remote_docs_signature
+
+ calls: list[str] = []
+
+ class FakeAsyncClient:
+ def __init__(self, *args, **kwargs):
+ pass
+
+ async def __aenter__(self):
+ return self
+
+ async def __aexit__(self, exc_type, exc, tb):
+ return False
+
+ async def head(self, _url):
+ calls.append("head")
+ return _FakeResponse(200, headers={}, text="")
+
+ async def get(self, _url):
+ calls.append("get")
+ return _FakeResponse(200, headers={}, text="docs-content-v1")
+
+ monkeypatch.setattr("main.httpx.AsyncClient", FakeAsyncClient)
+
+ signature = await _get_remote_docs_signature("https://docs.example")
+ expected = hashlib.sha256("docs-content-v1".encode("utf-8")).hexdigest()
+
+ assert signature == expected
+ assert calls == ["head", "get"]
+
+
+async def test_signature_changes_when_get_body_changes_without_cache_headers(monkeypatch):
+ """Without cache headers, different GET content should produce different signatures."""
+ from main import _get_remote_docs_signature
+
+ get_bodies = ["docs-content-v1", "docs-content-v2"]
+
+ class FakeAsyncClient:
+ def __init__(self, *args, **kwargs):
+ pass
+
+ async def __aenter__(self):
+ return self
+
+ async def __aexit__(self, exc_type, exc, tb):
+ return False
+
+ async def head(self, _url):
+ return _FakeResponse(200, headers={}, text="")
+
+ async def get(self, _url):
+ return _FakeResponse(200, headers={}, text=get_bodies.pop(0))
+
+ monkeypatch.setattr("main.httpx.AsyncClient", FakeAsyncClient)
+
+ signature_1 = await _get_remote_docs_signature("https://docs.example")
+ signature_2 = await _get_remote_docs_signature("https://docs.example")
+
+ assert signature_1 is not None
+ assert signature_2 is not None
+ assert signature_1 != signature_2