OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.
Python
fix: keep default docs title the same for every type of ingestion (#2118)
Lucas Oliveiracb822d1cb98f
diff --git a/frontend/tests/core/fetch_latest_docs.spec.ts b/frontend/tests/core/fetch_latest_docs.spec.tsindex 57bb90f89..93ea5e9c5 100644--- a/frontend/tests/core/fetch_latest_docs.spec.ts+++ b/frontend/tests/core/fetch_latest_docs.spec.ts@@ -3,29 +3,50 @@ import logger from "../utils/logger";import { navigateToKnowledge } from "../utils/navigation";test.describe("Fetch Latest Docs Functionality", () => {- test("Verify Fetch Latest Docs works and refreshes the document list @33219240", async ({+ test("Verify Fetch Latest Docs works and refreshes the document list with and without Langflow @33219240", async ({page,knowledge,+ settings,}) => {- test.setTimeout(180000); // 3 minutes timeout+ test.setTimeout(360000); // 6 minutes timeoutlogger.info("Starting Fetch Latest Docs E2E test");+ const expectedDocName = "What is OpenRAG? | OpenRAG";++ // Navigate initially to load the app UI (sidebar with Settings link)+ await navigateToKnowledge(page);+ logger.info("Successfully loaded the app UI");++ // --- Phase 1: Test with Langflow Ingestion DISABLED (Traditional OpenRAG processing) ---+ logger.info("PHASE 1: Testing with Langflow Ingestion DISABLED");+ await settings.setDisableLangflowIngestion(true);++ await navigateToKnowledge(page);+ logger.info("Successfully navigated to Knowledge page");++ logger.info("Triggering fetchLatestDocs()...");+ await knowledge.fetchLatestDocs();+ logger.info("fetchLatestDocs() execution completed successfully");++ logger.info(`Verifying default document '${expectedDocName}' is active`);+ await knowledge.verifyDocumentActive(expectedDocName);++ // --- Phase 2: Test with Langflow Ingestion ENABLED (Langflow processing) ---+ logger.info("PHASE 2: Testing with Langflow Ingestion ENABLED");+ await settings.setDisableLangflowIngestion(false);- // 1. Navigate to the Knowledge pageawait navigateToKnowledge(page);logger.info("Successfully navigated to Knowledge page");- // 2. Click the "Fetch latest docs" button and verify success toast is shownlogger.info("Triggering fetchLatestDocs()...");await knowledge.fetchLatestDocs();logger.info("fetchLatestDocs() execution completed successfully");- // 3. Verify that the OpenRAG docs URL document appears and becomes active- const expectedDocName =- process.env.DEFAULT_DOCS_URL || "https://docs.openr.ag/";logger.info(`Verifying default document '${expectedDocName}' is active`);await knowledge.verifyDocumentActive(expectedDocName);- logger.info("Fetch Latest Docs E2E test passed successfully!");+ logger.info(+ "Fetch Latest Docs E2E test passed successfully for both modes!",+ );});});diff --git a/frontend/tests/pages/Settings.ts b/frontend/tests/pages/Settings.tsindex 510cd5e76..d890b6a47 100644--- a/frontend/tests/pages/Settings.ts+++ b/frontend/tests/pages/Settings.ts@@ -29,6 +29,8 @@ export class Settings {private readonly chunkSizeInput = () => this.page.getByLabel(/chunk size/i);private readonly chunkOverlapInput = () =>this.page.getByLabel(/chunk overlap/i);+ private readonly disableLangflowIngestionToggle = () =>+ this.page.getByRole("switch", { name: /disable langflow ingestion/i });private readonly watsonxProjectIDInput = () =>this.page.locator("#project-id");private readonly apiKeyInput = () => this.page.locator("#api-key");@@ -236,6 +238,18 @@ export class Settings {}}+ async setDisableLangflowIngestion(enabled: boolean) {+ await this.clickTab("Langflow");+ const toggle = this.disableLangflowIngestionToggle();+ await toggle.scrollIntoViewIfNeeded();+ const state = await toggle.getAttribute("data-state");+ const isChecked = state === "checked";+ if (isChecked !== enabled) {+ await toggle.click();+ await this.saveIngestSettings();+ }+ }+async selectModel(section: string, model: string) {await this.open();const dropdown = this.getModelDropdown(section);diff --git a/src/services/default_docs_service.py b/src/services/default_docs_service.pyindex 7202a6bcd..06100711a 100644--- a/src/services/default_docs_service.py+++ b/src/services/default_docs_service.py@@ -316,7 +316,7 @@ async def _ingest_default_documents_url(docs_url=docs_url,crawl_depth=crawl_depth,)- temp_file_path = await materialize_url_as_text_file(+ temp_file_path, title = await materialize_url_as_text_file(docs_url=docs_url,crawl_depth=crawl_depth,)@@ -341,7 +341,7 @@ async def _ingest_default_documents_url(file_path=temp_file_path,file_hash=hash_id(temp_file_path),owner_user_id=None,- original_filename="openrag-url-default.txt",+ original_filename=title,jwt_token=jwt_token,owner_name=anonymous_user.name,owner_email=anonymous_user.email,diff --git a/src/services/document_index_writer.py b/src/services/document_index_writer.pyindex 0db09b860..11df4e804 100644--- a/src/services/document_index_writer.py+++ b/src/services/document_index_writer.py@@ -24,9 +24,9 @@ logger = get_logger(__name__)@dataclassclass DocumentIndexContext:document_id: str- filename: strmimetype: strembedding_model: str+ filename: str | None = Noneowner: str | None = Noneowner_name: str | None = Noneowner_email: str | None = Nonediff --git a/src/services/langflow_file_service.py b/src/services/langflow_file_service.pyindex 3be3c48dd..ff560aca7 100644--- a/src/services/langflow_file_service.py+++ b/src/services/langflow_file_service.py@@ -192,10 +192,10 @@ class LangflowFileService:self,*,document_id: str,- filename: str,mimetype: str,file_size: int,embedding_model: str,+ filename: str | None = None,owner: str | None,owner_name: str | None,owner_email: str | None,@@ -651,7 +651,6 @@ class LangflowFileService:}ingest_token, ingest_run_id = self._configure_ingest_callback(document_id=resolved_document_id,- filename=docs_url,mimetype="text/html",file_size=0,embedding_model=embedding_model,diff --git a/src/utils/url_content_fetcher.py b/src/utils/url_content_fetcher.pyindex e9bfb8071..7f9426016 100644--- a/src/utils/url_content_fetcher.py+++ b/src/utils/url_content_fetcher.py@@ -36,8 +36,8 @@ class _VisibleTextHTMLParser(HTMLParser):return " ".join(self._chunks)-async def materialize_url_as_text_file(docs_url: str, crawl_depth: int) -> str:- """Fetch URL content and write a temporary text file for OpenRAG ingestion."""+async def materialize_url_as_text_file(docs_url: str, crawl_depth: int) -> tuple[str, str]:+ """Fetch URL content and write a temporary text file for OpenRAG ingestion, returning path and title."""async with httpx.AsyncClient(timeout=30.0, follow_redirects=True) as client:response = await client.get(docs_url)response.raise_for_status()@@ -48,7 +48,8 @@ async def materialize_url_as_text_file(docs_url: str, crawl_depth: int) -> str:raw_html,flags=re.IGNORECASE | re.DOTALL,)- title = html.unescape(title_match.group(1).strip()) if title_match else "OpenRAG"+ title = html.unescape(title_match.group(1).strip()) if title_match else ""+ title = title or "OpenRAG"text_parser = _VisibleTextHTMLParser()text_parser.feed(raw_html)@@ -68,4 +69,4 @@ async def materialize_url_as_text_file(docs_url: str, crawl_depth: int) -> str:)with temp_file:temp_file.write(content)- return temp_file.name+ return temp_file.name, title