Skip to content

OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.

Python

fix: keep default docs title the same for every type of ingestion (#2118)

Lucas Oliveiracb822d1cb98f
diff --git a/frontend/tests/core/fetch_latest_docs.spec.ts b/frontend/tests/core/fetch_latest_docs.spec.ts
index 57bb90f89..93ea5e9c5 100644
--- a/frontend/tests/core/fetch_latest_docs.spec.ts
+++ b/frontend/tests/core/fetch_latest_docs.spec.ts
@@ -3,29 +3,50 @@ import logger from "../utils/logger";
import { navigateToKnowledge } from "../utils/navigation";
test.describe("Fetch Latest Docs Functionality", () => {
- test("Verify Fetch Latest Docs works and refreshes the document list @33219240", async ({
+ test("Verify Fetch Latest Docs works and refreshes the document list with and without Langflow @33219240", async ({
page,
knowledge,
+ settings,
}) => {
- test.setTimeout(180000); // 3 minutes timeout
+ test.setTimeout(360000); // 6 minutes timeout
logger.info("Starting Fetch Latest Docs E2E test");
+ const expectedDocName = "What is OpenRAG? | OpenRAG";
+
+ // Navigate initially to load the app UI (sidebar with Settings link)
+ await navigateToKnowledge(page);
+ logger.info("Successfully loaded the app UI");
+
+ // --- Phase 1: Test with Langflow Ingestion DISABLED (Traditional OpenRAG processing) ---
+ logger.info("PHASE 1: Testing with Langflow Ingestion DISABLED");
+ await settings.setDisableLangflowIngestion(true);
+
+ await navigateToKnowledge(page);
+ logger.info("Successfully navigated to Knowledge page");
+
+ logger.info("Triggering fetchLatestDocs()...");
+ await knowledge.fetchLatestDocs();
+ logger.info("fetchLatestDocs() execution completed successfully");
+
+ logger.info(`Verifying default document '${expectedDocName}' is active`);
+ await knowledge.verifyDocumentActive(expectedDocName);
+
+ // --- Phase 2: Test with Langflow Ingestion ENABLED (Langflow processing) ---
+ logger.info("PHASE 2: Testing with Langflow Ingestion ENABLED");
+ await settings.setDisableLangflowIngestion(false);
- // 1. Navigate to the Knowledge page
await navigateToKnowledge(page);
logger.info("Successfully navigated to Knowledge page");
- // 2. Click the "Fetch latest docs" button and verify success toast is shown
logger.info("Triggering fetchLatestDocs()...");
await knowledge.fetchLatestDocs();
logger.info("fetchLatestDocs() execution completed successfully");
- // 3. Verify that the OpenRAG docs URL document appears and becomes active
- const expectedDocName =
- process.env.DEFAULT_DOCS_URL || "https://docs.openr.ag/";
logger.info(`Verifying default document '${expectedDocName}' is active`);
await knowledge.verifyDocumentActive(expectedDocName);
- logger.info("Fetch Latest Docs E2E test passed successfully!");
+ logger.info(
+ "Fetch Latest Docs E2E test passed successfully for both modes!",
+ );
});
});
diff --git a/frontend/tests/pages/Settings.ts b/frontend/tests/pages/Settings.ts
index 510cd5e76..d890b6a47 100644
--- a/frontend/tests/pages/Settings.ts
+++ b/frontend/tests/pages/Settings.ts
@@ -29,6 +29,8 @@ export class Settings {
private readonly chunkSizeInput = () => this.page.getByLabel(/chunk size/i);
private readonly chunkOverlapInput = () =>
this.page.getByLabel(/chunk overlap/i);
+ private readonly disableLangflowIngestionToggle = () =>
+ this.page.getByRole("switch", { name: /disable langflow ingestion/i });
private readonly watsonxProjectIDInput = () =>
this.page.locator("#project-id");
private readonly apiKeyInput = () => this.page.locator("#api-key");
@@ -236,6 +238,18 @@ export class Settings {
}
}
+ async setDisableLangflowIngestion(enabled: boolean) {
+ await this.clickTab("Langflow");
+ const toggle = this.disableLangflowIngestionToggle();
+ await toggle.scrollIntoViewIfNeeded();
+ const state = await toggle.getAttribute("data-state");
+ const isChecked = state === "checked";
+ if (isChecked !== enabled) {
+ await toggle.click();
+ await this.saveIngestSettings();
+ }
+ }
+
async selectModel(section: string, model: string) {
await this.open();
const dropdown = this.getModelDropdown(section);
diff --git a/src/services/default_docs_service.py b/src/services/default_docs_service.py
index 7202a6bcd..06100711a 100644
--- a/src/services/default_docs_service.py
+++ b/src/services/default_docs_service.py
@@ -316,7 +316,7 @@ async def _ingest_default_documents_url(
docs_url=docs_url,
crawl_depth=crawl_depth,
)
- temp_file_path = await materialize_url_as_text_file(
+ temp_file_path, title = await materialize_url_as_text_file(
docs_url=docs_url,
crawl_depth=crawl_depth,
)
@@ -341,7 +341,7 @@ async def _ingest_default_documents_url(
file_path=temp_file_path,
file_hash=hash_id(temp_file_path),
owner_user_id=None,
- original_filename="openrag-url-default.txt",
+ original_filename=title,
jwt_token=jwt_token,
owner_name=anonymous_user.name,
owner_email=anonymous_user.email,
diff --git a/src/services/document_index_writer.py b/src/services/document_index_writer.py
index 0db09b860..11df4e804 100644
--- a/src/services/document_index_writer.py
+++ b/src/services/document_index_writer.py
@@ -24,9 +24,9 @@ logger = get_logger(__name__)
@dataclass
class DocumentIndexContext:
document_id: str
- filename: str
mimetype: str
embedding_model: str
+ filename: str | None = None
owner: str | None = None
owner_name: str | None = None
owner_email: str | None = None
diff --git a/src/services/langflow_file_service.py b/src/services/langflow_file_service.py
index 3be3c48dd..ff560aca7 100644
--- a/src/services/langflow_file_service.py
+++ b/src/services/langflow_file_service.py
@@ -192,10 +192,10 @@ class LangflowFileService:
self,
*,
document_id: str,
- filename: str,
mimetype: str,
file_size: int,
embedding_model: str,
+ filename: str | None = None,
owner: str | None,
owner_name: str | None,
owner_email: str | None,
@@ -651,7 +651,6 @@ class LangflowFileService:
}
ingest_token, ingest_run_id = self._configure_ingest_callback(
document_id=resolved_document_id,
- filename=docs_url,
mimetype="text/html",
file_size=0,
embedding_model=embedding_model,
diff --git a/src/utils/url_content_fetcher.py b/src/utils/url_content_fetcher.py
index e9bfb8071..7f9426016 100644
--- a/src/utils/url_content_fetcher.py
+++ b/src/utils/url_content_fetcher.py
@@ -36,8 +36,8 @@ class _VisibleTextHTMLParser(HTMLParser):
return " ".join(self._chunks)
-async def materialize_url_as_text_file(docs_url: str, crawl_depth: int) -> str:
- """Fetch URL content and write a temporary text file for OpenRAG ingestion."""
+async def materialize_url_as_text_file(docs_url: str, crawl_depth: int) -> tuple[str, str]:
+ """Fetch URL content and write a temporary text file for OpenRAG ingestion, returning path and title."""
async with httpx.AsyncClient(timeout=30.0, follow_redirects=True) as client:
response = await client.get(docs_url)
response.raise_for_status()
@@ -48,7 +48,8 @@ async def materialize_url_as_text_file(docs_url: str, crawl_depth: int) -> str:
raw_html,
flags=re.IGNORECASE | re.DOTALL,
)
- title = html.unescape(title_match.group(1).strip()) if title_match else "OpenRAG"
+ title = html.unescape(title_match.group(1).strip()) if title_match else ""
+ title = title or "OpenRAG"
text_parser = _VisibleTextHTMLParser()
text_parser.feed(raw_html)
@@ -68,4 +69,4 @@ async def materialize_url_as_text_file(docs_url: str, crawl_depth: int) -> str:
)
with temp_file:
temp_file.write(content)
- return temp_file.name
+ return temp_file.name, title