OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.
Python
refactor: isolate URL processor and scope OpenRAG doc filters
Olfa Maslah27b1213f844e
diff --git a/frontend/app/knowledge/page.tsx b/frontend/app/knowledge/page.tsxindex b027bad17..f07000088 100644--- a/frontend/app/knowledge/page.tsx+++ b/frontend/app/knowledge/page.tsx@@ -164,10 +164,10 @@ function SearchPage() {taskFilesAsFiles.map((file) => [file.filename, file]),);// Override backend files with task file status if they exist.- // Keep system_default rows sourced from indexed search results so+ // Keep openrag_docs rows sourced from indexed search results so// OpenRAG docs do not appear as pending in the table.const backendFiles = (searchData as File[]).map((file) => {- if (file.connector_type === "system_default") {+ if (file.connector_type === "openrag_docs") {return file;}const taskFile = taskFileMap.get(file.filename);@@ -187,9 +187,9 @@ function SearchPage() {) {return false;}- // Do not render task-only system_default placeholder rows in the table.+ // Do not render task-only openrag_docs placeholder rows in the table.// OpenRAG default docs should be represented only by indexed search results.- if (taskFile.connector_type === "system_default") {+ if (taskFile.connector_type === "openrag_docs") {return false;}return (@@ -379,7 +379,7 @@ function SearchPage() {cellRenderer: ({ data }: CustomCellRendererProps<File>) => {const status = data?.status || "active";const connectorType = data?.connector_type;- if (status !== "active" || connectorType === "system_default") {+ if (status !== "active" || connectorType === "openrag_docs") {return null;}return (diff --git a/frontend/app/onboarding/_components/onboarding-content.tsx b/frontend/app/onboarding/_components/onboarding-content.tsxindex c2b85a317..c07db632e 100644--- a/frontend/app/onboarding/_components/onboarding-content.tsx+++ b/frontend/app/onboarding/_components/onboarding-content.tsx@@ -22,7 +22,7 @@ const OPENRAG_DOCS_FILTERS: SelectedFilters = {data_sources: [],document_types: [],owners: [],- connector_types: ["system_default"],+ connector_types: ["openrag_docs"],};export function OnboardingContent({diff --git a/src/api/settings.py b/src/api/settings.pyindex ac5957f25..66fb56971 100644--- a/src/api/settings.py+++ b/src/api/settings.py@@ -1278,11 +1278,11 @@ async def _create_openrag_docs_filter("query": "","filters": {# URL-based docs ingestion produces many source URLs.- # Filter by connector type to target the OpenRAG system docs set.+ # Filter by connector type to target OpenRAG docs only."data_sources": ["*"],"document_types": ["*"],"owners": ["*"],- "connector_types": ["system_default"],+ "connector_types": ["openrag_docs"],},"limit": 10,"scoreThreshold": 0,diff --git a/src/models/processors.py b/src/models/processors.pyindex 2392ed712..d317a88fa 100644--- a/src/models/processors.py+++ b/src/models/processors.py@@ -819,148 +819,4 @@ class LangflowFileProcessor(TaskProcessor):raise-class LangflowUrlProcessor(TaskProcessor):- """Processor for Langflow URL ingestion flow as a tracked task."""-- def __init__(- self,- langflow_file_service,- session_manager,- docs_url: str,- crawl_depth: int,- owner_user_id: str = None,- jwt_token: str = None,- owner_name: str = None,- owner_email: str = None,- connector_type: str = "system_default",- prevent_outside: bool = True,- ):- super().__init__()- self.langflow_file_service = langflow_file_service- self.session_manager = session_manager- self.docs_url = docs_url- self.crawl_depth = crawl_depth- self.owner_user_id = owner_user_id- self.jwt_token = jwt_token- self.owner_name = owner_name- self.owner_email = owner_email- self.connector_type = connector_type- self.prevent_outside = prevent_outside-- async def _count_system_default_docs(self, opensearch_client) -> int:- """Count indexed OpenRAG docs for the current owner/context."""- from config.settings import get_index_name-- must_filters = [- {- "bool": {- "should": [- {"term": {"connector_type.keyword": self.connector_type}},- {"term": {"connector_type": self.connector_type}},- ],- "minimum_should_match": 1,- }- }- ]- if self.owner_email:- must_filters.append(- {- "bool": {- "should": [- {"term": {"owner_email.keyword": self.owner_email}},- {"match_phrase": {"owner_email": self.owner_email}},- ],- "minimum_should_match": 1,- }- }- )- if self.owner_user_id:- must_filters.append(- {- "bool": {- "should": [- {"term": {"owner.keyword": self.owner_user_id}},- {"match_phrase": {"owner": self.owner_user_id}},- ],- "minimum_should_match": 1,- }- }- )-- count_query = {- "query": {- "bool": {- "must": must_filters- }- }- }- response = await opensearch_client.count(- index=get_index_name(),- body=count_query,- )- return int(response.get("count", 0))-- async def process_item(- self, upload_task: UploadTask, item: str, file_task: FileTask- ) -> None:- """Process one URL ingestion item through Langflow."""- from models.tasks import TaskStatus- import asyncio- import time-- file_task.status = TaskStatus.RUNNING- file_task.updated_at = time.time()-- try:- effective_jwt = self.jwt_token- if self.session_manager and not effective_jwt:- self.session_manager.get_user_opensearch_client(- self.owner_user_id, self.jwt_token- )- if hasattr(self.session_manager, "_anonymous_jwt"):- effective_jwt = self.session_manager._anonymous_jwt-- opensearch_client = self.session_manager.get_user_opensearch_client(- self.owner_user_id, effective_jwt- )- docs_before = await self._count_system_default_docs(opensearch_client)-- result = await self.langflow_file_service.run_url_ingestion_flow(- docs_url=self.docs_url,- crawl_depth=self.crawl_depth,- jwt_token=effective_jwt,- owner=self.owner_user_id,- owner_name=self.owner_name,- owner_email=self.owner_email,- connector_type=self.connector_type,- prevent_outside=self.prevent_outside,- )- docs_after = await self._count_system_default_docs(opensearch_client)- # OpenSearch visibility can lag indexing by a refresh interval.- # Retry a few times before declaring ingestion failed.- for _ in range(5):- if docs_after > 0:- break- await asyncio.sleep(1)- docs_after = await self._count_system_default_docs(opensearch_client)- if docs_after <= 0:- raise ValueError(- "URL ingestion completed but no OpenRAG docs were indexed"- )- logger.info(- "URL ingestion indexed OpenRAG docs",- docs_before=docs_before,- docs_after=docs_after,- )-- file_task.status = TaskStatus.COMPLETED- file_task.result = result- file_task.updated_at = time.time()- upload_task.successful_files += 1-- except Exception as e:- file_task.status = TaskStatus.FAILED- file_task.error = str(e)- file_task.updated_at = time.time()- upload_task.failed_files += 1- raise+from .url import LangflowUrlProcessordiff --git a/src/models/url.py b/src/models/url.pynew file mode 100644index 000000000..cec18ec31--- /dev/null+++ b/src/models/url.py@@ -0,0 +1,145 @@+import asyncio+import time++from utils.logging_config import get_logger+from .processors import TaskProcessor+from .tasks import UploadTask, FileTask, TaskStatus++logger = get_logger(__name__)+++class LangflowUrlProcessor(TaskProcessor):+ """Processor for Langflow URL ingestion flow as a tracked task."""++ def __init__(+ self,+ langflow_file_service,+ session_manager,+ docs_url: str,+ crawl_depth: int,+ owner_user_id: str = None,+ jwt_token: str = None,+ owner_name: str = None,+ owner_email: str = None,+ connector_type: str = "system_default",+ prevent_outside: bool = True,+ ):+ super().__init__()+ self.langflow_file_service = langflow_file_service+ self.session_manager = session_manager+ self.docs_url = docs_url+ self.crawl_depth = crawl_depth+ self.owner_user_id = owner_user_id+ self.jwt_token = jwt_token+ self.owner_name = owner_name+ self.owner_email = owner_email+ self.connector_type = connector_type+ self.prevent_outside = prevent_outside++ async def _count_system_default_docs(self, opensearch_client) -> int:+ """Count indexed OpenRAG docs for the current owner/context."""+ from config.settings import get_index_name++ must_filters = [+ {+ "bool": {+ "should": [+ {"term": {"connector_type.keyword": self.connector_type}},+ {"term": {"connector_type": self.connector_type}},+ ],+ "minimum_should_match": 1,+ }+ }+ ]+ if self.owner_email:+ must_filters.append(+ {+ "bool": {+ "should": [+ {"term": {"owner_email.keyword": self.owner_email}},+ {"match_phrase": {"owner_email": self.owner_email}},+ ],+ "minimum_should_match": 1,+ }+ }+ )+ if self.owner_user_id:+ must_filters.append(+ {+ "bool": {+ "should": [+ {"term": {"owner.keyword": self.owner_user_id}},+ {"match_phrase": {"owner": self.owner_user_id}},+ ],+ "minimum_should_match": 1,+ }+ }+ )++ count_query = {"query": {"bool": {"must": must_filters}}}+ response = await opensearch_client.count(+ index=get_index_name(),+ body=count_query,+ )+ return int(response.get("count", 0))++ async def process_item(+ self, upload_task: UploadTask, item: str, file_task: FileTask+ ) -> None:+ """Process one URL ingestion item through Langflow."""+ file_task.status = TaskStatus.RUNNING+ file_task.updated_at = time.time()++ try:+ effective_jwt = self.jwt_token+ if self.session_manager and not effective_jwt:+ self.session_manager.get_user_opensearch_client(+ self.owner_user_id, self.jwt_token+ )+ if hasattr(self.session_manager, "_anonymous_jwt"):+ effective_jwt = self.session_manager._anonymous_jwt++ opensearch_client = self.session_manager.get_user_opensearch_client(+ self.owner_user_id, effective_jwt+ )+ docs_before = await self._count_system_default_docs(opensearch_client)++ result = await self.langflow_file_service.run_url_ingestion_flow(+ docs_url=self.docs_url,+ crawl_depth=self.crawl_depth,+ jwt_token=effective_jwt,+ owner=self.owner_user_id,+ owner_name=self.owner_name,+ owner_email=self.owner_email,+ connector_type=self.connector_type,+ prevent_outside=self.prevent_outside,+ )+ docs_after = await self._count_system_default_docs(opensearch_client)+ # OpenSearch visibility can lag indexing by a refresh interval.+ # Retry a few times before declaring ingestion failed.+ for _ in range(5):+ if docs_after > 0:+ break+ await asyncio.sleep(1)+ docs_after = await self._count_system_default_docs(opensearch_client)+ if docs_after <= 0:+ raise ValueError(+ "URL ingestion completed but no OpenRAG docs were indexed"+ )+ logger.info(+ "URL ingestion indexed OpenRAG docs",+ docs_before=docs_before,+ docs_after=docs_after,+ )++ file_task.status = TaskStatus.COMPLETED+ file_task.result = result+ file_task.updated_at = time.time()+ upload_task.successful_files += 1++ except Exception as e:+ file_task.status = TaskStatus.FAILED+ file_task.error = str(e)+ file_task.updated_at = time.time()+ upload_task.failed_files += 1+ raise