Skip to content

OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.

Python

refactor: isolate URL processor and scope OpenRAG doc filters

Olfa Maslah27b1213f844e
diff --git a/frontend/app/knowledge/page.tsx b/frontend/app/knowledge/page.tsx
index b027bad17..f07000088 100644
--- a/frontend/app/knowledge/page.tsx
+++ b/frontend/app/knowledge/page.tsx
@@ -164,10 +164,10 @@ function SearchPage() {
taskFilesAsFiles.map((file) => [file.filename, file]),
);
// Override backend files with task file status if they exist.
- // Keep system_default rows sourced from indexed search results so
+ // Keep openrag_docs rows sourced from indexed search results so
// OpenRAG docs do not appear as pending in the table.
const backendFiles = (searchData as File[]).map((file) => {
- if (file.connector_type === "system_default") {
+ if (file.connector_type === "openrag_docs") {
return file;
}
const taskFile = taskFileMap.get(file.filename);
@@ -187,9 +187,9 @@ function SearchPage() {
) {
return false;
}
- // Do not render task-only system_default placeholder rows in the table.
+ // Do not render task-only openrag_docs placeholder rows in the table.
// OpenRAG default docs should be represented only by indexed search results.
- if (taskFile.connector_type === "system_default") {
+ if (taskFile.connector_type === "openrag_docs") {
return false;
}
return (
@@ -379,7 +379,7 @@ function SearchPage() {
cellRenderer: ({ data }: CustomCellRendererProps<File>) => {
const status = data?.status || "active";
const connectorType = data?.connector_type;
- if (status !== "active" || connectorType === "system_default") {
+ if (status !== "active" || connectorType === "openrag_docs") {
return null;
}
return (
diff --git a/frontend/app/onboarding/_components/onboarding-content.tsx b/frontend/app/onboarding/_components/onboarding-content.tsx
index c2b85a317..c07db632e 100644
--- a/frontend/app/onboarding/_components/onboarding-content.tsx
+++ b/frontend/app/onboarding/_components/onboarding-content.tsx
@@ -22,7 +22,7 @@ const OPENRAG_DOCS_FILTERS: SelectedFilters = {
data_sources: [],
document_types: [],
owners: [],
- connector_types: ["system_default"],
+ connector_types: ["openrag_docs"],
};
export function OnboardingContent({
diff --git a/src/api/settings.py b/src/api/settings.py
index ac5957f25..66fb56971 100644
--- a/src/api/settings.py
+++ b/src/api/settings.py
@@ -1278,11 +1278,11 @@ async def _create_openrag_docs_filter(
"query": "",
"filters": {
# URL-based docs ingestion produces many source URLs.
- # Filter by connector type to target the OpenRAG system docs set.
+ # Filter by connector type to target OpenRAG docs only.
"data_sources": ["*"],
"document_types": ["*"],
"owners": ["*"],
- "connector_types": ["system_default"],
+ "connector_types": ["openrag_docs"],
},
"limit": 10,
"scoreThreshold": 0,
diff --git a/src/models/processors.py b/src/models/processors.py
index 2392ed712..d317a88fa 100644
--- a/src/models/processors.py
+++ b/src/models/processors.py
@@ -819,148 +819,4 @@ class LangflowFileProcessor(TaskProcessor):
raise
-class LangflowUrlProcessor(TaskProcessor):
- """Processor for Langflow URL ingestion flow as a tracked task."""
-
- def __init__(
- self,
- langflow_file_service,
- session_manager,
- docs_url: str,
- crawl_depth: int,
- owner_user_id: str = None,
- jwt_token: str = None,
- owner_name: str = None,
- owner_email: str = None,
- connector_type: str = "system_default",
- prevent_outside: bool = True,
- ):
- super().__init__()
- self.langflow_file_service = langflow_file_service
- self.session_manager = session_manager
- self.docs_url = docs_url
- self.crawl_depth = crawl_depth
- self.owner_user_id = owner_user_id
- self.jwt_token = jwt_token
- self.owner_name = owner_name
- self.owner_email = owner_email
- self.connector_type = connector_type
- self.prevent_outside = prevent_outside
-
- async def _count_system_default_docs(self, opensearch_client) -> int:
- """Count indexed OpenRAG docs for the current owner/context."""
- from config.settings import get_index_name
-
- must_filters = [
- {
- "bool": {
- "should": [
- {"term": {"connector_type.keyword": self.connector_type}},
- {"term": {"connector_type": self.connector_type}},
- ],
- "minimum_should_match": 1,
- }
- }
- ]
- if self.owner_email:
- must_filters.append(
- {
- "bool": {
- "should": [
- {"term": {"owner_email.keyword": self.owner_email}},
- {"match_phrase": {"owner_email": self.owner_email}},
- ],
- "minimum_should_match": 1,
- }
- }
- )
- if self.owner_user_id:
- must_filters.append(
- {
- "bool": {
- "should": [
- {"term": {"owner.keyword": self.owner_user_id}},
- {"match_phrase": {"owner": self.owner_user_id}},
- ],
- "minimum_should_match": 1,
- }
- }
- )
-
- count_query = {
- "query": {
- "bool": {
- "must": must_filters
- }
- }
- }
- response = await opensearch_client.count(
- index=get_index_name(),
- body=count_query,
- )
- return int(response.get("count", 0))
-
- async def process_item(
- self, upload_task: UploadTask, item: str, file_task: FileTask
- ) -> None:
- """Process one URL ingestion item through Langflow."""
- from models.tasks import TaskStatus
- import asyncio
- import time
-
- file_task.status = TaskStatus.RUNNING
- file_task.updated_at = time.time()
-
- try:
- effective_jwt = self.jwt_token
- if self.session_manager and not effective_jwt:
- self.session_manager.get_user_opensearch_client(
- self.owner_user_id, self.jwt_token
- )
- if hasattr(self.session_manager, "_anonymous_jwt"):
- effective_jwt = self.session_manager._anonymous_jwt
-
- opensearch_client = self.session_manager.get_user_opensearch_client(
- self.owner_user_id, effective_jwt
- )
- docs_before = await self._count_system_default_docs(opensearch_client)
-
- result = await self.langflow_file_service.run_url_ingestion_flow(
- docs_url=self.docs_url,
- crawl_depth=self.crawl_depth,
- jwt_token=effective_jwt,
- owner=self.owner_user_id,
- owner_name=self.owner_name,
- owner_email=self.owner_email,
- connector_type=self.connector_type,
- prevent_outside=self.prevent_outside,
- )
- docs_after = await self._count_system_default_docs(opensearch_client)
- # OpenSearch visibility can lag indexing by a refresh interval.
- # Retry a few times before declaring ingestion failed.
- for _ in range(5):
- if docs_after > 0:
- break
- await asyncio.sleep(1)
- docs_after = await self._count_system_default_docs(opensearch_client)
- if docs_after <= 0:
- raise ValueError(
- "URL ingestion completed but no OpenRAG docs were indexed"
- )
- logger.info(
- "URL ingestion indexed OpenRAG docs",
- docs_before=docs_before,
- docs_after=docs_after,
- )
-
- file_task.status = TaskStatus.COMPLETED
- file_task.result = result
- file_task.updated_at = time.time()
- upload_task.successful_files += 1
-
- except Exception as e:
- file_task.status = TaskStatus.FAILED
- file_task.error = str(e)
- file_task.updated_at = time.time()
- upload_task.failed_files += 1
- raise
+from .url import LangflowUrlProcessor
diff --git a/src/models/url.py b/src/models/url.py
new file mode 100644
index 000000000..cec18ec31
--- /dev/null
+++ b/src/models/url.py
@@ -0,0 +1,145 @@
+import asyncio
+import time
+
+from utils.logging_config import get_logger
+from .processors import TaskProcessor
+from .tasks import UploadTask, FileTask, TaskStatus
+
+logger = get_logger(__name__)
+
+
+class LangflowUrlProcessor(TaskProcessor):
+ """Processor for Langflow URL ingestion flow as a tracked task."""
+
+ def __init__(
+ self,
+ langflow_file_service,
+ session_manager,
+ docs_url: str,
+ crawl_depth: int,
+ owner_user_id: str = None,
+ jwt_token: str = None,
+ owner_name: str = None,
+ owner_email: str = None,
+ connector_type: str = "system_default",
+ prevent_outside: bool = True,
+ ):
+ super().__init__()
+ self.langflow_file_service = langflow_file_service
+ self.session_manager = session_manager
+ self.docs_url = docs_url
+ self.crawl_depth = crawl_depth
+ self.owner_user_id = owner_user_id
+ self.jwt_token = jwt_token
+ self.owner_name = owner_name
+ self.owner_email = owner_email
+ self.connector_type = connector_type
+ self.prevent_outside = prevent_outside
+
+ async def _count_system_default_docs(self, opensearch_client) -> int:
+ """Count indexed OpenRAG docs for the current owner/context."""
+ from config.settings import get_index_name
+
+ must_filters = [
+ {
+ "bool": {
+ "should": [
+ {"term": {"connector_type.keyword": self.connector_type}},
+ {"term": {"connector_type": self.connector_type}},
+ ],
+ "minimum_should_match": 1,
+ }
+ }
+ ]
+ if self.owner_email:
+ must_filters.append(
+ {
+ "bool": {
+ "should": [
+ {"term": {"owner_email.keyword": self.owner_email}},
+ {"match_phrase": {"owner_email": self.owner_email}},
+ ],
+ "minimum_should_match": 1,
+ }
+ }
+ )
+ if self.owner_user_id:
+ must_filters.append(
+ {
+ "bool": {
+ "should": [
+ {"term": {"owner.keyword": self.owner_user_id}},
+ {"match_phrase": {"owner": self.owner_user_id}},
+ ],
+ "minimum_should_match": 1,
+ }
+ }
+ )
+
+ count_query = {"query": {"bool": {"must": must_filters}}}
+ response = await opensearch_client.count(
+ index=get_index_name(),
+ body=count_query,
+ )
+ return int(response.get("count", 0))
+
+ async def process_item(
+ self, upload_task: UploadTask, item: str, file_task: FileTask
+ ) -> None:
+ """Process one URL ingestion item through Langflow."""
+ file_task.status = TaskStatus.RUNNING
+ file_task.updated_at = time.time()
+
+ try:
+ effective_jwt = self.jwt_token
+ if self.session_manager and not effective_jwt:
+ self.session_manager.get_user_opensearch_client(
+ self.owner_user_id, self.jwt_token
+ )
+ if hasattr(self.session_manager, "_anonymous_jwt"):
+ effective_jwt = self.session_manager._anonymous_jwt
+
+ opensearch_client = self.session_manager.get_user_opensearch_client(
+ self.owner_user_id, effective_jwt
+ )
+ docs_before = await self._count_system_default_docs(opensearch_client)
+
+ result = await self.langflow_file_service.run_url_ingestion_flow(
+ docs_url=self.docs_url,
+ crawl_depth=self.crawl_depth,
+ jwt_token=effective_jwt,
+ owner=self.owner_user_id,
+ owner_name=self.owner_name,
+ owner_email=self.owner_email,
+ connector_type=self.connector_type,
+ prevent_outside=self.prevent_outside,
+ )
+ docs_after = await self._count_system_default_docs(opensearch_client)
+ # OpenSearch visibility can lag indexing by a refresh interval.
+ # Retry a few times before declaring ingestion failed.
+ for _ in range(5):
+ if docs_after > 0:
+ break
+ await asyncio.sleep(1)
+ docs_after = await self._count_system_default_docs(opensearch_client)
+ if docs_after <= 0:
+ raise ValueError(
+ "URL ingestion completed but no OpenRAG docs were indexed"
+ )
+ logger.info(
+ "URL ingestion indexed OpenRAG docs",
+ docs_before=docs_before,
+ docs_after=docs_after,
+ )
+
+ file_task.status = TaskStatus.COMPLETED
+ file_task.result = result
+ file_task.updated_at = time.time()
+ upload_task.successful_files += 1
+
+ except Exception as e:
+ file_task.status = TaskStatus.FAILED
+ file_task.error = str(e)
+ file_task.updated_at = time.time()
+ upload_task.failed_files += 1
+ raise