OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.
Python
from typing import Any
from opensearchpy.exceptions import NotFoundError
from config.settings import get_index_name
from services.search_cache import invalidate_corpus_caches
from utils.file_utils import clean_connector_filename
from utils.logging_config import get_logger
from .base import (
CONTENT_ETAG_METADATA_KEY,
BaseConnector,
ConnectorDocument,
normalize_etag,
)
from .connection_manager import ConnectionManager
logger = get_logger(__name__)
def _next_page_token(file_list: dict[str, Any]) -> str | None:
"""Read the continuation token out of a ``list_files`` result.
Every connector in this repo returns ``next_page_token``; SharePoint is the
one that populates it with a real value (a Graph ``$skiptoken``). This used
to read ``nextPageToken`` only — a key no connector emits — so the token was
always None and paging stopped after the first call. Both spellings are
accepted now so a connector written against either convention still pages.
"""
return file_list.get("next_page_token") or file_list.get("nextPageToken")
class ConnectorService:
"""Service to manage document connectors and process files"""
def __init__(
self,
patched_async_client=None,
embed_model: str = "",
index_name: str = "",
task_service=None,
session_manager=None,
models_service=None,
document_service=None,
docling_service=None,
flows_service=None,
langflow_service=None,
):
self.clients = patched_async_client
self.embed_model = embed_model
self.index_name = index_name
self.task_service = task_service
self.session_manager = session_manager
self.connection_manager = ConnectionManager()
self.models_service = models_service
self.document_service = document_service
self.docling_service = docling_service
self.flows_service = flows_service
self.langflow_service = langflow_service
async def initialize(self):
"""Initialize the service by loading existing connections"""
await self.connection_manager.load_connections()
async def get_connector(self, connection_id: str) -> BaseConnector | None:
"""Get a connector by connection ID"""
return await self.connection_manager.get_connector(connection_id)
async def _get_effective_sync_jwt(
self,
user_id: str,
jwt_token: str | None = None,
) -> str | None:
"""Return a current OpenSearch JWT for connector sync work."""
if not self.session_manager:
return jwt_token
user = self.session_manager.get_user(user_id)
if user is None and user_id:
from session_manager import User
user = User(
user_id=user_id,
email=user_id,
name=user_id,
provider="connector",
)
self.session_manager.users[user_id] = user
if user is None:
return self.session_manager.get_effective_jwt_token(user_id, jwt_token)
effective_token = jwt_token or user.jwt_token
if (
effective_token is None
and getattr(self.session_manager, "private_key", None) is not None
):
return self.session_manager.create_jwt_token(user)
return self.session_manager.get_effective_jwt_token(user.user_id, effective_token)
async def _update_connector_metadata(
self,
document: ConnectorDocument,
owner_user_id: str,
connector_type: str,
jwt_token: str = None,
indexed_filename: str | None = None,
):
"""Update indexed chunks with connector-specific metadata"""
from utils.acl_utils import update_document_acl
logger.debug("Looking for chunks", document_id=document.id)
# Get user's OpenSearch client
opensearch_client = self.session_manager.get_user_opensearch_client(
owner_user_id, jwt_token
)
write_client = self.clients.opensearch
if write_client is None:
raise RuntimeError("Backend OpenSearch write client is unavailable")
# Update ACL if changed (hash-based skip optimization).
# Match both document_id and connector_file_id: both the Langflow and
# non-Langflow ingestion paths store the raw connector id in
# connector_file_id (document_id holds a content/id hash), except for
# pre-migration chunks indexed before that split existed, which only
# have document_id set to the raw connector id.
acl_result = await update_document_acl(
document_id=document.id,
acl=document.acl,
opensearch_client=opensearch_client,
write_opensearch_client=write_client,
id_fields=("document_id", "connector_file_id"),
)
# Log ACL update result
if acl_result["status"] == "unchanged":
logger.debug(f"ACL unchanged for {document.id}, skipped update")
elif acl_result["status"] == "updated":
logger.info(
f"Updated ACL for {document.id}, {acl_result['chunks_updated']} chunks updated"
)
elif acl_result["status"] == "error":
logger.error(f"ACL update error for {document.id}: {acl_result.get('error')}")
# Update other metadata fields (source_url, timestamps, etc.)
# Use the backend client for writes; the scoped client above is only
# used for DLS visibility/ACL-change checks.
try:
await write_client.update_by_query(
index=get_index_name(),
body={
# Match both fields: both ingestion paths carry the raw
# connector id in connector_file_id (document_id is a
# content/id hash); pre-migration chunks only have it in
# document_id.
"query": {
"bool": {
"should": [
{"term": {"document_id": document.id}},
{"term": {"connector_file_id": document.id}},
# See check_document_exists (models/processors.py):
# some indices predate the explicit keyword
# mapping for this field.
{"term": {"connector_file_id.keyword": document.id}},
],
"minimum_should_match": 1,
}
},
"script": {
"source": """
ctx._source.source_url = params.source_url;
ctx._source.connector_type = params.connector_type;
if (params.filename != null) {
ctx._source.filename = params.filename;
}
if (params.created_time != null) {
ctx._source.created_time = params.created_time;
}
if (params.modified_time != null) {
ctx._source.modified_time = params.modified_time;
}
if (params.content_etag != null) {
ctx._source.content_etag = params.content_etag;
}
if (params.metadata != null) {
ctx._source.metadata = params.metadata;
}
""",
"params": {
"source_url": document.source_url,
"connector_type": connector_type,
"filename": indexed_filename or document.filename,
"created_time": document.created_time.isoformat()
if document.created_time
else None,
"modified_time": document.modified_time.isoformat()
if document.modified_time
else None,
# Promoted out of metadata to a top-level keyword so
# sync can read it back with a terms aggregation (the
# same shape connector_file_id uses) instead of
# fetching every chunk's _source.
"content_etag": normalize_etag(
(document.metadata or {}).get(CONTENT_ETAG_METADATA_KEY)
),
"metadata": document.metadata,
},
},
},
refresh=True,
)
invalidate_corpus_caches()
logger.debug(f"Updated metadata for document {document.id}")
except Exception as e:
# A missing index means the chunks aren't where this write expects
# them (e.g. a residual index-name mismatch, issue 81583). The
# document is already indexed; metadata enrichment is best-effort
# and re-runs on the next sync, so don't fail the file over it —
# matching get_synced_file_ids_for_connector / should_update_acl.
if (
isinstance(e, NotFoundError)
and e.status_code == 404
and e.error == "index_not_found_exception"
):
logger.warning(
"Skipping connector metadata enrichment — index not found",
document_id=document.id,
index=get_index_name(),
error=str(e),
)
return
logger.error(
"OpenSearch metadata update failed",
document_id=document.id,
error=str(e),
)
raise
async def sync_connector_files(
self,
connection_id: str,
user_id: str,
max_files: int = None,
jwt_token: str = None,
filename_filter: set = None,
ingest_settings: dict[str, Any] | None = None,
replace_duplicates: bool = False,
shared: bool | None = False,
*,
allow_anonymous_delete: bool,
) -> str:
"""
Sync files from a connector connection using existing task tracking system.
Args:
connection_id: The connection ID
user_id: The user ID
max_files: Maximum number of files to sync
jwt_token: Optional JWT token
filename_filter: Optional set of filenames to filter - only files with names
in this set will be synced. Used to prevent deleted files
from being re-synced.
ingest_settings: Optional UI-style dict (``embeddingModel``, ``chunkSize``, …)
forwarded to ``ConnectorFileProcessor``.
allow_anonymous_delete: The syncing user's resolved
``knowledge:delete:anonymous``. No default; see
``TaskProcessor.delete_document_by_filename``.
"""
jwt_token = await self._get_effective_sync_jwt(user_id, jwt_token)
if not self.task_service:
raise ValueError(
"TaskService not available - connector sync requires task service dependency"
)
logger.debug(
"Starting sync for connection",
connection_id=connection_id,
max_files=max_files,
)
connector = await self.get_connector(connection_id)
if not connector:
raise ValueError(f"Connection '{connection_id}' not found or not authenticated")
logger.debug("Got connector", authenticated=connector.is_authenticated)
if not connector.is_authenticated:
raise ValueError(f"Connection '{connection_id}' not authenticated")
if shared and connector.CONNECTOR_TYPE != "ibm_cos":
raise ValueError("shared flag is only supported for the ibm_cos connector")
# Collect files to process (limited by max_files)
files_to_process: list[dict[str, Any]] = []
page_token = None
# A zero cap means "sync nothing" and has to short-circuit before the
# first list_files call: every cap below is spelled `if max_files and …`,
# so 0 would fall through as "no cap" and enumerate the whole source.
# None (no cap) and positive caps take the loop as before.
while max_files != 0:
# Pass max_files straight through — None means "no cap". Asking for a
# synthetic page size instead silently truncated every sync: the
# connectors that paginate internally (all three bucket ones, and
# Google Drive) honour the cap and then report next_page_token=None,
# so there was no token to continue with and everything past the
# first page was simply dropped.
logger.debug("Calling list_files", max_files=max_files, page_token=page_token)
file_list = await connector.list_files(page_token, max_files=max_files)
logger.debug("Got files from connector", file_count=len(file_list.get("files", [])))
files = file_list["files"]
page_token = _next_page_token(file_list)
if not files and not page_token:
break
for file_info in files:
if max_files and len(files_to_process) >= max_files:
break
# Filter by filename if filter is provided
if filename_filter is not None:
file_name = file_info.get("name", "")
if file_name not in filename_filter:
logger.debug(
"Skipping file not in filter",
filename=file_name,
)
continue
files_to_process.append(file_info)
# Stop if we have enough files or no more pages
if (max_files and len(files_to_process) >= max_files) or not page_token:
break
# Get user information
user = self.session_manager.get_user(user_id) if self.session_manager else None
owner_name = user.name if user else None
owner_email = user.email if user else None
# Create custom processor for connector files
from models.processors import ConnectorFileProcessor
from services.document_service import DocumentService
processor = ConnectorFileProcessor(
self,
connection_id,
files_to_process,
user_id,
jwt_token=jwt_token,
owner_name=owner_name,
owner_email=owner_email,
document_service=(
self.task_service.document_service
if self.task_service and self.task_service.document_service
else DocumentService(session_manager=self.session_manager)
),
models_service=self.models_service,
ingest_settings=ingest_settings,
replace_duplicates=replace_duplicates,
connector_type=connector.CONNECTOR_TYPE,
shared=shared,
allow_anonymous_delete=allow_anonymous_delete,
)
# Use file IDs as items (no more fake file paths!)
file_ids = [file_info["id"] for file_info in files_to_process]
original_filenames = {
file_info["id"]: clean_connector_filename(
file_info["name"], file_info.get("mimeType") or file_info.get("mimetype")
)
for file_info in files_to_process
if "name" in file_info
}
# Create custom task using TaskService
task_id = await self.task_service.create_custom_task(
user_id,
file_ids,
processor,
original_filenames=original_filenames,
)
return task_id
async def sync_specific_files(
self,
connection_id: str,
user_id: str,
file_ids: list[str],
jwt_token: str = None,
file_infos: list[dict[str, Any]] = None,
ingest_settings: dict[str, Any] | None = None,
replace_duplicates: bool = False,
preview_mode: bool = False,
shared: bool | None = False,
*,
allow_anonymous_delete: bool,
) -> str:
"""
Sync specific files by their IDs (used for webhook-triggered syncs or manual selection).
Automatically expands folders to their contents.
Args:
connection_id: The connection ID
user_id: The user ID
file_ids: List of file IDs to sync
jwt_token: Optional JWT token for authentication
file_infos: Optional list of file info dicts with {id, name, mimeType, downloadUrl, size}
When provided, download URLs can be used directly without Graph API calls.
ingest_settings: Optional UI-style dict (``embeddingModel``, ``chunkSize``, …) passed to
``ConnectorFileProcessor`` when Langflow ingest is disabled.
allow_anonymous_delete: The syncing user's resolved
``knowledge:delete:anonymous``. No default; see
``TaskProcessor.delete_document_by_filename``.
"""
jwt_token = await self._get_effective_sync_jwt(user_id, jwt_token)
if not self.task_service:
raise ValueError(
"TaskService not available - connector sync requires task service dependency"
)
connector = await self.get_connector(connection_id)
if not connector:
raise ValueError(f"Connection '{connection_id}' not found or not authenticated")
if not connector.is_authenticated:
raise ValueError(f"Connection '{connection_id}' not authenticated")
if shared and connector.CONNECTOR_TYPE != "ibm_cos":
raise ValueError("shared flag is only supported for the ibm_cos connector")
if not file_ids:
raise ValueError("No file IDs provided")
# Get user information
user = self.session_manager.get_user(user_id) if self.session_manager else None
owner_name = user.name if user else None
owner_email = user.email if user else None
# If file_infos provided, cache them in the connector for later use
# This allows get_file_content to use download URLs directly
if file_infos and hasattr(connector, "set_file_infos"):
connector.set_file_infos(file_infos)
logger.info(f"Cached {len(file_infos)} file infos with download URLs in connector")
expanded_file_ids = file_ids # Default to original IDs
expanded_files_info = []
try:
# cfg is None on bucket connectors (azure_blob/aws_s3/ibm_cos): they
# have no per-call file/folder selection to expand, and file_ids are
# already the exact ids to sync. Only cfg-backed connectors
# (Google Drive/OneDrive/SharePoint) expand folders here. Guarding on
# cfg-is-not-None rather than hasattr is deliberate: BaseConnector
# declares cfg=None as a class default, so hasattr is True for every
# connector and would route bucket syncs through list_selected_files
# -> list_files() (the whole account), discarding the selected ids.
if getattr(connector, "cfg", None) is not None:
result = await connector.list_selected_files(file_ids)
expanded_files = result.get("files", [])
expanded_file_ids = [f["id"] for f in expanded_files]
for f in expanded_files:
expanded_files_info.append(f)
# Requested IDs that vanished during expansion are either
# folders (replaced by their children) or gone at the source
# (deleted/trashed). Re-add the non-folder ones so the
# processor can run its deleted-at-source cleanup
# (get_file_content -> 404 -> delete indexed chunks).
expanded_set = set(expanded_file_ids)
known_folder_ids = {
f["id"] for f in (file_infos or []) if f.get("isFolder") and f.get("id")
}
missing_ids = [
fid
for fid in file_ids
if fid not in expanded_set and fid not in known_folder_ids
]
if missing_ids:
logger.info(
f"Re-adding {len(missing_ids)} file id(s) missing after expansion "
f"(possibly deleted at source)"
)
expanded_file_ids = expanded_file_ids + missing_ids
if not expanded_file_ids:
logger.warning(
f"No files found after expanding file_ids. "
f"Original IDs: {file_ids}. This may indicate all IDs were folders "
f"with no contents, or files that were filtered out."
)
if file_infos:
non_folder_infos = [f for f in file_infos if not f.get("isFolder")]
non_folder_ids = [f["id"] for f in non_folder_infos if f.get("id")]
if non_folder_ids:
logger.info(
"Using original file IDs with cached download URLs (folders excluded)"
)
expanded_file_ids = non_folder_ids
else:
raise ValueError("No files to sync after expanding folders")
else:
raise ValueError("No files to sync after expanding folders")
except Exception as e:
logger.error(f"Failed to expand file_ids via list_files(): {e}")
if isinstance(e, ValueError):
raise
if file_infos:
non_folder_ids = [
f["id"] for f in file_infos if f.get("id") and not f.get("isFolder")
]
expanded_file_ids = non_folder_ids or file_ids
else:
expanded_file_ids = file_ids
# Create custom processor for specific connector files
from models.processors import ConnectorFileProcessor
from services.document_service import DocumentService
# Use expanded_file_ids which has folders already expanded
processor = ConnectorFileProcessor(
self,
connection_id,
expanded_file_ids,
user_id,
jwt_token=jwt_token,
owner_name=owner_name,
owner_email=owner_email,
document_service=(
self.task_service.document_service
if self.task_service and self.task_service.document_service
else DocumentService(session_manager=self.session_manager)
),
models_service=self.models_service,
ingest_settings=ingest_settings,
replace_duplicates=replace_duplicates,
connector_type=connector.CONNECTOR_TYPE,
shared=shared,
allow_anonymous_delete=allow_anonymous_delete,
)
# Create custom task using TaskService
original_filenames = {}
# Combine file_infos and expanded_files_info
all_infos = (file_infos or []) + expanded_files_info
if all_infos:
original_filenames = {
f["id"]: clean_connector_filename(
f["name"], f.get("mimeType") or f.get("mime_type") or f.get("mimetype", "")
)
for f in all_infos
if "id" in f and "name" in f
}
task_id = await self.task_service.create_custom_task(
user_id,
expanded_file_ids,
processor,
original_filenames=original_filenames,
preview_mode=preview_mode,
)
return task_id
async def _get_connector(self, connection_id: str) -> BaseConnector | None:
"""Get a connector by connection ID (alias for get_connector)"""
return await self.get_connector(connection_id)