OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.
Python
import copy
import os
from abc import ABC, abstractmethod
from dataclasses import dataclass
from datetime import datetime
from typing import Any
# Key under which a connector reports its object's entity tag in
# ``ConnectorDocument.metadata``. Bucket connectors should set it from the
# store's ETag; sync change detection persists it and compares it against a
# fresh listing, which is how a modified object is noticed even when the two
# timestamps can't be compared. See api.connectors.classify_remote_file_change.
CONTENT_ETAG_METADATA_KEY = "content_etag"
def normalize_etag(value: Any) -> str | None:
"""Normalize an object-store entity tag for comparison, or None if empty.
Strips the weak-validator prefix and the surrounding quotes that S3-style
and HTTP entity tags carry, so a tag read back out of the index compares
equal to the one a fresh listing reports even if the two APIs quote it
differently.
The multipart suffix (``-12``) is deliberately kept: it is part of the
identity, so identical bytes uploaded with different part sizes compare as
different and get re-ingested, rather than being skipped on a guess.
"""
if not isinstance(value, str):
return None
tag = value.strip()
if tag[:2].upper() == "W/":
tag = tag[2:].strip()
return tag.strip('"').strip() or None
@dataclass
class DocumentACL:
"""Access Control List information for a document"""
owner: str = None
allowed_users: list[str] = None
allowed_groups: list[str] = None
allowed_principals: list[str] = None
allowed_principal_labels: list[dict[str, Any]] = None
def __post_init__(self):
if self.allowed_users is None:
self.allowed_users = []
if self.allowed_groups is None:
self.allowed_groups = []
if self.allowed_principals is None:
self.allowed_principals = []
if self.allowed_principal_labels is None:
self.allowed_principal_labels = []
@dataclass
class ConnectorDocument:
"""Document from a connector with metadata"""
id: str
filename: str
mimetype: str
content: bytes
source_url: str
acl: DocumentACL
modified_time: datetime
created_time: datetime
metadata: dict[str, Any] = None
def __post_init__(self):
if self.metadata is None:
self.metadata = {}
class BaseConnector(ABC):
"""Base class for all document connectors"""
# Each connector must define the environment variable names for OAuth credentials
CLIENT_ID_ENV_VAR: str = None
CLIENT_SECRET_ENV_VAR: str = None
# Key used to look up a workspace-level OAuth credential override
# (see services.connector_oauth_config_service). Falls back to
# CONNECTOR_TYPE when unset. Connectors that share one OAuth app
# registration (e.g. OneDrive + SharePoint both use "microsoft_graph")
# should set this explicitly so they resolve to the same override.
OAUTH_CREDENTIAL_KEY: str | None = None
# Stable identifier used in connections.json and on the wire (e.g. "google_drive").
CONNECTOR_TYPE: str = None
# "oauth" connectors authenticate per-user via OAuth env-var credentials.
# "bucket" connectors authenticate via per-connection config dict (HMAC, API key, etc).
# "managed" connectors own their persistent resources and do not create a
# ConnectionManager entry (the URL connector is the first example).
CONNECTOR_KIND: str = "oauth"
ALWAYS_CONNECTED: bool = False
# When a sync re-processes files that are already indexed, pass
# replace_duplicates so the indexed copy is replaced and content changes
# propagate, instead of being skipped by the duplicate-filename gate.
# Default True so a new connector gets change-propagating sync out of the
# box (the content-hash "unchanged" short-circuit still avoids re-indexing
# identical bytes on the non-Langflow path). Bucket connectors normally
# never consult this: their sync paths divert to modified_time change
# detection and replace only the blobs that actually changed.
SYNC_REPLACES_DUPLICATES: bool = True
# How sync decides whether an already-indexed file needs re-ingesting:
# "replace_always" — re-process every indexed file on sync, replacing the
# indexed copy (the content-hash short-circuit still
# skips identical bytes on the non-Langflow path).
# "timestamp" — list the source once and re-ingest only the files
# that actually differ: by entity tag where the
# listing and the index both have one, otherwise by a
# strictly newer remote modified_time. Connectors on
# this mode should report "etag" in list_files() and
# CONTENT_ETAG_METADATA_KEY in get_file_content()
# metadata; without them detection falls back to
# timestamps alone (see bucket_changed_file_ids).
CHANGE_DETECTION: str = "replace_always"
# Connector-specific keys in the config dict that must be encrypted at rest.
SECRET_CONFIG_KEYS: tuple = ()
# Connector metadata for UI
CONNECTOR_NAME: str = None
CONNECTOR_DESCRIPTION: str = None
CONNECTOR_ICON: str = None # Icon identifier or emoji
# Per-connection settings (file_ids/folder_ids selection, etc.). Declared
# here so subclasses that set a concrete cfg type still satisfy the base
# type; list_selected_files() scopes it via a per-call shallow copy.
cfg: Any = None
def __init__(self, config: dict[str, Any]):
self.config = config
self._authenticated = False
@classmethod
def register_routes(cls, app) -> None:
"""Register connector-specific FastAPI routes on the given app.
Default: no-op. Connectors with extra endpoints (bucket listing,
credential configuration, etc.) should override this and add their
own `app.add_api_route(...)` calls.
"""
return None
@classmethod
def is_available(cls, manager, user_id: str | None = None) -> bool:
"""Whether this connector should be offered to the given user.
Default (oauth-kind): env credentials present OR user has a saved connection.
Bucket-kind connectors should override to gate on their own feature flag.
"""
try:
instance = cls({})
instance.get_client_id()
instance.get_client_secret()
return True
except (ValueError, NotImplementedError, RuntimeError):
return manager._has_saved_credentials_for_user(cls.CONNECTOR_TYPE, user_id)
def _oauth_credential_key(self) -> str:
return self.OAUTH_CREDENTIAL_KEY or self.CONNECTOR_TYPE
def get_client_id(self) -> str:
"""Get the OAuth client ID.
Resolution order: per-connection config override, workspace-level
admin override (see services.connector_oauth_config_service), then
the environment variable.
"""
if not self.CLIENT_ID_ENV_VAR:
raise NotImplementedError(f"{self.__class__.__name__} must define CLIENT_ID_ENV_VAR")
config_client_id = self.config.get("client_id")
if isinstance(config_client_id, str) and config_client_id.strip():
return config_client_id
from services.connector_oauth_config_service import get_cached_client_id
override = get_cached_client_id(self._oauth_credential_key())
if override:
return override
client_id = os.getenv(self.CLIENT_ID_ENV_VAR)
if not client_id:
raise ValueError(f"Environment variable {self.CLIENT_ID_ENV_VAR} is not set")
return client_id
def get_client_secret(self) -> str:
"""Get the OAuth client secret.
Resolution order: per-connection config override, workspace-level
admin override (see services.connector_oauth_config_service), then
the environment variable.
"""
if not self.CLIENT_SECRET_ENV_VAR:
raise NotImplementedError(
f"{self.__class__.__name__} must define CLIENT_SECRET_ENV_VAR"
)
config_client_secret = self.config.get("client_secret")
if isinstance(config_client_secret, str) and config_client_secret.strip():
return config_client_secret
from services.connector_oauth_config_service import get_cached_client_secret
override = get_cached_client_secret(self._oauth_credential_key())
if override:
return override
secret = os.getenv(self.CLIENT_SECRET_ENV_VAR)
if not secret:
raise ValueError(f"Environment variable {self.CLIENT_SECRET_ENV_VAR} is not set")
return secret
@abstractmethod
async def authenticate(self) -> bool:
"""Authenticate with the service"""
pass
@abstractmethod
async def setup_subscription(self) -> str:
"""Set up real-time subscription for file changes. Returns subscription ID."""
pass
@abstractmethod
async def list_files(
self, page_token: str | None = None, max_files: int | None = None, **kwargs: Any
) -> dict[str, Any]:
"""List all files. Returns files and next_page_token if any."""
pass
async def list_selected_files(
self,
file_ids: list[str],
folder_ids: list[str] | None = None,
**kwargs: Any,
) -> dict[str, Any]:
"""List specific files/folders by scoping cfg to this call.
Callers that need list_files() to operate on a specific set of ids
previously had to manually save cfg.file_ids / cfg.folder_ids, overwrite
them, call list_files(), then restore in a finally block. This method
owns that scoping so the caller never has to.
Connectors are cached and shared across requests by ConnectionManager,
so we operate on a per-call shallow copy with its own cfg instead of
mutating (and racing on) the shared config.
"""
cfg = getattr(self, "cfg", None)
if cfg is None:
# No per-call selection state to scope by. Falling back to
# list_files() would list the connector's entire account and
# silently ignore file_ids, so refuse instead: callers must gate on
# `cfg is not None` and use file_ids directly for cfg-less
# (bucket) connectors.
raise NotImplementedError(
f"{type(self).__name__} has no cfg; list_selected_files is only "
"supported on cfg-backed connectors (Google Drive/OneDrive/SharePoint)."
)
scoped = copy.copy(self)
scoped.cfg = copy.copy(cfg)
scoped.cfg.file_ids = file_ids
scoped.cfg.folder_ids = folder_ids
return await scoped.list_files(**kwargs)
@abstractmethod
async def get_file_content(self, file_id: str) -> ConnectorDocument:
"""Get file content and metadata"""
pass
@abstractmethod
async def handle_webhook(self, payload: dict[str, Any]) -> list[str]:
"""Handle webhook notification. Returns list of affected file IDs."""
pass
def handle_webhook_validation(
self, request_method: str, headers: dict[str, str], query_params: dict[str, str]
) -> str | None:
"""Handle webhook validation (e.g., for subscription setup).
Returns validation response if applicable, None otherwise.
Default implementation returns None (no validation needed)."""
return None
def extract_webhook_channel_id(
self, payload: dict[str, Any], headers: dict[str, str]
) -> str | None:
"""Extract channel/subscription ID from webhook payload/headers.
Must be implemented by each connector."""
raise NotImplementedError(
f"{self.__class__.__name__} must implement extract_webhook_channel_id"
)
@abstractmethod
async def cleanup_subscription(self, subscription_id: str) -> bool:
"""Clean up subscription"""
pass
async def renew_subscription(self, subscription_id: str) -> str | None:
"""Extend an existing subscription in place.
Returns the new expiration (ISO-8601) on success, or None if the
connector does not support in-place renewal — the caller then falls
back to cleanup_subscription + setup_subscription."""
return None
@property
def is_authenticated(self) -> bool:
return self._authenticated
async def get_current_user_principal_labels(self) -> list[dict[str, Any]]:
"""Return non-authoritative display labels for current-user ACL principals."""
return []
async def _detect_base_url(self) -> str | None:
"""Auto-detect base URL for the connector.
Default implementation returns None.
Subclasses (OneDrive, SharePoint) should override this method.
"""
return None
async def get_current_user_group_roles(self) -> list[str]:
"""Return OpenSearch backend roles for the current connector user.
Connectors that support upstream group ACLs can override this hook.
The core ACL service calls it generically so new connectors only need
to implement their own provider-specific group lookup.
"""
return []
async def get_current_user_principals(self) -> list[str]:
"""Return provider-scoped ACL principals for the current connector user.
Connectors that store user ACLs in provider-specific identity spaces can
override this hook. The DLS principal service calls it generically so new
connectors only need to provide their own alias resolution.
"""
return []
@classmethod
def get_auth_user_principals(cls, user: Any) -> list[str]:
"""Return connector principals derivable from the OpenRAG auth user.
This hook covers cases where a document ACL names a provider user alias
but the current OpenRAG user has no saved connector connection to query.
Connectors should only return aliases when the auth provider gives enough
information to construct the same principal used during ingestion.
"""
return []