OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.
Python
from datetime import UTC, datetime, timedelta
from pathlib import Path
from typing import Any
from urllib.parse import urlparse
import httpx
from utils.group_acl import unique_acl_principal_labels
from utils.logging_config import get_logger
from ..base import BaseConnector, ConnectorDocument, DocumentACL
from ..microsoft_graph_acl import (
get_current_user_microsoft_group_roles,
get_current_user_microsoft_principal_labels,
get_current_user_microsoft_principals,
get_oauth_access_token,
microsoft_group_principal_label,
microsoft_group_role,
microsoft_user_principal,
microsoft_user_principal_label,
)
from .oauth import SharePointOAuth
logger = get_logger(__name__)
class SharePointConnector(BaseConnector):
"""SharePoint connector using MSAL-based OAuth for authentication"""
# Required BaseConnector class attributes
CLIENT_ID_ENV_VAR = "MICROSOFT_GRAPH_OAUTH_CLIENT_ID"
CLIENT_SECRET_ENV_VAR = "MICROSOFT_GRAPH_OAUTH_CLIENT_SECRET" # pragma: allowlist secret
# Shared with OneDriveConnector — both use one Microsoft Graph app registration.
OAUTH_CREDENTIAL_KEY = "microsoft_graph"
# Connector metadata
CONNECTOR_TYPE = "sharepoint"
CONNECTOR_KIND = "oauth"
CONNECTOR_NAME = "SharePoint"
CONNECTOR_DESCRIPTION = "Add knowledge from SharePoint"
CONNECTOR_ICON = "sharepoint"
@classmethod
def get_oauth_class(cls):
from .oauth import SharePointOAuth
return SharePointOAuth
def __init__(self, config: dict[str, Any]):
super().__init__(config)
logger.debug(f"SharePoint connector __init__ called with config type: {type(config)}")
logger.debug(f"SharePoint connector __init__ config value: {config}")
# Ensure we always pass a valid config to the base class
if config is None:
logger.debug("Config was None, using empty dict")
config = {}
try:
logger.debug("Calling super().__init__")
super().__init__(config) # Now safe to call with empty dict instead of None
logger.debug("super().__init__ completed successfully")
except Exception as e:
logger.error(f"super().__init__ failed: {e}")
raise
# Initialize with defaults that allow the connector to be listed
self.client_id = None
self.client_secret = None
self.tenant_id = config.get("tenant_id", "common")
# Graph delta link for webhook change tracking (in-memory per instance)
self._delta_link: str | None = None
# base_url is the generic field name, sharepoint_url is kept for backward compatibility
self.sharepoint_url = config.get("base_url") or config.get("sharepoint_url")
logger.debug(
f"SharePoint connector initialized with sharepoint_url from config: {self.sharepoint_url}"
)
self.redirect_uri = config.get("redirect_uri", "http://localhost")
# Try to get credentials, but don't fail if they're missing
try:
logger.debug("Attempting to get client_id")
self.client_id = self.get_client_id()
logger.debug(f"Got client_id: {self.client_id is not None}")
except Exception as e:
logger.debug(f"Failed to get client_id: {e}")
pass # Credentials not available, that's OK for listing
try:
logger.debug("Attempting to get client_secret")
self.client_secret = self.get_client_secret()
logger.debug(f"Got client_secret: {self.client_secret is not None}")
except Exception as e:
logger.debug(f"Failed to get client_secret: {e}")
pass # Credentials not available, that's OK for listing
# Token file setup - use data directory for persistence
from config.paths import get_data_file
token_file = config.get("token_file") or get_data_file("sharepoint_token.json")
Path(token_file).parent.mkdir(parents=True, exist_ok=True)
# Only initialize OAuth if we have credentials
if self.client_id and self.client_secret:
connection_id = config.get("connection_id", "default")
# Use token_file from config if provided, otherwise generate one
if config.get("token_file"):
oauth_token_file = config["token_file"]
else:
oauth_token_file = get_data_file(f"sharepoint_token_{connection_id}.json")
authority = (
f"https://login.microsoftonline.com/{self.tenant_id}"
if self.tenant_id != "common"
else "https://login.microsoftonline.com/common"
)
self.oauth = SharePointOAuth(
client_id=self.client_id,
client_secret=self.client_secret,
token_file=oauth_token_file,
authority=authority,
)
else:
self.oauth = None
# Track subscription ID for webhooks
self._subscription_id: str | None = None
# Set by setup_subscription/renew_subscription; read by the
# connection manager to persist
self.webhook_resource_id: str | None = None
self.webhook_expiration: str | None = None
# Add Graph API defaults similar to Google Drive flags
self._graph_api_version = "v1.0"
self._default_params = {
"$select": "id,name,size,lastModifiedDateTime,createdDateTime,webUrl,file,folder,@microsoft.graph.downloadUrl"
}
# Selective sync support (similar to Google Drive and OneDrive)
self.cfg = type(
"SharePointConfig",
(),
{
"file_ids": config.get("file_ids")
or config.get("selected_files")
or config.get("selected_file_ids"),
"folder_ids": config.get("folder_ids")
or config.get("selected_folders")
or config.get("selected_folder_ids"),
},
)()
# Cache for file metadata including download URLs
# This allows direct download without Graph API for sharing IDs
self._file_infos: dict[str, dict[str, Any]] = {}
@property
def _graph_base_url(self) -> str:
"""Base URL for Microsoft Graph API calls"""
return f"https://graph.microsoft.com/{self._graph_api_version}"
@property
def base_url(self) -> str | None:
"""Generic base URL property (returns sharepoint_url for SharePoint connector)"""
return self.sharepoint_url
@base_url.setter
def base_url(self, value: str):
"""Set base URL (updates sharepoint_url internally)"""
self.sharepoint_url = value
async def get_current_user_group_roles(self) -> list[str]:
"""Return canonical group ACL roles for the connected Microsoft user."""
return await get_current_user_microsoft_group_roles(
self.oauth,
self._graph_base_url,
tenant_id=self.tenant_id,
)
async def get_current_user_principals(self) -> list[str]:
"""Return canonical user ACL principals for the connected Microsoft user."""
return await get_current_user_microsoft_principals(
self.oauth,
self._graph_base_url,
tenant_id=self.tenant_id,
)
async def get_current_user_principal_labels(self) -> list[dict[str, Any]]:
"""Return display labels for current Microsoft user/group ACL principals."""
return await get_current_user_microsoft_principal_labels(
self.oauth,
self._graph_base_url,
tenant_id=self.tenant_id,
)
def set_file_infos(self, file_infos: list[dict[str, Any]]) -> None:
"""
Cache file metadata including download URLs for later use.
This allows direct download without Graph API calls for sharing IDs.
Args:
file_infos: List of file info dicts with {id, name, mimeType, downloadUrl, size}
"""
self._file_infos = {}
for info in file_infos:
file_id = info.get("id")
if file_id:
self._file_infos[file_id] = info
if info.get("downloadUrl"):
logger.debug(f"Cached download URL for file {file_id}: {info.get('name')}")
def get_cached_file_info(self, file_id: str) -> dict[str, Any] | None:
"""Get cached file info by ID."""
return self._file_infos.get(file_id)
def emit(self, doc: ConnectorDocument) -> None:
"""
Emit a ConnectorDocument instance.
"""
logger.debug(f"Emitting SharePoint document: {doc.id} ({doc.filename})")
async def authenticate(self) -> bool:
"""Test authentication - BaseConnector interface"""
logger.debug(f"SharePoint authenticate() called, oauth is None: {self.oauth is None}")
try:
if not self.oauth:
logger.debug("SharePoint authentication failed: OAuth not initialized")
self._authenticated = False
return False
logger.debug("Loading SharePoint credentials...")
# Try to load existing credentials first
load_result = await self.oauth.load_credentials()
logger.debug(f"Load credentials result: {load_result}")
logger.debug("Checking SharePoint authentication status...")
authenticated = await self.oauth.is_authenticated()
logger.debug(f"SharePoint is_authenticated result: {authenticated}")
self._authenticated = authenticated
return authenticated
except Exception:
logger.exception("[CONNECTOR] SharePoint authentication failed")
self._authenticated = False
return False
def get_auth_url(self) -> str:
"""Get OAuth authorization URL"""
if not self.oauth:
raise RuntimeError("SharePoint OAuth not initialized - missing credentials")
return self.oauth.create_authorization_url(self.redirect_uri)
async def handle_oauth_callback(self, auth_code: str) -> dict[str, Any]:
"""Handle OAuth callback"""
if not self.oauth:
raise RuntimeError("SharePoint OAuth not initialized - missing credentials")
try:
success = await self.oauth.handle_authorization_callback(auth_code, self.redirect_uri)
if success:
self._authenticated = True
# Auto-detect base URL from user's drive
detected_url = await self._detect_base_url()
if detected_url:
self.base_url = detected_url
logger.info(f"Auto-detected base URL: {detected_url}")
return {"status": "success", "base_url": self.base_url}
else:
raise ValueError("OAuth callback failed")
except Exception as e:
logger.error(f"OAuth callback failed: {e}")
raise
async def _detect_base_url(self) -> str | None:
"""Override base class method to detect SharePoint URL"""
return await self._detect_sharepoint_url()
async def _detect_sharepoint_url(self) -> str | None:
"""Auto-detect SharePoint URL from Microsoft Graph API"""
logger.info("_detect_sharepoint_url: Starting SharePoint URL detection")
try:
if not self.oauth:
logger.warning("_detect_sharepoint_url: OAuth not initialized")
return None
access_token = self.oauth.get_access_token()
logger.debug(
f"_detect_sharepoint_url: Got access token (length: {len(access_token) if access_token else 0})"
)
headers = {
"Authorization": f"Bearer {access_token}",
"Content-Type": "application/json",
}
async with httpx.AsyncClient() as client:
# Get user's default drive to extract SharePoint URL
url = f"{self._graph_base_url}/me/drive"
logger.info(f"_detect_sharepoint_url: Calling Graph API: {url}")
response = await client.get(url, headers=headers, timeout=30.0)
logger.info(
f"_detect_sharepoint_url: Graph API response status: {response.status_code}"
)
if response.status_code == 200:
data = response.json()
web_url = data.get("webUrl", "")
logger.info(f"_detect_sharepoint_url: webUrl from response: {web_url}")
# Extract the SharePoint domain from the webUrl
if web_url:
parsed = urlparse(web_url)
sharepoint_url = f"{parsed.scheme}://{parsed.netloc}"
logger.info(
f"_detect_sharepoint_url: Detected SharePoint URL: {sharepoint_url}"
)
return sharepoint_url
else:
logger.warning("_detect_sharepoint_url: webUrl is empty in response")
else:
logger.warning(
"[CONNECTOR] SharePoint detect URL failed", status_code=response.status_code
)
except Exception:
logger.exception("[CONNECTOR] SharePoint URL detection failed")
return None
def sync_once(self) -> None:
"""
Perform a one-shot sync of SharePoint files and emit documents.
This method mirrors the Google Drive connector's sync_once functionality.
"""
import asyncio
async def _async_sync():
try:
# Get list of files
file_list = await self.list_files(max_files=1000) # Adjust as needed
files = file_list.get("files", [])
for file_info in files:
try:
file_id = file_info.get("id")
if not file_id:
continue
# Get full document content
doc = await self.get_file_content(file_id)
self.emit(doc)
except Exception as e:
logger.error(
f"Failed to sync SharePoint file {file_info.get('name', 'unknown')}: {e}"
)
continue
except Exception as e:
logger.error(f"SharePoint sync_once failed: {e}")
raise
# Run the async sync
if hasattr(asyncio, "run"):
asyncio.run(_async_sync())
else:
# Python < 3.7 compatibility
loop = asyncio.get_event_loop()
loop.run_until_complete(_async_sync())
async def setup_subscription(self) -> str:
"""Set up real-time subscription for file changes - BaseConnector interface"""
webhook_url = self.config.get("webhook_url")
if not webhook_url:
logger.warning("No webhook URL configured, skipping SharePoint subscription setup")
return "no-webhook-configured"
try:
# Ensure we're authenticated
if not await self.authenticate():
raise RuntimeError("SharePoint authentication failed during subscription setup")
token = self.oauth.get_access_token()
# Microsoft Graph subscription for SharePoint site
site_info = self._parse_sharepoint_url()
if site_info:
resource = (
f"sites/{site_info['host_name']}:/sites/{site_info['site_name']}:/drive/root"
)
else:
resource = "/me/drive/root"
subscription_data = {
# Graph driveItem subscriptions only support "updated"; creates and
# deletes still surface through the delta query the webhook triggers.
"changeType": "updated",
# webhook_url is already the full endpoint
# ({WEBHOOK_BASE_URL}/connectors/sharepoint/webhook, set at connect time)
"notificationUrl": webhook_url,
"resource": resource,
"expirationDateTime": self._get_subscription_expiry(),
"clientState": f"sharepoint_{self.tenant_id}",
}
headers = {"Authorization": f"Bearer {token}", "Content-Type": "application/json"}
url = f"{self._graph_base_url}/subscriptions"
async with httpx.AsyncClient() as client:
response = await client.post(
url, json=subscription_data, headers=headers, timeout=30
)
if response.status_code >= 400:
logger.error(
f"Graph subscription request rejected: {response.status_code} {response.text}"
)
response.raise_for_status()
result = response.json()
subscription_id = result.get("id")
if subscription_id:
self._subscription_id = subscription_id
self.webhook_expiration = result.get("expirationDateTime")
logger.info(f"SharePoint subscription created: {subscription_id}")
return subscription_id
else:
raise ValueError("No subscription ID returned from Microsoft Graph")
except Exception as e:
logger.error(f"Failed to setup SharePoint subscription: {e}")
raise
def _get_subscription_expiry(self) -> str:
"""Get subscription expiry time (max 3 days for Graph API)"""
from datetime import datetime, timedelta
expiry = datetime.now(UTC) + timedelta(days=3) # 3 days max for Graph
return expiry.strftime("%Y-%m-%dT%H:%M:%S.%fZ")
def _parse_sharepoint_url(self) -> dict[str, str] | None:
"""Parse SharePoint URL to extract site information for Graph API"""
if not self.sharepoint_url:
return None
try:
parsed = urlparse(self.sharepoint_url)
# Extract hostname and site name from URL like: https://contoso.sharepoint.com/sites/teamsite
host_name = parsed.netloc
path_parts = parsed.path.strip("/").split("/")
if len(path_parts) >= 2 and path_parts[0] == "sites":
site_name = path_parts[1]
return {"host_name": host_name, "site_name": site_name}
except Exception as e:
logger.warning(f"Could not parse SharePoint URL {self.sharepoint_url}: {e}")
return None
async def list_files(
self, page_token: str | None = None, max_files: int | None = None, **kwargs
) -> dict[str, Any]:
"""List all files using Microsoft Graph API - BaseConnector interface"""
try:
# Ensure authentication
if not await self.authenticate():
raise RuntimeError("SharePoint authentication failed during file listing")
# If file_ids or folder_ids are specified in config, use selective sync
if self.cfg.file_ids or self.cfg.folder_ids:
return await self._list_selected_files()
files = []
max_files_value = max_files if max_files is not None else 100
# Build Graph API URL for the site or fallback to user's OneDrive
site_info = self._parse_sharepoint_url()
if site_info:
base_url = f"{self._graph_base_url}/sites/{site_info['host_name']}:/sites/{site_info['site_name']}:/drive/root/children"
else:
base_url = f"{self._graph_base_url}/me/drive/root/children"
params = dict(self._default_params)
params["$top"] = str(max_files_value)
if page_token:
params["$skiptoken"] = page_token
response = await self._make_graph_request(base_url, params=params)
data = response.json()
items = data.get("value", [])
for item in items:
# Only include files, not folders
if item.get("file"):
files.append(
{
"id": item.get("id", ""),
"name": item.get("name", ""),
"path": f"/drive/items/{item.get('id')}",
"size": int(item.get("size", 0)),
"modified": item.get("lastModifiedDateTime"),
"created": item.get("createdDateTime"),
"mime_type": item.get("file", {}).get(
"mimeType", self._get_mime_type(item.get("name", ""))
),
"url": item.get("webUrl", ""),
"download_url": item.get("@microsoft.graph.downloadUrl"),
}
)
# Check for next page
next_page_token = None
next_link = data.get("@odata.nextLink")
if next_link:
from urllib.parse import parse_qs, urlparse
parsed = urlparse(next_link)
query_params = parse_qs(parsed.query)
if "$skiptoken" in query_params:
next_page_token = query_params["$skiptoken"][0]
return {"files": files, "next_page_token": next_page_token}
except Exception as e:
logger.error(f"Failed to list SharePoint files: {e}")
return {"files": [], "next_page_token": None} # Return empty result instead of raising
async def _extract_sharepoint_acl(self, file_id: str, file_metadata: dict) -> DocumentACL:
"""
Extract ACL from SharePoint item.
Queries Microsoft Graph API permissions endpoint to get allowed users and groups.
Args:
file_id: SharePoint item ID
file_metadata: File metadata dict
Returns:
DocumentACL instance with extracted permissions
"""
try:
# Get access token - use same approach as _make_graph_request
access_token = await get_oauth_access_token(self.oauth)
if not access_token:
logger.warning(f"No access token available for ACL extraction: {file_id}")
return DocumentACL()
# Determine the correct path for permissions API call. Mirror
# _get_file_metadata_by_id: a composite "driveId!itemId" id must be
# split into /drives/{driveId}/items/{itemId}. Using the composite id
# verbatim against /drive/items/{id} yields a malformed URL → Graph
# error → empty ACL, which is why shared-user updates never landed.
if "!" in file_id and len(file_id.rsplit("!", 1)) == 2:
drive_id, item_id = file_id.rsplit("!", 1)
permissions_url = (
f"{self._graph_base_url}/drives/{drive_id}/items/{item_id}/permissions"
)
else:
site_info = self._parse_sharepoint_url()
if site_info:
permissions_url = f"{self._graph_base_url}/sites/{site_info['host_name']}:/sites/{site_info['site_name']}:/drive/items/{file_id}/permissions"
else:
# Fallback to user drive
permissions_url = f"{self._graph_base_url}/me/drive/items/{file_id}/permissions"
# Fetch permissions, following pagination so large share lists are
# captured in full (not just the first page).
permissions: list[dict[str, Any]] = []
async with httpx.AsyncClient() as client:
url: str | None = permissions_url
while url:
response = await client.get(
url, headers={"Authorization": f"Bearer {access_token}"}
)
if response.status_code != 200:
logger.warning(
f"Failed to fetch permissions for {file_id}: {response.status_code}"
)
return DocumentACL()
page = response.json()
permissions.extend(page.get("value", []))
url = page.get("@odata.nextLink")
permissions_data = {"value": permissions}
allowed_users = []
allowed_groups = []
allowed_principals = []
allowed_principal_labels = []
owner = None
for perm in permissions_data.get("value", []):
roles = perm.get("roles", []) # ["read", "write", "owner"]
# Granted to user (grantedTo or grantedToV2)
granted_to = perm.get("grantedToV2") or perm.get("grantedTo")
if granted_to:
user_info = granted_to.get("user", {})
email = user_info.get("email")
display_name = user_info.get("displayName")
user_identifier = email or display_name
if user_identifier:
allowed_users.append(user_identifier)
if "owner" in roles:
owner = user_identifier
for identifier in (
user_info.get("id"),
user_info.get("userPrincipalName"),
email,
):
user_principal = microsoft_user_principal(
identifier,
access_token=access_token,
tenant_id=self.tenant_id,
)
if user_principal:
allowed_principals.append(user_principal)
label = microsoft_user_principal_label(
identifier,
access_token=access_token,
tenant_id=self.tenant_id,
display_name=display_name or email,
email=email,
external_id=identifier,
)
if label:
allowed_principal_labels.append(label)
group_info = granted_to.get("group", {})
group_role = microsoft_group_role(
group_info.get("id"),
access_token=access_token,
tenant_id=self.tenant_id,
)
if group_role:
allowed_groups.append(group_role)
allowed_principals.append(group_role)
label = microsoft_group_principal_label(
group_info.get("id"),
access_token=access_token,
tenant_id=self.tenant_id,
display_name=group_info.get("displayName") or group_info.get("email"),
email=group_info.get("email"),
)
if label:
allowed_principal_labels.append(label)
# Granted to identities (can include users and groups)
if "grantedToIdentitiesV2" in perm or "grantedToIdentities" in perm:
identities = (
perm.get("grantedToIdentitiesV2") or perm.get("grantedToIdentities") or []
)
for identity in identities:
# User
if "user" in identity:
user_info = identity["user"]
email = user_info.get("email")
display_name = user_info.get("displayName")
user_identifier = email or display_name
if user_identifier:
allowed_users.append(user_identifier)
if "owner" in roles:
owner = user_identifier
for identifier in (
user_info.get("id"),
user_info.get("userPrincipalName"),
email,
):
user_principal = microsoft_user_principal(
identifier,
access_token=access_token,
tenant_id=self.tenant_id,
)
if user_principal:
allowed_principals.append(user_principal)
# Group
if "group" in identity:
group_info = identity["group"]
group_id = group_info.get("id")
group_role = microsoft_group_role(
group_id,
access_token=access_token,
tenant_id=self.tenant_id,
)
if group_role:
allowed_groups.append(group_role)
allowed_principals.append(group_role)
label = microsoft_group_principal_label(
group_id,
access_token=access_token,
tenant_id=self.tenant_id,
display_name=group_info.get("displayName")
or group_info.get("email"),
email=group_info.get("email"),
)
if label:
allowed_principal_labels.append(label)
return DocumentACL(
owner=owner,
allowed_users=allowed_users,
allowed_groups=allowed_groups,
allowed_principals=allowed_principals,
allowed_principal_labels=unique_acl_principal_labels(allowed_principal_labels),
)
except Exception as e:
logger.warning(f"Failed to extract ACL for SharePoint item {file_id}: {e}")
return DocumentACL()
async def get_file_content(self, file_id: str) -> ConnectorDocument:
"""Get file content and metadata - BaseConnector interface"""
try:
# Ensure authentication
if not await self.authenticate():
raise RuntimeError("SharePoint authentication failed during file content retrieval")
# First, check for cached file info with download URL
# This is used for SharePoint sharing IDs that can't be resolved via Graph API
cached_info = self.get_cached_file_info(file_id)
if cached_info and cached_info.get("downloadUrl"):
logger.info(f"Using cached download URL for file {file_id}")
content = await self._download_file_from_url(cached_info["downloadUrl"])
# Extract ACL even for cached files
acl = await self._extract_sharepoint_acl(file_id, cached_info)
return ConnectorDocument(
id=file_id,
filename=cached_info.get("name", "Unknown"),
mimetype=cached_info.get("mimeType", "application/octet-stream"),
content=content,
source_url=cached_info.get("webUrl", ""),
acl=acl,
modified_time=datetime.now(),
created_time=datetime.now(),
metadata={
"sharepoint_path": "",
"sharepoint_url": self.sharepoint_url,
"size": cached_info.get("size", 0),
},
)
# Fall back to Graph API for regular file IDs
file_metadata = await self._get_file_metadata_by_id(file_id)
if not file_metadata:
raise ValueError(f"File not found: {file_id}")
# Download file content
download_url = file_metadata.get("download_url")
if download_url:
content = await self._download_file_from_url(download_url)
else:
content = await self._download_file_content(file_id)
# Extract ACL from SharePoint item
acl = await self._extract_sharepoint_acl(file_id, file_metadata)
# Parse dates
modified_time = self._parse_graph_date(file_metadata.get("modified"))
created_time = self._parse_graph_date(file_metadata.get("created"))
return ConnectorDocument(
id=file_id,
filename=file_metadata.get("name", ""),
mimetype=file_metadata.get("mime_type", "application/octet-stream"),
content=content,
source_url=file_metadata.get("url", ""),
acl=acl,
modified_time=modified_time,
created_time=created_time,
metadata={
"sharepoint_path": file_metadata.get("path", ""),
"sharepoint_url": self.sharepoint_url,
"size": file_metadata.get("size", 0),
},
)
except Exception as e:
logger.error(f"Failed to get SharePoint file content {file_id}: {e}")
raise
async def _get_file_metadata_by_id(self, file_id: str) -> dict[str, Any] | None:
"""Get file metadata by ID using Graph API"""
try:
# Check if ID contains '!' which indicates driveId!itemId format
if "!" in file_id:
parts = file_id.rsplit("!", 1)
if len(parts) == 2:
drive_id, item_id = parts
url = f"{self._graph_base_url}/drives/{drive_id}/items/{item_id}"
else:
url = f"{self._graph_base_url}/me/drive/items/{file_id}"
else:
site_info = self._parse_sharepoint_url()
if site_info:
url = f"{self._graph_base_url}/sites/{site_info['host_name']}:/sites/{site_info['site_name']}:/drive/items/{file_id}"
else:
url = f"{self._graph_base_url}/me/drive/items/{file_id}"
params = dict(self._default_params)
response = await self._make_graph_request(url, params=params)
item = response.json()
if item.get("file"):
return {
"id": file_id,
"name": item.get("name", ""),
"path": f"/drive/items/{file_id}",
"size": int(item.get("size", 0)),
"modified": item.get("lastModifiedDateTime"),
"created": item.get("createdDateTime"),
"mime_type": item.get("file", {}).get(
"mimeType", self._get_mime_type(item.get("name", ""))
),
"url": item.get("webUrl", ""),
"download_url": item.get("@microsoft.graph.downloadUrl"),
}
# Check if it's a folder
if item.get("folder"):
return {
"id": file_id,
"name": item.get("name", ""),
"isFolder": True,
}
return None
except Exception as e:
logger.error(
f"Failed to get file metadata for {file_id}: {e}. "
f"Site info: {self._parse_sharepoint_url()}, "
f"SharePoint URL: {self.sharepoint_url}"
)
return None
async def _download_file_content(self, file_id: str) -> bytes:
"""Download file content by file ID using Graph API"""
try:
# Check if ID contains '!' which indicates driveId!itemId format
if "!" in file_id:
parts = file_id.rsplit("!", 1)
if len(parts) == 2:
drive_id, item_id = parts
if not item_id.startswith("s"):
url = f"{self._graph_base_url}/drives/{drive_id}/items/{item_id}/content"
else:
url = f"{self._graph_base_url}/me/drive/items/{file_id}/content"
else:
url = f"{self._graph_base_url}/me/drive/items/{file_id}/content"
else:
site_info = self._parse_sharepoint_url()
if site_info:
url = f"{self._graph_base_url}/sites/{site_info['host_name']}:/sites/{site_info['site_name']}:/drive/items/{file_id}/content"
else:
url = f"{self._graph_base_url}/me/drive/items/{file_id}/content"
token = self.oauth.get_access_token()
headers = {"Authorization": f"Bearer {token}"}
async with httpx.AsyncClient() as client:
response = await client.get(url, headers=headers, timeout=60, follow_redirects=True)
response.raise_for_status()
return response.content
except Exception as e:
logger.error(f"Failed to download file content for {file_id}: {e}")
raise
async def _list_selected_files(self) -> dict[str, Any]:
"""List only selected files/folders (selective sync)."""
files: list[dict[str, Any]] = []
# Process selected file IDs
if self.cfg.file_ids:
for file_id in self.cfg.file_ids:
try:
file_meta = await self._get_file_metadata_by_id(file_id)
if file_meta and not file_meta.get("isFolder", False):
files.append(file_meta)
elif file_meta and file_meta.get("isFolder", False):
# If it's a folder, expand its contents
folder_files = await self._list_folder_contents(file_id)
files.extend(folder_files)
except Exception as e:
logger.warning(f"Failed to get file {file_id}: {e}")
continue
# Process selected folder IDs
if self.cfg.folder_ids:
for folder_id in self.cfg.folder_ids:
try:
folder_files = await self._list_folder_contents(folder_id)
files.extend(folder_files)
except Exception as e:
logger.warning(f"Failed to list folder {folder_id}: {e}")
continue
return {"files": files, "next_page_token": None}
async def _list_folder_contents(self, folder_id: str) -> list[dict[str, Any]]:
"""List all files in a folder recursively."""
files: list[dict[str, Any]] = []
try:
drive_id = None
if "!" in folder_id:
parts = folder_id.rsplit("!", 1)
if len(parts) == 2:
potential_drive_id, item_id = parts
if not item_id.startswith("s"):
drive_id = potential_drive_id
url = f"{self._graph_base_url}/drives/{drive_id}/items/{item_id}/children"
if not drive_id:
site_info = self._parse_sharepoint_url()
if site_info:
url = f"{self._graph_base_url}/sites/{site_info['host_name']}:/sites/{site_info['site_name']}:/drive/items/{folder_id}/children"
else:
url = f"{self._graph_base_url}/me/drive/items/{folder_id}/children"
params = dict(self._default_params)
response = await self._make_graph_request(url, params=params)
data = response.json()
items = data.get("value", [])
for item in items:
parent_ref = item.get("parentReference", {})
item_drive_id = parent_ref.get("driveId") or drive_id
item_id = item.get("id")
if item_id and "!" in item_id:
final_item_id = item_id
else:
final_item_id = f"{item_drive_id}!{item_id}" if item_drive_id else item_id
if item.get("file"): # It's a file
file_meta = {
"id": final_item_id,
"name": item.get("name", ""),
"path": f"/drive/items/{item_id}",
"size": int(item.get("size") or 0),
"modified": item.get("lastModifiedDateTime"),
"created": item.get("createdDateTime"),
"mime_type": item.get("file", {}).get(
"mimeType", self._get_mime_type(item.get("name", ""))
),
"url": item.get("webUrl", ""),
"download_url": item.get("@microsoft.graph.downloadUrl"),
}
files.append(file_meta)
elif item.get("folder"): # It's a subfolder, recurse
subfolder_files = await self._list_folder_contents(final_item_id)
files.extend(subfolder_files)
except Exception as e:
import traceback
logger.error(
f"Failed to list folder contents for {folder_id}: {e}\n{traceback.format_exc()}"
)
return files
async def _download_file_from_url(self, download_url: str) -> bytes:
"""Download file content from direct download URL"""
try:
async with httpx.AsyncClient() as client:
response = await client.get(download_url, timeout=60, follow_redirects=True)
response.raise_for_status()
return response.content
except Exception as e:
logger.error(f"Failed to download from URL {download_url}: {e}")
raise
def _parse_graph_date(self, date_str: str | None) -> datetime:
"""Parse Microsoft Graph date string to datetime"""
if not date_str:
return datetime.now()
try:
if date_str.endswith("Z"):
return datetime.fromisoformat(date_str[:-1]).replace(tzinfo=None)
else:
return datetime.fromisoformat(date_str.replace("T", " "))
except (ValueError, AttributeError):
return datetime.now()
async def _make_graph_request(
self,
url: str,
method: str = "GET",
data: dict | None = None,
params: dict | None = None,
) -> httpx.Response:
"""Make authenticated API request to Microsoft Graph"""
token = self.oauth.get_access_token()
headers = {"Authorization": f"Bearer {token}", "Content-Type": "application/json"}
async with httpx.AsyncClient() as client:
if method.upper() == "GET":
response = await client.get(url, headers=headers, params=params, timeout=30)
elif method.upper() == "POST":
response = await client.post(url, headers=headers, json=data, timeout=30)
elif method.upper() == "DELETE":
response = await client.delete(url, headers=headers, timeout=30)
else:
raise ValueError(f"Unsupported HTTP method: {method}")
response.raise_for_status()
return response
def _get_mime_type(self, filename: str) -> str:
"""Get MIME type based on file extension"""
import mimetypes
mime_type, _ = mimetypes.guess_type(filename)
return mime_type or "application/octet-stream"
# Webhook methods - BaseConnector interface
def handle_webhook_validation(
self, request_method: str, headers: dict[str, str], query_params: dict[str, str]
) -> str | None:
"""Handle webhook validation (Graph API specific)"""
if request_method == "POST" and "validationToken" in query_params:
return query_params["validationToken"]
return None
def extract_webhook_channel_id(
self, payload: dict[str, Any], headers: dict[str, str]
) -> str | None:
"""Extract channel/subscription ID from webhook payload"""
notifications = payload.get("value", [])
if notifications:
return notifications[0].get("subscriptionId")
return None
@staticmethod
def _delta_item_file_id(item: dict[str, Any]) -> str:
"""Return the composite ``{driveId}!{itemId}`` id used at ingest time.
Selected-file listing stores ids as ``driveId!itemId`` (see
_list_folder_contents), so the webhook delta must emit the same shape
or the change can't be correlated with the indexed connector_file_id.
"""
item_id = item.get("id", "")
if item_id and "!" in item_id:
return item_id
drive_id = item.get("parentReference", {}).get("driveId")
return f"{drive_id}!{item_id}" if drive_id else item_id
async def handle_webhook(self, payload: dict[str, Any]) -> list[str]:
"""Handle webhook notification and return affected file IDs.
Graph driveItem notifications never identify the changed items — the
notification resource is the subscribed drive root — so run a delta
query against the drive to discover what changed.
"""
if not payload.get("value"):
return []
try:
if not await self.authenticate():
logger.error("SharePoint authentication failed during webhook handling")
return []
token = self.oauth.get_access_token()
headers = {"Authorization": f"Bearer {token}"}
site_info = self._parse_sharepoint_url()
if site_info:
resource = (
f"sites/{site_info['host_name']}:/sites/{site_info['site_name']}:/drive/root"
)
else:
resource = "me/drive/root"
# Without a stored delta link (first notification for this instance)
# the delta query enumerates the whole drive, so only keep recently
# modified files instead of re-syncing everything.
first_sweep = self._delta_link is None
url = self._delta_link or f"{self._graph_base_url}/{resource}/delta"
cutoff = datetime.now(UTC) - timedelta(minutes=10)
affected_files: list[str] = []
async with httpx.AsyncClient() as client:
while url:
response = await client.get(url, headers=headers, timeout=30)
response.raise_for_status()
data = response.json()
for item in data.get("value", []):
if "deleted" in item:
# Deleted at source: propagate the id so the processor
# runs its deleted-at-source cleanup
# (get_file_content -> 404 -> delete indexed chunks).
if "folder" not in item:
affected_files.append(self._delta_item_file_id(item))
continue
if "file" not in item:
continue
if first_sweep:
modified = item.get("lastModifiedDateTime")
if not modified:
continue
try:
modified_at = datetime.fromisoformat(
modified.replace("Z", "+00:00")
)
except ValueError:
continue
if modified_at < cutoff:
continue
affected_files.append(self._delta_item_file_id(item))
delta_link = data.get("@odata.deltaLink")
if delta_link:
self._delta_link = delta_link
url = data.get("@odata.nextLink")
return list(dict.fromkeys(affected_files))
except Exception as e:
logger.error(f"SharePoint webhook delta query failed: {e}")
return []
async def cleanup_subscription(self, subscription_id: str) -> bool:
"""Clean up subscription - BaseConnector interface"""
if subscription_id == "no-webhook-configured":
logger.info("No subscription to cleanup (webhook was not configured)")
return True
try:
# Ensure authentication
if not await self.authenticate():
logger.error("SharePoint authentication failed during subscription cleanup")
return False
token = self.oauth.get_access_token()
headers = {"Authorization": f"Bearer {token}"}
url = f"{self._graph_base_url}/subscriptions/{subscription_id}"
async with httpx.AsyncClient() as client:
response = await client.delete(url, headers=headers, timeout=30)
if response.status_code in [200, 204, 404]:
logger.info(
f"SharePoint subscription {subscription_id} cleaned up successfully"
)
return True
else:
logger.warning(
f"Unexpected response cleaning up subscription: {response.status_code}"
)
return False
except Exception as e:
logger.error(f"Failed to cleanup SharePoint subscription {subscription_id}: {e}")
return False
async def renew_subscription(self, subscription_id: str) -> str | None:
"""Extend the Graph subscription in place (PATCH avoids re-validation).
Returns the new expirationDateTime, or None to signal the caller to
fall back to delete + recreate."""
if subscription_id == "no-webhook-configured":
return None
try:
if not await self.authenticate():
logger.error("SharePoint authentication failed during subscription renewal")
return None
token = self.oauth.get_access_token()
headers = {"Authorization": f"Bearer {token}", "Content-Type": "application/json"}
url = f"{self._graph_base_url}/subscriptions/{subscription_id}"
body = {"expirationDateTime": self._get_subscription_expiry()}
async with httpx.AsyncClient() as client:
response = await client.patch(url, json=body, headers=headers, timeout=30)
if response.status_code == 404:
# Subscription already expired/deleted at Graph; recreate.
logger.info(
f"SharePoint subscription {subscription_id} not found, will recreate"
)
return None
if response.status_code not in [200, 201]:
logger.warning(
f"Unexpected response renewing SharePoint subscription: "
f"{response.status_code}"
)
return None
expiration = response.json().get("expirationDateTime")
self.webhook_expiration = expiration
logger.info(f"SharePoint subscription {subscription_id} renewed until {expiration}")
return expiration
except Exception as e:
logger.error(f"Failed to renew SharePoint subscription {subscription_id}: {e}")
return None