OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.
Python
fix: Address sharepoint folder handling (#1557)
Mike Fortman4b2d4cfcaee6
diff --git a/frontend/app/upload/[provider]/page.tsx b/frontend/app/upload/[provider]/page.tsxindex 6d315d85e..eae4389b5 100644--- a/frontend/app/upload/[provider]/page.tsx+++ b/frontend/app/upload/[provider]/page.tsx@@ -447,6 +447,7 @@ export default function UploadProviderPage() {mimeType: file.mimeType,downloadUrl: file.downloadUrl,size: file.size,+ isFolder: file.isFolder,})),settings: ingestSettings,},diff --git a/frontend/components/cloud-picker/file-item.tsx b/frontend/components/cloud-picker/file-item.tsxindex fc4573745..3f1306c26 100644--- a/frontend/components/cloud-picker/file-item.tsx+++ b/frontend/components/cloud-picker/file-item.tsx@@ -71,10 +71,18 @@ export const FileItem = ({ file, onRemove, provider }: FileItemProps) => (className="flex items-center justify-between p-1.5 rounded-md text-xs"><div className="flex items-center gap-2 flex-1 min-w-0">- {provider ? getProviderIcon(provider) : getFileIcon(file.mimeType)}+ {file.isFolder ? (+ <Folder className="h-6 w-6" />+ ) : provider ? (+ getProviderIcon(provider)+ ) : (+ getFileIcon(file.mimeType)+ )}<span className="truncate font-medium text-sm mr-2">{file.name}</span><span className="text-sm text-muted-foreground">- {getMimeTypeLabel(file.mimeType)}+ {file.isFolder+ ? "Folder — all contents will be ingested"+ : getMimeTypeLabel(file.mimeType)}</span></div><div className="flex items-center gap-1">diff --git a/frontend/components/cloud-picker/onedrive-v8-handler.ts b/frontend/components/cloud-picker/onedrive-v8-handler.tsindex 1227677f2..2afd4746d 100644--- a/frontend/components/cloud-picker/onedrive-v8-handler.ts+++ b/frontend/components/cloud-picker/onedrive-v8-handler.ts@@ -161,11 +161,6 @@ export class OneDriveV8Handler {commands: {pick: {action: "select",- select: {- urls: {- download: true, // Request download URLs - critical for ingestion!- },- },},},};diff --git a/frontend/components/cloud-picker/sharepoint-v8-handler.ts b/frontend/components/cloud-picker/sharepoint-v8-handler.tsindex 0a868f6b1..d309b9a3b 100644--- a/frontend/components/cloud-picker/sharepoint-v8-handler.ts+++ b/frontend/components/cloud-picker/sharepoint-v8-handler.ts@@ -204,11 +204,6 @@ export class SharePointV8Handler {commands: {pick: {action: "select",- select: {- urls: {- download: true, // Request download URLs- },- },},},};diff --git a/src/connectors/service.py b/src/connectors/service.pyindex 482a9aa54..fe463f590 100644--- a/src/connectors/service.py+++ b/src/connectors/service.py@@ -56,14 +56,17 @@ class ConnectorService:# Create temporary file from document contentfrom utils.file_utils import auto_cleanup_tempfile- with auto_cleanup_tempfile(- suffix=get_file_extension(document.mimetype)- ) as tmp_path:+ with auto_cleanup_tempfile(suffix=get_file_extension(document.mimetype)) as tmp_path:# Write document content to temp filewith open(tmp_path, "wb") as f:f.write(document.content)- logger.info("[CONNECTOR] Processing document", document_id=document.id, connector_type=connector_type, filename=document.filename)+ logger.info(+ "[CONNECTOR] Processing document",+ document_id=document.id,+ connector_type=connector_type,+ filename=document.filename,+ )# Process using consolidated processing pipelinefrom models.processors import TaskProcessor@@ -86,7 +89,11 @@ class ConnectorService:acl=document.acl,)- logger.info("[CONNECTOR] Document processed", document_id=document.id, status=result.get("status"))+ logger.info(+ "[CONNECTOR] Document processed",+ document_id=document.id,+ status=result.get("status"),+ )# If successfully indexed or already exists, update the indexed documents with connector metadataif result["status"] in ["indexed", "unchanged"]:@@ -130,8 +137,7 @@ class ConnectorService:logger.debug(f"ACL unchanged for {document.id}, skipped update")elif acl_result["status"] == "updated":logger.info(- f"Updated ACL for {document.id}, "- f"{acl_result['chunks_updated']} chunks updated"+ f"Updated ACL for {document.id}, {acl_result['chunks_updated']} chunks updated")elif acl_result["status"] == "error":logger.error(f"ACL update error for {document.id}: {acl_result.get('error')}")@@ -167,9 +173,9 @@ class ConnectorService:if document.modified_timeelse None,"metadata": document.metadata,- }- }- }+ },+ },+ },)logger.debug(f"Updated metadata for document {document.id}")except Exception as e:@@ -180,7 +186,6 @@ class ConnectorService:)raise-async def sync_connector_files(self,connection_id: str,@@ -191,7 +196,7 @@ class ConnectorService:) -> str:"""Sync files from a connector connection using existing task tracking system.-+Args:connection_id: The connection IDuser_id: The user ID@@ -214,9 +219,7 @@ class ConnectorService:connector = await self.get_connector(connection_id)if not connector:- raise ValueError(- f"Connection '{connection_id}' not found or not authenticated"- )+ raise ValueError(f"Connection '{connection_id}' not found or not authenticated")logger.debug("Got connector", authenticated=connector.is_authenticated)@@ -232,13 +235,9 @@ class ConnectorService:while True:# List files from connector with limit- logger.debug(- "Calling list_files", page_size=page_size, page_token=page_token- )+ logger.debug("Calling list_files", page_size=page_size, page_token=page_token)file_list = await connector.list_files(page_token, limit=page_size)- logger.debug(- "Got files from connector", file_count=len(file_list.get("files", []))- )+ logger.debug("Got files from connector", file_count=len(file_list.get("files", [])))files = file_list["files"]if not files:@@ -320,7 +319,7 @@ class ConnectorService:"""Sync specific files by their IDs (used for webhook-triggered syncs or manual selection).Automatically expands folders to their contents.-+Args:connection_id: The connection IDuser_id: The user ID@@ -338,9 +337,7 @@ class ConnectorService:connector = await self.get_connector(connection_id)if not connector:- raise ValueError(- f"Connection '{connection_id}' not found or not authenticated"- )+ raise ValueError(f"Connection '{connection_id}' not found or not authenticated")if not connector.is_authenticated:raise ValueError(f"Connection '{connection_id}' not authenticated")@@ -355,7 +352,7 @@ class ConnectorService:# If file_infos provided, cache them in the connector for later use# This allows get_file_content to use download URLs directly- if file_infos and hasattr(connector, 'set_file_infos'):+ if file_infos and hasattr(connector, "set_file_infos"):connector.set_file_infos(file_infos)logger.info(f"Cached {len(file_infos)} file infos with download URLs in connector")@@ -389,16 +386,33 @@ class ConnectorService:)# If we have file_infos with download URLs, use original file_ids# (OneDrive sharing IDs can't be expanded but can be downloaded directly)+ # Exclude folders — they have no downloadable content on their own.if file_infos:- logger.info("Using original file IDs with cached download URLs")- expanded_file_ids = file_ids+ non_folder_infos = [f for f in file_infos if not f.get("isFolder")]+ non_folder_ids = [f["id"] for f in non_folder_infos if f.get("id")]+ if non_folder_ids:+ logger.info(+ "Using original file IDs with cached download URLs (folders excluded)"+ )+ expanded_file_ids = non_folder_ids+ else:+ raise ValueError("No files to sync after expanding folders")else:raise ValueError("No files to sync after expanding folders")except Exception as e:logger.error(f"Failed to expand file_ids via list_files(): {e}")- # Fallback to original file_ids if expansion fails- expanded_file_ids = file_ids+ # Preserve intentional validation failures (e.g., folders-only selection)+ if isinstance(e, ValueError):+ raise+ # Fallback path: still exclude known folders when metadata is available+ if file_infos:+ non_folder_ids = [+ f["id"] for f in file_infos if f.get("id") and not f.get("isFolder")+ ]+ expanded_file_ids = non_folder_ids or file_ids+ else:+ expanded_file_ids = file_idsfinally:# Restore original config valuesif hasattr(connector, "cfg"):@@ -431,9 +445,7 @@ class ConnectorService:original_filenames = {}if file_infos:original_filenames = {- f["id"]: clean_connector_filename(- f["name"], f.get("mimeType") or f.get("mimetype")- )+ f["id"]: clean_connector_filename(f["name"], f.get("mimeType") or f.get("mimetype"))for f in file_infosif "id" in f and "name" in f}diff --git a/src/connectors/sharepoint/connector.py b/src/connectors/sharepoint/connector.pyindex e4f551c15..0a1b6cf61 100644--- a/src/connectors/sharepoint/connector.py+++ b/src/connectors/sharepoint/connector.py@@ -1,14 +1,16 @@-import logging+from datetime import datetimefrom pathlib import Path-from typing import List, Dict, Any, Optional+from typing import Any, Dict, List, Optionalfrom urllib.parse import urlparse-from datetime import datetime+import httpx+from utils.logging_config import get_logger+from ..base import BaseConnector, ConnectorDocument, DocumentACLfrom .oauth import SharePointOAuth-logger = logging.getLogger(__name__)+logger = get_logger(__name__)class SharePointConnector(BaseConnector):@@ -17,23 +19,23 @@ class SharePointConnector(BaseConnector):# Required BaseConnector class attributesCLIENT_ID_ENV_VAR = "MICROSOFT_GRAPH_OAUTH_CLIENT_ID"CLIENT_SECRET_ENV_VAR = "MICROSOFT_GRAPH_OAUTH_CLIENT_SECRET" # pragma: allowlist secret-+# Connector metadataCONNECTOR_NAME = "SharePoint"CONNECTOR_DESCRIPTION = "Add knowledge from SharePoint"CONNECTOR_ICON = "sharepoint"-+def __init__(self, config: Dict[str, Any]):super().__init__(config)logger.debug(f"SharePoint connector __init__ called with config type: {type(config)}")logger.debug(f"SharePoint connector __init__ config value: {config}")-+# Ensure we always pass a valid config to the base classif config is None:logger.debug("Config was None, using empty dict")config = {}-+try:logger.debug("Calling super().__init__")super().__init__(config) # Now safe to call with empty dict instead of None@@ -41,16 +43,18 @@ class SharePointConnector(BaseConnector):except Exception as e:logger.error(f"super().__init__ failed: {e}")raise-+# Initialize with defaults that allow the connector to be listedself.client_id = Noneself.client_secret = Noneself.tenant_id = config.get("tenant_id", "common")# base_url is the generic field name, sharepoint_url is kept for backward compatibilityself.sharepoint_url = config.get("base_url") or config.get("sharepoint_url")- logger.debug(f"SharePoint connector initialized with sharepoint_url from config: {self.sharepoint_url}")+ logger.debug(+ f"SharePoint connector initialized with sharepoint_url from config: {self.sharepoint_url}"+ )self.redirect_uri = config.get("redirect_uri", "http://localhost")-+# Try to get credentials, but don't fail if they're missingtry:logger.debug("Attempting to get client_id")@@ -59,7 +63,7 @@ class SharePointConnector(BaseConnector):except Exception as e:logger.debug(f"Failed to get client_id: {e}")pass # Credentials not available, that's OK for listing-+try:logger.debug("Attempting to get client_secret")self.client_secret = self.get_client_secret()@@ -70,6 +74,7 @@ class SharePointConnector(BaseConnector):# Token file setup - use data directory for persistencefrom config.paths import get_data_file+token_file = config.get("token_file") or get_data_file("sharepoint_token.json")Path(token_file).parent.mkdir(parents=True, exist_ok=True)@@ -82,47 +87,59 @@ class SharePointConnector(BaseConnector):oauth_token_file = config["token_file"]else:oauth_token_file = get_data_file(f"sharepoint_token_{connection_id}.json")-- authority = f"https://login.microsoftonline.com/{self.tenant_id}" if self.tenant_id != "common" else "https://login.microsoftonline.com/common"-++ authority = (+ f"https://login.microsoftonline.com/{self.tenant_id}"+ if self.tenant_id != "common"+ else "https://login.microsoftonline.com/common"+ )+self.oauth = SharePointOAuth(client_id=self.client_id,client_secret=self.client_secret,token_file=oauth_token_file,- authority=authority+ authority=authority,)else:self.oauth = None-+# Track subscription ID for webhooksself._subscription_id: Optional[str] = None-+# Add Graph API defaults similar to Google Drive flagsself._graph_api_version = "v1.0"self._default_params = {"$select": "id,name,size,lastModifiedDateTime,createdDateTime,webUrl,file,folder,@microsoft.graph.downloadUrl"}-+# Selective sync support (similar to Google Drive and OneDrive)- self.cfg = type('SharePointConfig', (), {- 'file_ids': config.get('file_ids') or config.get('selected_files') or config.get('selected_file_ids'),- 'folder_ids': config.get('folder_ids') or config.get('selected_folders') or config.get('selected_folder_ids'),- })()-+ self.cfg = type(+ "SharePointConfig",+ (),+ {+ "file_ids": config.get("file_ids")+ or config.get("selected_files")+ or config.get("selected_file_ids"),+ "folder_ids": config.get("folder_ids")+ or config.get("selected_folders")+ or config.get("selected_folder_ids"),+ },+ )()+# Cache for file metadata including download URLs# This allows direct download without Graph API for sharing IDsself._file_infos: Dict[str, Dict[str, Any]] = {}-+@propertydef _graph_base_url(self) -> str:"""Base URL for Microsoft Graph API calls"""return f"https://graph.microsoft.com/{self._graph_api_version}"-+@propertydef base_url(self) -> Optional[str]:"""Generic base URL property (returns sharepoint_url for SharePoint connector)"""return self.sharepoint_url-+@base_url.setterdef base_url(self, value: str):"""Set base URL (updates sharepoint_url internally)"""@@ -132,7 +149,7 @@ class SharePointConnector(BaseConnector):"""Cache file metadata including download URLs for later use.This allows direct download without Graph API calls for sharing IDs.-+Args:file_infos: List of file info dicts with {id, name, mimeType, downloadUrl, size}"""@@ -147,13 +164,13 @@ class SharePointConnector(BaseConnector):def get_cached_file_info(self, file_id: str) -> Optional[Dict[str, Any]]:"""Get cached file info by ID."""return self._file_infos.get(file_id)-+def emit(self, doc: ConnectorDocument) -> None:"""Emit a ConnectorDocument instance."""logger.debug(f"Emitting SharePoint document: {doc.id} ({doc.filename})")-+async def authenticate(self) -> bool:"""Test authentication - BaseConnector interface"""logger.debug(f"SharePoint authenticate() called, oauth is None: {self.oauth is None}")@@ -162,29 +179,29 @@ class SharePointConnector(BaseConnector):logger.debug("SharePoint authentication failed: OAuth not initialized")self._authenticated = Falsereturn False-+logger.debug("Loading SharePoint credentials...")# Try to load existing credentials firstload_result = await self.oauth.load_credentials()logger.debug(f"Load credentials result: {load_result}")-+logger.debug("Checking SharePoint authentication status...")authenticated = await self.oauth.is_authenticated()logger.debug(f"SharePoint is_authenticated result: {authenticated}")-+self._authenticated = authenticatedreturn authenticatedexcept Exception as e:logger.exception("[CONNECTOR] SharePoint authentication failed")self._authenticated = Falsereturn False-+def get_auth_url(self) -> str:"""Get OAuth authorization URL"""if not self.oauth:raise RuntimeError("SharePoint OAuth not initialized - missing credentials")return self.oauth.create_authorization_url(self.redirect_uri)-+async def handle_oauth_callback(self, auth_code: str) -> Dict[str, Any]:"""Handle OAuth callback"""if not self.oauth:@@ -193,24 +210,24 @@ class SharePointConnector(BaseConnector):success = await self.oauth.handle_authorization_callback(auth_code, self.redirect_uri)if success:self._authenticated = True-+# Auto-detect base URL from user's drivedetected_url = await self._detect_base_url()if detected_url:self.base_url = detected_urllogger.info(f"Auto-detected base URL: {detected_url}")-+return {"status": "success", "base_url": self.base_url}else:raise ValueError("OAuth callback failed")except Exception as e:logger.error(f"OAuth callback failed: {e}")raise-+async def _detect_base_url(self) -> Optional[str]:"""Override base class method to detect SharePoint URL"""return await self._detect_sharepoint_url()-+async def _detect_sharepoint_url(self) -> Optional[str]:"""Auto-detect SharePoint URL from Microsoft Graph API"""logger.info("_detect_sharepoint_url: Starting SharePoint URL detection")@@ -218,236 +235,244 @@ class SharePointConnector(BaseConnector):if not self.oauth:logger.warning("_detect_sharepoint_url: OAuth not initialized")return None-+access_token = self.oauth.get_access_token()- logger.debug(f"_detect_sharepoint_url: Got access token (length: {len(access_token) if access_token else 0})")-+ logger.debug(+ f"_detect_sharepoint_url: Got access token (length: {len(access_token) if access_token else 0})"+ )+headers = {"Authorization": f"Bearer {access_token}","Content-Type": "application/json",}-+async with httpx.AsyncClient() as client:# Get user's default drive to extract SharePoint URLurl = f"{self._graph_base_url}/me/drive"logger.info(f"_detect_sharepoint_url: Calling Graph API: {url}")-+response = await client.get(url, headers=headers, timeout=30.0)- logger.info(f"_detect_sharepoint_url: Graph API response status: {response.status_code}")-+ logger.info(+ f"_detect_sharepoint_url: Graph API response status: {response.status_code}"+ )+if response.status_code == 200:data = response.json()web_url = data.get("webUrl", "")logger.info(f"_detect_sharepoint_url: webUrl from response: {web_url}")-+# Extract the SharePoint domain from the webUrlif web_url:parsed = urlparse(web_url)sharepoint_url = f"{parsed.scheme}://{parsed.netloc}"- logger.info(f"_detect_sharepoint_url: Detected SharePoint URL: {sharepoint_url}")+ logger.info(+ f"_detect_sharepoint_url: Detected SharePoint URL: {sharepoint_url}"+ )return sharepoint_urlelse:logger.warning("_detect_sharepoint_url: webUrl is empty in response")else:- logger.warning("[CONNECTOR] SharePoint detect URL failed", status_code=response.status_code)-+ logger.warning(+ "[CONNECTOR] SharePoint detect URL failed", status_code=response.status_code+ )+except Exception as e:logger.exception("[CONNECTOR] SharePoint URL detection failed")return None-+def sync_once(self) -> None:"""Perform a one-shot sync of SharePoint files and emit documents.This method mirrors the Google Drive connector's sync_once functionality."""import asyncio-+async def _async_sync():try:# Get list of filesfile_list = await self.list_files(max_files=1000) # Adjust as neededfiles = file_list.get("files", [])-+for file_info in files:try:file_id = file_info.get("id")if not file_id:continue-+# Get full document contentdoc = await self.get_file_content(file_id)self.emit(doc)-+except Exception as e:- logger.error(f"Failed to sync SharePoint file {file_info.get('name', 'unknown')}: {e}")+ logger.error(+ f"Failed to sync SharePoint file {file_info.get('name', 'unknown')}: {e}"+ )continue-+except Exception as e:logger.error(f"SharePoint sync_once failed: {e}")raise-+# Run the async sync- if hasattr(asyncio, 'run'):+ if hasattr(asyncio, "run"):asyncio.run(_async_sync())else:# Python < 3.7 compatibilityloop = asyncio.get_event_loop()loop.run_until_complete(_async_sync())-+async def setup_subscription(self) -> str:"""Set up real-time subscription for file changes - BaseConnector interface"""- webhook_url = self.config.get('webhook_url')+ webhook_url = self.config.get("webhook_url")if not webhook_url:logger.warning("No webhook URL configured, skipping SharePoint subscription setup")return "no-webhook-configured"-+try:# Ensure we're authenticatedif not await self.authenticate():raise RuntimeError("SharePoint authentication failed during subscription setup")-+token = self.oauth.get_access_token()-+# Microsoft Graph subscription for SharePoint sitesite_info = self._parse_sharepoint_url()if site_info:- resource = f"sites/{site_info['host_name']}:/sites/{site_info['site_name']}:/drive/root"+ resource = (+ f"sites/{site_info['host_name']}:/sites/{site_info['site_name']}:/drive/root"+ )else:resource = "/me/drive/root"-+subscription_data = {"changeType": "created,updated,deleted","notificationUrl": f"{webhook_url}/webhook/sharepoint","resource": resource,"expirationDateTime": self._get_subscription_expiry(),- "clientState": f"sharepoint_{self.tenant_id}"- }-- headers = {- "Authorization": f"Bearer {token}",- "Content-Type": "application/json"+ "clientState": f"sharepoint_{self.tenant_id}",}-++ headers = {"Authorization": f"Bearer {token}", "Content-Type": "application/json"}+url = f"{self._graph_base_url}/subscriptions"-+async with httpx.AsyncClient() as client:- response = await client.post(url, json=subscription_data, headers=headers, timeout=30)+ response = await client.post(+ url, json=subscription_data, headers=headers, timeout=30+ )response.raise_for_status()-+result = response.json()subscription_id = result.get("id")-+if subscription_id:self._subscription_id = subscription_idlogger.info(f"SharePoint subscription created: {subscription_id}")return subscription_idelse:raise ValueError("No subscription ID returned from Microsoft Graph")-+except Exception as e:logger.error(f"Failed to setup SharePoint subscription: {e}")raise-+def _get_subscription_expiry(self) -> str:"""Get subscription expiry time (max 3 days for Graph API)"""from datetime import datetime, timedelta+expiry = datetime.utcnow() + timedelta(days=3) # 3 days max for Graphreturn expiry.strftime("%Y-%m-%dT%H:%M:%S.%fZ")-+def _parse_sharepoint_url(self) -> Optional[Dict[str, str]]:"""Parse SharePoint URL to extract site information for Graph API"""if not self.sharepoint_url:return None-+try:parsed = urlparse(self.sharepoint_url)# Extract hostname and site name from URL like: https://contoso.sharepoint.com/sites/teamsitehost_name = parsed.netloc- path_parts = parsed.path.strip('/').split('/')-- if len(path_parts) >= 2 and path_parts[0] == 'sites':+ path_parts = parsed.path.strip("/").split("/")++ if len(path_parts) >= 2 and path_parts[0] == "sites":site_name = path_parts[1]- return {- "host_name": host_name,- "site_name": site_name- }+ return {"host_name": host_name, "site_name": site_name}except Exception as e:logger.warning(f"Could not parse SharePoint URL {self.sharepoint_url}: {e}")-+return None-+async def list_files(- self,- page_token: Optional[str] = None,- max_files: Optional[int] = None,- **kwargs+ self, page_token: Optional[str] = None, max_files: Optional[int] = None, **kwargs) -> Dict[str, Any]:"""List all files using Microsoft Graph API - BaseConnector interface"""try:# Ensure authenticationif not await self.authenticate():raise RuntimeError("SharePoint authentication failed during file listing")-+# If file_ids or folder_ids are specified in config, use selective syncif self.cfg.file_ids or self.cfg.folder_ids:return await self._list_selected_files()-+files = []max_files_value = max_files if max_files is not None else 100-+# Build Graph API URL for the site or fallback to user's OneDrivesite_info = self._parse_sharepoint_url()if site_info:base_url = f"{self._graph_base_url}/sites/{site_info['host_name']}:/sites/{site_info['site_name']}:/drive/root/children"else:base_url = f"{self._graph_base_url}/me/drive/root/children"-+params = dict(self._default_params)params["$top"] = str(max_files_value)-+if page_token:params["$skiptoken"] = page_token-+response = await self._make_graph_request(base_url, params=params)data = response.json()-+items = data.get("value", [])for item in items:# Only include files, not foldersif item.get("file"):- files.append({- "id": item.get("id", ""),- "name": item.get("name", ""),- "path": f"/drive/items/{item.get('id')}",- "size": int(item.get("size", 0)),- "modified": item.get("lastModifiedDateTime"),- "created": item.get("createdDateTime"),- "mime_type": item.get("file", {}).get("mimeType", self._get_mime_type(item.get("name", ""))),- "url": item.get("webUrl", ""),- "download_url": item.get("@microsoft.graph.downloadUrl")- })+ files.append(+ {+ "id": item.get("id", ""),+ "name": item.get("name", ""),+ "path": f"/drive/items/{item.get('id')}",+ "size": int(item.get("size", 0)),+ "modified": item.get("lastModifiedDateTime"),+ "created": item.get("createdDateTime"),+ "mime_type": item.get("file", {}).get(+ "mimeType", self._get_mime_type(item.get("name", ""))+ ),+ "url": item.get("webUrl", ""),+ "download_url": item.get("@microsoft.graph.downloadUrl"),+ }+ )# Check for next pagenext_page_token = Nonenext_link = data.get("@odata.nextLink")if next_link:- from urllib.parse import urlparse, parse_qs+ from urllib.parse import parse_qs, urlparse+parsed = urlparse(next_link)query_params = parse_qs(parsed.query)if "$skiptoken" in query_params:next_page_token = query_params["$skiptoken"][0]- return {- "files": files,- "next_page_token": next_page_token- }+ return {"files": files, "next_page_token": next_page_token}except Exception as e:logger.error(f"Failed to list SharePoint files: {e}")return {"files": [], "next_page_token": None} # Return empty result instead of raising-+async def _extract_sharepoint_acl(self, file_id: str, file_metadata: Dict) -> DocumentACL:"""Extract ACL from SharePoint item.@@ -481,8 +506,7 @@ class SharePointConnector(BaseConnector):# Fetch permissionsasync with httpx.AsyncClient() as client:response = await client.get(- permissions_url,- headers={"Authorization": f"Bearer {access_token}"}+ permissions_url, headers={"Authorization": f"Bearer {access_token}"})if response.status_code != 200:@@ -512,7 +536,9 @@ class SharePointConnector(BaseConnector):# Granted to identities (can include users and groups)if "grantedToIdentitiesV2" in perm or "grantedToIdentities" in perm:- identities = perm.get("grantedToIdentitiesV2") or perm.get("grantedToIdentities") or []+ identities = (+ perm.get("grantedToIdentitiesV2") or perm.get("grantedToIdentities") or []+ )for identity in identities:# Userif "user" in identity:@@ -549,17 +575,17 @@ class SharePointConnector(BaseConnector):# Ensure authenticationif not await self.authenticate():raise RuntimeError("SharePoint authentication failed during file content retrieval")-+# First, check for cached file info with download URL# This is used for SharePoint sharing IDs that can't be resolved via Graph APIcached_info = self.get_cached_file_info(file_id)if cached_info and cached_info.get("downloadUrl"):logger.info(f"Using cached download URL for file {file_id}")content = await self._download_file_from_url(cached_info["downloadUrl"])-+# Extract ACL even for cached filesacl = await self._extract_sharepoint_acl(file_id, cached_info)-+return ConnectorDocument(id=file_id,filename=cached_info.get("name", "Unknown"),@@ -575,27 +601,27 @@ class SharePointConnector(BaseConnector):"size": cached_info.get("size", 0),},)-+# Fall back to Graph API for regular file IDsfile_metadata = await self._get_file_metadata_by_id(file_id)-+if not file_metadata:raise ValueError(f"File not found: {file_id}")-+# Download file contentdownload_url = file_metadata.get("download_url")if download_url:content = await self._download_file_from_url(download_url)else:content = await self._download_file_content(file_id)-+# Extract ACL from SharePoint itemacl = await self._extract_sharepoint_acl(file_id, file_metadata)-+# Parse datesmodified_time = self._parse_graph_date(file_metadata.get("modified"))created_time = self._parse_graph_date(file_metadata.get("created"))-+return ConnectorDocument(id=file_id,filename=file_metadata.get("name", ""),@@ -608,14 +634,14 @@ class SharePointConnector(BaseConnector):metadata={"sharepoint_path": file_metadata.get("path", ""),"sharepoint_url": self.sharepoint_url,- "size": file_metadata.get("size", 0)- }+ "size": file_metadata.get("size", 0),+ },)-+except Exception as e:logger.error(f"Failed to get SharePoint file content {file_id}: {e}")raise-+async def _get_file_metadata_by_id(self, file_id: str) -> Optional[Dict[str, Any]]:"""Get file metadata by ID using Graph API"""try:@@ -625,12 +651,12 @@ class SharePointConnector(BaseConnector):url = f"{self._graph_base_url}/sites/{site_info['host_name']}:/sites/{site_info['site_name']}:/drive/items/{file_id}"else:url = f"{self._graph_base_url}/me/drive/items/{file_id}"-+params = dict(self._default_params)-+response = await self._make_graph_request(url, params=params)item = response.json()-+if item.get("file"):return {"id": file_id,@@ -639,11 +665,13 @@ class SharePointConnector(BaseConnector):"size": int(item.get("size", 0)),"modified": item.get("lastModifiedDateTime"),"created": item.get("createdDateTime"),- "mime_type": item.get("file", {}).get("mimeType", self._get_mime_type(item.get("name", ""))),+ "mime_type": item.get("file", {}).get(+ "mimeType", self._get_mime_type(item.get("name", ""))+ ),"url": item.get("webUrl", ""),- "download_url": item.get("@microsoft.graph.downloadUrl")+ "download_url": item.get("@microsoft.graph.downloadUrl"),}-+# Check if it's a folderif item.get("folder"):return {@@ -651,13 +679,17 @@ class SharePointConnector(BaseConnector):"name": item.get("name", ""),"isFolder": True,}-+return None-+except Exception as e:- logger.error(f"Failed to get file metadata for {file_id}: {e}")+ logger.error(+ f"Failed to get file metadata for {file_id}: {e}. "+ f"Site info: {self._parse_sharepoint_url()}, "+ f"SharePoint URL: {self.sharepoint_url}"+ )return None-+async def _download_file_content(self, file_id: str) -> bytes:"""Download file content by file ID using Graph API"""try:@@ -666,38 +698,38 @@ class SharePointConnector(BaseConnector):url = f"{self._graph_base_url}/sites/{site_info['host_name']}:/sites/{site_info['site_name']}:/drive/items/{file_id}/content"else:url = f"{self._graph_base_url}/me/drive/items/{file_id}/content"-+token = self.oauth.get_access_token()headers = {"Authorization": f"Bearer {token}"}-+async with httpx.AsyncClient() as client:response = await client.get(url, headers=headers, timeout=60, follow_redirects=True)response.raise_for_status()return response.content-+except Exception as e:logger.error(f"Failed to download file content for {file_id}: {e}")raise-+async def _list_selected_files(self) -> Dict[str, Any]:"""List only selected files/folders (selective sync)."""files: List[Dict[str, Any]] = []-+# Process selected file IDsif self.cfg.file_ids:for file_id in self.cfg.file_ids:try:file_meta = await self._get_file_metadata_by_id(file_id)- if file_meta and not file_meta.get('isFolder', False):+ if file_meta and not file_meta.get("isFolder", False):files.append(file_meta)- elif file_meta and file_meta.get('isFolder', False):+ elif file_meta and file_meta.get("isFolder", False):# If it's a folder, expand its contentsfolder_files = await self._list_folder_contents(file_id)files.extend(folder_files)except Exception as e:logger.warning(f"Failed to get file {file_id}: {e}")continue-+# Process selected folder IDsif self.cfg.folder_ids:for folder_id in self.cfg.folder_ids:@@ -707,25 +739,25 @@ class SharePointConnector(BaseConnector):except Exception as e:logger.warning(f"Failed to list folder {folder_id}: {e}")continue-+return {"files": files, "next_page_token": None}-+async def _list_folder_contents(self, folder_id: str) -> List[Dict[str, Any]]:"""List all files in a folder recursively."""files: List[Dict[str, Any]] = []-+try:site_info = self._parse_sharepoint_url()if site_info:url = f"{self._graph_base_url}/sites/{site_info['host_name']}:/sites/{site_info['site_name']}:/drive/items/{folder_id}/children"else:url = f"{self._graph_base_url}/me/drive/items/{folder_id}/children"-+params = dict(self._default_params)-+response = await self._make_graph_request(url, params=params)data = response.json()-+items = data.get("value", [])for item in items:if item.get("file"): # It's a file@@ -737,9 +769,9 @@ class SharePointConnector(BaseConnector):files.extend(subfolder_files)except Exception as e:logger.error(f"Failed to list folder contents for {folder_id}: {e}")-+return files-+async def _download_file_from_url(self, download_url: str) -> bytes:"""Download file content from direct download URL"""try:@@ -750,29 +782,31 @@ class SharePointConnector(BaseConnector):except Exception as e:logger.error(f"Failed to download from URL {download_url}: {e}")raise-+def _parse_graph_date(self, date_str: Optional[str]) -> datetime:"""Parse Microsoft Graph date string to datetime"""if not date_str:return datetime.now()-+try:- if date_str.endswith('Z'):+ if date_str.endswith("Z"):return datetime.fromisoformat(date_str[:-1]).replace(tzinfo=None)else:- return datetime.fromisoformat(date_str.replace('T', ' '))+ return datetime.fromisoformat(date_str.replace("T", " "))except (ValueError, AttributeError):return datetime.now()-- async def _make_graph_request(self, url: str, method: str = "GET",- data: Optional[Dict] = None, params: Optional[Dict] = None) -> httpx.Response:++ async def _make_graph_request(+ self,+ url: str,+ method: str = "GET",+ data: Optional[Dict] = None,+ params: Optional[Dict] = None,+ ) -> httpx.Response:"""Make authenticated API request to Microsoft Graph"""token = self.oauth.get_access_token()- headers = {- "Authorization": f"Bearer {token}",- "Content-Type": "application/json"- }-+ headers = {"Authorization": f"Bearer {token}", "Content-Type": "application/json"}+async with httpx.AsyncClient() as client:if method.upper() == "GET":response = await client.get(url, headers=headers, params=params, timeout=30)@@ -782,36 +816,39 @@ class SharePointConnector(BaseConnector):response = await client.delete(url, headers=headers, timeout=30)else:raise ValueError(f"Unsupported HTTP method: {method}")-+response.raise_for_status()return response-+def _get_mime_type(self, filename: str) -> str:"""Get MIME type based on file extension"""import mimetypes+mime_type, _ = mimetypes.guess_type(filename)return mime_type or "application/octet-stream"-+# Webhook methods - BaseConnector interface- def handle_webhook_validation(self, request_method: str, headers: Dict[str, str],- query_params: Dict[str, str]) -> Optional[str]:+ def handle_webhook_validation(+ self, request_method: str, headers: Dict[str, str], query_params: Dict[str, str]+ ) -> Optional[str]:"""Handle webhook validation (Graph API specific)"""if request_method == "POST" and "validationToken" in query_params:return query_params["validationToken"]return None-- def extract_webhook_channel_id(self, payload: Dict[str, Any],- headers: Dict[str, str]) -> Optional[str]:++ def extract_webhook_channel_id(+ self, payload: Dict[str, Any], headers: Dict[str, str]+ ) -> Optional[str]:"""Extract channel/subscription ID from webhook payload"""notifications = payload.get("value", [])if notifications:return notifications[0].get("subscriptionId")return None-+async def handle_webhook(self, payload: Dict[str, Any]) -> List[str]:"""Handle webhook notification and return affected file IDs"""affected_files = []-+# Process Microsoft Graph webhook payloadnotifications = payload.get("value", [])for notification in notifications:@@ -819,36 +856,40 @@ class SharePointConnector(BaseConnector):if resource and "/drive/items/" in resource:file_id = resource.split("/drive/items/")[-1]affected_files.append(file_id)-+return affected_files-+async def cleanup_subscription(self, subscription_id: str) -> bool:"""Clean up subscription - BaseConnector interface"""if subscription_id == "no-webhook-configured":logger.info("No subscription to cleanup (webhook was not configured)")return True-+try:# Ensure authenticationif not await self.authenticate():logger.error("SharePoint authentication failed during subscription cleanup")return False-+token = self.oauth.get_access_token()headers = {"Authorization": f"Bearer {token}"}-+url = f"{self._graph_base_url}/subscriptions/{subscription_id}"-+async with httpx.AsyncClient() as client:response = await client.delete(url, headers=headers, timeout=30)-+if response.status_code in [200, 204, 404]:- logger.info(f"SharePoint subscription {subscription_id} cleaned up successfully")+ logger.info(+ f"SharePoint subscription {subscription_id} cleaned up successfully"+ )return Trueelse:- logger.warning(f"Unexpected response cleaning up subscription: {response.status_code}")+ logger.warning(+ f"Unexpected response cleaning up subscription: {response.status_code}"+ )return False-+except Exception as e:logger.error(f"Failed to cleanup SharePoint subscription {subscription_id}: {e}")return False