Skip to content

OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.

Python

fix: Address sharepoint folder handling (#1557)

Mike Fortman4b2d4cfcaee6
diff --git a/frontend/app/upload/[provider]/page.tsx b/frontend/app/upload/[provider]/page.tsx
index 6d315d85e..eae4389b5 100644
--- a/frontend/app/upload/[provider]/page.tsx
+++ b/frontend/app/upload/[provider]/page.tsx
@@ -447,6 +447,7 @@ export default function UploadProviderPage() {
mimeType: file.mimeType,
downloadUrl: file.downloadUrl,
size: file.size,
+ isFolder: file.isFolder,
})),
settings: ingestSettings,
},
diff --git a/frontend/components/cloud-picker/file-item.tsx b/frontend/components/cloud-picker/file-item.tsx
index fc4573745..3f1306c26 100644
--- a/frontend/components/cloud-picker/file-item.tsx
+++ b/frontend/components/cloud-picker/file-item.tsx
@@ -71,10 +71,18 @@ export const FileItem = ({ file, onRemove, provider }: FileItemProps) => (
className="flex items-center justify-between p-1.5 rounded-md text-xs"
>
<div className="flex items-center gap-2 flex-1 min-w-0">
- {provider ? getProviderIcon(provider) : getFileIcon(file.mimeType)}
+ {file.isFolder ? (
+ <Folder className="h-6 w-6" />
+ ) : provider ? (
+ getProviderIcon(provider)
+ ) : (
+ getFileIcon(file.mimeType)
+ )}
<span className="truncate font-medium text-sm mr-2">{file.name}</span>
<span className="text-sm text-muted-foreground">
- {getMimeTypeLabel(file.mimeType)}
+ {file.isFolder
+ ? "Folder — all contents will be ingested"
+ : getMimeTypeLabel(file.mimeType)}
</span>
</div>
<div className="flex items-center gap-1">
diff --git a/frontend/components/cloud-picker/onedrive-v8-handler.ts b/frontend/components/cloud-picker/onedrive-v8-handler.ts
index 1227677f2..2afd4746d 100644
--- a/frontend/components/cloud-picker/onedrive-v8-handler.ts
+++ b/frontend/components/cloud-picker/onedrive-v8-handler.ts
@@ -161,11 +161,6 @@ export class OneDriveV8Handler {
commands: {
pick: {
action: "select",
- select: {
- urls: {
- download: true, // Request download URLs - critical for ingestion!
- },
- },
},
},
};
diff --git a/frontend/components/cloud-picker/sharepoint-v8-handler.ts b/frontend/components/cloud-picker/sharepoint-v8-handler.ts
index 0a868f6b1..d309b9a3b 100644
--- a/frontend/components/cloud-picker/sharepoint-v8-handler.ts
+++ b/frontend/components/cloud-picker/sharepoint-v8-handler.ts
@@ -204,11 +204,6 @@ export class SharePointV8Handler {
commands: {
pick: {
action: "select",
- select: {
- urls: {
- download: true, // Request download URLs
- },
- },
},
},
};
diff --git a/src/connectors/service.py b/src/connectors/service.py
index 482a9aa54..fe463f590 100644
--- a/src/connectors/service.py
+++ b/src/connectors/service.py
@@ -56,14 +56,17 @@ class ConnectorService:
# Create temporary file from document content
from utils.file_utils import auto_cleanup_tempfile
- with auto_cleanup_tempfile(
- suffix=get_file_extension(document.mimetype)
- ) as tmp_path:
+ with auto_cleanup_tempfile(suffix=get_file_extension(document.mimetype)) as tmp_path:
# Write document content to temp file
with open(tmp_path, "wb") as f:
f.write(document.content)
- logger.info("[CONNECTOR] Processing document", document_id=document.id, connector_type=connector_type, filename=document.filename)
+ logger.info(
+ "[CONNECTOR] Processing document",
+ document_id=document.id,
+ connector_type=connector_type,
+ filename=document.filename,
+ )
# Process using consolidated processing pipeline
from models.processors import TaskProcessor
@@ -86,7 +89,11 @@ class ConnectorService:
acl=document.acl,
)
- logger.info("[CONNECTOR] Document processed", document_id=document.id, status=result.get("status"))
+ logger.info(
+ "[CONNECTOR] Document processed",
+ document_id=document.id,
+ status=result.get("status"),
+ )
# If successfully indexed or already exists, update the indexed documents with connector metadata
if result["status"] in ["indexed", "unchanged"]:
@@ -130,8 +137,7 @@ class ConnectorService:
logger.debug(f"ACL unchanged for {document.id}, skipped update")
elif acl_result["status"] == "updated":
logger.info(
- f"Updated ACL for {document.id}, "
- f"{acl_result['chunks_updated']} chunks updated"
+ f"Updated ACL for {document.id}, {acl_result['chunks_updated']} chunks updated"
)
elif acl_result["status"] == "error":
logger.error(f"ACL update error for {document.id}: {acl_result.get('error')}")
@@ -167,9 +173,9 @@ class ConnectorService:
if document.modified_time
else None,
"metadata": document.metadata,
- }
- }
- }
+ },
+ },
+ },
)
logger.debug(f"Updated metadata for document {document.id}")
except Exception as e:
@@ -180,7 +186,6 @@ class ConnectorService:
)
raise
-
async def sync_connector_files(
self,
connection_id: str,
@@ -191,7 +196,7 @@ class ConnectorService:
) -> str:
"""
Sync files from a connector connection using existing task tracking system.
-
+
Args:
connection_id: The connection ID
user_id: The user ID
@@ -214,9 +219,7 @@ class ConnectorService:
connector = await self.get_connector(connection_id)
if not connector:
- raise ValueError(
- f"Connection '{connection_id}' not found or not authenticated"
- )
+ raise ValueError(f"Connection '{connection_id}' not found or not authenticated")
logger.debug("Got connector", authenticated=connector.is_authenticated)
@@ -232,13 +235,9 @@ class ConnectorService:
while True:
# List files from connector with limit
- logger.debug(
- "Calling list_files", page_size=page_size, page_token=page_token
- )
+ logger.debug("Calling list_files", page_size=page_size, page_token=page_token)
file_list = await connector.list_files(page_token, limit=page_size)
- logger.debug(
- "Got files from connector", file_count=len(file_list.get("files", []))
- )
+ logger.debug("Got files from connector", file_count=len(file_list.get("files", [])))
files = file_list["files"]
if not files:
@@ -320,7 +319,7 @@ class ConnectorService:
"""
Sync specific files by their IDs (used for webhook-triggered syncs or manual selection).
Automatically expands folders to their contents.
-
+
Args:
connection_id: The connection ID
user_id: The user ID
@@ -338,9 +337,7 @@ class ConnectorService:
connector = await self.get_connector(connection_id)
if not connector:
- raise ValueError(
- f"Connection '{connection_id}' not found or not authenticated"
- )
+ raise ValueError(f"Connection '{connection_id}' not found or not authenticated")
if not connector.is_authenticated:
raise ValueError(f"Connection '{connection_id}' not authenticated")
@@ -355,7 +352,7 @@ class ConnectorService:
# If file_infos provided, cache them in the connector for later use
# This allows get_file_content to use download URLs directly
- if file_infos and hasattr(connector, 'set_file_infos'):
+ if file_infos and hasattr(connector, "set_file_infos"):
connector.set_file_infos(file_infos)
logger.info(f"Cached {len(file_infos)} file infos with download URLs in connector")
@@ -389,16 +386,33 @@ class ConnectorService:
)
# If we have file_infos with download URLs, use original file_ids
# (OneDrive sharing IDs can't be expanded but can be downloaded directly)
+ # Exclude folders — they have no downloadable content on their own.
if file_infos:
- logger.info("Using original file IDs with cached download URLs")
- expanded_file_ids = file_ids
+ non_folder_infos = [f for f in file_infos if not f.get("isFolder")]
+ non_folder_ids = [f["id"] for f in non_folder_infos if f.get("id")]
+ if non_folder_ids:
+ logger.info(
+ "Using original file IDs with cached download URLs (folders excluded)"
+ )
+ expanded_file_ids = non_folder_ids
+ else:
+ raise ValueError("No files to sync after expanding folders")
else:
raise ValueError("No files to sync after expanding folders")
except Exception as e:
logger.error(f"Failed to expand file_ids via list_files(): {e}")
- # Fallback to original file_ids if expansion fails
- expanded_file_ids = file_ids
+ # Preserve intentional validation failures (e.g., folders-only selection)
+ if isinstance(e, ValueError):
+ raise
+ # Fallback path: still exclude known folders when metadata is available
+ if file_infos:
+ non_folder_ids = [
+ f["id"] for f in file_infos if f.get("id") and not f.get("isFolder")
+ ]
+ expanded_file_ids = non_folder_ids or file_ids
+ else:
+ expanded_file_ids = file_ids
finally:
# Restore original config values
if hasattr(connector, "cfg"):
@@ -431,9 +445,7 @@ class ConnectorService:
original_filenames = {}
if file_infos:
original_filenames = {
- f["id"]: clean_connector_filename(
- f["name"], f.get("mimeType") or f.get("mimetype")
- )
+ f["id"]: clean_connector_filename(f["name"], f.get("mimeType") or f.get("mimetype"))
for f in file_infos
if "id" in f and "name" in f
}
diff --git a/src/connectors/sharepoint/connector.py b/src/connectors/sharepoint/connector.py
index e4f551c15..0a1b6cf61 100644
--- a/src/connectors/sharepoint/connector.py
+++ b/src/connectors/sharepoint/connector.py
@@ -1,14 +1,16 @@
-import logging
+from datetime import datetime
from pathlib import Path
-from typing import List, Dict, Any, Optional
+from typing import Any, Dict, List, Optional
from urllib.parse import urlparse
-from datetime import datetime
+
import httpx
+from utils.logging_config import get_logger
+
from ..base import BaseConnector, ConnectorDocument, DocumentACL
from .oauth import SharePointOAuth
-logger = logging.getLogger(__name__)
+logger = get_logger(__name__)
class SharePointConnector(BaseConnector):
@@ -17,23 +19,23 @@ class SharePointConnector(BaseConnector):
# Required BaseConnector class attributes
CLIENT_ID_ENV_VAR = "MICROSOFT_GRAPH_OAUTH_CLIENT_ID"
CLIENT_SECRET_ENV_VAR = "MICROSOFT_GRAPH_OAUTH_CLIENT_SECRET" # pragma: allowlist secret
-
+
# Connector metadata
CONNECTOR_NAME = "SharePoint"
CONNECTOR_DESCRIPTION = "Add knowledge from SharePoint"
CONNECTOR_ICON = "sharepoint"
-
+
def __init__(self, config: Dict[str, Any]):
super().__init__(config)
logger.debug(f"SharePoint connector __init__ called with config type: {type(config)}")
logger.debug(f"SharePoint connector __init__ config value: {config}")
-
+
# Ensure we always pass a valid config to the base class
if config is None:
logger.debug("Config was None, using empty dict")
config = {}
-
+
try:
logger.debug("Calling super().__init__")
super().__init__(config) # Now safe to call with empty dict instead of None
@@ -41,16 +43,18 @@ class SharePointConnector(BaseConnector):
except Exception as e:
logger.error(f"super().__init__ failed: {e}")
raise
-
+
# Initialize with defaults that allow the connector to be listed
self.client_id = None
self.client_secret = None
self.tenant_id = config.get("tenant_id", "common")
# base_url is the generic field name, sharepoint_url is kept for backward compatibility
self.sharepoint_url = config.get("base_url") or config.get("sharepoint_url")
- logger.debug(f"SharePoint connector initialized with sharepoint_url from config: {self.sharepoint_url}")
+ logger.debug(
+ f"SharePoint connector initialized with sharepoint_url from config: {self.sharepoint_url}"
+ )
self.redirect_uri = config.get("redirect_uri", "http://localhost")
-
+
# Try to get credentials, but don't fail if they're missing
try:
logger.debug("Attempting to get client_id")
@@ -59,7 +63,7 @@ class SharePointConnector(BaseConnector):
except Exception as e:
logger.debug(f"Failed to get client_id: {e}")
pass # Credentials not available, that's OK for listing
-
+
try:
logger.debug("Attempting to get client_secret")
self.client_secret = self.get_client_secret()
@@ -70,6 +74,7 @@ class SharePointConnector(BaseConnector):
# Token file setup - use data directory for persistence
from config.paths import get_data_file
+
token_file = config.get("token_file") or get_data_file("sharepoint_token.json")
Path(token_file).parent.mkdir(parents=True, exist_ok=True)
@@ -82,47 +87,59 @@ class SharePointConnector(BaseConnector):
oauth_token_file = config["token_file"]
else:
oauth_token_file = get_data_file(f"sharepoint_token_{connection_id}.json")
-
- authority = f"https://login.microsoftonline.com/{self.tenant_id}" if self.tenant_id != "common" else "https://login.microsoftonline.com/common"
-
+
+ authority = (
+ f"https://login.microsoftonline.com/{self.tenant_id}"
+ if self.tenant_id != "common"
+ else "https://login.microsoftonline.com/common"
+ )
+
self.oauth = SharePointOAuth(
client_id=self.client_id,
client_secret=self.client_secret,
token_file=oauth_token_file,
- authority=authority
+ authority=authority,
)
else:
self.oauth = None
-
+
# Track subscription ID for webhooks
self._subscription_id: Optional[str] = None
-
+
# Add Graph API defaults similar to Google Drive flags
self._graph_api_version = "v1.0"
self._default_params = {
"$select": "id,name,size,lastModifiedDateTime,createdDateTime,webUrl,file,folder,@microsoft.graph.downloadUrl"
}
-
+
# Selective sync support (similar to Google Drive and OneDrive)
- self.cfg = type('SharePointConfig', (), {
- 'file_ids': config.get('file_ids') or config.get('selected_files') or config.get('selected_file_ids'),
- 'folder_ids': config.get('folder_ids') or config.get('selected_folders') or config.get('selected_folder_ids'),
- })()
-
+ self.cfg = type(
+ "SharePointConfig",
+ (),
+ {
+ "file_ids": config.get("file_ids")
+ or config.get("selected_files")
+ or config.get("selected_file_ids"),
+ "folder_ids": config.get("folder_ids")
+ or config.get("selected_folders")
+ or config.get("selected_folder_ids"),
+ },
+ )()
+
# Cache for file metadata including download URLs
# This allows direct download without Graph API for sharing IDs
self._file_infos: Dict[str, Dict[str, Any]] = {}
-
+
@property
def _graph_base_url(self) -> str:
"""Base URL for Microsoft Graph API calls"""
return f"https://graph.microsoft.com/{self._graph_api_version}"
-
+
@property
def base_url(self) -> Optional[str]:
"""Generic base URL property (returns sharepoint_url for SharePoint connector)"""
return self.sharepoint_url
-
+
@base_url.setter
def base_url(self, value: str):
"""Set base URL (updates sharepoint_url internally)"""
@@ -132,7 +149,7 @@ class SharePointConnector(BaseConnector):
"""
Cache file metadata including download URLs for later use.
This allows direct download without Graph API calls for sharing IDs.
-
+
Args:
file_infos: List of file info dicts with {id, name, mimeType, downloadUrl, size}
"""
@@ -147,13 +164,13 @@ class SharePointConnector(BaseConnector):
def get_cached_file_info(self, file_id: str) -> Optional[Dict[str, Any]]:
"""Get cached file info by ID."""
return self._file_infos.get(file_id)
-
+
def emit(self, doc: ConnectorDocument) -> None:
"""
Emit a ConnectorDocument instance.
"""
logger.debug(f"Emitting SharePoint document: {doc.id} ({doc.filename})")
-
+
async def authenticate(self) -> bool:
"""Test authentication - BaseConnector interface"""
logger.debug(f"SharePoint authenticate() called, oauth is None: {self.oauth is None}")
@@ -162,29 +179,29 @@ class SharePointConnector(BaseConnector):
logger.debug("SharePoint authentication failed: OAuth not initialized")
self._authenticated = False
return False
-
+
logger.debug("Loading SharePoint credentials...")
# Try to load existing credentials first
load_result = await self.oauth.load_credentials()
logger.debug(f"Load credentials result: {load_result}")
-
+
logger.debug("Checking SharePoint authentication status...")
authenticated = await self.oauth.is_authenticated()
logger.debug(f"SharePoint is_authenticated result: {authenticated}")
-
+
self._authenticated = authenticated
return authenticated
except Exception as e:
logger.exception("[CONNECTOR] SharePoint authentication failed")
self._authenticated = False
return False
-
+
def get_auth_url(self) -> str:
"""Get OAuth authorization URL"""
if not self.oauth:
raise RuntimeError("SharePoint OAuth not initialized - missing credentials")
return self.oauth.create_authorization_url(self.redirect_uri)
-
+
async def handle_oauth_callback(self, auth_code: str) -> Dict[str, Any]:
"""Handle OAuth callback"""
if not self.oauth:
@@ -193,24 +210,24 @@ class SharePointConnector(BaseConnector):
success = await self.oauth.handle_authorization_callback(auth_code, self.redirect_uri)
if success:
self._authenticated = True
-
+
# Auto-detect base URL from user's drive
detected_url = await self._detect_base_url()
if detected_url:
self.base_url = detected_url
logger.info(f"Auto-detected base URL: {detected_url}")
-
+
return {"status": "success", "base_url": self.base_url}
else:
raise ValueError("OAuth callback failed")
except Exception as e:
logger.error(f"OAuth callback failed: {e}")
raise
-
+
async def _detect_base_url(self) -> Optional[str]:
"""Override base class method to detect SharePoint URL"""
return await self._detect_sharepoint_url()
-
+
async def _detect_sharepoint_url(self) -> Optional[str]:
"""Auto-detect SharePoint URL from Microsoft Graph API"""
logger.info("_detect_sharepoint_url: Starting SharePoint URL detection")
@@ -218,236 +235,244 @@ class SharePointConnector(BaseConnector):
if not self.oauth:
logger.warning("_detect_sharepoint_url: OAuth not initialized")
return None
-
+
access_token = self.oauth.get_access_token()
- logger.debug(f"_detect_sharepoint_url: Got access token (length: {len(access_token) if access_token else 0})")
-
+ logger.debug(
+ f"_detect_sharepoint_url: Got access token (length: {len(access_token) if access_token else 0})"
+ )
+
headers = {
"Authorization": f"Bearer {access_token}",
"Content-Type": "application/json",
}
-
+
async with httpx.AsyncClient() as client:
# Get user's default drive to extract SharePoint URL
url = f"{self._graph_base_url}/me/drive"
logger.info(f"_detect_sharepoint_url: Calling Graph API: {url}")
-
+
response = await client.get(url, headers=headers, timeout=30.0)
- logger.info(f"_detect_sharepoint_url: Graph API response status: {response.status_code}")
-
+ logger.info(
+ f"_detect_sharepoint_url: Graph API response status: {response.status_code}"
+ )
+
if response.status_code == 200:
data = response.json()
web_url = data.get("webUrl", "")
logger.info(f"_detect_sharepoint_url: webUrl from response: {web_url}")
-
+
# Extract the SharePoint domain from the webUrl
if web_url:
parsed = urlparse(web_url)
sharepoint_url = f"{parsed.scheme}://{parsed.netloc}"
- logger.info(f"_detect_sharepoint_url: Detected SharePoint URL: {sharepoint_url}")
+ logger.info(
+ f"_detect_sharepoint_url: Detected SharePoint URL: {sharepoint_url}"
+ )
return sharepoint_url
else:
logger.warning("_detect_sharepoint_url: webUrl is empty in response")
else:
- logger.warning("[CONNECTOR] SharePoint detect URL failed", status_code=response.status_code)
-
+ logger.warning(
+ "[CONNECTOR] SharePoint detect URL failed", status_code=response.status_code
+ )
+
except Exception as e:
logger.exception("[CONNECTOR] SharePoint URL detection failed")
return None
-
+
def sync_once(self) -> None:
"""
Perform a one-shot sync of SharePoint files and emit documents.
This method mirrors the Google Drive connector's sync_once functionality.
"""
import asyncio
-
+
async def _async_sync():
try:
# Get list of files
file_list = await self.list_files(max_files=1000) # Adjust as needed
files = file_list.get("files", [])
-
+
for file_info in files:
try:
file_id = file_info.get("id")
if not file_id:
continue
-
+
# Get full document content
doc = await self.get_file_content(file_id)
self.emit(doc)
-
+
except Exception as e:
- logger.error(f"Failed to sync SharePoint file {file_info.get('name', 'unknown')}: {e}")
+ logger.error(
+ f"Failed to sync SharePoint file {file_info.get('name', 'unknown')}: {e}"
+ )
continue
-
+
except Exception as e:
logger.error(f"SharePoint sync_once failed: {e}")
raise
-
+
# Run the async sync
- if hasattr(asyncio, 'run'):
+ if hasattr(asyncio, "run"):
asyncio.run(_async_sync())
else:
# Python < 3.7 compatibility
loop = asyncio.get_event_loop()
loop.run_until_complete(_async_sync())
-
+
async def setup_subscription(self) -> str:
"""Set up real-time subscription for file changes - BaseConnector interface"""
- webhook_url = self.config.get('webhook_url')
+ webhook_url = self.config.get("webhook_url")
if not webhook_url:
logger.warning("No webhook URL configured, skipping SharePoint subscription setup")
return "no-webhook-configured"
-
+
try:
# Ensure we're authenticated
if not await self.authenticate():
raise RuntimeError("SharePoint authentication failed during subscription setup")
-
+
token = self.oauth.get_access_token()
-
+
# Microsoft Graph subscription for SharePoint site
site_info = self._parse_sharepoint_url()
if site_info:
- resource = f"sites/{site_info['host_name']}:/sites/{site_info['site_name']}:/drive/root"
+ resource = (
+ f"sites/{site_info['host_name']}:/sites/{site_info['site_name']}:/drive/root"
+ )
else:
resource = "/me/drive/root"
-
+
subscription_data = {
"changeType": "created,updated,deleted",
"notificationUrl": f"{webhook_url}/webhook/sharepoint",
"resource": resource,
"expirationDateTime": self._get_subscription_expiry(),
- "clientState": f"sharepoint_{self.tenant_id}"
- }
-
- headers = {
- "Authorization": f"Bearer {token}",
- "Content-Type": "application/json"
+ "clientState": f"sharepoint_{self.tenant_id}",
}
-
+
+ headers = {"Authorization": f"Bearer {token}", "Content-Type": "application/json"}
+
url = f"{self._graph_base_url}/subscriptions"
-
+
async with httpx.AsyncClient() as client:
- response = await client.post(url, json=subscription_data, headers=headers, timeout=30)
+ response = await client.post(
+ url, json=subscription_data, headers=headers, timeout=30
+ )
response.raise_for_status()
-
+
result = response.json()
subscription_id = result.get("id")
-
+
if subscription_id:
self._subscription_id = subscription_id
logger.info(f"SharePoint subscription created: {subscription_id}")
return subscription_id
else:
raise ValueError("No subscription ID returned from Microsoft Graph")
-
+
except Exception as e:
logger.error(f"Failed to setup SharePoint subscription: {e}")
raise
-
+
def _get_subscription_expiry(self) -> str:
"""Get subscription expiry time (max 3 days for Graph API)"""
from datetime import datetime, timedelta
+
expiry = datetime.utcnow() + timedelta(days=3) # 3 days max for Graph
return expiry.strftime("%Y-%m-%dT%H:%M:%S.%fZ")
-
+
def _parse_sharepoint_url(self) -> Optional[Dict[str, str]]:
"""Parse SharePoint URL to extract site information for Graph API"""
if not self.sharepoint_url:
return None
-
+
try:
parsed = urlparse(self.sharepoint_url)
# Extract hostname and site name from URL like: https://contoso.sharepoint.com/sites/teamsite
host_name = parsed.netloc
- path_parts = parsed.path.strip('/').split('/')
-
- if len(path_parts) >= 2 and path_parts[0] == 'sites':
+ path_parts = parsed.path.strip("/").split("/")
+
+ if len(path_parts) >= 2 and path_parts[0] == "sites":
site_name = path_parts[1]
- return {
- "host_name": host_name,
- "site_name": site_name
- }
+ return {"host_name": host_name, "site_name": site_name}
except Exception as e:
logger.warning(f"Could not parse SharePoint URL {self.sharepoint_url}: {e}")
-
+
return None
-
+
async def list_files(
- self,
- page_token: Optional[str] = None,
- max_files: Optional[int] = None,
- **kwargs
+ self, page_token: Optional[str] = None, max_files: Optional[int] = None, **kwargs
) -> Dict[str, Any]:
"""List all files using Microsoft Graph API - BaseConnector interface"""
try:
# Ensure authentication
if not await self.authenticate():
raise RuntimeError("SharePoint authentication failed during file listing")
-
+
# If file_ids or folder_ids are specified in config, use selective sync
if self.cfg.file_ids or self.cfg.folder_ids:
return await self._list_selected_files()
-
+
files = []
max_files_value = max_files if max_files is not None else 100
-
+
# Build Graph API URL for the site or fallback to user's OneDrive
site_info = self._parse_sharepoint_url()
if site_info:
base_url = f"{self._graph_base_url}/sites/{site_info['host_name']}:/sites/{site_info['site_name']}:/drive/root/children"
else:
base_url = f"{self._graph_base_url}/me/drive/root/children"
-
+
params = dict(self._default_params)
params["$top"] = str(max_files_value)
-
+
if page_token:
params["$skiptoken"] = page_token
-
+
response = await self._make_graph_request(base_url, params=params)
data = response.json()
-
+
items = data.get("value", [])
for item in items:
# Only include files, not folders
if item.get("file"):
- files.append({
- "id": item.get("id", ""),
- "name": item.get("name", ""),
- "path": f"/drive/items/{item.get('id')}",
- "size": int(item.get("size", 0)),
- "modified": item.get("lastModifiedDateTime"),
- "created": item.get("createdDateTime"),
- "mime_type": item.get("file", {}).get("mimeType", self._get_mime_type(item.get("name", ""))),
- "url": item.get("webUrl", ""),
- "download_url": item.get("@microsoft.graph.downloadUrl")
- })
+ files.append(
+ {
+ "id": item.get("id", ""),
+ "name": item.get("name", ""),
+ "path": f"/drive/items/{item.get('id')}",
+ "size": int(item.get("size", 0)),
+ "modified": item.get("lastModifiedDateTime"),
+ "created": item.get("createdDateTime"),
+ "mime_type": item.get("file", {}).get(
+ "mimeType", self._get_mime_type(item.get("name", ""))
+ ),
+ "url": item.get("webUrl", ""),
+ "download_url": item.get("@microsoft.graph.downloadUrl"),
+ }
+ )
# Check for next page
next_page_token = None
next_link = data.get("@odata.nextLink")
if next_link:
- from urllib.parse import urlparse, parse_qs
+ from urllib.parse import parse_qs, urlparse
+
parsed = urlparse(next_link)
query_params = parse_qs(parsed.query)
if "$skiptoken" in query_params:
next_page_token = query_params["$skiptoken"][0]
- return {
- "files": files,
- "next_page_token": next_page_token
- }
+ return {"files": files, "next_page_token": next_page_token}
except Exception as e:
logger.error(f"Failed to list SharePoint files: {e}")
return {"files": [], "next_page_token": None} # Return empty result instead of raising
-
+
async def _extract_sharepoint_acl(self, file_id: str, file_metadata: Dict) -> DocumentACL:
"""
Extract ACL from SharePoint item.
@@ -481,8 +506,7 @@ class SharePointConnector(BaseConnector):
# Fetch permissions
async with httpx.AsyncClient() as client:
response = await client.get(
- permissions_url,
- headers={"Authorization": f"Bearer {access_token}"}
+ permissions_url, headers={"Authorization": f"Bearer {access_token}"}
)
if response.status_code != 200:
@@ -512,7 +536,9 @@ class SharePointConnector(BaseConnector):
# Granted to identities (can include users and groups)
if "grantedToIdentitiesV2" in perm or "grantedToIdentities" in perm:
- identities = perm.get("grantedToIdentitiesV2") or perm.get("grantedToIdentities") or []
+ identities = (
+ perm.get("grantedToIdentitiesV2") or perm.get("grantedToIdentities") or []
+ )
for identity in identities:
# User
if "user" in identity:
@@ -549,17 +575,17 @@ class SharePointConnector(BaseConnector):
# Ensure authentication
if not await self.authenticate():
raise RuntimeError("SharePoint authentication failed during file content retrieval")
-
+
# First, check for cached file info with download URL
# This is used for SharePoint sharing IDs that can't be resolved via Graph API
cached_info = self.get_cached_file_info(file_id)
if cached_info and cached_info.get("downloadUrl"):
logger.info(f"Using cached download URL for file {file_id}")
content = await self._download_file_from_url(cached_info["downloadUrl"])
-
+
# Extract ACL even for cached files
acl = await self._extract_sharepoint_acl(file_id, cached_info)
-
+
return ConnectorDocument(
id=file_id,
filename=cached_info.get("name", "Unknown"),
@@ -575,27 +601,27 @@ class SharePointConnector(BaseConnector):
"size": cached_info.get("size", 0),
},
)
-
+
# Fall back to Graph API for regular file IDs
file_metadata = await self._get_file_metadata_by_id(file_id)
-
+
if not file_metadata:
raise ValueError(f"File not found: {file_id}")
-
+
# Download file content
download_url = file_metadata.get("download_url")
if download_url:
content = await self._download_file_from_url(download_url)
else:
content = await self._download_file_content(file_id)
-
+
# Extract ACL from SharePoint item
acl = await self._extract_sharepoint_acl(file_id, file_metadata)
-
+
# Parse dates
modified_time = self._parse_graph_date(file_metadata.get("modified"))
created_time = self._parse_graph_date(file_metadata.get("created"))
-
+
return ConnectorDocument(
id=file_id,
filename=file_metadata.get("name", ""),
@@ -608,14 +634,14 @@ class SharePointConnector(BaseConnector):
metadata={
"sharepoint_path": file_metadata.get("path", ""),
"sharepoint_url": self.sharepoint_url,
- "size": file_metadata.get("size", 0)
- }
+ "size": file_metadata.get("size", 0),
+ },
)
-
+
except Exception as e:
logger.error(f"Failed to get SharePoint file content {file_id}: {e}")
raise
-
+
async def _get_file_metadata_by_id(self, file_id: str) -> Optional[Dict[str, Any]]:
"""Get file metadata by ID using Graph API"""
try:
@@ -625,12 +651,12 @@ class SharePointConnector(BaseConnector):
url = f"{self._graph_base_url}/sites/{site_info['host_name']}:/sites/{site_info['site_name']}:/drive/items/{file_id}"
else:
url = f"{self._graph_base_url}/me/drive/items/{file_id}"
-
+
params = dict(self._default_params)
-
+
response = await self._make_graph_request(url, params=params)
item = response.json()
-
+
if item.get("file"):
return {
"id": file_id,
@@ -639,11 +665,13 @@ class SharePointConnector(BaseConnector):
"size": int(item.get("size", 0)),
"modified": item.get("lastModifiedDateTime"),
"created": item.get("createdDateTime"),
- "mime_type": item.get("file", {}).get("mimeType", self._get_mime_type(item.get("name", ""))),
+ "mime_type": item.get("file", {}).get(
+ "mimeType", self._get_mime_type(item.get("name", ""))
+ ),
"url": item.get("webUrl", ""),
- "download_url": item.get("@microsoft.graph.downloadUrl")
+ "download_url": item.get("@microsoft.graph.downloadUrl"),
}
-
+
# Check if it's a folder
if item.get("folder"):
return {
@@ -651,13 +679,17 @@ class SharePointConnector(BaseConnector):
"name": item.get("name", ""),
"isFolder": True,
}
-
+
return None
-
+
except Exception as e:
- logger.error(f"Failed to get file metadata for {file_id}: {e}")
+ logger.error(
+ f"Failed to get file metadata for {file_id}: {e}. "
+ f"Site info: {self._parse_sharepoint_url()}, "
+ f"SharePoint URL: {self.sharepoint_url}"
+ )
return None
-
+
async def _download_file_content(self, file_id: str) -> bytes:
"""Download file content by file ID using Graph API"""
try:
@@ -666,38 +698,38 @@ class SharePointConnector(BaseConnector):
url = f"{self._graph_base_url}/sites/{site_info['host_name']}:/sites/{site_info['site_name']}:/drive/items/{file_id}/content"
else:
url = f"{self._graph_base_url}/me/drive/items/{file_id}/content"
-
+
token = self.oauth.get_access_token()
headers = {"Authorization": f"Bearer {token}"}
-
+
async with httpx.AsyncClient() as client:
response = await client.get(url, headers=headers, timeout=60, follow_redirects=True)
response.raise_for_status()
return response.content
-
+
except Exception as e:
logger.error(f"Failed to download file content for {file_id}: {e}")
raise
-
+
async def _list_selected_files(self) -> Dict[str, Any]:
"""List only selected files/folders (selective sync)."""
files: List[Dict[str, Any]] = []
-
+
# Process selected file IDs
if self.cfg.file_ids:
for file_id in self.cfg.file_ids:
try:
file_meta = await self._get_file_metadata_by_id(file_id)
- if file_meta and not file_meta.get('isFolder', False):
+ if file_meta and not file_meta.get("isFolder", False):
files.append(file_meta)
- elif file_meta and file_meta.get('isFolder', False):
+ elif file_meta and file_meta.get("isFolder", False):
# If it's a folder, expand its contents
folder_files = await self._list_folder_contents(file_id)
files.extend(folder_files)
except Exception as e:
logger.warning(f"Failed to get file {file_id}: {e}")
continue
-
+
# Process selected folder IDs
if self.cfg.folder_ids:
for folder_id in self.cfg.folder_ids:
@@ -707,25 +739,25 @@ class SharePointConnector(BaseConnector):
except Exception as e:
logger.warning(f"Failed to list folder {folder_id}: {e}")
continue
-
+
return {"files": files, "next_page_token": None}
-
+
async def _list_folder_contents(self, folder_id: str) -> List[Dict[str, Any]]:
"""List all files in a folder recursively."""
files: List[Dict[str, Any]] = []
-
+
try:
site_info = self._parse_sharepoint_url()
if site_info:
url = f"{self._graph_base_url}/sites/{site_info['host_name']}:/sites/{site_info['site_name']}:/drive/items/{folder_id}/children"
else:
url = f"{self._graph_base_url}/me/drive/items/{folder_id}/children"
-
+
params = dict(self._default_params)
-
+
response = await self._make_graph_request(url, params=params)
data = response.json()
-
+
items = data.get("value", [])
for item in items:
if item.get("file"): # It's a file
@@ -737,9 +769,9 @@ class SharePointConnector(BaseConnector):
files.extend(subfolder_files)
except Exception as e:
logger.error(f"Failed to list folder contents for {folder_id}: {e}")
-
+
return files
-
+
async def _download_file_from_url(self, download_url: str) -> bytes:
"""Download file content from direct download URL"""
try:
@@ -750,29 +782,31 @@ class SharePointConnector(BaseConnector):
except Exception as e:
logger.error(f"Failed to download from URL {download_url}: {e}")
raise
-
+
def _parse_graph_date(self, date_str: Optional[str]) -> datetime:
"""Parse Microsoft Graph date string to datetime"""
if not date_str:
return datetime.now()
-
+
try:
- if date_str.endswith('Z'):
+ if date_str.endswith("Z"):
return datetime.fromisoformat(date_str[:-1]).replace(tzinfo=None)
else:
- return datetime.fromisoformat(date_str.replace('T', ' '))
+ return datetime.fromisoformat(date_str.replace("T", " "))
except (ValueError, AttributeError):
return datetime.now()
-
- async def _make_graph_request(self, url: str, method: str = "GET",
- data: Optional[Dict] = None, params: Optional[Dict] = None) -> httpx.Response:
+
+ async def _make_graph_request(
+ self,
+ url: str,
+ method: str = "GET",
+ data: Optional[Dict] = None,
+ params: Optional[Dict] = None,
+ ) -> httpx.Response:
"""Make authenticated API request to Microsoft Graph"""
token = self.oauth.get_access_token()
- headers = {
- "Authorization": f"Bearer {token}",
- "Content-Type": "application/json"
- }
-
+ headers = {"Authorization": f"Bearer {token}", "Content-Type": "application/json"}
+
async with httpx.AsyncClient() as client:
if method.upper() == "GET":
response = await client.get(url, headers=headers, params=params, timeout=30)
@@ -782,36 +816,39 @@ class SharePointConnector(BaseConnector):
response = await client.delete(url, headers=headers, timeout=30)
else:
raise ValueError(f"Unsupported HTTP method: {method}")
-
+
response.raise_for_status()
return response
-
+
def _get_mime_type(self, filename: str) -> str:
"""Get MIME type based on file extension"""
import mimetypes
+
mime_type, _ = mimetypes.guess_type(filename)
return mime_type or "application/octet-stream"
-
+
# Webhook methods - BaseConnector interface
- def handle_webhook_validation(self, request_method: str, headers: Dict[str, str],
- query_params: Dict[str, str]) -> Optional[str]:
+ def handle_webhook_validation(
+ self, request_method: str, headers: Dict[str, str], query_params: Dict[str, str]
+ ) -> Optional[str]:
"""Handle webhook validation (Graph API specific)"""
if request_method == "POST" and "validationToken" in query_params:
return query_params["validationToken"]
return None
-
- def extract_webhook_channel_id(self, payload: Dict[str, Any],
- headers: Dict[str, str]) -> Optional[str]:
+
+ def extract_webhook_channel_id(
+ self, payload: Dict[str, Any], headers: Dict[str, str]
+ ) -> Optional[str]:
"""Extract channel/subscription ID from webhook payload"""
notifications = payload.get("value", [])
if notifications:
return notifications[0].get("subscriptionId")
return None
-
+
async def handle_webhook(self, payload: Dict[str, Any]) -> List[str]:
"""Handle webhook notification and return affected file IDs"""
affected_files = []
-
+
# Process Microsoft Graph webhook payload
notifications = payload.get("value", [])
for notification in notifications:
@@ -819,36 +856,40 @@ class SharePointConnector(BaseConnector):
if resource and "/drive/items/" in resource:
file_id = resource.split("/drive/items/")[-1]
affected_files.append(file_id)
-
+
return affected_files
-
+
async def cleanup_subscription(self, subscription_id: str) -> bool:
"""Clean up subscription - BaseConnector interface"""
if subscription_id == "no-webhook-configured":
logger.info("No subscription to cleanup (webhook was not configured)")
return True
-
+
try:
# Ensure authentication
if not await self.authenticate():
logger.error("SharePoint authentication failed during subscription cleanup")
return False
-
+
token = self.oauth.get_access_token()
headers = {"Authorization": f"Bearer {token}"}
-
+
url = f"{self._graph_base_url}/subscriptions/{subscription_id}"
-
+
async with httpx.AsyncClient() as client:
response = await client.delete(url, headers=headers, timeout=30)
-
+
if response.status_code in [200, 204, 404]:
- logger.info(f"SharePoint subscription {subscription_id} cleaned up successfully")
+ logger.info(
+ f"SharePoint subscription {subscription_id} cleaned up successfully"
+ )
return True
else:
- logger.warning(f"Unexpected response cleaning up subscription: {response.status_code}")
+ logger.warning(
+ f"Unexpected response cleaning up subscription: {response.status_code}"
+ )
return False
-
+
except Exception as e:
logger.error(f"Failed to cleanup SharePoint subscription {subscription_id}: {e}")
return False