Skip to content

OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.

Python

fix: display file names and fix ingestion for onedrive (#1609)

Lucas Oliveira62efe56491a4
diff --git a/frontend/app/api/queries/useGetSearchQuery.ts b/frontend/app/api/queries/useGetSearchQuery.ts
index 94e55af8f..0ccb88eb3 100644
--- a/frontend/app/api/queries/useGetSearchQuery.ts
+++ b/frontend/app/api/queries/useGetSearchQuery.ts
@@ -85,7 +85,10 @@ export { EMPTY_SEARCH_RESULT };
export const useGetSearchQuery = (
query: string,
queryData?: ParsedQueryData | null,
- options?: Omit<UseQueryOptions, "queryKey" | "queryFn">,
+ options?: Omit<
+ UseQueryOptions<SearchResult, Error, SearchResult, any[]>,
+ "queryKey" | "queryFn"
+ >,
) => {
const queryClient = useQueryClient();
const getFileIdentity = (chunk: ChunkResult): string => {
diff --git a/frontend/components/cloud-picker/file-item.tsx b/frontend/components/cloud-picker/file-item.tsx
index 3f1306c26..f4d7d4257 100644
--- a/frontend/components/cloud-picker/file-item.tsx
+++ b/frontend/components/cloud-picker/file-item.tsx
@@ -31,10 +31,24 @@ const getMimeTypeLabel = (mimeType: string) => {
"application/vnd.google-apps.folder": "Folder",
"application/pdf": "PDF",
"text/plain": "Text",
+ "text/csv": "CSV",
+ "text/html": "HTML",
+ "application/xml": "XML",
+ "application/json": "JSON",
+ "application/msword": "Word Doc",
+ "application/vnd.ms-excel": "Excel",
+ "application/vnd.ms-powerpoint": "PowerPoint",
"application/vnd.openxmlformats-officedocument.wordprocessingml.document":
"Word Doc",
+ "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet":
+ "Excel",
"application/vnd.openxmlformats-officedocument.presentationml.presentation":
"PowerPoint",
+ "application/octet-stream": "File",
+ "image/jpeg": "JPEG",
+ "image/png": "PNG",
+ "image/gif": "GIF",
+ "image/svg+xml": "SVG",
};
return typeMap[mimeType] || mimeType?.split("/").pop() || "Document";
diff --git a/frontend/components/cloud-picker/provider-handlers.ts b/frontend/components/cloud-picker/provider-handlers.ts
index b64808918..2966c4083 100644
--- a/frontend/components/cloud-picker/provider-handlers.ts
+++ b/frontend/components/cloud-picker/provider-handlers.ts
@@ -159,17 +159,20 @@ export class OneDriveHandler {
private clientId: string;
private provider: CloudProvider;
private baseUrl?: string;
+ private onPickerStateChange?: (isOpen: boolean) => void;
constructor(
accessToken: string,
clientId: string,
provider: CloudProvider = "onedrive",
baseUrl?: string,
+ onPickerStateChange?: (isOpen: boolean) => void,
) {
this.accessToken = accessToken;
this.clientId = clientId;
this.provider = provider;
this.baseUrl = baseUrl;
+ this.onPickerStateChange = onPickerStateChange;
}
async loadPickerApi(): Promise<boolean> {
@@ -206,57 +209,111 @@ export class OneDriveHandler {
console.log("OneDrive picker success callback:", response);
if (!response || !response.value) {
console.warn("OneDrive picker returned no value");
+ this.onPickerStateChange?.(false);
return;
}
- const newFiles: CloudFile[] =
- response.value?.map((item: any) => {
- // Extract mimeType from file object or infer from name
- let mimeType = item.file?.mimeType;
- if (!mimeType && item.name) {
- // Infer from extension if mimeType not provided
- const ext = item.name.split(".").pop()?.toLowerCase();
- const mimeTypes: { [key: string]: string } = {
- pdf: "application/pdf",
- doc: "application/msword",
- docx: "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
- xls: "application/vnd.ms-excel",
- xlsx: "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
- ppt: "application/vnd.ms-powerpoint",
- pptx: "application/vnd.openxmlformats-officedocument.presentationml.presentation",
- txt: "text/plain",
- csv: "text/csv",
- json: "application/json",
- xml: "application/xml",
- html: "text/html",
- jpg: "image/jpeg",
- jpeg: "image/jpeg",
- png: "image/png",
- gif: "image/gif",
- svg: "image/svg+xml",
- };
- mimeType = mimeTypes[ext || ""] || "application/octet-stream";
- }
+ // v7.2 action:"query" returns stubs with only id/endpoint/parentReference.
+ // Enrich each item by fetching full metadata from the Graph API.
+ const enrichItems = async () => {
+ const enriched = await Promise.all(
+ response.value.map(async (item: any) => {
+ const driveId =
+ item.parentReference?.driveId || item.id?.split("!")[0];
+ const itemId = item.id;
- return {
- id: item.id,
- name: item.name || `${this.getProviderName()} File`,
- mimeType: mimeType || "application/octet-stream",
- webUrl: item.webUrl || "",
- downloadUrl: item["@microsoft.graph.downloadUrl"] || "",
- size: item.size,
- modifiedTime: item.lastModifiedDateTime,
- isFolder: !!item.folder,
- };
- }) || [];
+ if (driveId && itemId) {
+ try {
+ const url = `https://graph.microsoft.com/v1.0/drives/${driveId}/items/${itemId}`;
+ const res = await fetch(url, {
+ headers: { Authorization: `Bearer ${this.accessToken}` },
+ });
+ if (res.ok) {
+ const meta = await res.json();
+ console.log(
+ "OneDrive enriched metadata:",
+ meta.name,
+ meta.file?.mimeType,
+ );
+
+ let mimeType = meta.file?.mimeType;
+ if (!mimeType && meta.name) {
+ const ext = meta.name.split(".").pop()?.toLowerCase();
+ const mimeTypes: { [key: string]: string } = {
+ pdf: "application/pdf",
+ doc: "application/msword",
+ docx: "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
+ xls: "application/vnd.ms-excel",
+ xlsx: "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
+ ppt: "application/vnd.ms-powerpoint",
+ pptx: "application/vnd.openxmlformats-officedocument.presentationml.presentation",
+ txt: "text/plain",
+ csv: "text/csv",
+ json: "application/json",
+ xml: "application/xml",
+ html: "text/html",
+ jpg: "image/jpeg",
+ jpeg: "image/jpeg",
+ png: "image/png",
+ gif: "image/gif",
+ svg: "image/svg+xml",
+ };
+ mimeType =
+ mimeTypes[ext || ""] || "application/octet-stream";
+ }
- onFileSelected(newFiles);
+ return {
+ id: meta.id,
+ name: meta.name || `${this.getProviderName()} File`,
+ mimeType: mimeType || "application/octet-stream",
+ webUrl: meta.webUrl || "",
+ downloadUrl: meta["@microsoft.graph.downloadUrl"] || "",
+ size: meta.size,
+ modifiedTime: meta.lastModifiedDateTime,
+ isFolder: !!meta.folder,
+ } as CloudFile;
+ } else {
+ console.warn(
+ "Graph API metadata fetch failed:",
+ res.status,
+ await res.text(),
+ );
+ }
+ } catch (e) {
+ console.warn("Graph API metadata fetch error:", e);
+ }
+ }
+
+ // Fallback: use stub data if Graph call fails
+ return {
+ id: item.id,
+ name: item.name || `${this.getProviderName()} File`,
+ mimeType: "application/octet-stream",
+ webUrl: item.webUrl || "",
+ downloadUrl: item["@microsoft.graph.downloadUrl"] || "",
+ size: item.size,
+ modifiedTime: item.lastModifiedDateTime,
+ isFolder: !!item.folder,
+ } as CloudFile;
+ }),
+ );
+
+ onFileSelected(enriched);
+ this.onPickerStateChange?.(false);
+ };
+
+ enrichItems().catch((e) => {
+ console.error("Failed to enrich OneDrive items:", e);
+ this.onPickerStateChange?.(false);
+ });
},
cancel: () => {
console.log("Picker cancelled");
+ this.onPickerStateChange?.(false);
},
error: (error: any) => {
console.error("Picker error callback:", error);
+ this.onPickerStateChange?.(false);
},
});
}
@@ -310,7 +367,13 @@ export const createProviderHandler = (
if (!clientId) {
throw new Error("Client ID required for OneDrive");
}
- return new OneDriveHandler(accessToken, clientId, provider, baseUrl);
+ return new OneDriveHandler(
+ accessToken,
+ clientId,
+ provider,
+ baseUrl,
+ onPickerStateChange,
+ );
default:
throw new Error(`Unsupported provider: ${provider}`);
}
diff --git a/src/connectors/onedrive/connector.py b/src/connectors/onedrive/connector.py
index cadb910f5..316369f6b 100644
--- a/src/connectors/onedrive/connector.py
+++ b/src/connectors/onedrive/connector.py
@@ -1,8 +1,9 @@
import logging
-from pathlib import Path
-from typing import List, Dict, Any, Optional
from datetime import datetime
+from pathlib import Path
+from typing import Any
from urllib.parse import urlparse
+
import httpx
from ..base import BaseConnector, ConnectorDocument, DocumentACL
@@ -23,7 +24,7 @@ class OneDriveConnector(BaseConnector):
CONNECTOR_DESCRIPTION = "Add knowledge from OneDrive"
CONNECTOR_ICON = "onedrive"
- def __init__(self, config: Dict[str, Any]):
+ def __init__(self, config: dict[str, Any]):
super().__init__(config)
logger.debug(f"OneDrive connector __init__ called with config type: {type(config)}")
@@ -63,6 +64,7 @@ class OneDriveConnector(BaseConnector):
# Token file setup - use data directory for persistence
from config.paths import get_data_file
+
token_file = config.get("token_file") or get_data_file("onedrive_token.json")
Path(token_file).parent.mkdir(parents=True, exist_ok=True)
@@ -91,23 +93,29 @@ class OneDriveConnector(BaseConnector):
self.oauth = None
# Track subscription ID for webhooks (note: change notifications might not be available for personal accounts)
- self._subscription_id: Optional[str] = None
+ self._subscription_id: str | None = None
# Graph API defaults
self._graph_api_version = "v1.0"
- self._default_params = {
- "$select": "id,name,size,lastModifiedDateTime,createdDateTime,webUrl,file,folder,@microsoft.graph.downloadUrl"
- }
-
+ self._default_params: dict[str, Any] = {}
+
# Selective sync support (similar to Google Drive)
- self.cfg = type('OneDriveConfig', (), {
- 'file_ids': config.get('file_ids') or config.get('selected_files') or config.get('selected_file_ids'),
- 'folder_ids': config.get('folder_ids') or config.get('selected_folders') or config.get('selected_folder_ids'),
- })()
-
+ self.cfg = type(
+ "OneDriveConfig",
+ (),
+ {
+ "file_ids": config.get("file_ids")
+ or config.get("selected_files")
+ or config.get("selected_file_ids"),
+ "folder_ids": config.get("folder_ids")
+ or config.get("selected_folders")
+ or config.get("selected_folder_ids"),
+ },
+ )()
+
# Cache for file metadata including download URLs
# This allows direct download without Graph API for sharing IDs
- self._file_infos: Dict[str, Dict[str, Any]] = {}
+ self._file_infos: dict[str, dict[str, Any]] = {}
@property
def _graph_base_url(self) -> str:
@@ -115,20 +123,20 @@ class OneDriveConnector(BaseConnector):
return f"https://graph.microsoft.com/{self._graph_api_version}"
@property
- def base_url(self) -> Optional[str]:
+ def base_url(self) -> str | None:
"""Generic base URL property (OneDrive/SharePoint domain)"""
return self._base_url
-
+
@base_url.setter
def base_url(self, value: str):
"""Set base URL"""
self._base_url = value
- def set_file_infos(self, file_infos: List[Dict[str, Any]]) -> None:
+ def set_file_infos(self, file_infos: list[dict[str, Any]]) -> None:
"""
Cache file metadata including download URLs for later use.
This allows direct download without Graph API calls for sharing IDs.
-
+
Args:
file_infos: List of file info dicts with {id, name, mimeType, downloadUrl, size}
"""
@@ -140,7 +148,7 @@ class OneDriveConnector(BaseConnector):
if info.get("downloadUrl"):
logger.debug(f"Cached download URL for file {file_id}: {info.get('name')}")
- def get_cached_file_info(self, file_id: str) -> Optional[Dict[str, Any]]:
+ def get_cached_file_info(self, file_id: str) -> dict[str, Any] | None:
"""Get cached file info by ID."""
return self._file_infos.get(file_id)
@@ -167,7 +175,7 @@ class OneDriveConnector(BaseConnector):
self._authenticated = authenticated
return authenticated
- except Exception as e:
+ except Exception:
logger.exception("[CONNECTOR] OneDrive authentication failed")
self._authenticated = False
return False
@@ -178,7 +186,7 @@ class OneDriveConnector(BaseConnector):
raise RuntimeError("OneDrive OAuth not initialized - missing credentials")
return self.oauth.create_authorization_url(self.redirect_uri)
- async def handle_oauth_callback(self, auth_code: str) -> Dict[str, Any]:
+ async def handle_oauth_callback(self, auth_code: str) -> dict[str, Any]:
"""Handle OAuth callback."""
if not self.oauth:
raise RuntimeError("OneDrive OAuth not initialized - missing credentials")
@@ -193,39 +201,43 @@ class OneDriveConnector(BaseConnector):
logger.error(f"OAuth callback failed: {e}")
raise
- async def _detect_base_url(self) -> Optional[str]:
+ async def _detect_base_url(self) -> str | None:
"""Override base class method to detect OneDrive URL"""
return await self._detect_onedrive_url()
-
- async def _detect_onedrive_url(self) -> Optional[str]:
+
+ async def _detect_onedrive_url(self) -> str | None:
"""Auto-detect OneDrive URL from Microsoft Graph API"""
logger.info("_detect_onedrive_url: Starting OneDrive URL detection")
try:
if not self.oauth:
logger.warning("_detect_onedrive_url: OAuth not initialized")
return None
-
+
access_token = self.oauth.get_access_token()
- logger.debug(f"_detect_onedrive_url: Got access token (length: {len(access_token) if access_token else 0})")
-
+ logger.debug(
+ f"_detect_onedrive_url: Got access token (length: {len(access_token) if access_token else 0})"
+ )
+
headers = {
"Authorization": f"Bearer {access_token}",
"Content-Type": "application/json",
}
-
+
async with httpx.AsyncClient() as client:
# Get user's default drive to extract OneDrive URL
url = f"{self._graph_base_url}/me/drive"
logger.info(f"_detect_onedrive_url: Calling Graph API: {url}")
-
+
response = await client.get(url, headers=headers, timeout=30.0)
- logger.info(f"_detect_onedrive_url: Graph API response status: {response.status_code}")
-
+ logger.info(
+ f"_detect_onedrive_url: Graph API response status: {response.status_code}"
+ )
+
if response.status_code == 200:
data = response.json()
web_url = data.get("webUrl", "")
logger.info(f"_detect_onedrive_url: webUrl from response: {web_url}")
-
+
# Extract the domain from the webUrl
# e.g., "https://onedrive.live.com/..." or "https://company-my.sharepoint.com/..."
if web_url:
@@ -236,9 +248,11 @@ class OneDriveConnector(BaseConnector):
else:
logger.warning("_detect_onedrive_url: webUrl is empty in response")
else:
- logger.warning("[CONNECTOR] OneDrive detect URL failed", status_code=response.status_code)
-
- except Exception as e:
+ logger.warning(
+ f"[CONNECTOR] OneDrive detect URL failed, status_code: {response.status_code}"
+ )
+
+ except Exception:
logger.exception("[CONNECTOR] OneDrive URL detection failed")
return None
@@ -261,13 +275,15 @@ class OneDriveConnector(BaseConnector):
doc = await self.get_file_content(file_id)
self.emit(doc)
except Exception as e:
- logger.error(f"Failed to sync OneDrive file {file_info.get('name', 'unknown')}: {e}")
+ logger.error(
+ f"Failed to sync OneDrive file {file_info.get('name', 'unknown')}: {e}"
+ )
continue
except Exception as e:
logger.error(f"OneDrive sync_once failed: {e}")
raise
- if hasattr(asyncio, 'run'):
+ if hasattr(asyncio, "run"):
asyncio.run(_async_sync())
else:
loop = asyncio.get_event_loop()
@@ -278,7 +294,7 @@ class OneDriveConnector(BaseConnector):
Set up real-time subscription for file changes.
NOTE: Change notifications may not be available for personal OneDrive accounts.
"""
- webhook_url = self.config.get('webhook_url')
+ webhook_url = self.config.get("webhook_url")
if not webhook_url:
logger.warning("No webhook URL configured, skipping OneDrive subscription setup")
return "no-webhook-configured"
@@ -308,7 +324,9 @@ class OneDriveConnector(BaseConnector):
url = f"{self._graph_base_url}/subscriptions"
async with httpx.AsyncClient() as client:
- response = await client.post(url, json=subscription_data, headers=headers, timeout=30)
+ response = await client.post(
+ url, json=subscription_data, headers=headers, timeout=30
+ )
response.raise_for_status()
result = response.json()
@@ -328,15 +346,13 @@ class OneDriveConnector(BaseConnector):
def _get_subscription_expiry(self) -> str:
"""Get subscription expiry time (Graph caps duration; often <= 3 days)."""
from datetime import datetime, timedelta
+
expiry = datetime.utcnow() + timedelta(days=3)
return expiry.strftime("%Y-%m-%dT%H:%M:%S.%fZ")
async def list_files(
- self,
- page_token: Optional[str] = None,
- max_files: Optional[int] = None,
- **kwargs
- ) -> Dict[str, Any]:
+ self, page_token: str | None = None, max_files: int | None = None, **kwargs
+ ) -> dict[str, Any]:
"""List files from OneDrive using Microsoft Graph."""
try:
if not await self.authenticate():
@@ -346,7 +362,7 @@ class OneDriveConnector(BaseConnector):
if self.cfg.file_ids or self.cfg.folder_ids:
return await self._list_selected_files()
- files: List[Dict[str, Any]] = []
+ files: list[dict[str, Any]] = []
max_files_value = max_files if max_files is not None else 100
base_url = f"{self._graph_base_url}/me/drive/root/children"
@@ -363,23 +379,28 @@ class OneDriveConnector(BaseConnector):
items = data.get("value", [])
for item in items:
if item.get("file"): # include files only
- files.append({
- "id": item.get("id", ""),
- "name": item.get("name", ""),
- "path": f"/drive/items/{item.get('id')}",
- "size": int(item.get("size", 0)),
- "modified": item.get("lastModifiedDateTime"),
- "created": item.get("createdDateTime"),
- "mime_type": item.get("file", {}).get("mimeType", self._get_mime_type(item.get("name", ""))),
- "url": item.get("webUrl", ""),
- "download_url": item.get("@microsoft.graph.downloadUrl"),
- })
+ files.append(
+ {
+ "id": item.get("id", ""),
+ "name": item.get("name", ""),
+ "path": f"/drive/items/{item.get('id')}",
+ "size": int(item.get("size", 0)),
+ "modified": item.get("lastModifiedDateTime"),
+ "created": item.get("createdDateTime"),
+ "mime_type": item.get("file", {}).get(
+ "mimeType", self._get_mime_type(item.get("name", ""))
+ ),
+ "url": item.get("webUrl", ""),
+ "download_url": item.get("@microsoft.graph.downloadUrl"),
+ }
+ )
# Next page
next_page_token = None
next_link = data.get("@odata.nextLink")
if next_link:
- from urllib.parse import urlparse, parse_qs
+ from urllib.parse import parse_qs, urlparse
+
parsed = urlparse(next_link)
query_params = parse_qs(parsed.query)
if "$skiptoken" in query_params:
@@ -391,7 +412,7 @@ class OneDriveConnector(BaseConnector):
logger.error(f"Failed to list OneDrive files: {e}")
return {"files": [], "next_page_token": None}
- async def _extract_onedrive_acl(self, file_id: str, file_metadata: Dict) -> DocumentACL:
+ async def _extract_onedrive_acl(self, file_id: str, file_metadata: dict) -> DocumentACL:
"""
Extract ACL from OneDrive item.
@@ -406,8 +427,7 @@ class OneDriveConnector(BaseConnector):
"""
try:
# Get access token
- token_data = await self.oauth.get_access_token()
- access_token = token_data.get("access_token")
+ access_token = self.oauth.get_access_token()
if not access_token:
logger.warning(f"No access token available for ACL extraction: {file_id}")
@@ -419,8 +439,7 @@ class OneDriveConnector(BaseConnector):
# Fetch permissions
async with httpx.AsyncClient() as client:
response = await client.get(
- permissions_url,
- headers={"Authorization": f"Bearer {access_token}"}
+ permissions_url, headers={"Authorization": f"Bearer {access_token}"}
)
if response.status_code != 200:
@@ -487,13 +506,11 @@ class OneDriveConnector(BaseConnector):
if cached_info and cached_info.get("downloadUrl"):
logger.info(f"Using cached download URL for file {file_id}")
content = await self._download_file_from_url(cached_info["downloadUrl"])
-
+
acl = DocumentACL(
owner="",
- user_permissions={},
- group_permissions={},
)
-
+
return ConnectorDocument(
id=file_id,
filename=cached_info.get("name", "Unknown"),
@@ -513,13 +530,15 @@ class OneDriveConnector(BaseConnector):
file_metadata = await self._get_file_metadata_by_id(file_id)
if not file_metadata:
# Last-resort: try shares endpoint download directly if this is a sharing ID
- if '!' in file_id and file_id.split('!', 1)[1].startswith('s'):
- logger.info(f"No metadata for sharing ID {file_id}, attempting direct shares download")
+ if "!" in file_id and file_id.split("!", 1)[1].startswith("s"):
+ logger.info(
+ f"No metadata for sharing ID {file_id}, attempting direct shares download"
+ )
token = self.oauth.get_access_token()
headers = {"Authorization": f"Bearer {token}"}
shares_content = await self._download_via_shares_endpoint(file_id, headers)
if shares_content is not None:
- acl = DocumentACL(owner="", user_permissions={}, group_permissions={})
+ acl = DocumentACL(owner="")
return ConnectorDocument(
id=file_id,
filename="Unknown",
@@ -564,9 +583,9 @@ class OneDriveConnector(BaseConnector):
logger.error(f"Failed to get OneDrive file content {file_id}: {e}")
raise
- async def _get_file_metadata_by_id(self, file_id: str) -> Optional[Dict[str, Any]]:
+ async def _get_file_metadata_by_id(self, file_id: str) -> dict[str, Any] | None:
"""Get file metadata by ID using Graph API.
-
+
Handles multiple ID formats:
- Standard item ID: uses /me/drive/items/{id}
- Personal OneDrive format (driveId!itemId): uses /drives/{driveId}/items/{itemId}
@@ -575,7 +594,7 @@ class OneDriveConnector(BaseConnector):
try:
# Try different endpoints based on ID format
item = await self._fetch_item_metadata(file_id)
-
+
if not item:
return None
@@ -595,7 +614,9 @@ class OneDriveConnector(BaseConnector):
"size": int(item.get("size", 0)),
"modified": item.get("lastModifiedDateTime"),
"created": item.get("createdDateTime"),
- "mime_type": item.get("file", {}).get("mimeType", self._get_mime_type(item.get("name", ""))),
+ "mime_type": item.get("file", {}).get(
+ "mimeType", self._get_mime_type(item.get("name", ""))
+ ),
"url": item.get("webUrl", ""),
"download_url": item.get("@microsoft.graph.downloadUrl"),
"isFolder": False,
@@ -607,69 +628,79 @@ class OneDriveConnector(BaseConnector):
logger.error(f"Failed to get file metadata for {file_id}: {e}")
return None
- async def _fetch_item_metadata(self, file_id: str) -> Optional[Dict[str, Any]]:
+ async def _fetch_item_metadata(self, file_id: str) -> dict[str, Any] | None:
"""Fetch item metadata, trying multiple endpoints for different ID formats."""
import base64
+
params = dict(self._default_params)
-
+
# Check if ID contains '!' which indicates driveId!itemId format
- if '!' in file_id:
- parts = file_id.split('!', 1)
+ if "!" in file_id:
+ parts = file_id.split("!", 1)
if len(parts) == 2:
drive_id = parts[0]
item_id = parts[1]
-
+
# Handle sharing IDs (item ID starts with 's')
- if item_id.startswith('s'):
+ if item_id.startswith("s"):
logger.info(f"Detected sharing ID format for {file_id}")
-
+
# Try multiple encoding approaches for the shares endpoint
share_encodings = [
# Approach 1: Encode the full ID with "u!" prefix
- base64.urlsafe_b64encode(f"u!{file_id}".encode()).decode().rstrip('='),
+ base64.urlsafe_b64encode(f"u!{file_id}".encode()).decode().rstrip("="),
# Approach 2: Encode just the share token with "s!" prefix
- base64.urlsafe_b64encode(f"s!{item_id}".encode()).decode().rstrip('='),
+ base64.urlsafe_b64encode(f"s!{item_id}".encode()).decode().rstrip("="),
# Approach 3: Encode the ID directly
- base64.urlsafe_b64encode(file_id.encode()).decode().rstrip('='),
+ base64.urlsafe_b64encode(file_id.encode()).decode().rstrip("="),
# Approach 4: Use the ID as-is (some APIs accept this)
f"u!{file_id}",
]
-
+
for i, encoded in enumerate(share_encodings):
try:
url = f"{self._graph_base_url}/shares/{encoded}/driveItem"
- logger.debug(f"Trying shares endpoint approach {i+1}: {url}")
+ logger.debug(f"Trying shares endpoint approach {i + 1}: {url}")
response = await self._make_graph_request(url, params=params)
if response.status_code == 200:
- logger.info(f"Shares endpoint approach {i+1} succeeded")
+ logger.info(f"Shares endpoint approach {i + 1} succeeded")
return response.json()
else:
- logger.debug(f"Shares approach {i+1} failed with status {response.status_code}")
+ logger.debug(
+ f"Shares approach {i + 1} failed with status {response.status_code}"
+ )
except Exception as e:
- logger.debug(f"Shares approach {i+1} failed: {e}")
-
- # Try: /drives/{driveId}/items/{itemId} with full item ID (including 's' prefix)
- logger.info(f"Trying drives endpoint: /drives/{drive_id}/items/{item_id}")
+ logger.debug(f"Shares approach {i + 1} failed: {e}")
+
+ # Try: /drives/{driveId}/items/{file_id} with full file ID
+ logger.info(f"Trying drives endpoint: /drives/{drive_id}/items/{file_id}")
try:
- url = f"{self._graph_base_url}/drives/{drive_id}/items/{item_id}"
+ url = f"{self._graph_base_url}/drives/{drive_id}/items/{file_id}"
response = await self._make_graph_request(url, params=params)
if response.status_code == 200:
return response.json()
+ else:
+ logger.warning(
+ f"Drives endpoint failed with status {response.status_code}: {response.text}"
+ )
except Exception as e:
- logger.debug(f"Drives endpoint failed: {e}")
-
- # Try: /drives/{driveId}/items/{itemId} without 's' prefix
- if item_id.startswith('s'):
+ logger.debug(f"Drives endpoint exception: {e}")
+
+ # Try: /drives/{driveId}/items/{clean_file_id} without 's' prefix
+ if item_id.startswith("s"):
clean_item_id = item_id[1:] # Remove 's' prefix
- logger.info(f"Trying drives endpoint without 's' prefix: /drives/{drive_id}/items/{clean_item_id}")
+ clean_file_id = f"{drive_id}!{clean_item_id}"
+ logger.info(
+ f"Trying drives endpoint without 's' prefix: /drives/{drive_id}/items/{clean_file_id}"
+ )
try:
- url = f"{self._graph_base_url}/drives/{drive_id}/items/{clean_item_id}"
+ url = f"{self._graph_base_url}/drives/{drive_id}/items/{clean_file_id}"
response = await self._make_graph_request(url, params=params)
if response.status_code == 200:
return response.json()
except Exception as e:
logger.debug(f"Drives endpoint (no prefix) failed: {e}")
-
+
# Try: /me/drive/items/{full_id} as fallback
logger.info(f"Trying standard endpoint: /me/drive/items/{file_id}")
try:
@@ -677,42 +708,46 @@ class OneDriveConnector(BaseConnector):
response = await self._make_graph_request(url, params=params)
if response.status_code == 200:
return response.json()
+ else:
+ logger.warning(
+ f"Standard endpoint failed with status {response.status_code}: {response.text}"
+ )
except Exception as e:
- logger.debug(f"Standard endpoint failed: {e}")
+ logger.debug(f"Standard endpoint exception: {e}")
else:
# Standard item ID without '!'
url = f"{self._graph_base_url}/me/drive/items/{file_id}"
response = await self._make_graph_request(url, params=params)
if response.status_code == 200:
return response.json()
-
+
logger.error(f"All endpoints failed for file_id: {file_id}")
return None
async def _download_file_content(self, file_id: str) -> bytes:
"""Download file content by file ID using Graph API.
-
+
Handles multiple ID formats like _get_file_metadata_by_id.
"""
try:
token = self.oauth.get_access_token()
headers = {"Authorization": f"Bearer {token}"}
-
+
# Build URL based on ID format
- if '!' in file_id:
- parts = file_id.split('!', 1)
+ if "!" in file_id:
+ parts = file_id.split("!", 1)
if len(parts) == 2:
drive_id = parts[0]
item_id = parts[1]
-
+
# If this looks like a sharing ID (starts with 's'), try shares endpoint first
- if item_id.startswith('s'):
+ if item_id.startswith("s"):
content = await self._download_via_shares_endpoint(file_id, headers)
if content is not None:
return content
- # Try drives endpoint for driveId!itemId format (including the 's' prefix)
- url = f"{self._graph_base_url}/drives/{drive_id}/items/{item_id}/content"
+ # Try drives endpoint for driveId!itemId format
+ url = f"{self._graph_base_url}/drives/{drive_id}/items/{file_id}/content"
logger.info(f"Downloading via drives endpoint: {url}")
else:
url = f"{self._graph_base_url}/me/drive/items/{file_id}/content"
@@ -728,31 +763,37 @@ class OneDriveConnector(BaseConnector):
logger.error(f"Failed to download file content for {file_id}: {e}")
raise
- async def _download_via_shares_endpoint(self, file_id: str, headers: Dict[str, str]) -> Optional[bytes]:
+ async def _download_via_shares_endpoint(
+ self, file_id: str, headers: dict[str, str]
+ ) -> bytes | None:
"""
Attempt to download content using the Graph /shares endpoint for sharing IDs.
"""
import base64
share_encodings = [
- base64.urlsafe_b64encode(f"u!{file_id}".encode()).decode().rstrip('='),
- base64.urlsafe_b64encode(f"s!{file_id}".encode()).decode().rstrip('='),
- base64.urlsafe_b64encode(file_id.encode()).decode().rstrip('='),
+ base64.urlsafe_b64encode(f"u!{file_id}".encode()).decode().rstrip("="),
+ base64.urlsafe_b64encode(f"s!{file_id}".encode()).decode().rstrip("="),
+ base64.urlsafe_b64encode(file_id.encode()).decode().rstrip("="),
f"u!{file_id}",
]
for i, encoded in enumerate(share_encodings):
try:
url = f"{self._graph_base_url}/shares/{encoded}/driveItem/content"
- logger.info(f"Attempting shares download (approach {i+1}): {url}")
+ logger.info(f"Attempting shares download (approach {i + 1}): {url}")
async with httpx.AsyncClient() as client:
- response = await client.get(url, headers=headers, timeout=60, follow_redirects=True)
+ response = await client.get(
+ url, headers=headers, timeout=60, follow_redirects=True
+ )
if response.status_code == 200:
return response.content
else:
- logger.debug(f"Shares download approach {i+1} failed with status {response.status_code}")
+ logger.debug(
+ f"Shares download approach {i + 1} failed with status {response.status_code}"
+ )
except Exception as e:
- logger.debug(f"Shares download approach {i+1} failed: {e}")
+ logger.debug(f"Shares download approach {i + 1} failed: {e}")
return None
@@ -767,20 +808,25 @@ class OneDriveConnector(BaseConnector):
logger.error(f"Failed to download from URL {download_url}: {e}")
raise
- def _parse_graph_date(self, date_str: Optional[str]) -> datetime:
+ def _parse_graph_date(self, date_str: str | None) -> datetime:
"""Parse Microsoft Graph date string to datetime."""
if not date_str:
return datetime.now()
try:
- if date_str.endswith('Z'):
+ if date_str.endswith("Z"):
return datetime.fromisoformat(date_str[:-1]).replace(tzinfo=None)
else:
- return datetime.fromisoformat(date_str.replace('T', ' '))
+ return datetime.fromisoformat(date_str.replace("T", " "))
except (ValueError, AttributeError):
return datetime.now()
- async def _make_graph_request(self, url: str, method: str = "GET",
- data: Optional[Dict] = None, params: Optional[Dict] = None) -> httpx.Response:
+ async def _make_graph_request(
+ self,
+ url: str,
+ method: str = "GET",
+ data: dict | None = None,
+ params: dict | None = None,
+ ) -> httpx.Response:
"""Make authenticated API request to Microsoft Graph."""
token = self.oauth.get_access_token()
headers = {
@@ -801,25 +847,25 @@ class OneDriveConnector(BaseConnector):
response.raise_for_status()
return response
- async def _list_selected_files(self) -> Dict[str, Any]:
+ async def _list_selected_files(self) -> dict[str, Any]:
"""List only selected files/folders (selective sync)."""
- files: List[Dict[str, Any]] = []
-
+ files: list[dict[str, Any]] = []
+
# Process selected file IDs
if self.cfg.file_ids:
for file_id in self.cfg.file_ids:
try:
file_meta = await self._get_file_metadata_by_id(file_id)
- if file_meta and not file_meta.get('isFolder', False):
+ if file_meta and not file_meta.get("isFolder", False):
files.append(file_meta)
- elif file_meta and file_meta.get('isFolder', False):
+ elif file_meta and file_meta.get("isFolder", False):
# If it's a folder, expand its contents
folder_files = await self._list_folder_contents(file_id)
files.extend(folder_files)
except Exception as e:
logger.warning(f"Failed to get file {file_id}: {e}")
continue
-
+
# Process selected folder IDs
if self.cfg.folder_ids:
for folder_id in self.cfg.folder_ids:
@@ -829,20 +875,20 @@ class OneDriveConnector(BaseConnector):
except Exception as e:
logger.warning(f"Failed to list folder {folder_id}: {e}")
continue
-
+
return {"files": files, "next_page_token": None}
-
- async def _list_folder_contents(self, folder_id: str) -> List[Dict[str, Any]]:
+
+ async def _list_folder_contents(self, folder_id: str) -> list[dict[str, Any]]:
"""List all files in a folder recursively."""
- files: List[Dict[str, Any]] = []
-
+ files: list[dict[str, Any]] = []
+
try:
url = f"{self._graph_base_url}/me/drive/items/{folder_id}/children"
params = dict(self._default_params)
-
+
response = await self._make_graph_request(url, params=params)
data = response.json()
-
+
items = data.get("value", [])
for item in items:
if item.get("file"): # It's a file
@@ -854,35 +900,37 @@ class OneDriveConnector(BaseConnector):
files.extend(subfolder_files)
except Exception as e:
logger.error(f"Failed to list folder contents for {folder_id}: {e}")
-
+
return files
def _get_mime_type(self, filename: str) -> str:
"""Get MIME type based on file extension."""
import mimetypes
+
mime_type, _ = mimetypes.guess_type(filename)
return mime_type or "application/octet-stream"
# Webhook methods - BaseConnector interface
- def handle_webhook_validation(self, request_method: str,
- headers: Dict[str, str],
- query_params: Dict[str, str]) -> Optional[str]:
+ def handle_webhook_validation(
+ self, request_method: str, headers: dict[str, str], query_params: dict[str, str]
+ ) -> str | None:
"""Handle webhook validation (Graph API specific)."""
if request_method == "POST" and "validationToken" in query_params:
return query_params["validationToken"]
return None
- def extract_webhook_channel_id(self, payload: Dict[str, Any],
- headers: Dict[str, str]) -> Optional[str]:
+ def extract_webhook_channel_id(
+ self, payload: dict[str, Any], headers: dict[str, str]
+ ) -> str | None:
"""Extract channel/subscription ID from webhook payload."""
notifications = payload.get("value", [])
if notifications:
return notifications[0].get("subscriptionId")
return None
- async def handle_webhook(self, payload: Dict[str, Any]) -> List[str]:
+ async def handle_webhook(self, payload: dict[str, Any]) -> list[str]:
"""Handle webhook notification and return affected file IDs."""
- affected_files: List[str] = []
+ affected_files: list[str] = []
notifications = payload.get("value", [])
for notification in notifications:
resource = notification.get("resource")
@@ -914,7 +962,9 @@ class OneDriveConnector(BaseConnector):
logger.info(f"OneDrive subscription {subscription_id} cleaned up successfully")
return True
else:
- logger.warning(f"Unexpected response cleaning up subscription: {response.status_code}")
+ logger.warning(
+ f"Unexpected response cleaning up subscription: {response.status_code}"
+ )
return False
except Exception as e:
diff --git a/src/connectors/onedrive/oauth.py b/src/connectors/onedrive/oauth.py
index b275561e2..898f16673 100644
--- a/src/connectors/onedrive/oauth.py
+++ b/src/connectors/onedrive/oauth.py
@@ -1,8 +1,7 @@
-import os
import json
import logging
-from typing import Optional, Dict, Any
-
+import os
+from typing import Any
import msal
@@ -25,16 +24,9 @@ class OneDriveOAuth:
"Files.Read",
"Files.Read.All", # Access all files user can access
"Files.Read.Selected",
- "Sites.Read.All", # Read SharePoint sites (for File Picker and OneDrive for Business)
- "offline_access"
- ]
- RESOURCE_SCOPES = [
- "User.Read",
- "Files.Read",
- "Files.Read.All",
- "Files.Read.Selected",
- "Sites.Read.All"
+ "offline_access",
]
+ RESOURCE_SCOPES = ["User.Read", "Files.Read", "Files.Read.All", "Files.Read.Selected"]
SCOPES = AUTH_SCOPES # Backward-compat alias if something references .SCOPES
# Kept for reference; MSAL derives endpoints from `authority`
@@ -67,7 +59,7 @@ class OneDriveOAuth:
self.authority = authority
self.allow_json_refresh = allow_json_refresh
self.token_cache = msal.SerializableTokenCache()
- self._current_account = None
+ self._current_account: dict[str, Any] | None = None
# Initialize MSAL Confidential Client
self.app = msal.ConfidentialClientApplication(
@@ -81,10 +73,10 @@ class OneDriveOAuth:
"""Load existing credentials from token file (async)."""
try:
from utils.encryption import read_encrypted_file
-
+
logger.debug(f"OneDrive OAuth loading credentials from: {self.token_file}")
cache_data, needs_upgrade = await read_encrypted_file(self.token_file)
-
+
if cache_data and cache_data.strip():
# 1) Try legacy flat JSON first
try:
@@ -112,15 +104,20 @@ class OneDriveOAuth:
accounts = self.app.get_accounts()
logger.debug(f"Found {len(accounts)} accounts in MSAL cache")
if accounts:
- self._current_account = accounts[0]
- logger.debug(f"Set current account: {self._current_account.get('username', 'no username')}")
-
+ account = accounts[0]
+ self._current_account = account
+ logger.debug(f"Set current account: {account.get('username', 'no username')}")
+
if needs_upgrade:
await self.save_cache()
# Use RESOURCE_SCOPES (no reserved scopes) for silent acquisition
- result = self.app.acquire_token_silent(self.RESOURCE_SCOPES, account=self._current_account)
- logger.debug(f"Silent token acquisition result keys: {list(result.keys()) if result else 'None'}")
+ result = self.app.acquire_token_silent(
+ self.RESOURCE_SCOPES, account=self._current_account
+ )
+ logger.debug(
+ f"Silent token acquisition result keys: {list(result.keys()) if result else 'None'}"
+ )
if result and "access_token" in result:
logger.debug("Silent token acquisition successful")
if getattr(self.token_cache, "has_state_changed", False):
@@ -134,7 +131,7 @@ class OneDriveOAuth:
return False
- except Exception as e:
+ except Exception:
logger.exception("[CONNECTOR] OneDrive credential load failed")
return False
@@ -166,15 +163,24 @@ class OneDriveOAuth:
accounts = self.app.get_accounts()
logger.debug(f"After refresh, found {len(accounts)} accounts")
if accounts:
- self._current_account = accounts[0]
- logger.debug(f"Set current account after refresh: {self._current_account.get('username', 'no username')}")
+ account = accounts[0]
+ self._current_account = account
+ logger.debug(
+ f"Set current account after refresh: {account.get('username', 'no username')}"
+ )
return True
# Error handling
- err = (result or {}).get("error_description") or (result or {}).get("error") or "Unknown error"
+ err = (
+ (result or {}).get("error_description")
+ or (result or {}).get("error")
+ or "Unknown error"
+ )
logger.error(f"Refresh token failed: {err}")
- if any(code in err for code in ("AADSTS70000", "invalid_grant", "interaction_required")):
+ if any(
+ code in err for code in ("AADSTS70000", "invalid_grant", "interaction_required")
+ ):
logger.warning(
"Refresh denied due to unauthorized/expired scopes or invalid grant. "
"Delete the token file and perform interactive sign-in with correct scopes."
@@ -182,7 +188,7 @@ class OneDriveOAuth:
return False
- except Exception as e:
+ except Exception:
logger.exception("[CONNECTOR] OneDrive JSON token refresh failed")
return False
@@ -197,18 +203,19 @@ class OneDriveOAuth:
cache_data = self.token_cache.serialize()
if cache_data:
from utils.encryption import write_encrypted_file
+
await write_encrypted_file(self.token_file, cache_data)
-
+
logger.debug(f"Token cache saved to {self.token_file}")
except Exception as e:
logger.error(f"Failed to save token cache: {e}")
- def create_authorization_url(self, redirect_uri: str, state: Optional[str] = None) -> str:
+ def create_authorization_url(self, redirect_uri: str, state: str | None = None) -> str:
"""Create authorization URL for OAuth flow."""
# Store redirect URI for later use in callback
self._redirect_uri = redirect_uri
- kwargs: Dict[str, Any] = {
+ kwargs: dict[str, Any] = {
# Interactive auth includes offline_access
"scopes": self.AUTH_SCOPES,
"redirect_uri": redirect_uri,
@@ -231,7 +238,7 @@ class OneDriveOAuth:
try:
result = self.app.acquire_token_by_authorization_code(
authorization_code,
- scopes=self.AUTH_SCOPES, # same as authorize step
+ scopes=self.AUTH_SCOPES, # same as authorize step
redirect_uri=redirect_uri,
)
@@ -244,7 +251,11 @@ class OneDriveOAuth:
logger.info("OneDrive OAuth authorization successful")
return True
- error_msg = (result or {}).get("error_description") or (result or {}).get("error") or "Unknown error"
+ error_msg = (
+ (result or {}).get("error_description")
+ or (result or {}).get("error")
+ or "Unknown error"
+ )
logger.error(f"OneDrive OAuth authorization failed: {error_msg}")
return False
@@ -262,29 +273,47 @@ class OneDriveOAuth:
load_result = await self.load_credentials()
logger.info(f"OneDrive is_authenticated: load_credentials returned {load_result}")
- logger.info(f"OneDrive is_authenticated: current_account={self._current_account is not None}")
-
+ logger.info(
+ f"OneDrive is_authenticated: current_account={self._current_account is not None}"
+ )
+
# Try to get a token (MSAL will refresh if needed)
if self._current_account:
- logger.info(f"OneDrive is_authenticated: Trying acquire_token_silent with account {self._current_account.get('username', 'unknown')}")
+ logger.info(
+ f"OneDrive is_authenticated: Trying acquire_token_silent with account {self._current_account.get('username', 'unknown')}"
+ )
logger.info(f"OneDrive is_authenticated: RESOURCE_SCOPES={self.RESOURCE_SCOPES}")
- result = self.app.acquire_token_silent(self.RESOURCE_SCOPES, account=self._current_account)
+ result = self.app.acquire_token_silent(
+ self.RESOURCE_SCOPES, account=self._current_account
+ )
if result and "access_token" in result:
- logger.info("OneDrive is_authenticated: Successfully acquired token with account")
+ logger.info(
+ "OneDrive is_authenticated: Successfully acquired token with account"
+ )
if getattr(self.token_cache, "has_state_changed", False):
await self.save_cache()
return True
else:
- error_msg = (result or {}).get("error") or (result or {}).get("error_description") or "No result returned"
- logger.warning(f"OneDrive is_authenticated: Token acquisition failed for current account: {error_msg}")
+ error_msg = (
+ (result or {}).get("error")
+ or (result or {}).get("error_description")
+ or "No result returned"
+ )
+ logger.warning(
+ f"OneDrive is_authenticated: Token acquisition failed for current account: {error_msg}"
+ )
logger.warning(f"OneDrive is_authenticated: Full result: {result}")
# Fallback: try without specific account
- logger.info("OneDrive is_authenticated: Fallback - trying acquire_token_silent without account")
+ logger.info(
+ "OneDrive is_authenticated: Fallback - trying acquire_token_silent without account"
+ )
result = self.app.acquire_token_silent(self.RESOURCE_SCOPES, account=None)
if result and "access_token" in result:
accounts = self.app.get_accounts()
- logger.info(f"OneDrive is_authenticated: Fallback succeeded, found {len(accounts)} accounts")
+ logger.info(
+ f"OneDrive is_authenticated: Fallback succeeded, found {len(accounts)} accounts"
+ )
if accounts:
self._current_account = accounts[0]
if getattr(self.token_cache, "has_state_changed", False):
@@ -294,23 +323,31 @@ class OneDriveOAuth:
logger.warning(f"OneDrive is_authenticated: Fallback also failed, result: {result}")
return False
- except Exception as e:
+ except Exception:
logger.exception("[CONNECTOR] OneDrive is_authenticated failed")
return False
def get_access_token(self) -> str:
"""Get an access token for Microsoft Graph."""
- logger.info(f"OneDrive get_access_token: Starting, current_account={self._current_account is not None}")
+ logger.info(
+ f"OneDrive get_access_token: Starting, current_account={self._current_account is not None}"
+ )
try:
# Try with current account first
if self._current_account:
- logger.info(f"OneDrive get_access_token: Trying with account {self._current_account.get('username', 'unknown')}")
- result = self.app.acquire_token_silent(self.RESOURCE_SCOPES, account=self._current_account)
+ logger.info(
+ f"OneDrive get_access_token: Trying with account {self._current_account.get('username', 'unknown')}"
+ )
+ result = self.app.acquire_token_silent(
+ self.RESOURCE_SCOPES, account=self._current_account
+ )
if result and "access_token" in result:
logger.info("OneDrive get_access_token: Success with current account")
return result["access_token"]
else:
- logger.warning(f"OneDrive get_access_token: Failed with account, result: {result}")
+ logger.warning(
+ f"OneDrive get_access_token: Failed with account, result: {result}"
+ )
# Fallback: try without specific account
logger.info("OneDrive get_access_token: Fallback - trying without account")
@@ -320,11 +357,15 @@ class OneDriveOAuth:
return result["access_token"]
# If we get here, authentication has failed
- error_msg = (result or {}).get("error_description") or (result or {}).get("error") or "No valid authentication"
+ error_msg = (
+ (result or {}).get("error_description")
+ or (result or {}).get("error")
+ or "No valid authentication"
+ )
logger.error(f"OneDrive get_access_token: All attempts failed, error: {error_msg}")
raise ValueError(f"Failed to acquire access token: {error_msg}")
- except Exception as e:
+ except Exception:
logger.exception("[CONNECTOR] OneDrive get_access_token failed")
raise