OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.
Python
fix: display file names and fix ingestion for onedrive (#1609)
Lucas Oliveira62efe56491a4
diff --git a/frontend/app/api/queries/useGetSearchQuery.ts b/frontend/app/api/queries/useGetSearchQuery.tsindex 94e55af8f..0ccb88eb3 100644--- a/frontend/app/api/queries/useGetSearchQuery.ts+++ b/frontend/app/api/queries/useGetSearchQuery.ts@@ -85,7 +85,10 @@ export { EMPTY_SEARCH_RESULT };export const useGetSearchQuery = (query: string,queryData?: ParsedQueryData | null,- options?: Omit<UseQueryOptions, "queryKey" | "queryFn">,+ options?: Omit<+ UseQueryOptions<SearchResult, Error, SearchResult, any[]>,+ "queryKey" | "queryFn"+ >,) => {const queryClient = useQueryClient();const getFileIdentity = (chunk: ChunkResult): string => {diff --git a/frontend/components/cloud-picker/file-item.tsx b/frontend/components/cloud-picker/file-item.tsxindex 3f1306c26..f4d7d4257 100644--- a/frontend/components/cloud-picker/file-item.tsx+++ b/frontend/components/cloud-picker/file-item.tsx@@ -31,10 +31,24 @@ const getMimeTypeLabel = (mimeType: string) => {"application/vnd.google-apps.folder": "Folder","application/pdf": "PDF","text/plain": "Text",+ "text/csv": "CSV",+ "text/html": "HTML",+ "application/xml": "XML",+ "application/json": "JSON",+ "application/msword": "Word Doc",+ "application/vnd.ms-excel": "Excel",+ "application/vnd.ms-powerpoint": "PowerPoint","application/vnd.openxmlformats-officedocument.wordprocessingml.document":"Word Doc",+ "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet":+ "Excel","application/vnd.openxmlformats-officedocument.presentationml.presentation":"PowerPoint",+ "application/octet-stream": "File",+ "image/jpeg": "JPEG",+ "image/png": "PNG",+ "image/gif": "GIF",+ "image/svg+xml": "SVG",};return typeMap[mimeType] || mimeType?.split("/").pop() || "Document";diff --git a/frontend/components/cloud-picker/provider-handlers.ts b/frontend/components/cloud-picker/provider-handlers.tsindex b64808918..2966c4083 100644--- a/frontend/components/cloud-picker/provider-handlers.ts+++ b/frontend/components/cloud-picker/provider-handlers.ts@@ -159,17 +159,20 @@ export class OneDriveHandler {private clientId: string;private provider: CloudProvider;private baseUrl?: string;+ private onPickerStateChange?: (isOpen: boolean) => void;constructor(accessToken: string,clientId: string,provider: CloudProvider = "onedrive",baseUrl?: string,+ onPickerStateChange?: (isOpen: boolean) => void,) {this.accessToken = accessToken;this.clientId = clientId;this.provider = provider;this.baseUrl = baseUrl;+ this.onPickerStateChange = onPickerStateChange;}async loadPickerApi(): Promise<boolean> {@@ -206,57 +209,111 @@ export class OneDriveHandler {console.log("OneDrive picker success callback:", response);if (!response || !response.value) {console.warn("OneDrive picker returned no value");+ this.onPickerStateChange?.(false);return;}- const newFiles: CloudFile[] =- response.value?.map((item: any) => {- // Extract mimeType from file object or infer from name- let mimeType = item.file?.mimeType;- if (!mimeType && item.name) {- // Infer from extension if mimeType not provided- const ext = item.name.split(".").pop()?.toLowerCase();- const mimeTypes: { [key: string]: string } = {- pdf: "application/pdf",- doc: "application/msword",- docx: "application/vnd.openxmlformats-officedocument.wordprocessingml.document",- xls: "application/vnd.ms-excel",- xlsx: "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",- ppt: "application/vnd.ms-powerpoint",- pptx: "application/vnd.openxmlformats-officedocument.presentationml.presentation",- txt: "text/plain",- csv: "text/csv",- json: "application/json",- xml: "application/xml",- html: "text/html",- jpg: "image/jpeg",- jpeg: "image/jpeg",- png: "image/png",- gif: "image/gif",- svg: "image/svg+xml",- };- mimeType = mimeTypes[ext || ""] || "application/octet-stream";- }+ // v7.2 action:"query" returns stubs with only id/endpoint/parentReference.+ // Enrich each item by fetching full metadata from the Graph API.+ const enrichItems = async () => {+ const enriched = await Promise.all(+ response.value.map(async (item: any) => {+ const driveId =+ item.parentReference?.driveId || item.id?.split("!")[0];+ const itemId = item.id;- return {- id: item.id,- name: item.name || `${this.getProviderName()} File`,- mimeType: mimeType || "application/octet-stream",- webUrl: item.webUrl || "",- downloadUrl: item["@microsoft.graph.downloadUrl"] || "",- size: item.size,- modifiedTime: item.lastModifiedDateTime,- isFolder: !!item.folder,- };- }) || [];+ if (driveId && itemId) {+ try {+ const url = `https://graph.microsoft.com/v1.0/drives/${driveId}/items/${itemId}`;+ const res = await fetch(url, {+ headers: { Authorization: `Bearer ${this.accessToken}` },+ });+ if (res.ok) {+ const meta = await res.json();+ console.log(+ "OneDrive enriched metadata:",+ meta.name,+ meta.file?.mimeType,+ );++ let mimeType = meta.file?.mimeType;+ if (!mimeType && meta.name) {+ const ext = meta.name.split(".").pop()?.toLowerCase();+ const mimeTypes: { [key: string]: string } = {+ pdf: "application/pdf",+ doc: "application/msword",+ docx: "application/vnd.openxmlformats-officedocument.wordprocessingml.document",+ xls: "application/vnd.ms-excel",+ xlsx: "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",+ ppt: "application/vnd.ms-powerpoint",+ pptx: "application/vnd.openxmlformats-officedocument.presentationml.presentation",+ txt: "text/plain",+ csv: "text/csv",+ json: "application/json",+ xml: "application/xml",+ html: "text/html",+ jpg: "image/jpeg",+ jpeg: "image/jpeg",+ png: "image/png",+ gif: "image/gif",+ svg: "image/svg+xml",+ };+ mimeType =+ mimeTypes[ext || ""] || "application/octet-stream";+ }- onFileSelected(newFiles);+ return {+ id: meta.id,+ name: meta.name || `${this.getProviderName()} File`,+ mimeType: mimeType || "application/octet-stream",+ webUrl: meta.webUrl || "",+ downloadUrl: meta["@microsoft.graph.downloadUrl"] || "",+ size: meta.size,+ modifiedTime: meta.lastModifiedDateTime,+ isFolder: !!meta.folder,+ } as CloudFile;+ } else {+ console.warn(+ "Graph API metadata fetch failed:",+ res.status,+ await res.text(),+ );+ }+ } catch (e) {+ console.warn("Graph API metadata fetch error:", e);+ }+ }++ // Fallback: use stub data if Graph call fails+ return {+ id: item.id,+ name: item.name || `${this.getProviderName()} File`,+ mimeType: "application/octet-stream",+ webUrl: item.webUrl || "",+ downloadUrl: item["@microsoft.graph.downloadUrl"] || "",+ size: item.size,+ modifiedTime: item.lastModifiedDateTime,+ isFolder: !!item.folder,+ } as CloudFile;+ }),+ );++ onFileSelected(enriched);+ this.onPickerStateChange?.(false);+ };++ enrichItems().catch((e) => {+ console.error("Failed to enrich OneDrive items:", e);+ this.onPickerStateChange?.(false);+ });},cancel: () => {console.log("Picker cancelled");+ this.onPickerStateChange?.(false);},error: (error: any) => {console.error("Picker error callback:", error);+ this.onPickerStateChange?.(false);},});}@@ -310,7 +367,13 @@ export const createProviderHandler = (if (!clientId) {throw new Error("Client ID required for OneDrive");}- return new OneDriveHandler(accessToken, clientId, provider, baseUrl);+ return new OneDriveHandler(+ accessToken,+ clientId,+ provider,+ baseUrl,+ onPickerStateChange,+ );default:throw new Error(`Unsupported provider: ${provider}`);}diff --git a/src/connectors/onedrive/connector.py b/src/connectors/onedrive/connector.pyindex cadb910f5..316369f6b 100644--- a/src/connectors/onedrive/connector.py+++ b/src/connectors/onedrive/connector.py@@ -1,8 +1,9 @@import logging-from pathlib import Path-from typing import List, Dict, Any, Optionalfrom datetime import datetime+from pathlib import Path+from typing import Anyfrom urllib.parse import urlparse+import httpxfrom ..base import BaseConnector, ConnectorDocument, DocumentACL@@ -23,7 +24,7 @@ class OneDriveConnector(BaseConnector):CONNECTOR_DESCRIPTION = "Add knowledge from OneDrive"CONNECTOR_ICON = "onedrive"- def __init__(self, config: Dict[str, Any]):+ def __init__(self, config: dict[str, Any]):super().__init__(config)logger.debug(f"OneDrive connector __init__ called with config type: {type(config)}")@@ -63,6 +64,7 @@ class OneDriveConnector(BaseConnector):# Token file setup - use data directory for persistencefrom config.paths import get_data_file+token_file = config.get("token_file") or get_data_file("onedrive_token.json")Path(token_file).parent.mkdir(parents=True, exist_ok=True)@@ -91,23 +93,29 @@ class OneDriveConnector(BaseConnector):self.oauth = None# Track subscription ID for webhooks (note: change notifications might not be available for personal accounts)- self._subscription_id: Optional[str] = None+ self._subscription_id: str | None = None# Graph API defaultsself._graph_api_version = "v1.0"- self._default_params = {- "$select": "id,name,size,lastModifiedDateTime,createdDateTime,webUrl,file,folder,@microsoft.graph.downloadUrl"- }-+ self._default_params: dict[str, Any] = {}+# Selective sync support (similar to Google Drive)- self.cfg = type('OneDriveConfig', (), {- 'file_ids': config.get('file_ids') or config.get('selected_files') or config.get('selected_file_ids'),- 'folder_ids': config.get('folder_ids') or config.get('selected_folders') or config.get('selected_folder_ids'),- })()-+ self.cfg = type(+ "OneDriveConfig",+ (),+ {+ "file_ids": config.get("file_ids")+ or config.get("selected_files")+ or config.get("selected_file_ids"),+ "folder_ids": config.get("folder_ids")+ or config.get("selected_folders")+ or config.get("selected_folder_ids"),+ },+ )()+# Cache for file metadata including download URLs# This allows direct download without Graph API for sharing IDs- self._file_infos: Dict[str, Dict[str, Any]] = {}+ self._file_infos: dict[str, dict[str, Any]] = {}@propertydef _graph_base_url(self) -> str:@@ -115,20 +123,20 @@ class OneDriveConnector(BaseConnector):return f"https://graph.microsoft.com/{self._graph_api_version}"@property- def base_url(self) -> Optional[str]:+ def base_url(self) -> str | None:"""Generic base URL property (OneDrive/SharePoint domain)"""return self._base_url-+@base_url.setterdef base_url(self, value: str):"""Set base URL"""self._base_url = value- def set_file_infos(self, file_infos: List[Dict[str, Any]]) -> None:+ def set_file_infos(self, file_infos: list[dict[str, Any]]) -> None:"""Cache file metadata including download URLs for later use.This allows direct download without Graph API calls for sharing IDs.-+Args:file_infos: List of file info dicts with {id, name, mimeType, downloadUrl, size}"""@@ -140,7 +148,7 @@ class OneDriveConnector(BaseConnector):if info.get("downloadUrl"):logger.debug(f"Cached download URL for file {file_id}: {info.get('name')}")- def get_cached_file_info(self, file_id: str) -> Optional[Dict[str, Any]]:+ def get_cached_file_info(self, file_id: str) -> dict[str, Any] | None:"""Get cached file info by ID."""return self._file_infos.get(file_id)@@ -167,7 +175,7 @@ class OneDriveConnector(BaseConnector):self._authenticated = authenticatedreturn authenticated- except Exception as e:+ except Exception:logger.exception("[CONNECTOR] OneDrive authentication failed")self._authenticated = Falsereturn False@@ -178,7 +186,7 @@ class OneDriveConnector(BaseConnector):raise RuntimeError("OneDrive OAuth not initialized - missing credentials")return self.oauth.create_authorization_url(self.redirect_uri)- async def handle_oauth_callback(self, auth_code: str) -> Dict[str, Any]:+ async def handle_oauth_callback(self, auth_code: str) -> dict[str, Any]:"""Handle OAuth callback."""if not self.oauth:raise RuntimeError("OneDrive OAuth not initialized - missing credentials")@@ -193,39 +201,43 @@ class OneDriveConnector(BaseConnector):logger.error(f"OAuth callback failed: {e}")raise- async def _detect_base_url(self) -> Optional[str]:+ async def _detect_base_url(self) -> str | None:"""Override base class method to detect OneDrive URL"""return await self._detect_onedrive_url()-- async def _detect_onedrive_url(self) -> Optional[str]:++ async def _detect_onedrive_url(self) -> str | None:"""Auto-detect OneDrive URL from Microsoft Graph API"""logger.info("_detect_onedrive_url: Starting OneDrive URL detection")try:if not self.oauth:logger.warning("_detect_onedrive_url: OAuth not initialized")return None-+access_token = self.oauth.get_access_token()- logger.debug(f"_detect_onedrive_url: Got access token (length: {len(access_token) if access_token else 0})")-+ logger.debug(+ f"_detect_onedrive_url: Got access token (length: {len(access_token) if access_token else 0})"+ )+headers = {"Authorization": f"Bearer {access_token}","Content-Type": "application/json",}-+async with httpx.AsyncClient() as client:# Get user's default drive to extract OneDrive URLurl = f"{self._graph_base_url}/me/drive"logger.info(f"_detect_onedrive_url: Calling Graph API: {url}")-+response = await client.get(url, headers=headers, timeout=30.0)- logger.info(f"_detect_onedrive_url: Graph API response status: {response.status_code}")-+ logger.info(+ f"_detect_onedrive_url: Graph API response status: {response.status_code}"+ )+if response.status_code == 200:data = response.json()web_url = data.get("webUrl", "")logger.info(f"_detect_onedrive_url: webUrl from response: {web_url}")-+# Extract the domain from the webUrl# e.g., "https://onedrive.live.com/..." or "https://company-my.sharepoint.com/..."if web_url:@@ -236,9 +248,11 @@ class OneDriveConnector(BaseConnector):else:logger.warning("_detect_onedrive_url: webUrl is empty in response")else:- logger.warning("[CONNECTOR] OneDrive detect URL failed", status_code=response.status_code)-- except Exception as e:+ logger.warning(+ f"[CONNECTOR] OneDrive detect URL failed, status_code: {response.status_code}"+ )++ except Exception:logger.exception("[CONNECTOR] OneDrive URL detection failed")return None@@ -261,13 +275,15 @@ class OneDriveConnector(BaseConnector):doc = await self.get_file_content(file_id)self.emit(doc)except Exception as e:- logger.error(f"Failed to sync OneDrive file {file_info.get('name', 'unknown')}: {e}")+ logger.error(+ f"Failed to sync OneDrive file {file_info.get('name', 'unknown')}: {e}"+ )continueexcept Exception as e:logger.error(f"OneDrive sync_once failed: {e}")raise- if hasattr(asyncio, 'run'):+ if hasattr(asyncio, "run"):asyncio.run(_async_sync())else:loop = asyncio.get_event_loop()@@ -278,7 +294,7 @@ class OneDriveConnector(BaseConnector):Set up real-time subscription for file changes.NOTE: Change notifications may not be available for personal OneDrive accounts."""- webhook_url = self.config.get('webhook_url')+ webhook_url = self.config.get("webhook_url")if not webhook_url:logger.warning("No webhook URL configured, skipping OneDrive subscription setup")return "no-webhook-configured"@@ -308,7 +324,9 @@ class OneDriveConnector(BaseConnector):url = f"{self._graph_base_url}/subscriptions"async with httpx.AsyncClient() as client:- response = await client.post(url, json=subscription_data, headers=headers, timeout=30)+ response = await client.post(+ url, json=subscription_data, headers=headers, timeout=30+ )response.raise_for_status()result = response.json()@@ -328,15 +346,13 @@ class OneDriveConnector(BaseConnector):def _get_subscription_expiry(self) -> str:"""Get subscription expiry time (Graph caps duration; often <= 3 days)."""from datetime import datetime, timedelta+expiry = datetime.utcnow() + timedelta(days=3)return expiry.strftime("%Y-%m-%dT%H:%M:%S.%fZ")async def list_files(- self,- page_token: Optional[str] = None,- max_files: Optional[int] = None,- **kwargs- ) -> Dict[str, Any]:+ self, page_token: str | None = None, max_files: int | None = None, **kwargs+ ) -> dict[str, Any]:"""List files from OneDrive using Microsoft Graph."""try:if not await self.authenticate():@@ -346,7 +362,7 @@ class OneDriveConnector(BaseConnector):if self.cfg.file_ids or self.cfg.folder_ids:return await self._list_selected_files()- files: List[Dict[str, Any]] = []+ files: list[dict[str, Any]] = []max_files_value = max_files if max_files is not None else 100base_url = f"{self._graph_base_url}/me/drive/root/children"@@ -363,23 +379,28 @@ class OneDriveConnector(BaseConnector):items = data.get("value", [])for item in items:if item.get("file"): # include files only- files.append({- "id": item.get("id", ""),- "name": item.get("name", ""),- "path": f"/drive/items/{item.get('id')}",- "size": int(item.get("size", 0)),- "modified": item.get("lastModifiedDateTime"),- "created": item.get("createdDateTime"),- "mime_type": item.get("file", {}).get("mimeType", self._get_mime_type(item.get("name", ""))),- "url": item.get("webUrl", ""),- "download_url": item.get("@microsoft.graph.downloadUrl"),- })+ files.append(+ {+ "id": item.get("id", ""),+ "name": item.get("name", ""),+ "path": f"/drive/items/{item.get('id')}",+ "size": int(item.get("size", 0)),+ "modified": item.get("lastModifiedDateTime"),+ "created": item.get("createdDateTime"),+ "mime_type": item.get("file", {}).get(+ "mimeType", self._get_mime_type(item.get("name", ""))+ ),+ "url": item.get("webUrl", ""),+ "download_url": item.get("@microsoft.graph.downloadUrl"),+ }+ )# Next pagenext_page_token = Nonenext_link = data.get("@odata.nextLink")if next_link:- from urllib.parse import urlparse, parse_qs+ from urllib.parse import parse_qs, urlparse+parsed = urlparse(next_link)query_params = parse_qs(parsed.query)if "$skiptoken" in query_params:@@ -391,7 +412,7 @@ class OneDriveConnector(BaseConnector):logger.error(f"Failed to list OneDrive files: {e}")return {"files": [], "next_page_token": None}- async def _extract_onedrive_acl(self, file_id: str, file_metadata: Dict) -> DocumentACL:+ async def _extract_onedrive_acl(self, file_id: str, file_metadata: dict) -> DocumentACL:"""Extract ACL from OneDrive item.@@ -406,8 +427,7 @@ class OneDriveConnector(BaseConnector):"""try:# Get access token- token_data = await self.oauth.get_access_token()- access_token = token_data.get("access_token")+ access_token = self.oauth.get_access_token()if not access_token:logger.warning(f"No access token available for ACL extraction: {file_id}")@@ -419,8 +439,7 @@ class OneDriveConnector(BaseConnector):# Fetch permissionsasync with httpx.AsyncClient() as client:response = await client.get(- permissions_url,- headers={"Authorization": f"Bearer {access_token}"}+ permissions_url, headers={"Authorization": f"Bearer {access_token}"})if response.status_code != 200:@@ -487,13 +506,11 @@ class OneDriveConnector(BaseConnector):if cached_info and cached_info.get("downloadUrl"):logger.info(f"Using cached download URL for file {file_id}")content = await self._download_file_from_url(cached_info["downloadUrl"])-+acl = DocumentACL(owner="",- user_permissions={},- group_permissions={},)-+return ConnectorDocument(id=file_id,filename=cached_info.get("name", "Unknown"),@@ -513,13 +530,15 @@ class OneDriveConnector(BaseConnector):file_metadata = await self._get_file_metadata_by_id(file_id)if not file_metadata:# Last-resort: try shares endpoint download directly if this is a sharing ID- if '!' in file_id and file_id.split('!', 1)[1].startswith('s'):- logger.info(f"No metadata for sharing ID {file_id}, attempting direct shares download")+ if "!" in file_id and file_id.split("!", 1)[1].startswith("s"):+ logger.info(+ f"No metadata for sharing ID {file_id}, attempting direct shares download"+ )token = self.oauth.get_access_token()headers = {"Authorization": f"Bearer {token}"}shares_content = await self._download_via_shares_endpoint(file_id, headers)if shares_content is not None:- acl = DocumentACL(owner="", user_permissions={}, group_permissions={})+ acl = DocumentACL(owner="")return ConnectorDocument(id=file_id,filename="Unknown",@@ -564,9 +583,9 @@ class OneDriveConnector(BaseConnector):logger.error(f"Failed to get OneDrive file content {file_id}: {e}")raise- async def _get_file_metadata_by_id(self, file_id: str) -> Optional[Dict[str, Any]]:+ async def _get_file_metadata_by_id(self, file_id: str) -> dict[str, Any] | None:"""Get file metadata by ID using Graph API.-+Handles multiple ID formats:- Standard item ID: uses /me/drive/items/{id}- Personal OneDrive format (driveId!itemId): uses /drives/{driveId}/items/{itemId}@@ -575,7 +594,7 @@ class OneDriveConnector(BaseConnector):try:# Try different endpoints based on ID formatitem = await self._fetch_item_metadata(file_id)-+if not item:return None@@ -595,7 +614,9 @@ class OneDriveConnector(BaseConnector):"size": int(item.get("size", 0)),"modified": item.get("lastModifiedDateTime"),"created": item.get("createdDateTime"),- "mime_type": item.get("file", {}).get("mimeType", self._get_mime_type(item.get("name", ""))),+ "mime_type": item.get("file", {}).get(+ "mimeType", self._get_mime_type(item.get("name", ""))+ ),"url": item.get("webUrl", ""),"download_url": item.get("@microsoft.graph.downloadUrl"),"isFolder": False,@@ -607,69 +628,79 @@ class OneDriveConnector(BaseConnector):logger.error(f"Failed to get file metadata for {file_id}: {e}")return None- async def _fetch_item_metadata(self, file_id: str) -> Optional[Dict[str, Any]]:+ async def _fetch_item_metadata(self, file_id: str) -> dict[str, Any] | None:"""Fetch item metadata, trying multiple endpoints for different ID formats."""import base64+params = dict(self._default_params)-+# Check if ID contains '!' which indicates driveId!itemId format- if '!' in file_id:- parts = file_id.split('!', 1)+ if "!" in file_id:+ parts = file_id.split("!", 1)if len(parts) == 2:drive_id = parts[0]item_id = parts[1]-+# Handle sharing IDs (item ID starts with 's')- if item_id.startswith('s'):+ if item_id.startswith("s"):logger.info(f"Detected sharing ID format for {file_id}")-+# Try multiple encoding approaches for the shares endpointshare_encodings = [# Approach 1: Encode the full ID with "u!" prefix- base64.urlsafe_b64encode(f"u!{file_id}".encode()).decode().rstrip('='),+ base64.urlsafe_b64encode(f"u!{file_id}".encode()).decode().rstrip("="),# Approach 2: Encode just the share token with "s!" prefix- base64.urlsafe_b64encode(f"s!{item_id}".encode()).decode().rstrip('='),+ base64.urlsafe_b64encode(f"s!{item_id}".encode()).decode().rstrip("="),# Approach 3: Encode the ID directly- base64.urlsafe_b64encode(file_id.encode()).decode().rstrip('='),+ base64.urlsafe_b64encode(file_id.encode()).decode().rstrip("="),# Approach 4: Use the ID as-is (some APIs accept this)f"u!{file_id}",]-+for i, encoded in enumerate(share_encodings):try:url = f"{self._graph_base_url}/shares/{encoded}/driveItem"- logger.debug(f"Trying shares endpoint approach {i+1}: {url}")+ logger.debug(f"Trying shares endpoint approach {i + 1}: {url}")response = await self._make_graph_request(url, params=params)if response.status_code == 200:- logger.info(f"Shares endpoint approach {i+1} succeeded")+ logger.info(f"Shares endpoint approach {i + 1} succeeded")return response.json()else:- logger.debug(f"Shares approach {i+1} failed with status {response.status_code}")+ logger.debug(+ f"Shares approach {i + 1} failed with status {response.status_code}"+ )except Exception as e:- logger.debug(f"Shares approach {i+1} failed: {e}")-- # Try: /drives/{driveId}/items/{itemId} with full item ID (including 's' prefix)- logger.info(f"Trying drives endpoint: /drives/{drive_id}/items/{item_id}")+ logger.debug(f"Shares approach {i + 1} failed: {e}")++ # Try: /drives/{driveId}/items/{file_id} with full file ID+ logger.info(f"Trying drives endpoint: /drives/{drive_id}/items/{file_id}")try:- url = f"{self._graph_base_url}/drives/{drive_id}/items/{item_id}"+ url = f"{self._graph_base_url}/drives/{drive_id}/items/{file_id}"response = await self._make_graph_request(url, params=params)if response.status_code == 200:return response.json()+ else:+ logger.warning(+ f"Drives endpoint failed with status {response.status_code}: {response.text}"+ )except Exception as e:- logger.debug(f"Drives endpoint failed: {e}")-- # Try: /drives/{driveId}/items/{itemId} without 's' prefix- if item_id.startswith('s'):+ logger.debug(f"Drives endpoint exception: {e}")++ # Try: /drives/{driveId}/items/{clean_file_id} without 's' prefix+ if item_id.startswith("s"):clean_item_id = item_id[1:] # Remove 's' prefix- logger.info(f"Trying drives endpoint without 's' prefix: /drives/{drive_id}/items/{clean_item_id}")+ clean_file_id = f"{drive_id}!{clean_item_id}"+ logger.info(+ f"Trying drives endpoint without 's' prefix: /drives/{drive_id}/items/{clean_file_id}"+ )try:- url = f"{self._graph_base_url}/drives/{drive_id}/items/{clean_item_id}"+ url = f"{self._graph_base_url}/drives/{drive_id}/items/{clean_file_id}"response = await self._make_graph_request(url, params=params)if response.status_code == 200:return response.json()except Exception as e:logger.debug(f"Drives endpoint (no prefix) failed: {e}")-+# Try: /me/drive/items/{full_id} as fallbacklogger.info(f"Trying standard endpoint: /me/drive/items/{file_id}")try:@@ -677,42 +708,46 @@ class OneDriveConnector(BaseConnector):response = await self._make_graph_request(url, params=params)if response.status_code == 200:return response.json()+ else:+ logger.warning(+ f"Standard endpoint failed with status {response.status_code}: {response.text}"+ )except Exception as e:- logger.debug(f"Standard endpoint failed: {e}")+ logger.debug(f"Standard endpoint exception: {e}")else:# Standard item ID without '!'url = f"{self._graph_base_url}/me/drive/items/{file_id}"response = await self._make_graph_request(url, params=params)if response.status_code == 200:return response.json()-+logger.error(f"All endpoints failed for file_id: {file_id}")return Noneasync def _download_file_content(self, file_id: str) -> bytes:"""Download file content by file ID using Graph API.-+Handles multiple ID formats like _get_file_metadata_by_id."""try:token = self.oauth.get_access_token()headers = {"Authorization": f"Bearer {token}"}-+# Build URL based on ID format- if '!' in file_id:- parts = file_id.split('!', 1)+ if "!" in file_id:+ parts = file_id.split("!", 1)if len(parts) == 2:drive_id = parts[0]item_id = parts[1]-+# If this looks like a sharing ID (starts with 's'), try shares endpoint first- if item_id.startswith('s'):+ if item_id.startswith("s"):content = await self._download_via_shares_endpoint(file_id, headers)if content is not None:return content- # Try drives endpoint for driveId!itemId format (including the 's' prefix)- url = f"{self._graph_base_url}/drives/{drive_id}/items/{item_id}/content"+ # Try drives endpoint for driveId!itemId format+ url = f"{self._graph_base_url}/drives/{drive_id}/items/{file_id}/content"logger.info(f"Downloading via drives endpoint: {url}")else:url = f"{self._graph_base_url}/me/drive/items/{file_id}/content"@@ -728,31 +763,37 @@ class OneDriveConnector(BaseConnector):logger.error(f"Failed to download file content for {file_id}: {e}")raise- async def _download_via_shares_endpoint(self, file_id: str, headers: Dict[str, str]) -> Optional[bytes]:+ async def _download_via_shares_endpoint(+ self, file_id: str, headers: dict[str, str]+ ) -> bytes | None:"""Attempt to download content using the Graph /shares endpoint for sharing IDs."""import base64share_encodings = [- base64.urlsafe_b64encode(f"u!{file_id}".encode()).decode().rstrip('='),- base64.urlsafe_b64encode(f"s!{file_id}".encode()).decode().rstrip('='),- base64.urlsafe_b64encode(file_id.encode()).decode().rstrip('='),+ base64.urlsafe_b64encode(f"u!{file_id}".encode()).decode().rstrip("="),+ base64.urlsafe_b64encode(f"s!{file_id}".encode()).decode().rstrip("="),+ base64.urlsafe_b64encode(file_id.encode()).decode().rstrip("="),f"u!{file_id}",]for i, encoded in enumerate(share_encodings):try:url = f"{self._graph_base_url}/shares/{encoded}/driveItem/content"- logger.info(f"Attempting shares download (approach {i+1}): {url}")+ logger.info(f"Attempting shares download (approach {i + 1}): {url}")async with httpx.AsyncClient() as client:- response = await client.get(url, headers=headers, timeout=60, follow_redirects=True)+ response = await client.get(+ url, headers=headers, timeout=60, follow_redirects=True+ )if response.status_code == 200:return response.contentelse:- logger.debug(f"Shares download approach {i+1} failed with status {response.status_code}")+ logger.debug(+ f"Shares download approach {i + 1} failed with status {response.status_code}"+ )except Exception as e:- logger.debug(f"Shares download approach {i+1} failed: {e}")+ logger.debug(f"Shares download approach {i + 1} failed: {e}")return None@@ -767,20 +808,25 @@ class OneDriveConnector(BaseConnector):logger.error(f"Failed to download from URL {download_url}: {e}")raise- def _parse_graph_date(self, date_str: Optional[str]) -> datetime:+ def _parse_graph_date(self, date_str: str | None) -> datetime:"""Parse Microsoft Graph date string to datetime."""if not date_str:return datetime.now()try:- if date_str.endswith('Z'):+ if date_str.endswith("Z"):return datetime.fromisoformat(date_str[:-1]).replace(tzinfo=None)else:- return datetime.fromisoformat(date_str.replace('T', ' '))+ return datetime.fromisoformat(date_str.replace("T", " "))except (ValueError, AttributeError):return datetime.now()- async def _make_graph_request(self, url: str, method: str = "GET",- data: Optional[Dict] = None, params: Optional[Dict] = None) -> httpx.Response:+ async def _make_graph_request(+ self,+ url: str,+ method: str = "GET",+ data: dict | None = None,+ params: dict | None = None,+ ) -> httpx.Response:"""Make authenticated API request to Microsoft Graph."""token = self.oauth.get_access_token()headers = {@@ -801,25 +847,25 @@ class OneDriveConnector(BaseConnector):response.raise_for_status()return response- async def _list_selected_files(self) -> Dict[str, Any]:+ async def _list_selected_files(self) -> dict[str, Any]:"""List only selected files/folders (selective sync)."""- files: List[Dict[str, Any]] = []-+ files: list[dict[str, Any]] = []+# Process selected file IDsif self.cfg.file_ids:for file_id in self.cfg.file_ids:try:file_meta = await self._get_file_metadata_by_id(file_id)- if file_meta and not file_meta.get('isFolder', False):+ if file_meta and not file_meta.get("isFolder", False):files.append(file_meta)- elif file_meta and file_meta.get('isFolder', False):+ elif file_meta and file_meta.get("isFolder", False):# If it's a folder, expand its contentsfolder_files = await self._list_folder_contents(file_id)files.extend(folder_files)except Exception as e:logger.warning(f"Failed to get file {file_id}: {e}")continue-+# Process selected folder IDsif self.cfg.folder_ids:for folder_id in self.cfg.folder_ids:@@ -829,20 +875,20 @@ class OneDriveConnector(BaseConnector):except Exception as e:logger.warning(f"Failed to list folder {folder_id}: {e}")continue-+return {"files": files, "next_page_token": None}-- async def _list_folder_contents(self, folder_id: str) -> List[Dict[str, Any]]:++ async def _list_folder_contents(self, folder_id: str) -> list[dict[str, Any]]:"""List all files in a folder recursively."""- files: List[Dict[str, Any]] = []-+ files: list[dict[str, Any]] = []+try:url = f"{self._graph_base_url}/me/drive/items/{folder_id}/children"params = dict(self._default_params)-+response = await self._make_graph_request(url, params=params)data = response.json()-+items = data.get("value", [])for item in items:if item.get("file"): # It's a file@@ -854,35 +900,37 @@ class OneDriveConnector(BaseConnector):files.extend(subfolder_files)except Exception as e:logger.error(f"Failed to list folder contents for {folder_id}: {e}")-+return filesdef _get_mime_type(self, filename: str) -> str:"""Get MIME type based on file extension."""import mimetypes+mime_type, _ = mimetypes.guess_type(filename)return mime_type or "application/octet-stream"# Webhook methods - BaseConnector interface- def handle_webhook_validation(self, request_method: str,- headers: Dict[str, str],- query_params: Dict[str, str]) -> Optional[str]:+ def handle_webhook_validation(+ self, request_method: str, headers: dict[str, str], query_params: dict[str, str]+ ) -> str | None:"""Handle webhook validation (Graph API specific)."""if request_method == "POST" and "validationToken" in query_params:return query_params["validationToken"]return None- def extract_webhook_channel_id(self, payload: Dict[str, Any],- headers: Dict[str, str]) -> Optional[str]:+ def extract_webhook_channel_id(+ self, payload: dict[str, Any], headers: dict[str, str]+ ) -> str | None:"""Extract channel/subscription ID from webhook payload."""notifications = payload.get("value", [])if notifications:return notifications[0].get("subscriptionId")return None- async def handle_webhook(self, payload: Dict[str, Any]) -> List[str]:+ async def handle_webhook(self, payload: dict[str, Any]) -> list[str]:"""Handle webhook notification and return affected file IDs."""- affected_files: List[str] = []+ affected_files: list[str] = []notifications = payload.get("value", [])for notification in notifications:resource = notification.get("resource")@@ -914,7 +962,9 @@ class OneDriveConnector(BaseConnector):logger.info(f"OneDrive subscription {subscription_id} cleaned up successfully")return Trueelse:- logger.warning(f"Unexpected response cleaning up subscription: {response.status_code}")+ logger.warning(+ f"Unexpected response cleaning up subscription: {response.status_code}"+ )return Falseexcept Exception as e:diff --git a/src/connectors/onedrive/oauth.py b/src/connectors/onedrive/oauth.pyindex b275561e2..898f16673 100644--- a/src/connectors/onedrive/oauth.py+++ b/src/connectors/onedrive/oauth.py@@ -1,8 +1,7 @@-import osimport jsonimport logging-from typing import Optional, Dict, Any-+import os+from typing import Anyimport msal@@ -25,16 +24,9 @@ class OneDriveOAuth:"Files.Read","Files.Read.All", # Access all files user can access"Files.Read.Selected",- "Sites.Read.All", # Read SharePoint sites (for File Picker and OneDrive for Business)- "offline_access"- ]- RESOURCE_SCOPES = [- "User.Read",- "Files.Read",- "Files.Read.All",- "Files.Read.Selected",- "Sites.Read.All"+ "offline_access",]+ RESOURCE_SCOPES = ["User.Read", "Files.Read", "Files.Read.All", "Files.Read.Selected"]SCOPES = AUTH_SCOPES # Backward-compat alias if something references .SCOPES# Kept for reference; MSAL derives endpoints from `authority`@@ -67,7 +59,7 @@ class OneDriveOAuth:self.authority = authorityself.allow_json_refresh = allow_json_refreshself.token_cache = msal.SerializableTokenCache()- self._current_account = None+ self._current_account: dict[str, Any] | None = None# Initialize MSAL Confidential Clientself.app = msal.ConfidentialClientApplication(@@ -81,10 +73,10 @@ class OneDriveOAuth:"""Load existing credentials from token file (async)."""try:from utils.encryption import read_encrypted_file-+logger.debug(f"OneDrive OAuth loading credentials from: {self.token_file}")cache_data, needs_upgrade = await read_encrypted_file(self.token_file)-+if cache_data and cache_data.strip():# 1) Try legacy flat JSON firsttry:@@ -112,15 +104,20 @@ class OneDriveOAuth:accounts = self.app.get_accounts()logger.debug(f"Found {len(accounts)} accounts in MSAL cache")if accounts:- self._current_account = accounts[0]- logger.debug(f"Set current account: {self._current_account.get('username', 'no username')}")-+ account = accounts[0]+ self._current_account = account+ logger.debug(f"Set current account: {account.get('username', 'no username')}")+if needs_upgrade:await self.save_cache()# Use RESOURCE_SCOPES (no reserved scopes) for silent acquisition- result = self.app.acquire_token_silent(self.RESOURCE_SCOPES, account=self._current_account)- logger.debug(f"Silent token acquisition result keys: {list(result.keys()) if result else 'None'}")+ result = self.app.acquire_token_silent(+ self.RESOURCE_SCOPES, account=self._current_account+ )+ logger.debug(+ f"Silent token acquisition result keys: {list(result.keys()) if result else 'None'}"+ )if result and "access_token" in result:logger.debug("Silent token acquisition successful")if getattr(self.token_cache, "has_state_changed", False):@@ -134,7 +131,7 @@ class OneDriveOAuth:return False- except Exception as e:+ except Exception:logger.exception("[CONNECTOR] OneDrive credential load failed")return False@@ -166,15 +163,24 @@ class OneDriveOAuth:accounts = self.app.get_accounts()logger.debug(f"After refresh, found {len(accounts)} accounts")if accounts:- self._current_account = accounts[0]- logger.debug(f"Set current account after refresh: {self._current_account.get('username', 'no username')}")+ account = accounts[0]+ self._current_account = account+ logger.debug(+ f"Set current account after refresh: {account.get('username', 'no username')}"+ )return True# Error handling- err = (result or {}).get("error_description") or (result or {}).get("error") or "Unknown error"+ err = (+ (result or {}).get("error_description")+ or (result or {}).get("error")+ or "Unknown error"+ )logger.error(f"Refresh token failed: {err}")- if any(code in err for code in ("AADSTS70000", "invalid_grant", "interaction_required")):+ if any(+ code in err for code in ("AADSTS70000", "invalid_grant", "interaction_required")+ ):logger.warning("Refresh denied due to unauthorized/expired scopes or invalid grant. ""Delete the token file and perform interactive sign-in with correct scopes."@@ -182,7 +188,7 @@ class OneDriveOAuth:return False- except Exception as e:+ except Exception:logger.exception("[CONNECTOR] OneDrive JSON token refresh failed")return False@@ -197,18 +203,19 @@ class OneDriveOAuth:cache_data = self.token_cache.serialize()if cache_data:from utils.encryption import write_encrypted_file+await write_encrypted_file(self.token_file, cache_data)-+logger.debug(f"Token cache saved to {self.token_file}")except Exception as e:logger.error(f"Failed to save token cache: {e}")- def create_authorization_url(self, redirect_uri: str, state: Optional[str] = None) -> str:+ def create_authorization_url(self, redirect_uri: str, state: str | None = None) -> str:"""Create authorization URL for OAuth flow."""# Store redirect URI for later use in callbackself._redirect_uri = redirect_uri- kwargs: Dict[str, Any] = {+ kwargs: dict[str, Any] = {# Interactive auth includes offline_access"scopes": self.AUTH_SCOPES,"redirect_uri": redirect_uri,@@ -231,7 +238,7 @@ class OneDriveOAuth:try:result = self.app.acquire_token_by_authorization_code(authorization_code,- scopes=self.AUTH_SCOPES, # same as authorize step+ scopes=self.AUTH_SCOPES, # same as authorize stepredirect_uri=redirect_uri,)@@ -244,7 +251,11 @@ class OneDriveOAuth:logger.info("OneDrive OAuth authorization successful")return True- error_msg = (result or {}).get("error_description") or (result or {}).get("error") or "Unknown error"+ error_msg = (+ (result or {}).get("error_description")+ or (result or {}).get("error")+ or "Unknown error"+ )logger.error(f"OneDrive OAuth authorization failed: {error_msg}")return False@@ -262,29 +273,47 @@ class OneDriveOAuth:load_result = await self.load_credentials()logger.info(f"OneDrive is_authenticated: load_credentials returned {load_result}")- logger.info(f"OneDrive is_authenticated: current_account={self._current_account is not None}")-+ logger.info(+ f"OneDrive is_authenticated: current_account={self._current_account is not None}"+ )+# Try to get a token (MSAL will refresh if needed)if self._current_account:- logger.info(f"OneDrive is_authenticated: Trying acquire_token_silent with account {self._current_account.get('username', 'unknown')}")+ logger.info(+ f"OneDrive is_authenticated: Trying acquire_token_silent with account {self._current_account.get('username', 'unknown')}"+ )logger.info(f"OneDrive is_authenticated: RESOURCE_SCOPES={self.RESOURCE_SCOPES}")- result = self.app.acquire_token_silent(self.RESOURCE_SCOPES, account=self._current_account)+ result = self.app.acquire_token_silent(+ self.RESOURCE_SCOPES, account=self._current_account+ )if result and "access_token" in result:- logger.info("OneDrive is_authenticated: Successfully acquired token with account")+ logger.info(+ "OneDrive is_authenticated: Successfully acquired token with account"+ )if getattr(self.token_cache, "has_state_changed", False):await self.save_cache()return Trueelse:- error_msg = (result or {}).get("error") or (result or {}).get("error_description") or "No result returned"- logger.warning(f"OneDrive is_authenticated: Token acquisition failed for current account: {error_msg}")+ error_msg = (+ (result or {}).get("error")+ or (result or {}).get("error_description")+ or "No result returned"+ )+ logger.warning(+ f"OneDrive is_authenticated: Token acquisition failed for current account: {error_msg}"+ )logger.warning(f"OneDrive is_authenticated: Full result: {result}")# Fallback: try without specific account- logger.info("OneDrive is_authenticated: Fallback - trying acquire_token_silent without account")+ logger.info(+ "OneDrive is_authenticated: Fallback - trying acquire_token_silent without account"+ )result = self.app.acquire_token_silent(self.RESOURCE_SCOPES, account=None)if result and "access_token" in result:accounts = self.app.get_accounts()- logger.info(f"OneDrive is_authenticated: Fallback succeeded, found {len(accounts)} accounts")+ logger.info(+ f"OneDrive is_authenticated: Fallback succeeded, found {len(accounts)} accounts"+ )if accounts:self._current_account = accounts[0]if getattr(self.token_cache, "has_state_changed", False):@@ -294,23 +323,31 @@ class OneDriveOAuth:logger.warning(f"OneDrive is_authenticated: Fallback also failed, result: {result}")return False- except Exception as e:+ except Exception:logger.exception("[CONNECTOR] OneDrive is_authenticated failed")return Falsedef get_access_token(self) -> str:"""Get an access token for Microsoft Graph."""- logger.info(f"OneDrive get_access_token: Starting, current_account={self._current_account is not None}")+ logger.info(+ f"OneDrive get_access_token: Starting, current_account={self._current_account is not None}"+ )try:# Try with current account firstif self._current_account:- logger.info(f"OneDrive get_access_token: Trying with account {self._current_account.get('username', 'unknown')}")- result = self.app.acquire_token_silent(self.RESOURCE_SCOPES, account=self._current_account)+ logger.info(+ f"OneDrive get_access_token: Trying with account {self._current_account.get('username', 'unknown')}"+ )+ result = self.app.acquire_token_silent(+ self.RESOURCE_SCOPES, account=self._current_account+ )if result and "access_token" in result:logger.info("OneDrive get_access_token: Success with current account")return result["access_token"]else:- logger.warning(f"OneDrive get_access_token: Failed with account, result: {result}")+ logger.warning(+ f"OneDrive get_access_token: Failed with account, result: {result}"+ )# Fallback: try without specific accountlogger.info("OneDrive get_access_token: Fallback - trying without account")@@ -320,11 +357,15 @@ class OneDriveOAuth:return result["access_token"]# If we get here, authentication has failed- error_msg = (result or {}).get("error_description") or (result or {}).get("error") or "No valid authentication"+ error_msg = (+ (result or {}).get("error_description")+ or (result or {}).get("error")+ or "No valid authentication"+ )logger.error(f"OneDrive get_access_token: All attempts failed, error: {error_msg}")raise ValueError(f"Failed to acquire access token: {error_msg}")- except Exception as e:+ except Exception:logger.exception("[CONNECTOR] OneDrive get_access_token failed")raise