Skip to content

OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.

Python
54.0 KB
Raw Blame
from datetime import UTC, datetime, timedelta
from pathlib import Path
from typing import Any
from urllib.parse import urlparse

import httpx

from utils.group_acl import unique_acl_principal_labels
from utils.logging_config import get_logger

from ..base import BaseConnector, ConnectorDocument, DocumentACL
from ..microsoft_graph_acl import (
    get_current_user_microsoft_group_roles,
    get_current_user_microsoft_principal_labels,
    get_current_user_microsoft_principals,
    get_oauth_access_token,
    microsoft_group_principal_label,
    microsoft_group_role,
    microsoft_user_principal,
    microsoft_user_principal_label,
)
from .oauth import OneDriveOAuth

logger = get_logger(__name__)


class OneDriveConnector(BaseConnector):
    """OneDrive connector using MSAL-based OAuth for authentication."""

    # Required BaseConnector class attributes
    CLIENT_ID_ENV_VAR = "MICROSOFT_GRAPH_OAUTH_CLIENT_ID"
    CLIENT_SECRET_ENV_VAR = "MICROSOFT_GRAPH_OAUTH_CLIENT_SECRET"  # pragma: allowlist secret
    # Shared with SharePointConnector — both use one Microsoft Graph app registration.
    OAUTH_CREDENTIAL_KEY = "microsoft_graph"

    # Connector metadata
    CONNECTOR_TYPE = "onedrive"
    CONNECTOR_KIND = "oauth"
    CONNECTOR_NAME = "OneDrive"
    CONNECTOR_DESCRIPTION = "Add knowledge from OneDrive"
    CONNECTOR_ICON = "onedrive"

    @classmethod
    def get_oauth_class(cls):
        from .oauth import OneDriveOAuth

        return OneDriveOAuth

    def __init__(self, config: dict[str, Any]):
        super().__init__(config)

        logger.debug(f"OneDrive connector __init__ called with config type: {type(config)}")
        logger.debug(f"OneDrive connector __init__ config value: {config}")

        if config is None:
            logger.debug("Config was None, using empty dict")
            config = {}

        try:
            logger.debug("Calling super().__init__")
            super().__init__(config)
            logger.debug("super().__init__ completed successfully")
        except Exception as e:
            logger.error(f"super().__init__ failed: {e}")
            raise

        # Initialize with defaults that allow the connector to be listed
        self.client_id = None
        self.client_secret = None
        self.redirect_uri = config.get("redirect_uri", "http://localhost")
        # Graph delta link for webhook change tracking (in-memory per instance)
        self._delta_link: str | None = None
        self._base_url = config.get("base_url")  # Generic URL field for OneDrive/SharePoint domain
        logger.debug(f"OneDrive connector initialized with base_url from config: {self._base_url}")

        # Try to get credentials, but don't fail if they're missing
        try:
            self.client_id = self.get_client_id()
            logger.debug(f"Got client_id: {self.client_id is not None}")
        except Exception as e:
            logger.debug(f"Failed to get client_id: {e}")

        try:
            self.client_secret = self.get_client_secret()
            logger.debug(f"Got client_secret: {self.client_secret is not None}")
        except Exception as e:
            logger.debug(f"Failed to get client_secret: {e}")

        # Token file setup - use data directory for persistence
        from config.paths import get_data_file

        token_file = config.get("token_file") or get_data_file("onedrive_token.json")
        Path(token_file).parent.mkdir(parents=True, exist_ok=True)

        # Only initialize OAuth if we have credentials
        if self.client_id and self.client_secret:
            connection_id = config.get("connection_id", "default")

            # Use token_file from config if provided, otherwise generate one
            if config.get("token_file"):
                oauth_token_file = config["token_file"]
            else:
                # Use a per-connection cache file to avoid collisions with other connectors
                oauth_token_file = get_data_file(f"onedrive_token_{connection_id}.json")

            # MSA & org both work via /common for OneDrive personal testing
            authority = "https://login.microsoftonline.com/common"

            self.oauth = OneDriveOAuth(
                client_id=self.client_id,
                client_secret=self.client_secret,
                token_file=oauth_token_file,
                authority=authority,
                allow_json_refresh=True,  # allows one-time migration from legacy JSON if present
            )
        else:
            self.oauth = None

        # Track subscription ID for webhooks (note: change notifications might not be available for personal accounts)
        self._subscription_id: str | None = None

        # Set by setup_subscription/renew_subscription; read by the
        # connection manager to persist
        self.webhook_resource_id: str | None = None
        self.webhook_expiration: str | None = None

        # Graph API defaults
        self._graph_api_version = "v1.0"
        self._default_params: dict[str, Any] = {
            "$select": "id,name,size,lastModifiedDateTime,createdDateTime,webUrl,file,folder,@microsoft.graph.downloadUrl"
        }

        # Selective sync support (similar to Google Drive)
        self.cfg = type(
            "OneDriveConfig",
            (),
            {
                "file_ids": config.get("file_ids")
                or config.get("selected_files")
                or config.get("selected_file_ids"),
                "folder_ids": config.get("folder_ids")
                or config.get("selected_folders")
                or config.get("selected_folder_ids"),
            },
        )()

        # Cache for file metadata including download URLs
        # This allows direct download without Graph API for sharing IDs
        self._file_infos: dict[str, dict[str, Any]] = {}

    @property
    def _graph_base_url(self) -> str:
        """Base URL for Microsoft Graph API calls."""
        return f"https://graph.microsoft.com/{self._graph_api_version}"

    @property
    def base_url(self) -> str | None:
        """Generic base URL property (OneDrive/SharePoint domain)"""
        return self._base_url

    @base_url.setter
    def base_url(self, value: str):
        """Set base URL"""
        self._base_url = value

    async def get_current_user_group_roles(self) -> list[str]:
        """Return canonical group ACL roles for the connected Microsoft user."""
        return await get_current_user_microsoft_group_roles(
            self.oauth,
            self._graph_base_url,
        )

    async def get_current_user_principals(self) -> list[str]:
        """Return canonical user ACL principals for the connected Microsoft user."""
        return await get_current_user_microsoft_principals(
            self.oauth,
            self._graph_base_url,
        )

    async def get_current_user_principal_labels(self) -> list[dict[str, Any]]:
        """Return display labels for current Microsoft user/group ACL principals."""
        return await get_current_user_microsoft_principal_labels(
            self.oauth,
            self._graph_base_url,
        )

    def set_file_infos(self, file_infos: list[dict[str, Any]]) -> None:
        """
        Cache file metadata including download URLs for later use.
        This allows direct download without Graph API calls for sharing IDs.

        Args:
            file_infos: List of file info dicts with {id, name, mimeType, downloadUrl, size}
        """
        self._file_infos = {}
        for info in file_infos:
            file_id = info.get("id")
            if file_id:
                self._file_infos[file_id] = info
                if info.get("downloadUrl"):
                    logger.debug(f"Cached download URL for file {file_id}: {info.get('name')}")

    def get_cached_file_info(self, file_id: str) -> dict[str, Any] | None:
        """Get cached file info by ID."""
        return self._file_infos.get(file_id)

    def emit(self, doc: ConnectorDocument) -> None:
        """Emit a ConnectorDocument instance."""
        logger.debug(f"Emitting OneDrive document: {doc.id} ({doc.filename})")

    async def authenticate(self) -> bool:
        """Test authentication - BaseConnector interface."""
        logger.debug(f"OneDrive authenticate() called, oauth is None: {self.oauth is None}")
        try:
            if not self.oauth:
                logger.debug("OneDrive authentication failed: OAuth not initialized")
                self._authenticated = False
                return False

            logger.debug("Loading OneDrive credentials...")
            load_result = await self.oauth.load_credentials()
            logger.debug(f"Load credentials result: {load_result}")

            logger.debug("Checking OneDrive authentication status...")
            authenticated = await self.oauth.is_authenticated()
            logger.debug(f"OneDrive is_authenticated result: {authenticated}")

            self._authenticated = authenticated
            return authenticated
        except Exception:
            logger.exception("[CONNECTOR] OneDrive authentication failed")
            self._authenticated = False
            return False

    def get_auth_url(self) -> str:
        """Get OAuth authorization URL."""
        if not self.oauth:
            raise RuntimeError("OneDrive OAuth not initialized - missing credentials")
        return self.oauth.create_authorization_url(self.redirect_uri)

    async def handle_oauth_callback(self, auth_code: str) -> dict[str, Any]:
        """Handle OAuth callback."""
        if not self.oauth:
            raise RuntimeError("OneDrive OAuth not initialized - missing credentials")
        try:
            success = await self.oauth.handle_authorization_callback(auth_code, self.redirect_uri)
            if success:
                self._authenticated = True
                return {"status": "success"}
            else:
                raise ValueError("OAuth callback failed")
        except Exception as e:
            logger.error(f"OAuth callback failed: {e}")
            raise

    async def _detect_base_url(self) -> str | None:
        """Override base class method to detect OneDrive URL"""
        return await self._detect_onedrive_url()

    async def _detect_onedrive_url(self) -> str | None:
        """Auto-detect OneDrive URL from Microsoft Graph API"""
        logger.info("_detect_onedrive_url: Starting OneDrive URL detection")
        try:
            if not self.oauth:
                logger.warning("_detect_onedrive_url: OAuth not initialized")
                return None

            access_token = self.oauth.get_access_token()
            logger.debug(
                f"_detect_onedrive_url: Got access token (length: {len(access_token) if access_token else 0})"
            )

            headers = {
                "Authorization": f"Bearer {access_token}",
                "Content-Type": "application/json",
            }

            async with httpx.AsyncClient() as client:
                # Get user's default drive to extract OneDrive URL
                url = f"{self._graph_base_url}/me/drive"
                logger.info(f"_detect_onedrive_url: Calling Graph API: {url}")

                response = await client.get(url, headers=headers, timeout=30.0)
                logger.info(
                    f"_detect_onedrive_url: Graph API response status: {response.status_code}"
                )

                if response.status_code == 200:
                    data = response.json()
                    web_url = data.get("webUrl", "")
                    logger.info(f"_detect_onedrive_url: webUrl from response: {web_url}")

                    # Extract the domain from the webUrl
                    # e.g., "https://onedrive.live.com/..." or "https://company-my.sharepoint.com/..."
                    if web_url:
                        parsed = urlparse(web_url)
                        onedrive_url = f"{parsed.scheme}://{parsed.netloc}"
                        logger.info(f"_detect_onedrive_url: Detected OneDrive URL: {onedrive_url}")
                        return onedrive_url
                    else:
                        logger.warning("_detect_onedrive_url: webUrl is empty in response")
                else:
                    logger.warning(
                        f"[CONNECTOR] OneDrive detect URL failed, status_code: {response.status_code}"
                    )

        except Exception:
            logger.exception("[CONNECTOR] OneDrive URL detection failed")

        return None

    def sync_once(self) -> None:
        """
        Perform a one-shot sync of OneDrive files and emit documents.
        """
        import asyncio

        async def _async_sync():
            try:
                file_list = await self.list_files(max_files=1000)
                files = file_list.get("files", [])
                for file_info in files:
                    try:
                        file_id = file_info.get("id")
                        if not file_id:
                            continue
                        doc = await self.get_file_content(file_id)
                        self.emit(doc)
                    except Exception as e:
                        logger.error(
                            f"Failed to sync OneDrive file {file_info.get('name', 'unknown')}: {e}"
                        )
                        continue
            except Exception as e:
                logger.error(f"OneDrive sync_once failed: {e}")
                raise

        if hasattr(asyncio, "run"):
            asyncio.run(_async_sync())
        else:
            loop = asyncio.get_event_loop()
            loop.run_until_complete(_async_sync())

    async def setup_subscription(self) -> str:
        """
        Set up real-time subscription for file changes.
        NOTE: Change notifications may not be available for personal OneDrive accounts.
        """
        webhook_url = self.config.get("webhook_url")
        if not webhook_url:
            logger.warning("No webhook URL configured, skipping OneDrive subscription setup")
            return "no-webhook-configured"

        try:
            if not await self.authenticate():
                raise RuntimeError("OneDrive authentication failed during subscription setup")

            token = self.oauth.get_access_token()

            # For OneDrive personal we target the user's drive
            resource = "/me/drive/root"

            subscription_data = {
                # Graph driveItem subscriptions only support "updated"; creates and
                # deletes still surface through the delta query the webhook triggers.
                "changeType": "updated",
                # webhook_url is already the full endpoint
                # ({WEBHOOK_BASE_URL}/connectors/onedrive/webhook, set at connect time)
                "notificationUrl": webhook_url,
                "resource": resource,
                "expirationDateTime": self._get_subscription_expiry(),
                "clientState": "onedrive_personal",
            }

            headers = {
                "Authorization": f"Bearer {token}",
                "Content-Type": "application/json",
            }

            url = f"{self._graph_base_url}/subscriptions"

            async with httpx.AsyncClient() as client:
                response = await client.post(
                    url, json=subscription_data, headers=headers, timeout=30
                )
                if response.status_code >= 400:
                    logger.error(
                        f"Graph subscription request rejected: {response.status_code} {response.text}"
                    )
                response.raise_for_status()

                result = response.json()
                subscription_id = result.get("id")

                if subscription_id:
                    self._subscription_id = subscription_id
                    self.webhook_expiration = result.get("expirationDateTime")
                    logger.info(f"OneDrive subscription created: {subscription_id}")
                    return subscription_id
                else:
                    raise ValueError("No subscription ID returned from Microsoft Graph")

        except Exception as e:
            logger.error(f"Failed to setup OneDrive subscription: {e}")
            raise

    def _get_subscription_expiry(self) -> str:
        """Get subscription expiry time (Graph caps duration; often <= 3 days)."""
        from datetime import datetime, timedelta

        expiry = datetime.now(UTC) + timedelta(days=3)
        return expiry.strftime("%Y-%m-%dT%H:%M:%S.%fZ")

    async def list_files(
        self, page_token: str | None = None, max_files: int | None = None, **kwargs
    ) -> dict[str, Any]:
        """List files from OneDrive using Microsoft Graph."""
        try:
            if not await self.authenticate():
                raise RuntimeError("OneDrive authentication failed during file listing")

            # If file_ids or folder_ids are specified in config, use selective sync
            if self.cfg.file_ids or self.cfg.folder_ids:
                return await self._list_selected_files()

            files: list[dict[str, Any]] = []
            max_files_value = max_files if max_files is not None else 100

            base_url = f"{self._graph_base_url}/me/drive/root/children"

            params = dict(self._default_params)
            params["$top"] = str(max_files_value)

            if page_token:
                params["$skiptoken"] = page_token

            response = await self._make_graph_request(base_url, params=params)
            data = response.json()

            items = data.get("value", [])
            for item in items:
                if item.get("file"):  # include files only
                    files.append(
                        {
                            "id": item.get("id", ""),
                            "name": item.get("name", ""),
                            "path": f"/drive/items/{item.get('id')}",
                            "size": int(item.get("size", 0)),
                            "modified": item.get("lastModifiedDateTime"),
                            "created": item.get("createdDateTime"),
                            "mime_type": item.get("file", {}).get(
                                "mimeType", self._get_mime_type(item.get("name", ""))
                            ),
                            "url": item.get("webUrl", ""),
                            "download_url": item.get("@microsoft.graph.downloadUrl"),
                        }
                    )

            # Next page
            next_page_token = None
            next_link = data.get("@odata.nextLink")
            if next_link:
                from urllib.parse import parse_qs, urlparse

                parsed = urlparse(next_link)
                query_params = parse_qs(parsed.query)
                if "$skiptoken" in query_params:
                    next_page_token = query_params["$skiptoken"][0]

            return {"files": files, "next_page_token": next_page_token}

        except Exception as e:
            logger.error(f"Failed to list OneDrive files: {e}")
            return {"files": [], "next_page_token": None}

    async def _extract_onedrive_acl(self, file_id: str, file_metadata: dict) -> DocumentACL:
        """
        Extract ACL from OneDrive item.

        Queries Microsoft Graph API permissions endpoint to get allowed users and groups.

        Args:
            file_id: OneDrive item ID
            file_metadata: File metadata dict

        Returns:
            DocumentACL instance with extracted permissions
        """
        try:
            # Get access token
            access_token = await get_oauth_access_token(self.oauth)

            if not access_token:
                logger.warning(f"No access token available for ACL extraction: {file_id}")
                return DocumentACL()

            # OneDrive permissions API endpoint. A composite "driveId!itemId"
            # must be split into /drives/{driveId}/items/{itemId}; using it
            # verbatim against /me/drive/items/{id} is malformed → empty ACL.
            if "!" in file_id and len(file_id.rsplit("!", 1)) == 2:
                drive_id, item_id = file_id.rsplit("!", 1)
                permissions_url = (
                    f"{self._graph_base_url}/drives/{drive_id}/items/{item_id}/permissions"
                )
            else:
                permissions_url = f"{self._graph_base_url}/me/drive/items/{file_id}/permissions"

            # Fetch permissions, following pagination for full share lists.
            permissions: list[dict[str, Any]] = []
            async with httpx.AsyncClient() as client:
                url: str | None = permissions_url
                while url:
                    response = await client.get(
                        url, headers={"Authorization": f"Bearer {access_token}"}
                    )
                    if response.status_code != 200:
                        logger.warning(
                            f"Failed to fetch permissions for {file_id}: {response.status_code}"
                        )
                        return DocumentACL()
                    page = response.json()
                    permissions.extend(page.get("value", []))
                    url = page.get("@odata.nextLink")

            permissions_data = {"value": permissions}

            allowed_users = []
            allowed_groups = []
            allowed_principals = []
            allowed_principal_labels = []
            owner = None

            for perm in permissions_data.get("value", []):
                roles = perm.get("roles", [])  # ["read", "write", "owner"]

                # Granted to user/group (grantedToV2 is the current Graph shape;
                # grantedTo is retained for older responses).
                granted_to = perm.get("grantedToV2") or perm.get("grantedTo")
                if granted_to:
                    user_info = granted_to.get("user", {})
                    email = user_info.get("email")
                    if email:
                        allowed_users.append(email)
                        if "owner" in roles:
                            owner = email
                    for identifier in (
                        user_info.get("id"),
                        user_info.get("userPrincipalName"),
                        email,
                    ):
                        user_principal = microsoft_user_principal(
                            identifier,
                            access_token=access_token,
                        )
                        if user_principal:
                            allowed_principals.append(user_principal)
                            label = microsoft_user_principal_label(
                                identifier,
                                access_token=access_token,
                                display_name=user_info.get("displayName") or email,
                                email=email,
                                external_id=identifier,
                            )
                            if label:
                                allowed_principal_labels.append(label)
                    group_info = granted_to.get("group", {})
                    group_role = microsoft_group_role(
                        group_info.get("id"),
                        access_token=access_token,
                    )
                    if group_role:
                        allowed_groups.append(group_role)
                        allowed_principals.append(group_role)
                        label = microsoft_group_principal_label(
                            group_info.get("id"),
                            access_token=access_token,
                            display_name=group_info.get("displayName") or group_info.get("email"),
                            email=group_info.get("email"),
                        )
                        if label:
                            allowed_principal_labels.append(label)

                # Granted to identities (can include users and groups)
                identities = (
                    perm.get("grantedToIdentitiesV2") or perm.get("grantedToIdentities") or []
                )
                if identities:
                    for identity in identities:
                        # User
                        if "user" in identity:
                            user_info = identity["user"]
                            email = user_info.get("email")
                            if email:
                                allowed_users.append(email)
                                if "owner" in roles:
                                    owner = email
                            for identifier in (
                                user_info.get("id"),
                                user_info.get("userPrincipalName"),
                                email,
                            ):
                                user_principal = microsoft_user_principal(
                                    identifier,
                                    access_token=access_token,
                                )
                                if user_principal:
                                    allowed_principals.append(user_principal)

                        # Group
                        if "group" in identity:
                            group_info = identity["group"]
                            group_id = group_info.get("id")
                            group_role = microsoft_group_role(
                                group_id,
                                access_token=access_token,
                            )
                            if group_role:
                                allowed_groups.append(group_role)
                                allowed_principals.append(group_role)
                                label = microsoft_group_principal_label(
                                    group_id,
                                    access_token=access_token,
                                    display_name=group_info.get("displayName")
                                    or group_info.get("email"),
                                    email=group_info.get("email"),
                                )
                                if label:
                                    allowed_principal_labels.append(label)

            return DocumentACL(
                owner=owner,
                allowed_users=allowed_users,
                allowed_groups=allowed_groups,
                allowed_principals=allowed_principals,
                allowed_principal_labels=unique_acl_principal_labels(allowed_principal_labels),
            )

        except Exception as e:
            logger.warning(f"Failed to extract ACL for OneDrive item {file_id}: {e}")
            return DocumentACL()

    async def get_file_content(self, file_id: str) -> ConnectorDocument:
        """Get file content and metadata."""
        try:
            if not await self.authenticate():
                raise RuntimeError("OneDrive authentication failed during file content retrieval")

            # First, check for cached file info with download URL
            # This is used for OneDrive sharing IDs that can't be resolved via Graph API
            cached_info = self.get_cached_file_info(file_id)
            if cached_info and cached_info.get("downloadUrl"):
                logger.info(f"Using cached download URL for file {file_id}")
                content = await self._download_file_from_url(cached_info["downloadUrl"])

                acl = DocumentACL(owner="")

                return ConnectorDocument(
                    id=file_id,
                    filename=cached_info.get("name", "Unknown"),
                    mimetype=cached_info.get("mimeType", "application/octet-stream"),
                    content=content,
                    source_url=cached_info.get("webUrl", ""),
                    acl=acl,
                    modified_time=datetime.now(),
                    created_time=datetime.now(),
                    metadata={
                        "onedrive_path": "",
                        "size": cached_info.get("size", 0),
                    },
                )

            # Fall back to Graph API for regular file IDs
            file_metadata = await self._get_file_metadata_by_id(file_id)
            if not file_metadata:
                # Last-resort: try shares endpoint download directly if this is a sharing ID
                if "!" in file_id and file_id.split("!", 1)[1].startswith("s"):
                    logger.info(
                        f"No metadata for sharing ID {file_id}, attempting direct shares download"
                    )
                    token = self.oauth.get_access_token()
                    headers = {"Authorization": f"Bearer {token}"}
                    shares_content = await self._download_via_shares_endpoint(file_id, headers)
                    if shares_content is not None:
                        acl = DocumentACL(owner="")
                        return ConnectorDocument(
                            id=file_id,
                            filename="Unknown",
                            mimetype="application/octet-stream",
                            content=shares_content,
                            source_url="",
                            acl=acl,
                            modified_time=datetime.now(),
                            created_time=datetime.now(),
                            metadata={"onedrive_path": "", "size": 0},
                        )
                raise ValueError(f"File not found: {file_id}")

            download_url = file_metadata.get("download_url")
            if download_url:
                content = await self._download_file_from_url(download_url)
            else:
                content = await self._download_file_content(file_id)

            # Extract ACL from OneDrive item
            acl = await self._extract_onedrive_acl(file_id, file_metadata)

            modified_time = self._parse_graph_date(file_metadata.get("modified"))
            created_time = self._parse_graph_date(file_metadata.get("created"))

            return ConnectorDocument(
                id=file_id,
                filename=file_metadata.get("name", ""),
                mimetype=file_metadata.get("mime_type", "application/octet-stream"),
                content=content,
                source_url=file_metadata.get("url", ""),
                acl=acl,
                modified_time=modified_time,
                created_time=created_time,
                metadata={
                    "onedrive_path": file_metadata.get("path", ""),
                    "size": file_metadata.get("size", 0),
                },
            )

        except Exception as e:
            logger.error(f"Failed to get OneDrive file content {file_id}: {e}")
            raise

    async def _get_file_metadata_by_id(self, file_id: str) -> dict[str, Any] | None:
        """Get file metadata by ID using Graph API.

        Handles multiple ID formats:
        - Standard item ID: uses /me/drive/items/{id}
        - Personal OneDrive format (driveId!itemId): uses /drives/{driveId}/items/{itemId}
        - Sharing IDs (with !s prefix): uses /shares endpoint
        """
        try:
            # Try different endpoints based on ID format
            item = await self._fetch_item_metadata(file_id)

            if not item:
                return None

            # Check if it's a folder
            if item.get("folder"):
                return {
                    "id": file_id,
                    "name": item.get("name", ""),
                    "isFolder": True,
                }

            if item.get("file"):
                return {
                    "id": file_id,
                    "name": item.get("name", ""),
                    "path": f"/drive/items/{file_id}",
                    "size": int(item.get("size", 0)),
                    "modified": item.get("lastModifiedDateTime"),
                    "created": item.get("createdDateTime"),
                    "mime_type": item.get("file", {}).get(
                        "mimeType", self._get_mime_type(item.get("name", ""))
                    ),
                    "url": item.get("webUrl", ""),
                    "download_url": item.get("@microsoft.graph.downloadUrl"),
                    "isFolder": False,
                }

            return None

        except Exception as e:
            logger.error(f"Failed to get file metadata for {file_id}: {e}")
            return None

    async def _fetch_item_metadata(self, file_id: str) -> dict[str, Any] | None:
        """Fetch item metadata, trying multiple endpoints for different ID formats."""
        import base64

        params = dict(self._default_params)

        # Check if ID contains '!' which indicates driveId!itemId format
        if "!" in file_id:
            parts = file_id.rsplit("!", 1)
            if len(parts) == 2:
                drive_id = parts[0]
                item_id = parts[1]

                # Handle sharing IDs (item ID starts with 's')
                if item_id.startswith("s"):
                    logger.info(f"Detected sharing ID format for {file_id}")

                    # Try multiple encoding approaches for the shares endpoint
                    share_encodings = [
                        # Approach 1: Encode the full ID with "u!" prefix
                        base64.urlsafe_b64encode(f"u!{file_id}".encode()).decode().rstrip("="),
                        # Approach 2: Encode just the share token with "s!" prefix
                        base64.urlsafe_b64encode(f"s!{item_id}".encode()).decode().rstrip("="),
                        # Approach 3: Encode the ID directly
                        base64.urlsafe_b64encode(file_id.encode()).decode().rstrip("="),
                        # Approach 4: Use the ID as-is (some APIs accept this)
                        f"u!{file_id}",
                    ]

                    for i, encoded in enumerate(share_encodings):
                        try:
                            url = f"{self._graph_base_url}/shares/{encoded}/driveItem"
                            logger.debug(f"Trying shares endpoint approach {i + 1}: {url}")
                            response = await self._make_graph_request(url, params=params)
                            if response.status_code == 200:
                                logger.info(f"Shares endpoint approach {i + 1} succeeded")
                                return response.json()
                            else:
                                logger.debug(
                                    f"Shares approach {i + 1} failed with status {response.status_code}"
                                )
                        except Exception as e:
                            logger.debug(f"Shares approach {i + 1} failed: {e}")

                # Try: /drives/{driveId}/items/{itemId} with full item ID (including 's' prefix)
                logger.info(f"Trying drives endpoint: /drives/{drive_id}/items/{item_id}")
                try:
                    url = f"{self._graph_base_url}/drives/{drive_id}/items/{item_id}"
                    response = await self._make_graph_request(url, params=params)
                    if response.status_code == 200:
                        return response.json()
                    else:
                        logger.warning(
                            f"Drives endpoint failed with status {response.status_code}: {response.text}"
                        )
                except Exception as e:
                    logger.debug(f"Drives endpoint failed: {e}")

                # Try: /drives/{driveId}/items/{itemId} without 's' prefix
                if item_id.startswith("s"):
                    clean_item_id = item_id[1:]  # Remove 's' prefix
                    logger.info(
                        f"Trying drives endpoint without 's' prefix: /drives/{drive_id}/items/{clean_item_id}"
                    )
                    try:
                        url = f"{self._graph_base_url}/drives/{drive_id}/items/{clean_item_id}"
                        response = await self._make_graph_request(url, params=params)
                        if response.status_code == 200:
                            return response.json()
                        else:
                            logger.warning(
                                f"Drives endpoint without 's' prefix failed with status {response.status_code}: {response.text}"
                            )
                    except Exception as e:
                        logger.debug(f"Drives endpoint (no prefix) failed: {e}")

                # Try: /me/drive/items/{full_id} as fallback
                logger.info(f"Trying standard endpoint: /me/drive/items/{file_id}")
                try:
                    url = f"{self._graph_base_url}/me/drive/items/{file_id}"
                    response = await self._make_graph_request(url, params=params)
                    if response.status_code == 200:
                        return response.json()
                    else:
                        logger.warning(
                            f"Standard endpoint failed with status {response.status_code}: {response.text}"
                        )
                except Exception as e:
                    logger.debug(f"Standard endpoint exception: {e}")
        else:
            # Standard item ID without '!'
            url = f"{self._graph_base_url}/me/drive/items/{file_id}"
            response = await self._make_graph_request(url, params=params)
            if response.status_code == 200:
                return response.json()

        logger.error(f"All endpoints failed for file_id: {file_id}")
        return None

    async def _download_file_content(self, file_id: str) -> bytes:
        """Download file content by file ID using Graph API.

        Handles multiple ID formats like _get_file_metadata_by_id.
        """
        try:
            token = self.oauth.get_access_token()
            headers = {"Authorization": f"Bearer {token}"}

            # Build URL based on ID format
            if "!" in file_id:
                parts = file_id.rsplit("!", 1)
                if len(parts) == 2:
                    drive_id = parts[0]
                    item_id = parts[1]

                    # If this looks like a sharing ID (starts with 's'), try shares endpoint first
                    if item_id.startswith("s"):
                        content = await self._download_via_shares_endpoint(file_id, headers)
                        if content is not None:
                            return content

                    # Try drives endpoint for driveId!itemId format
                    if not item_id.startswith("s"):
                        url = f"{self._graph_base_url}/drives/{drive_id}/items/{item_id}/content"
                        logger.info(f"Downloading via drives endpoint: {url}")
                    else:
                        url = f"{self._graph_base_url}/me/drive/items/{file_id}/content"
                else:
                    url = f"{self._graph_base_url}/me/drive/items/{file_id}/content"
            else:
                url = f"{self._graph_base_url}/me/drive/items/{file_id}/content"

            async with httpx.AsyncClient() as client:
                response = await client.get(url, headers=headers, timeout=60, follow_redirects=True)
                response.raise_for_status()
                return response.content

        except Exception as e:
            logger.error(f"Failed to download file content for {file_id}: {e}")
            raise

    async def _download_via_shares_endpoint(
        self, file_id: str, headers: dict[str, str]
    ) -> bytes | None:
        """
        Attempt to download content using the Graph /shares endpoint for sharing IDs.
        """
        import base64

        share_encodings = [
            base64.urlsafe_b64encode(f"u!{file_id}".encode()).decode().rstrip("="),
            base64.urlsafe_b64encode(f"s!{file_id}".encode()).decode().rstrip("="),
            base64.urlsafe_b64encode(file_id.encode()).decode().rstrip("="),
            f"u!{file_id}",
        ]

        for i, encoded in enumerate(share_encodings):
            try:
                url = f"{self._graph_base_url}/shares/{encoded}/driveItem/content"
                logger.info(f"Attempting shares download (approach {i + 1}): {url}")
                async with httpx.AsyncClient() as client:
                    response = await client.get(
                        url, headers=headers, timeout=60, follow_redirects=True
                    )
                    if response.status_code == 200:
                        return response.content
                    else:
                        logger.debug(
                            f"Shares download approach {i + 1} failed with status {response.status_code}"
                        )
            except Exception as e:
                logger.debug(f"Shares download approach {i + 1} failed: {e}")

        return None

    async def _download_file_from_url(self, download_url: str) -> bytes:
        """Download file content from direct download URL."""
        try:
            async with httpx.AsyncClient() as client:
                response = await client.get(download_url, timeout=60, follow_redirects=True)
                response.raise_for_status()
                return response.content
        except Exception as e:
            logger.error(f"Failed to download from URL {download_url}: {e}")
            raise

    def _parse_graph_date(self, date_str: str | None) -> datetime:
        """Parse Microsoft Graph date string to datetime."""
        if not date_str:
            return datetime.now()
        try:
            if date_str.endswith("Z"):
                return datetime.fromisoformat(date_str[:-1]).replace(tzinfo=None)
            else:
                return datetime.fromisoformat(date_str.replace("T", " "))
        except (ValueError, AttributeError):
            return datetime.now()

    async def _make_graph_request(
        self,
        url: str,
        method: str = "GET",
        data: dict | None = None,
        params: dict | None = None,
    ) -> httpx.Response:
        """Make authenticated API request to Microsoft Graph."""
        token = self.oauth.get_access_token()
        headers = {
            "Authorization": f"Bearer {token}",
            "Content-Type": "application/json",
        }

        async with httpx.AsyncClient() as client:
            if method.upper() == "GET":
                response = await client.get(url, headers=headers, params=params, timeout=30)
            elif method.upper() == "POST":
                response = await client.post(url, headers=headers, json=data, timeout=30)
            elif method.upper() == "DELETE":
                response = await client.delete(url, headers=headers, timeout=30)
            else:
                raise ValueError(f"Unsupported HTTP method: {method}")

            response.raise_for_status()
            return response

    async def _list_selected_files(self) -> dict[str, Any]:
        """List only selected files/folders (selective sync)."""
        files: list[dict[str, Any]] = []

        # Process selected file IDs
        if self.cfg.file_ids:
            for file_id in self.cfg.file_ids:
                try:
                    file_meta = await self._get_file_metadata_by_id(file_id)
                    if file_meta and not file_meta.get("isFolder", False):
                        files.append(file_meta)
                    elif file_meta and file_meta.get("isFolder", False):
                        # If it's a folder, expand its contents
                        folder_files = await self._list_folder_contents(file_id)
                        files.extend(folder_files)
                except Exception as e:
                    logger.warning(f"Failed to get file {file_id}: {e}")
                    continue

        # Process selected folder IDs
        if self.cfg.folder_ids:
            for folder_id in self.cfg.folder_ids:
                try:
                    folder_files = await self._list_folder_contents(folder_id)
                    files.extend(folder_files)
                except Exception as e:
                    logger.warning(f"Failed to list folder {folder_id}: {e}")
                    continue

        return {"files": files, "next_page_token": None}

    async def _list_folder_contents(self, folder_id: str) -> list[dict[str, Any]]:
        """List all files in a folder recursively."""
        files: list[dict[str, Any]] = []

        try:
            drive_id = None
            if "!" in folder_id:
                parts = folder_id.rsplit("!", 1)
                if len(parts) == 2:
                    potential_drive_id, item_id = parts
                    if not item_id.startswith("s"):
                        drive_id = potential_drive_id
                        url = f"{self._graph_base_url}/drives/{drive_id}/items/{item_id}/children"

            if not drive_id:
                url = f"{self._graph_base_url}/me/drive/items/{folder_id}/children"

            params = dict(self._default_params)

            response = await self._make_graph_request(url, params=params)
            data = response.json()

            items = data.get("value", [])
            for item in items:
                parent_ref = item.get("parentReference", {})
                item_drive_id = parent_ref.get("driveId") or drive_id
                item_id = item.get("id")
                if item_id and "!" in item_id:
                    final_item_id = item_id
                else:
                    final_item_id = f"{item_drive_id}!{item_id}" if item_drive_id else item_id

                if item.get("file"):  # It's a file
                    file_meta = {
                        "id": final_item_id,
                        "name": item.get("name", ""),
                        "path": f"/drive/items/{item_id}",
                        "size": int(item.get("size") or 0),
                        "modified": item.get("lastModifiedDateTime"),
                        "created": item.get("createdDateTime"),
                        "mime_type": item.get("file", {}).get(
                            "mimeType", self._get_mime_type(item.get("name", ""))
                        ),
                        "url": item.get("webUrl", ""),
                        "download_url": item.get("@microsoft.graph.downloadUrl"),
                        "isFolder": False,
                    }
                    files.append(file_meta)
                elif item.get("folder"):  # It's a subfolder, recurse
                    subfolder_files = await self._list_folder_contents(final_item_id)
                    files.extend(subfolder_files)
        except Exception as e:
            logger.error(f"Failed to list folder contents for {folder_id}: {e}")

        return files

    def _get_mime_type(self, filename: str) -> str:
        """Get MIME type based on file extension."""
        import mimetypes

        mime_type, _ = mimetypes.guess_type(filename)
        return mime_type or "application/octet-stream"

    # Webhook methods - BaseConnector interface
    def handle_webhook_validation(
        self, request_method: str, headers: dict[str, str], query_params: dict[str, str]
    ) -> str | None:
        """Handle webhook validation (Graph API specific)."""
        if request_method == "POST" and "validationToken" in query_params:
            return query_params["validationToken"]
        return None

    def extract_webhook_channel_id(
        self, payload: dict[str, Any], headers: dict[str, str]
    ) -> str | None:
        """Extract channel/subscription ID from webhook payload."""
        notifications = payload.get("value", [])
        if notifications:
            return notifications[0].get("subscriptionId")
        return None

    @staticmethod
    def _delta_item_file_id(item: dict[str, Any]) -> str:
        """Return the composite ``{driveId}!{itemId}`` id used at ingest time.

        Selected-file listing stores ids as ``driveId!itemId`` (see
        _list_folder_contents), so the webhook delta must emit the same shape
        or the change can't be correlated with the indexed connector_file_id.
        """
        item_id = item.get("id", "")
        if item_id and "!" in item_id:
            return item_id
        drive_id = item.get("parentReference", {}).get("driveId")
        return f"{drive_id}!{item_id}" if drive_id else item_id

    async def handle_webhook(self, payload: dict[str, Any]) -> list[str]:
        """Handle webhook notification and return affected file IDs.

        Graph driveItem notifications never identify the changed items — the
        notification resource is the subscribed drive root — so run a delta
        query against the drive to discover what changed.
        """
        if not payload.get("value"):
            return []

        try:
            if not await self.authenticate():
                logger.error("OneDrive authentication failed during webhook handling")
                return []

            token = self.oauth.get_access_token()
            headers = {"Authorization": f"Bearer {token}"}

            # Without a stored delta link (first notification for this instance)
            # the delta query enumerates the whole drive, so only keep recently
            # modified files instead of re-syncing everything.
            first_sweep = self._delta_link is None
            url = self._delta_link or f"{self._graph_base_url}/me/drive/root/delta"
            cutoff = datetime.now(UTC) - timedelta(minutes=10)

            affected_files: list[str] = []
            async with httpx.AsyncClient() as client:
                while url:
                    response = await client.get(url, headers=headers, timeout=30)
                    response.raise_for_status()
                    data = response.json()

                    for item in data.get("value", []):
                        if "deleted" in item:
                            # Deleted at source: propagate the id so the processor
                            # runs its deleted-at-source cleanup
                            # (get_file_content -> 404 -> delete indexed chunks).
                            if "folder" not in item:
                                affected_files.append(self._delta_item_file_id(item))
                            continue
                        if "file" not in item:
                            continue
                        if first_sweep:
                            modified = item.get("lastModifiedDateTime")
                            if not modified:
                                continue
                            try:
                                modified_at = datetime.fromisoformat(
                                    modified.replace("Z", "+00:00")
                                )
                            except ValueError:
                                continue
                            if modified_at < cutoff:
                                continue
                        affected_files.append(self._delta_item_file_id(item))

                    delta_link = data.get("@odata.deltaLink")
                    if delta_link:
                        self._delta_link = delta_link
                    url = data.get("@odata.nextLink")

            return list(dict.fromkeys(affected_files))

        except Exception as e:
            logger.error(f"OneDrive webhook delta query failed: {e}")
            return []

    async def cleanup_subscription(self, subscription_id: str) -> bool:
        """Clean up subscription - BaseConnector interface."""
        if subscription_id == "no-webhook-configured":
            logger.info("No subscription to cleanup (webhook was not configured)")
            return True

        try:
            if not await self.authenticate():
                logger.error("OneDrive authentication failed during subscription cleanup")
                return False

            token = self.oauth.get_access_token()
            headers = {"Authorization": f"Bearer {token}"}

            url = f"{self._graph_base_url}/subscriptions/{subscription_id}"

            async with httpx.AsyncClient() as client:
                response = await client.delete(url, headers=headers, timeout=30)

                if response.status_code in [200, 204, 404]:
                    logger.info(f"OneDrive subscription {subscription_id} cleaned up successfully")
                    return True
                else:
                    logger.warning(
                        f"Unexpected response cleaning up subscription: {response.status_code}"
                    )
                    return False

        except Exception as e:
            logger.error(f"Failed to cleanup OneDrive subscription {subscription_id}: {e}")
            return False

    async def renew_subscription(self, subscription_id: str) -> str | None:
        """Extend the Graph subscription in place (PATCH avoids re-validation).

        Returns the new expirationDateTime, or None to signal the caller to
        fall back to delete + recreate."""
        if subscription_id == "no-webhook-configured":
            return None

        try:
            if not await self.authenticate():
                logger.error("OneDrive authentication failed during subscription renewal")
                return None

            token = self.oauth.get_access_token()
            headers = {"Authorization": f"Bearer {token}", "Content-Type": "application/json"}

            url = f"{self._graph_base_url}/subscriptions/{subscription_id}"
            body = {"expirationDateTime": self._get_subscription_expiry()}

            async with httpx.AsyncClient() as client:
                response = await client.patch(url, json=body, headers=headers, timeout=30)

                if response.status_code == 404:
                    # Subscription already expired/deleted at Graph; recreate.
                    logger.info(f"OneDrive subscription {subscription_id} not found, will recreate")
                    return None
                if response.status_code not in [200, 201]:
                    logger.warning(
                        f"Unexpected response renewing OneDrive subscription: "
                        f"{response.status_code}"
                    )
                    return None

                expiration = response.json().get("expirationDateTime")
                self.webhook_expiration = expiration
                logger.info(f"OneDrive subscription {subscription_id} renewed until {expiration}")
                return expiration

        except Exception as e:
            logger.error(f"Failed to renew OneDrive subscription {subscription_id}: {e}")
            return None