OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.
Python
Add logging and async/timeout handling for Drive
Edwin Josedeecd664c29d
diff --git a/src/connectors/google_drive/connector.py b/src/connectors/google_drive/connector.pyindex 874d0be69..e55f62619 100644--- a/src/connectors/google_drive/connector.py+++ b/src/connectors/google_drive/connector.py@@ -1,3 +1,4 @@+import asyncioimport ioimport osimport time@@ -342,38 +343,41 @@ class GoogleDriveConnector(BaseConnector):- items inside folder_ids (with optional recursion)Shortcuts are resolved to their targets automatically."""- # Clear shortcut cache to ensure fresh data+ logger.debug(+ "[GoogleDrive] _iter_selected_items: entry (file_ids=%s, folder_ids=%s)",+ self.cfg.file_ids,+ self.cfg.folder_ids,+ )self._clear_shortcut_cache()seen: Set[str] = set()items: List[Dict[str, Any]] = []folders_to_expand: List[str] = []- # Process file_ids: separate actual files from foldersif self.cfg.file_ids:+ logger.debug("[GoogleDrive] _iter_selected_items: processing %d file_id(s)", len(self.cfg.file_ids))for fid in self.cfg.file_ids:meta = self._get_file_meta_by_id(fid)if not meta:+ logger.debug("[GoogleDrive] _iter_selected_items: no metadata for file_id=%s", fid)continue- # If it's a folder, add to folders_to_expand insteadif meta.get("mimeType") == "application/vnd.google-apps.folder":logger.debug(- f"Item {fid} ({meta.get('name')}) is a folder, "- f"will expand to contents"+ "[GoogleDrive] _iter_selected_items: %s (%s) is a folder, will expand",+ fid,+ meta.get("name"),)folders_to_expand.append(fid)elif meta["id"] not in seen:- # It's a regular file, add it directlyseen.add(meta["id"])items.append(meta)- # Collect all folders to expand (from both file_ids and folder_ids)if self.cfg.folder_ids:folders_to_expand.extend(self.cfg.folder_ids)- # Expand all folders to their contentsif folders_to_expand:+ logger.debug("[GoogleDrive] _iter_selected_items: expanding %d folder(s)", len(folders_to_expand))folder_children = self._bfs_expand_folders(folders_to_expand)for meta in folder_children:meta = self._resolve_shortcut(meta)@@ -382,34 +386,28 @@ class GoogleDriveConnector(BaseConnector):seen.add(meta["id"])items.append(meta)- # If neither file_ids nor folder_ids are set, you could:- # - return [] to force explicit selection- # - OR default to entire drive.- # Here we choose to require explicit selection:if not self.cfg.file_ids and not self.cfg.folder_ids:logger.warning(- "No file_ids or folder_ids specified - returning empty result. "- "Explicit selection is required."+ "[GoogleDrive] _iter_selected_items: no file_ids or folder_ids specified, returning empty")return []items = self._filter_by_mime(items)- # Exclude folders from final emits:items = [mfor m in itemsif m.get("mimeType") != "application/vnd.google-apps.folder"]- # Log a warning if we ended up with no files after expansion/filteringif not items and (self.cfg.file_ids or self.cfg.folder_ids):logger.warning(- f"No files found after expanding and filtering. "- f"file_ids={self.cfg.file_ids}, folder_ids={self.cfg.folder_ids}. "- f"This could mean: (1) folders are empty, (2) all files were filtered by mime types, "- f"or (3) permissions prevent access to the files."+ "[GoogleDrive] _iter_selected_items: no files after filtering. "+ "file_ids=%s, folder_ids=%s",+ self.cfg.file_ids,+ self.cfg.folder_ids,)+ logger.debug("[GoogleDrive] _iter_selected_items: returning %d file(s)", len(items))return items# -------------------------@@ -453,7 +451,10 @@ class GoogleDriveConnector(BaseConnector):mime_type = file_meta.get("mimeType") or ""logger.debug(- f"Downloading file {file_id} ({file_name}) with mimetype: {mime_type}"+ "[GoogleDrive] _download_file_bytes: file_id=%s, name=%s, mimeType=%s",+ file_id,+ file_name,+ mime_type,)# Folders cannot be downloaded or exported - this should never be reached@@ -484,7 +485,9 @@ class GoogleDriveConnector(BaseConnector):export_mime = "application/pdf"logger.debug(- f"Using export_media for {file_id} ({mime_type} -> {export_mime})"+ "[GoogleDrive] _download_file_bytes: using export_media (%s -> %s)",+ mime_type,+ export_mime,)# NOTE: export_media does not accept supportsAllDrives/includeItemsFromAllDrivesrequest = self.service.files().export_media(@@ -493,7 +496,7 @@ class GoogleDriveConnector(BaseConnector):else:# This is a regular uploaded file (PDF, image, video, etc.) - use get_media# Also handles non-exportable Google Apps files (Forms, Sites, Maps, etc.)- logger.debug(f"Using get_media for {file_id} ({mime_type})")+ logger.debug("[GoogleDrive] _download_file_bytes: using get_media (%s)", mime_type)# Binary download (get_media also doesn't accept the Drive flags)request = self.service.files().get_media(fileId=file_id)@@ -526,7 +529,9 @@ class GoogleDriveConnector(BaseConnector):else:raise- return fh.getvalue()+ data = fh.getvalue()+ logger.debug("[GoogleDrive] _download_file_bytes: done, %d bytes", len(data))+ return data# -------------------------# Public sync surface@@ -538,27 +543,30 @@ class GoogleDriveConnector(BaseConnector):Returns True if ready to use; False otherwise."""try:- # Load/refresh creds from token file (async)+ logger.debug("[GoogleDrive] authenticate: loading credentials")self.creds = await self.oauth.load_credentials()- # If still not authenticated, bail (caller should kick off OAuth init)+ logger.debug("[GoogleDrive] authenticate: checking is_authenticated")if not await self.oauth.is_authenticated():logger.debug(- "authenticate: no valid credentials; run OAuth init/callback first."+ "[GoogleDrive] authenticate: no valid credentials; run OAuth init/callback first.")return False- # Build Drive service from OAuth helper+ logger.debug("[GoogleDrive] authenticate: building Drive service")self.service = self.oauth.get_service()- # Optional sanity check (small, fast request)- _ = self.service.files().get(fileId="root", fields="id").execute()+ logger.debug("[GoogleDrive] authenticate: running sanity check (files.get root)")+ req = self.service.files().get(fileId="root", fields="id")+ await asyncio.to_thread(req.execute)+ logger.debug("[GoogleDrive] authenticate: sanity check passed")+self._authenticated = Truereturn Trueexcept Exception as e:self._authenticated = False- logger.error(f"GoogleDriveConnector.authenticate failed: {e}")+ logger.error("[GoogleDrive] authenticate failed: %s", e)return Falseasync def list_files(@@ -575,34 +583,29 @@ class GoogleDriveConnector(BaseConnector):- If page_token is None: return all files in one batch.- Otherwise: return {} and no next_page_token."""- # Ensure service is initializedif self.service is None:raise RuntimeError("Google Drive service is not initialized. Please authenticate first.")+ logger.debug("[GoogleDrive] list_files: entry (page_token=%s, max_files=%s)", page_token, max_files)+try:- items = self._iter_selected_items()+ items = await asyncio.to_thread(self._iter_selected_items)- # Optionally honor a request-scoped max_files (e.g., from your API payload)if isinstance(max_files, int) and max_files > 0:items = items[:max_files]- # Simplest: ignore page_token and just dump all- # If you want real pagination, slice items hereif page_token:return {"files": [], "next_page_token": None}+ logger.debug("[GoogleDrive] list_files: returning %d file(s)", len(items))return {"files": items,- "next_page_token": None, # no more pages+ "next_page_token": None,}except Exception as e:- # Log the error and re-raise to surface authentication/permission issues- logger.error(- f"GoogleDriveConnector.list_files failed: {e}",- exc_info=True- )+ logger.error("[GoogleDrive] list_files failed: %s", e, exc_info=True)raisedef _extract_google_drive_acl(self, file_meta: Dict) -> DocumentACL:@@ -676,11 +679,11 @@ class GoogleDriveConnector(BaseConnector):Fetch a file's metadata and content from Google Drive and wrap it in a ConnectorDocument.Raises FileNotFoundError if the ID is a folder (folders cannot be downloaded)."""- meta = self._get_file_meta_by_id(file_id)+ logger.debug("[GoogleDrive] get_file_content: fetching metadata for file_id=%s", file_id)+ meta = await asyncio.to_thread(self._get_file_meta_by_id, file_id)if not meta:raise FileNotFoundError(f"Google Drive file not found: {file_id}")- # Check if this is a folder - folders cannot be downloadedif meta.get("mimeType") == "application/vnd.google-apps.folder":raise FileNotFoundError(f"Cannot download folder {file_id} ({meta.get('name')}). "@@ -688,13 +691,15 @@ class GoogleDriveConnector(BaseConnector):f"This ID should not have been passed to get_file_content().")+ logger.debug(+ "[GoogleDrive] get_file_content: downloading file_id=%s, name=%s",+ file_id,+ meta.get("name"),+ )try:- blob = self._download_file_bytes(meta)+ blob = await asyncio.to_thread(self._download_file_bytes, meta)except Exception as e:- try:- logger.error(f"Download failed for {file_id}: {e}")- except Exception:- pass+ logger.error("[GoogleDrive] get_file_content: download failed for %s: %s", file_id, e)raisefrom datetime import datetime@@ -703,7 +708,6 @@ class GoogleDriveConnector(BaseConnector):if not dt_str:return Nonetry:- # Google Drive returns RFC3339 formatreturn datetime.strptime(dt_str, "%Y-%m-%dT%H:%M:%S.%fZ")except ValueError:try:@@ -711,7 +715,6 @@ class GoogleDriveConnector(BaseConnector):except ValueError:return None- # Extract ACL from file metadataacl = self._extract_google_drive_acl(meta)doc = ConnectorDocument(@@ -731,6 +734,7 @@ class GoogleDriveConnector(BaseConnector):else None,},)+ logger.debug("[GoogleDrive] get_file_content: done for file_id=%s (%d bytes)", file_id, len(blob))return docasync def setup_subscription(self) -> str:diff --git a/src/connectors/google_drive/oauth.py b/src/connectors/google_drive/oauth.pyindex d77ddfb63..fda96b898 100644--- a/src/connectors/google_drive/oauth.py+++ b/src/connectors/google_drive/oauth.py@@ -1,11 +1,19 @@+import asyncioimport osimport jsonfrom typing import Optional++import requests as req_libfrom google.auth.transport.requests import Requestfrom google.oauth2.credentials import Credentialsfrom google_auth_oauthlib.flow import Flowfrom googleapiclient.discovery import build+from utils.logging_config import get_logger++logger = get_logger(__name__)++_REFRESH_TIMEOUT_SECONDS = 30class GoogleDriveOAuth:@@ -33,30 +41,39 @@ class GoogleDriveOAuth:self.token_file = token_fileself.creds: Optional[Credentials] = None+ def _make_timeout_request(self) -> Request:+ """Build a google-auth Request transport with a bounded timeout."""+ session = req_lib.Session()+ session.timeout = _REFRESH_TIMEOUT_SECONDS+ return Request(session=session)+async def load_credentials(self) -> Optional[Credentials]:"""Load existing credentials from token file"""from utils.encryption import read_encrypted_file-++ logger.debug("[GoogleDrive] load_credentials: reading token file %s", self.token_file)raw_data, needs_upgrade = await read_encrypted_file(self.token_file)if raw_data is None:+ logger.debug("[GoogleDrive] load_credentials: no token file found")return None-+try:token_data = json.loads(raw_data)except Exception:- # Corrupted file, fallback to missing+ logger.debug("[GoogleDrive] load_credentials: corrupted token file, removing")if os.path.exists(self.token_file):os.remove(self.token_file)return None- # Create credentials from token data+ logger.debug("[GoogleDrive] load_credentials: token data loaded, creating Credentials object")+self.creds = Credentials(token=token_data.get("token"),refresh_token=token_data.get("refresh_token"),id_token=token_data.get("id_token"),token_uri="https://oauth2.googleapis.com/token",client_id=self.client_id,- client_secret=self.client_secret, # Need for refresh+ client_secret=self.client_secret,scopes=token_data.get("scopes", self.SCOPES),)@@ -65,22 +82,24 @@ class GoogleDriveOAuth:from datetime import datetimeexpiry_dt = datetime.fromisoformat(token_data["expiry"])- # Remove timezone info to make it naive (Google auth expects naive datetimes)self.creds.expiry = expiry_dt.replace(tzinfo=None)-+ logger.debug("[GoogleDrive] load_credentials: token expiry=%s", self.creds.expiry)+if needs_upgrade and self.creds:await self.save_credentials()- # If credentials are expired, refresh themif self.creds and self.creds.expired and self.creds.refresh_token:+ logger.debug(+ "[GoogleDrive] load_credentials: token expired, refreshing (timeout=%ss)",+ _REFRESH_TIMEOUT_SECONDS,+ )try:- self.creds.refresh(Request())+ await asyncio.to_thread(self.creds.refresh, self._make_timeout_request())+ logger.debug("[GoogleDrive] load_credentials: token refresh succeeded")await self.save_credentials()except Exception as e:- # Refresh failed - likely refresh token expired or revoked- # Clear credentials and raise a clear error+ logger.debug("[GoogleDrive] load_credentials: token refresh failed: %s", e)self.creds = None- # Try to clean up the invalid token fileif os.path.exists(self.token_file):try:os.remove(self.token_file)@@ -91,6 +110,13 @@ class GoogleDriveOAuth:f"The refresh token may have expired or been revoked. "f"Please re-authenticate: {str(e)}") from e+ else:+ logger.debug(+ "[GoogleDrive] load_credentials: token valid=%s, expired=%s, has_refresh=%s",+ self.creds.valid if self.creds else None,+ self.creds.expired if self.creds else None,+ bool(self.creds.refresh_token) if self.creds else None,+ )return self.creds@@ -153,13 +179,12 @@ class GoogleDriveOAuth:if not hasattr(self, "_flow") or self._flow_state != state:raise ValueError("Invalid OAuth state")- # Exchange authorization code for credentials- self._flow.fetch_token(code=authorization_code)+ logger.debug("[GoogleDrive] handle_authorization_callback: exchanging auth code for tokens")+ await asyncio.to_thread(self._flow.fetch_token, code=authorization_code)self.creds = self._flow.credentials+ logger.debug("[GoogleDrive] handle_authorization_callback: token exchange complete")- # Save credentialsawait self.save_credentials()-return Trueasync def is_authenticated(self) -> bool:@@ -167,21 +192,26 @@ class GoogleDriveOAuth:if not self.creds:await self.load_credentials()- return bool(self.creds and self.creds.valid)+ result = bool(self.creds and self.creds.valid)+ logger.debug("[GoogleDrive] is_authenticated: %s", result)+ return resultdef get_service(self):"""Get authenticated Google Drive service"""if not self.creds or not self.creds.valid:raise ValueError("Not authenticated")+ logger.debug("[GoogleDrive] get_service: building Drive v3 service")return build("drive", "v3", credentials=self.creds)async def revoke_credentials(self):"""Revoke credentials and delete token file"""if self.creds:- self.creds.revoke(Request())+ logger.debug("[GoogleDrive] revoke_credentials: revoking token")+ await asyncio.to_thread(self.creds.revoke, self._make_timeout_request())if os.path.exists(self.token_file):os.remove(self.token_file)self.creds = None+ logger.debug("[GoogleDrive] revoke_credentials: done")