Skip to content

OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.

Python

Add logging and async/timeout handling for Drive

Edwin Josedeecd664c29d
diff --git a/src/connectors/google_drive/connector.py b/src/connectors/google_drive/connector.py
index 874d0be69..e55f62619 100644
--- a/src/connectors/google_drive/connector.py
+++ b/src/connectors/google_drive/connector.py
@@ -1,3 +1,4 @@
+import asyncio
import io
import os
import time
@@ -342,38 +343,41 @@ class GoogleDriveConnector(BaseConnector):
- items inside folder_ids (with optional recursion)
Shortcuts are resolved to their targets automatically.
"""
- # Clear shortcut cache to ensure fresh data
+ logger.debug(
+ "[GoogleDrive] _iter_selected_items: entry (file_ids=%s, folder_ids=%s)",
+ self.cfg.file_ids,
+ self.cfg.folder_ids,
+ )
self._clear_shortcut_cache()
seen: Set[str] = set()
items: List[Dict[str, Any]] = []
folders_to_expand: List[str] = []
- # Process file_ids: separate actual files from folders
if self.cfg.file_ids:
+ logger.debug("[GoogleDrive] _iter_selected_items: processing %d file_id(s)", len(self.cfg.file_ids))
for fid in self.cfg.file_ids:
meta = self._get_file_meta_by_id(fid)
if not meta:
+ logger.debug("[GoogleDrive] _iter_selected_items: no metadata for file_id=%s", fid)
continue
- # If it's a folder, add to folders_to_expand instead
if meta.get("mimeType") == "application/vnd.google-apps.folder":
logger.debug(
- f"Item {fid} ({meta.get('name')}) is a folder, "
- f"will expand to contents"
+ "[GoogleDrive] _iter_selected_items: %s (%s) is a folder, will expand",
+ fid,
+ meta.get("name"),
)
folders_to_expand.append(fid)
elif meta["id"] not in seen:
- # It's a regular file, add it directly
seen.add(meta["id"])
items.append(meta)
- # Collect all folders to expand (from both file_ids and folder_ids)
if self.cfg.folder_ids:
folders_to_expand.extend(self.cfg.folder_ids)
- # Expand all folders to their contents
if folders_to_expand:
+ logger.debug("[GoogleDrive] _iter_selected_items: expanding %d folder(s)", len(folders_to_expand))
folder_children = self._bfs_expand_folders(folders_to_expand)
for meta in folder_children:
meta = self._resolve_shortcut(meta)
@@ -382,34 +386,28 @@ class GoogleDriveConnector(BaseConnector):
seen.add(meta["id"])
items.append(meta)
- # If neither file_ids nor folder_ids are set, you could:
- # - return [] to force explicit selection
- # - OR default to entire drive.
- # Here we choose to require explicit selection:
if not self.cfg.file_ids and not self.cfg.folder_ids:
logger.warning(
- "No file_ids or folder_ids specified - returning empty result. "
- "Explicit selection is required."
+ "[GoogleDrive] _iter_selected_items: no file_ids or folder_ids specified, returning empty"
)
return []
items = self._filter_by_mime(items)
- # Exclude folders from final emits:
items = [
m
for m in items
if m.get("mimeType") != "application/vnd.google-apps.folder"
]
- # Log a warning if we ended up with no files after expansion/filtering
if not items and (self.cfg.file_ids or self.cfg.folder_ids):
logger.warning(
- f"No files found after expanding and filtering. "
- f"file_ids={self.cfg.file_ids}, folder_ids={self.cfg.folder_ids}. "
- f"This could mean: (1) folders are empty, (2) all files were filtered by mime types, "
- f"or (3) permissions prevent access to the files."
+ "[GoogleDrive] _iter_selected_items: no files after filtering. "
+ "file_ids=%s, folder_ids=%s",
+ self.cfg.file_ids,
+ self.cfg.folder_ids,
)
+ logger.debug("[GoogleDrive] _iter_selected_items: returning %d file(s)", len(items))
return items
# -------------------------
@@ -453,7 +451,10 @@ class GoogleDriveConnector(BaseConnector):
mime_type = file_meta.get("mimeType") or ""
logger.debug(
- f"Downloading file {file_id} ({file_name}) with mimetype: {mime_type}"
+ "[GoogleDrive] _download_file_bytes: file_id=%s, name=%s, mimeType=%s",
+ file_id,
+ file_name,
+ mime_type,
)
# Folders cannot be downloaded or exported - this should never be reached
@@ -484,7 +485,9 @@ class GoogleDriveConnector(BaseConnector):
export_mime = "application/pdf"
logger.debug(
- f"Using export_media for {file_id} ({mime_type} -> {export_mime})"
+ "[GoogleDrive] _download_file_bytes: using export_media (%s -> %s)",
+ mime_type,
+ export_mime,
)
# NOTE: export_media does not accept supportsAllDrives/includeItemsFromAllDrives
request = self.service.files().export_media(
@@ -493,7 +496,7 @@ class GoogleDriveConnector(BaseConnector):
else:
# This is a regular uploaded file (PDF, image, video, etc.) - use get_media
# Also handles non-exportable Google Apps files (Forms, Sites, Maps, etc.)
- logger.debug(f"Using get_media for {file_id} ({mime_type})")
+ logger.debug("[GoogleDrive] _download_file_bytes: using get_media (%s)", mime_type)
# Binary download (get_media also doesn't accept the Drive flags)
request = self.service.files().get_media(fileId=file_id)
@@ -526,7 +529,9 @@ class GoogleDriveConnector(BaseConnector):
else:
raise
- return fh.getvalue()
+ data = fh.getvalue()
+ logger.debug("[GoogleDrive] _download_file_bytes: done, %d bytes", len(data))
+ return data
# -------------------------
# Public sync surface
@@ -538,27 +543,30 @@ class GoogleDriveConnector(BaseConnector):
Returns True if ready to use; False otherwise.
"""
try:
- # Load/refresh creds from token file (async)
+ logger.debug("[GoogleDrive] authenticate: loading credentials")
self.creds = await self.oauth.load_credentials()
- # If still not authenticated, bail (caller should kick off OAuth init)
+ logger.debug("[GoogleDrive] authenticate: checking is_authenticated")
if not await self.oauth.is_authenticated():
logger.debug(
- "authenticate: no valid credentials; run OAuth init/callback first."
+ "[GoogleDrive] authenticate: no valid credentials; run OAuth init/callback first."
)
return False
- # Build Drive service from OAuth helper
+ logger.debug("[GoogleDrive] authenticate: building Drive service")
self.service = self.oauth.get_service()
- # Optional sanity check (small, fast request)
- _ = self.service.files().get(fileId="root", fields="id").execute()
+ logger.debug("[GoogleDrive] authenticate: running sanity check (files.get root)")
+ req = self.service.files().get(fileId="root", fields="id")
+ await asyncio.to_thread(req.execute)
+ logger.debug("[GoogleDrive] authenticate: sanity check passed")
+
self._authenticated = True
return True
except Exception as e:
self._authenticated = False
- logger.error(f"GoogleDriveConnector.authenticate failed: {e}")
+ logger.error("[GoogleDrive] authenticate failed: %s", e)
return False
async def list_files(
@@ -575,34 +583,29 @@ class GoogleDriveConnector(BaseConnector):
- If page_token is None: return all files in one batch.
- Otherwise: return {} and no next_page_token.
"""
- # Ensure service is initialized
if self.service is None:
raise RuntimeError(
"Google Drive service is not initialized. Please authenticate first."
)
+ logger.debug("[GoogleDrive] list_files: entry (page_token=%s, max_files=%s)", page_token, max_files)
+
try:
- items = self._iter_selected_items()
+ items = await asyncio.to_thread(self._iter_selected_items)
- # Optionally honor a request-scoped max_files (e.g., from your API payload)
if isinstance(max_files, int) and max_files > 0:
items = items[:max_files]
- # Simplest: ignore page_token and just dump all
- # If you want real pagination, slice items here
if page_token:
return {"files": [], "next_page_token": None}
+ logger.debug("[GoogleDrive] list_files: returning %d file(s)", len(items))
return {
"files": items,
- "next_page_token": None, # no more pages
+ "next_page_token": None,
}
except Exception as e:
- # Log the error and re-raise to surface authentication/permission issues
- logger.error(
- f"GoogleDriveConnector.list_files failed: {e}",
- exc_info=True
- )
+ logger.error("[GoogleDrive] list_files failed: %s", e, exc_info=True)
raise
def _extract_google_drive_acl(self, file_meta: Dict) -> DocumentACL:
@@ -676,11 +679,11 @@ class GoogleDriveConnector(BaseConnector):
Fetch a file's metadata and content from Google Drive and wrap it in a ConnectorDocument.
Raises FileNotFoundError if the ID is a folder (folders cannot be downloaded).
"""
- meta = self._get_file_meta_by_id(file_id)
+ logger.debug("[GoogleDrive] get_file_content: fetching metadata for file_id=%s", file_id)
+ meta = await asyncio.to_thread(self._get_file_meta_by_id, file_id)
if not meta:
raise FileNotFoundError(f"Google Drive file not found: {file_id}")
- # Check if this is a folder - folders cannot be downloaded
if meta.get("mimeType") == "application/vnd.google-apps.folder":
raise FileNotFoundError(
f"Cannot download folder {file_id} ({meta.get('name')}). "
@@ -688,13 +691,15 @@ class GoogleDriveConnector(BaseConnector):
f"This ID should not have been passed to get_file_content()."
)
+ logger.debug(
+ "[GoogleDrive] get_file_content: downloading file_id=%s, name=%s",
+ file_id,
+ meta.get("name"),
+ )
try:
- blob = self._download_file_bytes(meta)
+ blob = await asyncio.to_thread(self._download_file_bytes, meta)
except Exception as e:
- try:
- logger.error(f"Download failed for {file_id}: {e}")
- except Exception:
- pass
+ logger.error("[GoogleDrive] get_file_content: download failed for %s: %s", file_id, e)
raise
from datetime import datetime
@@ -703,7 +708,6 @@ class GoogleDriveConnector(BaseConnector):
if not dt_str:
return None
try:
- # Google Drive returns RFC3339 format
return datetime.strptime(dt_str, "%Y-%m-%dT%H:%M:%S.%fZ")
except ValueError:
try:
@@ -711,7 +715,6 @@ class GoogleDriveConnector(BaseConnector):
except ValueError:
return None
- # Extract ACL from file metadata
acl = self._extract_google_drive_acl(meta)
doc = ConnectorDocument(
@@ -731,6 +734,7 @@ class GoogleDriveConnector(BaseConnector):
else None,
},
)
+ logger.debug("[GoogleDrive] get_file_content: done for file_id=%s (%d bytes)", file_id, len(blob))
return doc
async def setup_subscription(self) -> str:
diff --git a/src/connectors/google_drive/oauth.py b/src/connectors/google_drive/oauth.py
index d77ddfb63..fda96b898 100644
--- a/src/connectors/google_drive/oauth.py
+++ b/src/connectors/google_drive/oauth.py
@@ -1,11 +1,19 @@
+import asyncio
import os
import json
from typing import Optional
+
+import requests as req_lib
from google.auth.transport.requests import Request
from google.oauth2.credentials import Credentials
from google_auth_oauthlib.flow import Flow
from googleapiclient.discovery import build
+from utils.logging_config import get_logger
+
+logger = get_logger(__name__)
+
+_REFRESH_TIMEOUT_SECONDS = 30
class GoogleDriveOAuth:
@@ -33,30 +41,39 @@ class GoogleDriveOAuth:
self.token_file = token_file
self.creds: Optional[Credentials] = None
+ def _make_timeout_request(self) -> Request:
+ """Build a google-auth Request transport with a bounded timeout."""
+ session = req_lib.Session()
+ session.timeout = _REFRESH_TIMEOUT_SECONDS
+ return Request(session=session)
+
async def load_credentials(self) -> Optional[Credentials]:
"""Load existing credentials from token file"""
from utils.encryption import read_encrypted_file
-
+
+ logger.debug("[GoogleDrive] load_credentials: reading token file %s", self.token_file)
raw_data, needs_upgrade = await read_encrypted_file(self.token_file)
if raw_data is None:
+ logger.debug("[GoogleDrive] load_credentials: no token file found")
return None
-
+
try:
token_data = json.loads(raw_data)
except Exception:
- # Corrupted file, fallback to missing
+ logger.debug("[GoogleDrive] load_credentials: corrupted token file, removing")
if os.path.exists(self.token_file):
os.remove(self.token_file)
return None
- # Create credentials from token data
+ logger.debug("[GoogleDrive] load_credentials: token data loaded, creating Credentials object")
+
self.creds = Credentials(
token=token_data.get("token"),
refresh_token=token_data.get("refresh_token"),
id_token=token_data.get("id_token"),
token_uri="https://oauth2.googleapis.com/token",
client_id=self.client_id,
- client_secret=self.client_secret, # Need for refresh
+ client_secret=self.client_secret,
scopes=token_data.get("scopes", self.SCOPES),
)
@@ -65,22 +82,24 @@ class GoogleDriveOAuth:
from datetime import datetime
expiry_dt = datetime.fromisoformat(token_data["expiry"])
- # Remove timezone info to make it naive (Google auth expects naive datetimes)
self.creds.expiry = expiry_dt.replace(tzinfo=None)
-
+ logger.debug("[GoogleDrive] load_credentials: token expiry=%s", self.creds.expiry)
+
if needs_upgrade and self.creds:
await self.save_credentials()
- # If credentials are expired, refresh them
if self.creds and self.creds.expired and self.creds.refresh_token:
+ logger.debug(
+ "[GoogleDrive] load_credentials: token expired, refreshing (timeout=%ss)",
+ _REFRESH_TIMEOUT_SECONDS,
+ )
try:
- self.creds.refresh(Request())
+ await asyncio.to_thread(self.creds.refresh, self._make_timeout_request())
+ logger.debug("[GoogleDrive] load_credentials: token refresh succeeded")
await self.save_credentials()
except Exception as e:
- # Refresh failed - likely refresh token expired or revoked
- # Clear credentials and raise a clear error
+ logger.debug("[GoogleDrive] load_credentials: token refresh failed: %s", e)
self.creds = None
- # Try to clean up the invalid token file
if os.path.exists(self.token_file):
try:
os.remove(self.token_file)
@@ -91,6 +110,13 @@ class GoogleDriveOAuth:
f"The refresh token may have expired or been revoked. "
f"Please re-authenticate: {str(e)}"
) from e
+ else:
+ logger.debug(
+ "[GoogleDrive] load_credentials: token valid=%s, expired=%s, has_refresh=%s",
+ self.creds.valid if self.creds else None,
+ self.creds.expired if self.creds else None,
+ bool(self.creds.refresh_token) if self.creds else None,
+ )
return self.creds
@@ -153,13 +179,12 @@ class GoogleDriveOAuth:
if not hasattr(self, "_flow") or self._flow_state != state:
raise ValueError("Invalid OAuth state")
- # Exchange authorization code for credentials
- self._flow.fetch_token(code=authorization_code)
+ logger.debug("[GoogleDrive] handle_authorization_callback: exchanging auth code for tokens")
+ await asyncio.to_thread(self._flow.fetch_token, code=authorization_code)
self.creds = self._flow.credentials
+ logger.debug("[GoogleDrive] handle_authorization_callback: token exchange complete")
- # Save credentials
await self.save_credentials()
-
return True
async def is_authenticated(self) -> bool:
@@ -167,21 +192,26 @@ class GoogleDriveOAuth:
if not self.creds:
await self.load_credentials()
- return bool(self.creds and self.creds.valid)
+ result = bool(self.creds and self.creds.valid)
+ logger.debug("[GoogleDrive] is_authenticated: %s", result)
+ return result
def get_service(self):
"""Get authenticated Google Drive service"""
if not self.creds or not self.creds.valid:
raise ValueError("Not authenticated")
+ logger.debug("[GoogleDrive] get_service: building Drive v3 service")
return build("drive", "v3", credentials=self.creds)
async def revoke_credentials(self):
"""Revoke credentials and delete token file"""
if self.creds:
- self.creds.revoke(Request())
+ logger.debug("[GoogleDrive] revoke_credentials: revoking token")
+ await asyncio.to_thread(self.creds.revoke, self._make_timeout_request())
if os.path.exists(self.token_file):
os.remove(self.token_file)
self.creds = None
+ logger.debug("[GoogleDrive] revoke_credentials: done")