OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.
Python
fix: make google drive ingestion work (#1672)
Lucas Oliveira48e49119206d
diff --git a/src/connectors/google_drive/connector.py b/src/connectors/google_drive/connector.pyindex 53b3e8eea..a656730a1 100644--- a/src/connectors/google_drive/connector.py+++ b/src/connectors/google_drive/connector.py@@ -3,9 +3,10 @@ import ioimport osimport timefrom collections import deque+from collections.abc import Iterablefrom dataclasses import dataclassfrom pathlib import Path-from typing import Any, Dict, Iterable, List, Optional, Set+from typing import Anyfrom googleapiclient.errors import HttpErrorfrom googleapiclient.http import MediaIoBaseDownload@@ -17,6 +18,7 @@ from .oauth import GoogleDriveOAuthlogger = get_logger(__name__)+# -------------------------# Config model# -------------------------@@ -27,26 +29,26 @@ class GoogleDriveConfig:token_file: str# Selective sync- file_ids: Optional[List[str]] = None- folder_ids: Optional[List[str]] = None+ file_ids: list[str] | None = None+ folder_ids: list[str] | None = Nonerecursive: bool = True# Shared Drives control- drive_id: Optional[str] = None # when set, we use corpora='drive'- corpora: Optional[str] = None # 'user' | 'drive' | 'domain'; auto-picked if None+ drive_id: str | None = None # when set, we use corpora='drive'+ corpora: str | None = None # 'user' | 'drive' | 'domain'; auto-picked if None# Optional filtering- include_mime_types: Optional[List[str]] = None- exclude_mime_types: Optional[List[str]] = None+ include_mime_types: list[str] | None = None+ exclude_mime_types: list[str] | None = None# Export overrides for Google-native types- export_format_overrides: Optional[Dict[str, str]] = None # mime -> export-mime+ export_format_overrides: dict[str, str] | None = None # mime -> export-mime# Changes API state persistence (store these in your DB/kv if needed)- changes_page_token: Optional[str] = None+ changes_page_token: str | None = None# Optional: resource_id for webhook cleanup- resource_id: Optional[str] = None+ resource_id: str | None = None# -------------------------@@ -89,7 +91,7 @@ class GoogleDriveConnector(BaseConnector):# Otherwise, implement your custom logic here.logger.debug(f"Emitting document: {doc.id} ({doc.filename})")- def __init__(self, config: Dict[str, Any]) -> None:+ def __init__(self, config: dict[str, Any]) -> None:# Read from config OR env (backend env, not NEXT_PUBLIC_*):env_client_id = os.getenv(self.CLIENT_ID_ENV_VAR)env_client_secret = os.getenv(self.CLIENT_SECRET_ENV_VAR)@@ -99,6 +101,7 @@ class GoogleDriveConnector(BaseConnector):# Token file default - use data directory for persistencefrom config.paths import get_data_file+token_file = config.get("token_file") or get_data_file("google_drive_token.json")Path(token_file).parent.mkdir(parents=True, exist_ok=True)@@ -114,9 +117,7 @@ class GoogleDriveConnector(BaseConnector):)# Normalize incoming IDs from any of the supported alias keys- def _first_present_list(- cfg: Dict[str, Any], keys: Iterable[str]- ) -> Optional[List[str]]:+ def _first_present_list(cfg: dict[str, Any], keys: Iterable[str]) -> list[str] | None:for k in keys:v = cfg.get(k)if v: # accept non-empty list@@ -153,11 +154,15 @@ class GoogleDriveConnector(BaseConnector):# Drive client is built in authenticate()from google.oauth2.credentials import Credentials- self.creds: Optional[Credentials] = None+ self.creds: Credentials | None = Noneself.service: Any = None# cache of resolved shortcutId -> target file metadata- self._shortcut_cache: Dict[str, Dict[str, Any]] = {}+ self._shortcut_cache: dict[str, dict[str, Any]] = {}++ import threading++ self._lock = threading.Lock()# Authentication stateself._authenticated: bool = False@@ -170,20 +175,20 @@ class GoogleDriveConnector(BaseConnector):self._shortcut_cache.clear()@property- def _drives_get_flags(self) -> Dict[str, Any]:+ def _drives_get_flags(self) -> dict[str, Any]:"""Flags valid for GET-like calls (files.get, changes.getStartPageToken)."""return {"supportsAllDrives": True}@property- def _drives_list_flags(self) -> Dict[str, Any]:+ def _drives_list_flags(self) -> dict[str, Any]:"""Flags valid for LIST-like calls (files.list, changes.list)."""return {"supportsAllDrives": True, "includeItemsFromAllDrives": True}- def _pick_corpora_args(self) -> Dict[str, Any]:+ def _pick_corpora_args(self) -> dict[str, Any]:"""Decide corpora/driveId based on config.@@ -197,7 +202,7 @@ class GoogleDriveConnector(BaseConnector):# Default to allDrives so Picker selections from Shared Drives work without explicit drive_idreturn {"corpora": "allDrives"}- def _resolve_shortcut(self, file_obj: Dict[str, Any]) -> Dict[str, Any]:+ def _resolve_shortcut(self, file_obj: dict[str, Any]) -> dict[str, Any]:"""If a file is a shortcut, fetch and return the real target metadata."""@@ -234,7 +239,7 @@ class GoogleDriveConnector(BaseConnector):# shortcut target not accessiblereturn file_obj- def _list_children(self, folder_id: str) -> List[Dict[str, Any]]:+ def _list_children(self, folder_id: str) -> list[dict[str, Any]]:"""List immediate children of a folder."""@@ -245,7 +250,7 @@ class GoogleDriveConnector(BaseConnector):query = f"'{folder_id}' in parents and trashed = false"page_token = None- results: List[Dict[str, Any]] = []+ results: list[dict[str, Any]] = []while True:resp = (@@ -272,13 +277,13 @@ class GoogleDriveConnector(BaseConnector):return results- def _bfs_expand_folders(self, folder_ids: Iterable[str]) -> List[Dict[str, Any]]:+ def _bfs_expand_folders(self, folder_ids: Iterable[str]) -> list[dict[str, Any]]:"""Breadth-first traversal to expand folders to all descendant files (if recursive),or just immediate children (if not recursive). Folders themselves are returnedas items too, but filtered later."""- out: List[Dict[str, Any]] = []+ out: list[dict[str, Any]] = []queue = deque(folder_ids)while queue:@@ -295,7 +300,7 @@ class GoogleDriveConnector(BaseConnector):return out- def _get_file_meta_by_id(self, file_id: str) -> Optional[Dict[str, Any]]:+ def _get_file_meta_by_id(self, file_id: str) -> dict[str, Any] | None:"""Fetch metadata for a file by ID (resolving shortcuts)."""@@ -304,30 +309,31 @@ class GoogleDriveConnector(BaseConnector):"Google Drive service is not initialized. Please authenticate first.")try:- meta = (- self.service.files()- .get(- fileId=file_id,- fields=(- "id, name, mimeType, modifiedTime, createdTime, size, "- "webViewLink, parents, shortcutDetails, driveId"- ),- **self._drives_get_flags,+ with self._lock:+ meta = (+ self.service.files()+ .get(+ fileId=file_id,+ fields=(+ "id, name, mimeType, modifiedTime, createdTime, size, "+ "webViewLink, parents, shortcutDetails, driveId"+ ),+ **self._drives_get_flags,+ )+ .execute())- .execute()- )return self._resolve_shortcut(meta)except HttpError:return None- def _filter_by_mime(self, items: Iterable[Dict[str, Any]]) -> List[Dict[str, Any]]:+ def _filter_by_mime(self, items: Iterable[dict[str, Any]]) -> list[dict[str, Any]]:"""Apply include/exclude mime filters if configured."""include = set(self.cfg.include_mime_types or [])exclude = set(self.cfg.exclude_mime_types or [])- def keep(m: Dict[str, Any]) -> bool:+ def keep(m: dict[str, Any]) -> bool:mt = m.get("mimeType")if exclude and mt in exclude:return False@@ -337,7 +343,7 @@ class GoogleDriveConnector(BaseConnector):return [m for m in items if keep(m)]- def _iter_selected_items(self) -> List[Dict[str, Any]]:+ def _iter_selected_items(self) -> list[dict[str, Any]]:"""Return a de-duplicated list of file metadata for the selected scope:- explicit file_ids (automatically expands folders to their contents)@@ -351,16 +357,21 @@ class GoogleDriveConnector(BaseConnector):)self._clear_shortcut_cache()- seen: Set[str] = set()- items: List[Dict[str, Any]] = []- folders_to_expand: List[str] = []+ seen: set[str] = set()+ items: list[dict[str, Any]] = []+ folders_to_expand: list[str] = []if self.cfg.file_ids:- logger.debug("[GoogleDrive] _iter_selected_items: processing %d file_id(s)", len(self.cfg.file_ids))+ logger.debug(+ "[GoogleDrive] _iter_selected_items: processing %d file_id(s)",+ len(self.cfg.file_ids),+ )for fid in self.cfg.file_ids:meta = self._get_file_meta_by_id(fid)if not meta:- logger.debug("[GoogleDrive] _iter_selected_items: no metadata for file_id=%s", fid)+ logger.debug(+ "[GoogleDrive] _iter_selected_items: no metadata for file_id=%s", fid+ )continueif meta.get("mimeType") == "application/vnd.google-apps.folder":@@ -378,7 +389,9 @@ class GoogleDriveConnector(BaseConnector):folders_to_expand.extend(self.cfg.folder_ids)if folders_to_expand:- logger.debug("[GoogleDrive] _iter_selected_items: expanding %d folder(s)", len(folders_to_expand))+ logger.debug(+ "[GoogleDrive] _iter_selected_items: expanding %d folder(s)", len(folders_to_expand)+ )folder_children = self._bfs_expand_folders(folders_to_expand)for meta in folder_children:meta = self._resolve_shortcut(meta)@@ -394,11 +407,7 @@ class GoogleDriveConnector(BaseConnector):return []items = self._filter_by_mime(items)- items = [- m- for m in items- if m.get("mimeType") != "application/vnd.google-apps.folder"- ]+ items = [m for m in items if m.get("mimeType") != "application/vnd.google-apps.folder"]if not items and (self.cfg.file_ids or self.cfg.folder_ids):logger.warning(@@ -414,7 +423,7 @@ class GoogleDriveConnector(BaseConnector):# -------------------------# Download logic# -------------------------- def _pick_export_mime(self, source_mime: str) -> Optional[str]:+ def _pick_export_mime(self, source_mime: str) -> str | None:"""Choose export mime for Google-native docs if needed."""@@ -437,7 +446,7 @@ class GoogleDriveConnector(BaseConnector):# Return None for non-Google-native or unsupported typesreturn overrides.get(source_mime)- def _download_file_bytes(self, file_meta: Dict[str, Any]) -> bytes:+ def _download_file_bytes(self, file_meta: dict[str, Any]) -> bytes:"""Download bytes for a given file (exporting if Google-native).Raises ValueError if the item is a folder (folders cannot be downloaded).@@ -478,57 +487,56 @@ class GoogleDriveConnector(BaseConnector):"application/vnd.google-apps.drawing", # Google Drawings}- if mime_type in exportable_types:- # This is an exportable Google Workspace file - must use export_media- export_mime = self._pick_export_mime(mime_type)- if not export_mime:- # Default fallback for unsupported Google native types- export_mime = "application/pdf"+ with self._lock:+ if mime_type in exportable_types:+ # This is an exportable Google Workspace file - must use export_media+ export_mime = self._pick_export_mime(mime_type)+ if not export_mime:+ # Default fallback for unsupported Google native types+ export_mime = "application/pdf"- logger.debug(- "[GoogleDrive] _download_file_bytes: using export_media (%s -> %s)",- mime_type,- export_mime,- )- # NOTE: export_media does not accept supportsAllDrives/includeItemsFromAllDrives- request = self.service.files().export_media(- fileId=file_id, mimeType=export_mime- )- else:- # This is a regular uploaded file (PDF, image, video, etc.) - use get_media- # Also handles non-exportable Google Apps files (Forms, Sites, Maps, etc.)- logger.debug("[GoogleDrive] _download_file_bytes: using get_media (%s)", mime_type)- # Binary download (get_media also doesn't accept the Drive flags)- request = self.service.files().get_media(fileId=file_id)-- # Download the file with error handling for misclassified Google Docs- fh = io.BytesIO()- downloader = MediaIoBaseDownload(fh, request, chunksize=1024 * 1024)- done = False-- try:- while not done:- status, done = downloader.next_chunk()- # Optional: you can log progress via status.progress()- except HttpError as e:- # If download fails with "fileNotDownloadable", it's a Docs Editor file- # that wasn't properly detected. Retry with export_media.- if "fileNotDownloadable" in str(e) and mime_type not in exportable_types:- logger.warning(- f"Download failed for {file_id} ({mime_type}) with fileNotDownloadable error. "- f"Retrying with export_media (file might be a Google Doc)"- )- export_mime = "application/pdf"- request = self.service.files().export_media(- fileId=file_id, mimeType=export_mime+ logger.debug(+ "[GoogleDrive] _download_file_bytes: using export_media (%s -> %s)",+ mime_type,+ export_mime,)- fh = io.BytesIO()- downloader = MediaIoBaseDownload(fh, request, chunksize=1024 * 1024)- done = False+ # NOTE: export_media does not accept supportsAllDrives/includeItemsFromAllDrives+ request = self.service.files().export_media(fileId=file_id, mimeType=export_mime)+ else:+ # This is a regular uploaded file (PDF, image, video, etc.) - use get_media+ # Also handles non-exportable Google Apps files (Forms, Sites, Maps, etc.)+ logger.debug("[GoogleDrive] _download_file_bytes: using get_media (%s)", mime_type)+ # Binary download (get_media also doesn't accept the Drive flags)+ request = self.service.files().get_media(fileId=file_id)++ # Download the file with error handling for misclassified Google Docs+ fh = io.BytesIO()+ downloader = MediaIoBaseDownload(fh, request, chunksize=1024 * 1024)+ done = False++ try:while not done:status, done = downloader.next_chunk()- else:- raise+ # Optional: you can log progress via status.progress()+ except HttpError as e:+ # If download fails with "fileNotDownloadable", it's a Docs Editor file+ # that wasn't properly detected. Retry with export_media.+ if "fileNotDownloadable" in str(e) and mime_type not in exportable_types:+ logger.warning(+ f"Download failed for {file_id} ({mime_type}) with fileNotDownloadable error. "+ f"Retrying with export_media (file might be a Google Doc)"+ )+ export_mime = "application/pdf"+ request = self.service.files().export_media(+ fileId=file_id, mimeType=export_mime+ )+ fh = io.BytesIO()+ downloader = MediaIoBaseDownload(fh, request, chunksize=1024 * 1024)+ done = False+ while not done:+ status, done = downloader.next_chunk()+ else:+ raisedata = fh.getvalue()logger.debug("[GoogleDrive] _download_file_bytes: done, %d bytes", len(data))@@ -572,10 +580,10 @@ class GoogleDriveConnector(BaseConnector):async def list_files(self,- page_token: Optional[str] = None,- max_files: Optional[int] = None,+ page_token: str | None = None,+ max_files: int | None = None,**kwargs,- ) -> Dict[str, Any]:+ ) -> dict[str, Any]:"""List files in the currently selected scope (file_ids/folder_ids/recursive).Returns a dict with 'files' and 'next_page_token'.@@ -589,7 +597,9 @@ class GoogleDriveConnector(BaseConnector):"Google Drive service is not initialized. Please authenticate first.")- logger.debug("[GoogleDrive] list_files: entry (page_token=%s, max_files=%s)", page_token, max_files)+ logger.debug(+ "[GoogleDrive] list_files: entry (page_token=%s, max_files=%s)", page_token, max_files+ )try:items = await asyncio.to_thread(self._iter_selected_items)@@ -609,7 +619,7 @@ class GoogleDriveConnector(BaseConnector):logger.error("[GoogleDrive] list_files failed: %s", e, exc_info=True)raise- def _extract_google_drive_acl(self, file_meta: Dict) -> DocumentACL:+ def _extract_google_drive_acl(self, file_meta: dict) -> DocumentACL:"""Extract ACL from Google Drive file metadata.@@ -624,10 +634,15 @@ class GoogleDriveConnector(BaseConnector):"""try:# Fetch permissions (requires additional API call)- permissions_list = self.service.permissions().list(- fileId=file_meta["id"],- fields="permissions(emailAddress,role,type,deleted,displayName)"- ).execute()+ with self._lock:+ permissions_list = (+ self.service.permissions()+ .list(+ fileId=file_meta["id"],+ fields="permissions(emailAddress,role,type,deleted,displayName)",+ )+ .execute()+ )allowed_users = []allowed_groups = []@@ -730,12 +745,12 @@ class GoogleDriveConnector(BaseConnector):metadata={"parents": meta.get("parents"),"driveId": meta.get("driveId"),- "size": int(meta.get("size", 0))- if str(meta.get("size", "")).isdigit()- else None,+ "size": int(meta.get("size", 0)) if str(meta.get("size", "")).isdigit() else None,},)- logger.debug("[GoogleDrive] get_file_content: done for file_id=%s (%d bytes)", file_id, len(blob))+ logger.debug(+ "[GoogleDrive] get_file_content: done for file_id=%s (%d bytes)", file_id, len(blob)+ )return docasync def setup_subscription(self) -> str:@@ -752,9 +767,7 @@ class GoogleDriveConnector(BaseConnector):# 1) Ensure we are authenticated and have a live Drive serviceok = await self.authenticate()if not ok:- raise RuntimeError(- "GoogleDriveConnector.setup_subscription: not authenticated"- )+ raise RuntimeError("GoogleDriveConnector.setup_subscription: not authenticated")# 2) Resolve webhook address (no param in ABC, so pull from config/env)webhook_address = getattr(self.cfg, "webhook_address", None) or os.getenv(@@ -810,9 +823,7 @@ class GoogleDriveConnector(BaseConnector):}if not isinstance(channel_id, str) or not channel_id:- raise RuntimeError(- f"Drive watch returned invalid channel id: {channel_id!r}"- )+ raise RuntimeError(f"Drive watch returned invalid channel id: {channel_id!r}")return channel_id@@ -850,13 +861,13 @@ class GoogleDriveConnector(BaseConnector):# Single-channel memoryif getattr(self, "_active_channel", None):- ch = getattr(self, "_active_channel")+ ch = self._active_channelif isinstance(ch, dict) and ch.get("channel_id") == subscription_id:resource_id = ch.get("resource_id")# Multi-channel memoryif resource_id is None and hasattr(self, "_subscriptions"):- subs = getattr(self, "_subscriptions")+ subs = self._subscriptionsif isinstance(subs, dict):entry = subs.get(subscription_id)if isinstance(entry, dict):@@ -888,9 +899,7 @@ class GoogleDriveConnector(BaseConnector):):self._active_channel = {}- if hasattr(self, "_subscriptions") and isinstance(- self._subscriptions, dict- ):+ if hasattr(self, "_subscriptions") and isinstance(self._subscriptions, dict):self._subscriptions.pop(subscription_id, None)return True@@ -902,7 +911,7 @@ class GoogleDriveConnector(BaseConnector):passreturn False- async def handle_webhook(self, payload: Dict[str, Any]) -> List[str]:+ async def handle_webhook(self, payload: dict[str, Any]) -> list[str]:"""Process a Google Drive Changes webhook.Drive push notifications do NOT include the changed files themselves; they merely tell us@@ -915,7 +924,7 @@ class GoogleDriveConnector(BaseConnector):Returns:List[str]: unique list of affected file IDs (filtered to our selected scope)."""- affected: List[str] = []+ affected: list[str] = []try:# 1) Ensure we're authenticated / service readyok = await self.authenticate()@@ -941,9 +950,7 @@ class GoogleDriveConnector(BaseConnector):except Exception as e:selected_ids = set()try:- logger.error(- f"handle_webhook: scope build failed, proceeding unfiltered: {e}"- )+ logger.error(f"handle_webhook: scope build failed, proceeding unfiltered: {e}")except Exception:pass@@ -980,11 +987,7 @@ class GoogleDriveConnector(BaseConnector):# Filter to our selected scope if we have one; otherwise accept allif selected_ids and (rid not in selected_ids):# Shortcut target might be in scope even if the shortcut isn't- tgt = (- fobj.get("shortcutDetails", {}).get("targetId")- if fobj- else None- )+ tgt = fobj.get("shortcutDetails", {}).get("targetId") if fobj else Noneif not (tgt and tgt in selected_ids):continue@@ -1007,7 +1010,7 @@ class GoogleDriveConnector(BaseConnector):# Deduplicate while preserving orderseen = set()- deduped: List[str] = []+ deduped: list[str] = []for x in affected:if x not in seen:seen.add(x)@@ -1033,9 +1036,7 @@ class GoogleDriveConnector(BaseConnector):blob = self._download_file_bytes(meta)except HttpError as e:# Skip/record failures- logger.error(- f"Failed to download {meta.get('name')} ({meta.get('id')}): {e}"- )+ logger.error(f"Failed to download {meta.get('name')} ({meta.get('id')}): {e}")continuefrom datetime import datetime@@ -1081,12 +1082,10 @@ class GoogleDriveConnector(BaseConnector):# -------------------------def get_start_page_token(self) -> str:# getStartPageToken accepts supportsAllDrives (not includeItemsFromAllDrives)- resp = (- self.service.changes().getStartPageToken(**self._drives_get_flags).execute()- )+ resp = self.service.changes().getStartPageToken(**self._drives_get_flags).execute()return resp["startPageToken"]- def poll_changes_and_sync(self) -> Optional[str]:+ def poll_changes_and_sync(self) -> str | None:"""Incrementally process changes since the last page token in cfg.changes_page_token.@@ -1122,10 +1121,7 @@ class GoogleDriveConnector(BaseConnector):# Match scopeif fid not in selected_ids:# also consider shortcut target- if (- file_obj.get("mimeType")- == "application/vnd.google-apps.shortcut"- ):+ if file_obj.get("mimeType") == "application/vnd.google-apps.shortcut":tgt = file_obj.get("shortcutDetails", {}).get("targetId")if tgt and tgt in selected_ids:pass@@ -1189,8 +1185,8 @@ class GoogleDriveConnector(BaseConnector):# Optional: webhook stubs# -------------------------def build_watch_body(- self, webhook_address: str, channel_id: Optional[str] = None- ) -> Dict[str, Any]:+ self, webhook_address: str, channel_id: str | None = None+ ) -> dict[str, Any]:"""Prepare the request body for changes.watch if you use webhooks."""@@ -1200,7 +1196,7 @@ class GoogleDriveConnector(BaseConnector):"address": webhook_address,}- def start_watch(self, webhook_address: str) -> Dict[str, Any]:+ def start_watch(self, webhook_address: str) -> dict[str, Any]:"""Start a webhook watch on changes using the current page token.Persist the returned resourceId/expiration on your side.diff --git a/src/connectors/langflow_connector_service.py b/src/connectors/langflow_connector_service.pyindex 3f2ba93a0..6b5b9e01c 100644--- a/src/connectors/langflow_connector_service.py+++ b/src/connectors/langflow_connector_service.py@@ -56,9 +56,13 @@ class LangflowConnectorService:filename=document.filename,)+ import os+from utils.file_utils import auto_cleanup_tempfile- suffix = get_file_extension(document.mimetype)+ suffix = os.path.splitext(document.filename)[1]+ if not suffix:+ suffix = get_file_extension(document.mimetype)# Create temporary file from document contentwith auto_cleanup_tempfile(suffix=suffix) as tmp_path:diff --git a/src/connectors/service.py b/src/connectors/service.pyindex ed9b385b0..cfdba9735 100644--- a/src/connectors/service.py+++ b/src/connectors/service.py@@ -53,9 +53,15 @@ class ConnectorService:"""Process a document from a connector using existing processing pipeline"""# Create temporary file from document content+ import os+from utils.file_utils import auto_cleanup_tempfile- with auto_cleanup_tempfile(suffix=get_file_extension(document.mimetype)) as tmp_path:+ suffix = os.path.splitext(document.filename)[1]+ if not suffix:+ suffix = get_file_extension(document.mimetype)++ with auto_cleanup_tempfile(suffix=suffix) as tmp_path:# Write document content to temp filewith open(tmp_path, "wb") as f:f.write(document.content)@@ -370,6 +376,7 @@ class ConnectorService:original_folder_ids = getattr(connector.cfg, "folder_ids", None)expanded_file_ids = file_ids # Default to original IDs+ expanded_files_info = []try:# Set the file_ids we want to sync in the connector's config@@ -380,7 +387,12 @@ class ConnectorService:# Get the expanded list of file IDs (folders will be expanded to their contents)# This uses the connector's list_files() which calls _iter_selected_items()result = await connector.list_files()- expanded_file_ids = [f["id"] for f in result.get("files", [])]+ expanded_files = result.get("files", [])+ expanded_file_ids = [f["id"] for f in expanded_files]++ # Save the expanded files info so we can set correct names in the task UI+ for f in expanded_files:+ expanded_files_info.append(f)if not expanded_file_ids:logger.warning(@@ -448,10 +460,15 @@ class ConnectorService:# Create custom task using TaskServiceoriginal_filenames = {}- if file_infos:++ # Combine file_infos and expanded_files_info+ all_infos = (file_infos or []) + expanded_files_info+ if all_infos:original_filenames = {- f["id"]: clean_connector_filename(f["name"], f.get("mimeType") or f.get("mimetype"))- for f in file_infos+ f["id"]: clean_connector_filename(+ f["name"], f.get("mimeType") or f.get("mimetype", "")+ )+ for f in all_infosif "id" in f and "name" in f}diff --git a/src/models/processors.py b/src/models/processors.pyindex 05ffaf4a1..ba7ff205b 100644--- a/src/models/processors.py+++ b/src/models/processors.py@@ -241,15 +241,16 @@ class TaskProcessor:file_hash=file_hash,)slim_doc = process_text_file(file_path)- # Override filename with original_filename if provided- if original_filename:- slim_doc["filename"] = original_filenameelse:full_doc = await self.docling_service.convert_file(file_path, user_id=owner_user_id, auth_header=jwt_token)slim_doc = extract_relevant(full_doc)+ # Override filename with original_filename if provided+ if original_filename:+ slim_doc["filename"] = original_filename+if chunk_size is not None:try:cs = int(chunk_size)@@ -562,8 +563,11 @@ class ConnectorFileProcessor(TaskProcessor):returnawait self.delete_document_by_filename(document.filename, opensearch_client)- # Create temporary file from document content- suffix = get_file_extension(document.mimetype)+ # Create temporary file from document content import os++ suffix = os.path.splitext(document.filename)[1]+ if not suffix:+ suffix = get_file_extension(document.mimetype)with auto_cleanup_tempfile(suffix=suffix) as tmp_path:# Write content to temp filewith open(tmp_path, "wb") as f:@@ -698,8 +702,11 @@ class LangflowConnectorFileProcessor(TaskProcessor):returnawait self.delete_document_by_filename(document.filename, opensearch_client)- # Create temporary file and compute hash to check for duplicates- suffix = get_file_extension(document.mimetype)+ # Create temporary file and compute hash to check for duplicates import os++ suffix = os.path.splitext(document.filename)[1]+ if not suffix:+ suffix = get_file_extension(document.mimetype)with auto_cleanup_tempfile(suffix=suffix) as tmp_path:# Write content to temp filewith open(tmp_path, "wb") as f:@@ -774,7 +781,8 @@ class S3FileProcessor(TaskProcessor):file_task.updated_at = time.time()try:- with auto_cleanup_tempfile() as tmp_path:+ suffix = os.path.splitext(item)[1]+ with auto_cleanup_tempfile(suffix=suffix) as tmp_path:# Download object to temporary filewith open(tmp_path, "wb") as tmp_file:self.s3_client.download_fileobj(self.bucket, item, tmp_file)diff --git a/src/services/auth_service.py b/src/services/auth_service.pyindex 6d0676ed7..c14e0a73c 100644--- a/src/services/auth_service.py+++ b/src/services/auth_service.py@@ -306,7 +306,7 @@ class AuthService:# Add expiry if providedif token_data.get("expires_in"):- expiry = datetime.now() + timedelta(seconds=int(token_data["expires_in"]))+ expiry = datetime.utcnow() + timedelta(seconds=int(token_data["expires_in"]))token_file_data["expiry"] = expiry.isoformat()# Save tokens to file