Skip to content

OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.

Python

fix: make google drive ingestion work (#1672)

Lucas Oliveira48e49119206d
diff --git a/src/connectors/google_drive/connector.py b/src/connectors/google_drive/connector.py
index 53b3e8eea..a656730a1 100644
--- a/src/connectors/google_drive/connector.py
+++ b/src/connectors/google_drive/connector.py
@@ -3,9 +3,10 @@ import io
import os
import time
from collections import deque
+from collections.abc import Iterable
from dataclasses import dataclass
from pathlib import Path
-from typing import Any, Dict, Iterable, List, Optional, Set
+from typing import Any
from googleapiclient.errors import HttpError
from googleapiclient.http import MediaIoBaseDownload
@@ -17,6 +18,7 @@ from .oauth import GoogleDriveOAuth
logger = get_logger(__name__)
+
# -------------------------
# Config model
# -------------------------
@@ -27,26 +29,26 @@ class GoogleDriveConfig:
token_file: str
# Selective sync
- file_ids: Optional[List[str]] = None
- folder_ids: Optional[List[str]] = None
+ file_ids: list[str] | None = None
+ folder_ids: list[str] | None = None
recursive: bool = True
# Shared Drives control
- drive_id: Optional[str] = None # when set, we use corpora='drive'
- corpora: Optional[str] = None # 'user' | 'drive' | 'domain'; auto-picked if None
+ drive_id: str | None = None # when set, we use corpora='drive'
+ corpora: str | None = None # 'user' | 'drive' | 'domain'; auto-picked if None
# Optional filtering
- include_mime_types: Optional[List[str]] = None
- exclude_mime_types: Optional[List[str]] = None
+ include_mime_types: list[str] | None = None
+ exclude_mime_types: list[str] | None = None
# Export overrides for Google-native types
- export_format_overrides: Optional[Dict[str, str]] = None # mime -> export-mime
+ export_format_overrides: dict[str, str] | None = None # mime -> export-mime
# Changes API state persistence (store these in your DB/kv if needed)
- changes_page_token: Optional[str] = None
+ changes_page_token: str | None = None
# Optional: resource_id for webhook cleanup
- resource_id: Optional[str] = None
+ resource_id: str | None = None
# -------------------------
@@ -89,7 +91,7 @@ class GoogleDriveConnector(BaseConnector):
# Otherwise, implement your custom logic here.
logger.debug(f"Emitting document: {doc.id} ({doc.filename})")
- def __init__(self, config: Dict[str, Any]) -> None:
+ def __init__(self, config: dict[str, Any]) -> None:
# Read from config OR env (backend env, not NEXT_PUBLIC_*):
env_client_id = os.getenv(self.CLIENT_ID_ENV_VAR)
env_client_secret = os.getenv(self.CLIENT_SECRET_ENV_VAR)
@@ -99,6 +101,7 @@ class GoogleDriveConnector(BaseConnector):
# Token file default - use data directory for persistence
from config.paths import get_data_file
+
token_file = config.get("token_file") or get_data_file("google_drive_token.json")
Path(token_file).parent.mkdir(parents=True, exist_ok=True)
@@ -114,9 +117,7 @@ class GoogleDriveConnector(BaseConnector):
)
# Normalize incoming IDs from any of the supported alias keys
- def _first_present_list(
- cfg: Dict[str, Any], keys: Iterable[str]
- ) -> Optional[List[str]]:
+ def _first_present_list(cfg: dict[str, Any], keys: Iterable[str]) -> list[str] | None:
for k in keys:
v = cfg.get(k)
if v: # accept non-empty list
@@ -153,11 +154,15 @@ class GoogleDriveConnector(BaseConnector):
# Drive client is built in authenticate()
from google.oauth2.credentials import Credentials
- self.creds: Optional[Credentials] = None
+ self.creds: Credentials | None = None
self.service: Any = None
# cache of resolved shortcutId -> target file metadata
- self._shortcut_cache: Dict[str, Dict[str, Any]] = {}
+ self._shortcut_cache: dict[str, dict[str, Any]] = {}
+
+ import threading
+
+ self._lock = threading.Lock()
# Authentication state
self._authenticated: bool = False
@@ -170,20 +175,20 @@ class GoogleDriveConnector(BaseConnector):
self._shortcut_cache.clear()
@property
- def _drives_get_flags(self) -> Dict[str, Any]:
+ def _drives_get_flags(self) -> dict[str, Any]:
"""
Flags valid for GET-like calls (files.get, changes.getStartPageToken).
"""
return {"supportsAllDrives": True}
@property
- def _drives_list_flags(self) -> Dict[str, Any]:
+ def _drives_list_flags(self) -> dict[str, Any]:
"""
Flags valid for LIST-like calls (files.list, changes.list).
"""
return {"supportsAllDrives": True, "includeItemsFromAllDrives": True}
- def _pick_corpora_args(self) -> Dict[str, Any]:
+ def _pick_corpora_args(self) -> dict[str, Any]:
"""
Decide corpora/driveId based on config.
@@ -197,7 +202,7 @@ class GoogleDriveConnector(BaseConnector):
# Default to allDrives so Picker selections from Shared Drives work without explicit drive_id
return {"corpora": "allDrives"}
- def _resolve_shortcut(self, file_obj: Dict[str, Any]) -> Dict[str, Any]:
+ def _resolve_shortcut(self, file_obj: dict[str, Any]) -> dict[str, Any]:
"""
If a file is a shortcut, fetch and return the real target metadata.
"""
@@ -234,7 +239,7 @@ class GoogleDriveConnector(BaseConnector):
# shortcut target not accessible
return file_obj
- def _list_children(self, folder_id: str) -> List[Dict[str, Any]]:
+ def _list_children(self, folder_id: str) -> list[dict[str, Any]]:
"""
List immediate children of a folder.
"""
@@ -245,7 +250,7 @@ class GoogleDriveConnector(BaseConnector):
query = f"'{folder_id}' in parents and trashed = false"
page_token = None
- results: List[Dict[str, Any]] = []
+ results: list[dict[str, Any]] = []
while True:
resp = (
@@ -272,13 +277,13 @@ class GoogleDriveConnector(BaseConnector):
return results
- def _bfs_expand_folders(self, folder_ids: Iterable[str]) -> List[Dict[str, Any]]:
+ def _bfs_expand_folders(self, folder_ids: Iterable[str]) -> list[dict[str, Any]]:
"""
Breadth-first traversal to expand folders to all descendant files (if recursive),
or just immediate children (if not recursive). Folders themselves are returned
as items too, but filtered later.
"""
- out: List[Dict[str, Any]] = []
+ out: list[dict[str, Any]] = []
queue = deque(folder_ids)
while queue:
@@ -295,7 +300,7 @@ class GoogleDriveConnector(BaseConnector):
return out
- def _get_file_meta_by_id(self, file_id: str) -> Optional[Dict[str, Any]]:
+ def _get_file_meta_by_id(self, file_id: str) -> dict[str, Any] | None:
"""
Fetch metadata for a file by ID (resolving shortcuts).
"""
@@ -304,30 +309,31 @@ class GoogleDriveConnector(BaseConnector):
"Google Drive service is not initialized. Please authenticate first."
)
try:
- meta = (
- self.service.files()
- .get(
- fileId=file_id,
- fields=(
- "id, name, mimeType, modifiedTime, createdTime, size, "
- "webViewLink, parents, shortcutDetails, driveId"
- ),
- **self._drives_get_flags,
+ with self._lock:
+ meta = (
+ self.service.files()
+ .get(
+ fileId=file_id,
+ fields=(
+ "id, name, mimeType, modifiedTime, createdTime, size, "
+ "webViewLink, parents, shortcutDetails, driveId"
+ ),
+ **self._drives_get_flags,
+ )
+ .execute()
)
- .execute()
- )
return self._resolve_shortcut(meta)
except HttpError:
return None
- def _filter_by_mime(self, items: Iterable[Dict[str, Any]]) -> List[Dict[str, Any]]:
+ def _filter_by_mime(self, items: Iterable[dict[str, Any]]) -> list[dict[str, Any]]:
"""
Apply include/exclude mime filters if configured.
"""
include = set(self.cfg.include_mime_types or [])
exclude = set(self.cfg.exclude_mime_types or [])
- def keep(m: Dict[str, Any]) -> bool:
+ def keep(m: dict[str, Any]) -> bool:
mt = m.get("mimeType")
if exclude and mt in exclude:
return False
@@ -337,7 +343,7 @@ class GoogleDriveConnector(BaseConnector):
return [m for m in items if keep(m)]
- def _iter_selected_items(self) -> List[Dict[str, Any]]:
+ def _iter_selected_items(self) -> list[dict[str, Any]]:
"""
Return a de-duplicated list of file metadata for the selected scope:
- explicit file_ids (automatically expands folders to their contents)
@@ -351,16 +357,21 @@ class GoogleDriveConnector(BaseConnector):
)
self._clear_shortcut_cache()
- seen: Set[str] = set()
- items: List[Dict[str, Any]] = []
- folders_to_expand: List[str] = []
+ seen: set[str] = set()
+ items: list[dict[str, Any]] = []
+ folders_to_expand: list[str] = []
if self.cfg.file_ids:
- logger.debug("[GoogleDrive] _iter_selected_items: processing %d file_id(s)", len(self.cfg.file_ids))
+ logger.debug(
+ "[GoogleDrive] _iter_selected_items: processing %d file_id(s)",
+ len(self.cfg.file_ids),
+ )
for fid in self.cfg.file_ids:
meta = self._get_file_meta_by_id(fid)
if not meta:
- logger.debug("[GoogleDrive] _iter_selected_items: no metadata for file_id=%s", fid)
+ logger.debug(
+ "[GoogleDrive] _iter_selected_items: no metadata for file_id=%s", fid
+ )
continue
if meta.get("mimeType") == "application/vnd.google-apps.folder":
@@ -378,7 +389,9 @@ class GoogleDriveConnector(BaseConnector):
folders_to_expand.extend(self.cfg.folder_ids)
if folders_to_expand:
- logger.debug("[GoogleDrive] _iter_selected_items: expanding %d folder(s)", len(folders_to_expand))
+ logger.debug(
+ "[GoogleDrive] _iter_selected_items: expanding %d folder(s)", len(folders_to_expand)
+ )
folder_children = self._bfs_expand_folders(folders_to_expand)
for meta in folder_children:
meta = self._resolve_shortcut(meta)
@@ -394,11 +407,7 @@ class GoogleDriveConnector(BaseConnector):
return []
items = self._filter_by_mime(items)
- items = [
- m
- for m in items
- if m.get("mimeType") != "application/vnd.google-apps.folder"
- ]
+ items = [m for m in items if m.get("mimeType") != "application/vnd.google-apps.folder"]
if not items and (self.cfg.file_ids or self.cfg.folder_ids):
logger.warning(
@@ -414,7 +423,7 @@ class GoogleDriveConnector(BaseConnector):
# -------------------------
# Download logic
# -------------------------
- def _pick_export_mime(self, source_mime: str) -> Optional[str]:
+ def _pick_export_mime(self, source_mime: str) -> str | None:
"""
Choose export mime for Google-native docs if needed.
"""
@@ -437,7 +446,7 @@ class GoogleDriveConnector(BaseConnector):
# Return None for non-Google-native or unsupported types
return overrides.get(source_mime)
- def _download_file_bytes(self, file_meta: Dict[str, Any]) -> bytes:
+ def _download_file_bytes(self, file_meta: dict[str, Any]) -> bytes:
"""
Download bytes for a given file (exporting if Google-native).
Raises ValueError if the item is a folder (folders cannot be downloaded).
@@ -478,57 +487,56 @@ class GoogleDriveConnector(BaseConnector):
"application/vnd.google-apps.drawing", # Google Drawings
}
- if mime_type in exportable_types:
- # This is an exportable Google Workspace file - must use export_media
- export_mime = self._pick_export_mime(mime_type)
- if not export_mime:
- # Default fallback for unsupported Google native types
- export_mime = "application/pdf"
+ with self._lock:
+ if mime_type in exportable_types:
+ # This is an exportable Google Workspace file - must use export_media
+ export_mime = self._pick_export_mime(mime_type)
+ if not export_mime:
+ # Default fallback for unsupported Google native types
+ export_mime = "application/pdf"
- logger.debug(
- "[GoogleDrive] _download_file_bytes: using export_media (%s -> %s)",
- mime_type,
- export_mime,
- )
- # NOTE: export_media does not accept supportsAllDrives/includeItemsFromAllDrives
- request = self.service.files().export_media(
- fileId=file_id, mimeType=export_mime
- )
- else:
- # This is a regular uploaded file (PDF, image, video, etc.) - use get_media
- # Also handles non-exportable Google Apps files (Forms, Sites, Maps, etc.)
- logger.debug("[GoogleDrive] _download_file_bytes: using get_media (%s)", mime_type)
- # Binary download (get_media also doesn't accept the Drive flags)
- request = self.service.files().get_media(fileId=file_id)
-
- # Download the file with error handling for misclassified Google Docs
- fh = io.BytesIO()
- downloader = MediaIoBaseDownload(fh, request, chunksize=1024 * 1024)
- done = False
-
- try:
- while not done:
- status, done = downloader.next_chunk()
- # Optional: you can log progress via status.progress()
- except HttpError as e:
- # If download fails with "fileNotDownloadable", it's a Docs Editor file
- # that wasn't properly detected. Retry with export_media.
- if "fileNotDownloadable" in str(e) and mime_type not in exportable_types:
- logger.warning(
- f"Download failed for {file_id} ({mime_type}) with fileNotDownloadable error. "
- f"Retrying with export_media (file might be a Google Doc)"
- )
- export_mime = "application/pdf"
- request = self.service.files().export_media(
- fileId=file_id, mimeType=export_mime
+ logger.debug(
+ "[GoogleDrive] _download_file_bytes: using export_media (%s -> %s)",
+ mime_type,
+ export_mime,
)
- fh = io.BytesIO()
- downloader = MediaIoBaseDownload(fh, request, chunksize=1024 * 1024)
- done = False
+ # NOTE: export_media does not accept supportsAllDrives/includeItemsFromAllDrives
+ request = self.service.files().export_media(fileId=file_id, mimeType=export_mime)
+ else:
+ # This is a regular uploaded file (PDF, image, video, etc.) - use get_media
+ # Also handles non-exportable Google Apps files (Forms, Sites, Maps, etc.)
+ logger.debug("[GoogleDrive] _download_file_bytes: using get_media (%s)", mime_type)
+ # Binary download (get_media also doesn't accept the Drive flags)
+ request = self.service.files().get_media(fileId=file_id)
+
+ # Download the file with error handling for misclassified Google Docs
+ fh = io.BytesIO()
+ downloader = MediaIoBaseDownload(fh, request, chunksize=1024 * 1024)
+ done = False
+
+ try:
while not done:
status, done = downloader.next_chunk()
- else:
- raise
+ # Optional: you can log progress via status.progress()
+ except HttpError as e:
+ # If download fails with "fileNotDownloadable", it's a Docs Editor file
+ # that wasn't properly detected. Retry with export_media.
+ if "fileNotDownloadable" in str(e) and mime_type not in exportable_types:
+ logger.warning(
+ f"Download failed for {file_id} ({mime_type}) with fileNotDownloadable error. "
+ f"Retrying with export_media (file might be a Google Doc)"
+ )
+ export_mime = "application/pdf"
+ request = self.service.files().export_media(
+ fileId=file_id, mimeType=export_mime
+ )
+ fh = io.BytesIO()
+ downloader = MediaIoBaseDownload(fh, request, chunksize=1024 * 1024)
+ done = False
+ while not done:
+ status, done = downloader.next_chunk()
+ else:
+ raise
data = fh.getvalue()
logger.debug("[GoogleDrive] _download_file_bytes: done, %d bytes", len(data))
@@ -572,10 +580,10 @@ class GoogleDriveConnector(BaseConnector):
async def list_files(
self,
- page_token: Optional[str] = None,
- max_files: Optional[int] = None,
+ page_token: str | None = None,
+ max_files: int | None = None,
**kwargs,
- ) -> Dict[str, Any]:
+ ) -> dict[str, Any]:
"""
List files in the currently selected scope (file_ids/folder_ids/recursive).
Returns a dict with 'files' and 'next_page_token'.
@@ -589,7 +597,9 @@ class GoogleDriveConnector(BaseConnector):
"Google Drive service is not initialized. Please authenticate first."
)
- logger.debug("[GoogleDrive] list_files: entry (page_token=%s, max_files=%s)", page_token, max_files)
+ logger.debug(
+ "[GoogleDrive] list_files: entry (page_token=%s, max_files=%s)", page_token, max_files
+ )
try:
items = await asyncio.to_thread(self._iter_selected_items)
@@ -609,7 +619,7 @@ class GoogleDriveConnector(BaseConnector):
logger.error("[GoogleDrive] list_files failed: %s", e, exc_info=True)
raise
- def _extract_google_drive_acl(self, file_meta: Dict) -> DocumentACL:
+ def _extract_google_drive_acl(self, file_meta: dict) -> DocumentACL:
"""
Extract ACL from Google Drive file metadata.
@@ -624,10 +634,15 @@ class GoogleDriveConnector(BaseConnector):
"""
try:
# Fetch permissions (requires additional API call)
- permissions_list = self.service.permissions().list(
- fileId=file_meta["id"],
- fields="permissions(emailAddress,role,type,deleted,displayName)"
- ).execute()
+ with self._lock:
+ permissions_list = (
+ self.service.permissions()
+ .list(
+ fileId=file_meta["id"],
+ fields="permissions(emailAddress,role,type,deleted,displayName)",
+ )
+ .execute()
+ )
allowed_users = []
allowed_groups = []
@@ -730,12 +745,12 @@ class GoogleDriveConnector(BaseConnector):
metadata={
"parents": meta.get("parents"),
"driveId": meta.get("driveId"),
- "size": int(meta.get("size", 0))
- if str(meta.get("size", "")).isdigit()
- else None,
+ "size": int(meta.get("size", 0)) if str(meta.get("size", "")).isdigit() else None,
},
)
- logger.debug("[GoogleDrive] get_file_content: done for file_id=%s (%d bytes)", file_id, len(blob))
+ logger.debug(
+ "[GoogleDrive] get_file_content: done for file_id=%s (%d bytes)", file_id, len(blob)
+ )
return doc
async def setup_subscription(self) -> str:
@@ -752,9 +767,7 @@ class GoogleDriveConnector(BaseConnector):
# 1) Ensure we are authenticated and have a live Drive service
ok = await self.authenticate()
if not ok:
- raise RuntimeError(
- "GoogleDriveConnector.setup_subscription: not authenticated"
- )
+ raise RuntimeError("GoogleDriveConnector.setup_subscription: not authenticated")
# 2) Resolve webhook address (no param in ABC, so pull from config/env)
webhook_address = getattr(self.cfg, "webhook_address", None) or os.getenv(
@@ -810,9 +823,7 @@ class GoogleDriveConnector(BaseConnector):
}
if not isinstance(channel_id, str) or not channel_id:
- raise RuntimeError(
- f"Drive watch returned invalid channel id: {channel_id!r}"
- )
+ raise RuntimeError(f"Drive watch returned invalid channel id: {channel_id!r}")
return channel_id
@@ -850,13 +861,13 @@ class GoogleDriveConnector(BaseConnector):
# Single-channel memory
if getattr(self, "_active_channel", None):
- ch = getattr(self, "_active_channel")
+ ch = self._active_channel
if isinstance(ch, dict) and ch.get("channel_id") == subscription_id:
resource_id = ch.get("resource_id")
# Multi-channel memory
if resource_id is None and hasattr(self, "_subscriptions"):
- subs = getattr(self, "_subscriptions")
+ subs = self._subscriptions
if isinstance(subs, dict):
entry = subs.get(subscription_id)
if isinstance(entry, dict):
@@ -888,9 +899,7 @@ class GoogleDriveConnector(BaseConnector):
):
self._active_channel = {}
- if hasattr(self, "_subscriptions") and isinstance(
- self._subscriptions, dict
- ):
+ if hasattr(self, "_subscriptions") and isinstance(self._subscriptions, dict):
self._subscriptions.pop(subscription_id, None)
return True
@@ -902,7 +911,7 @@ class GoogleDriveConnector(BaseConnector):
pass
return False
- async def handle_webhook(self, payload: Dict[str, Any]) -> List[str]:
+ async def handle_webhook(self, payload: dict[str, Any]) -> list[str]:
"""
Process a Google Drive Changes webhook.
Drive push notifications do NOT include the changed files themselves; they merely tell us
@@ -915,7 +924,7 @@ class GoogleDriveConnector(BaseConnector):
Returns:
List[str]: unique list of affected file IDs (filtered to our selected scope).
"""
- affected: List[str] = []
+ affected: list[str] = []
try:
# 1) Ensure we're authenticated / service ready
ok = await self.authenticate()
@@ -941,9 +950,7 @@ class GoogleDriveConnector(BaseConnector):
except Exception as e:
selected_ids = set()
try:
- logger.error(
- f"handle_webhook: scope build failed, proceeding unfiltered: {e}"
- )
+ logger.error(f"handle_webhook: scope build failed, proceeding unfiltered: {e}")
except Exception:
pass
@@ -980,11 +987,7 @@ class GoogleDriveConnector(BaseConnector):
# Filter to our selected scope if we have one; otherwise accept all
if selected_ids and (rid not in selected_ids):
# Shortcut target might be in scope even if the shortcut isn't
- tgt = (
- fobj.get("shortcutDetails", {}).get("targetId")
- if fobj
- else None
- )
+ tgt = fobj.get("shortcutDetails", {}).get("targetId") if fobj else None
if not (tgt and tgt in selected_ids):
continue
@@ -1007,7 +1010,7 @@ class GoogleDriveConnector(BaseConnector):
# Deduplicate while preserving order
seen = set()
- deduped: List[str] = []
+ deduped: list[str] = []
for x in affected:
if x not in seen:
seen.add(x)
@@ -1033,9 +1036,7 @@ class GoogleDriveConnector(BaseConnector):
blob = self._download_file_bytes(meta)
except HttpError as e:
# Skip/record failures
- logger.error(
- f"Failed to download {meta.get('name')} ({meta.get('id')}): {e}"
- )
+ logger.error(f"Failed to download {meta.get('name')} ({meta.get('id')}): {e}")
continue
from datetime import datetime
@@ -1081,12 +1082,10 @@ class GoogleDriveConnector(BaseConnector):
# -------------------------
def get_start_page_token(self) -> str:
# getStartPageToken accepts supportsAllDrives (not includeItemsFromAllDrives)
- resp = (
- self.service.changes().getStartPageToken(**self._drives_get_flags).execute()
- )
+ resp = self.service.changes().getStartPageToken(**self._drives_get_flags).execute()
return resp["startPageToken"]
- def poll_changes_and_sync(self) -> Optional[str]:
+ def poll_changes_and_sync(self) -> str | None:
"""
Incrementally process changes since the last page token in cfg.changes_page_token.
@@ -1122,10 +1121,7 @@ class GoogleDriveConnector(BaseConnector):
# Match scope
if fid not in selected_ids:
# also consider shortcut target
- if (
- file_obj.get("mimeType")
- == "application/vnd.google-apps.shortcut"
- ):
+ if file_obj.get("mimeType") == "application/vnd.google-apps.shortcut":
tgt = file_obj.get("shortcutDetails", {}).get("targetId")
if tgt and tgt in selected_ids:
pass
@@ -1189,8 +1185,8 @@ class GoogleDriveConnector(BaseConnector):
# Optional: webhook stubs
# -------------------------
def build_watch_body(
- self, webhook_address: str, channel_id: Optional[str] = None
- ) -> Dict[str, Any]:
+ self, webhook_address: str, channel_id: str | None = None
+ ) -> dict[str, Any]:
"""
Prepare the request body for changes.watch if you use webhooks.
"""
@@ -1200,7 +1196,7 @@ class GoogleDriveConnector(BaseConnector):
"address": webhook_address,
}
- def start_watch(self, webhook_address: str) -> Dict[str, Any]:
+ def start_watch(self, webhook_address: str) -> dict[str, Any]:
"""
Start a webhook watch on changes using the current page token.
Persist the returned resourceId/expiration on your side.
diff --git a/src/connectors/langflow_connector_service.py b/src/connectors/langflow_connector_service.py
index 3f2ba93a0..6b5b9e01c 100644
--- a/src/connectors/langflow_connector_service.py
+++ b/src/connectors/langflow_connector_service.py
@@ -56,9 +56,13 @@ class LangflowConnectorService:
filename=document.filename,
)
+ import os
+
from utils.file_utils import auto_cleanup_tempfile
- suffix = get_file_extension(document.mimetype)
+ suffix = os.path.splitext(document.filename)[1]
+ if not suffix:
+ suffix = get_file_extension(document.mimetype)
# Create temporary file from document content
with auto_cleanup_tempfile(suffix=suffix) as tmp_path:
diff --git a/src/connectors/service.py b/src/connectors/service.py
index ed9b385b0..cfdba9735 100644
--- a/src/connectors/service.py
+++ b/src/connectors/service.py
@@ -53,9 +53,15 @@ class ConnectorService:
"""Process a document from a connector using existing processing pipeline"""
# Create temporary file from document content
+ import os
+
from utils.file_utils import auto_cleanup_tempfile
- with auto_cleanup_tempfile(suffix=get_file_extension(document.mimetype)) as tmp_path:
+ suffix = os.path.splitext(document.filename)[1]
+ if not suffix:
+ suffix = get_file_extension(document.mimetype)
+
+ with auto_cleanup_tempfile(suffix=suffix) as tmp_path:
# Write document content to temp file
with open(tmp_path, "wb") as f:
f.write(document.content)
@@ -370,6 +376,7 @@ class ConnectorService:
original_folder_ids = getattr(connector.cfg, "folder_ids", None)
expanded_file_ids = file_ids # Default to original IDs
+ expanded_files_info = []
try:
# Set the file_ids we want to sync in the connector's config
@@ -380,7 +387,12 @@ class ConnectorService:
# Get the expanded list of file IDs (folders will be expanded to their contents)
# This uses the connector's list_files() which calls _iter_selected_items()
result = await connector.list_files()
- expanded_file_ids = [f["id"] for f in result.get("files", [])]
+ expanded_files = result.get("files", [])
+ expanded_file_ids = [f["id"] for f in expanded_files]
+
+ # Save the expanded files info so we can set correct names in the task UI
+ for f in expanded_files:
+ expanded_files_info.append(f)
if not expanded_file_ids:
logger.warning(
@@ -448,10 +460,15 @@ class ConnectorService:
# Create custom task using TaskService
original_filenames = {}
- if file_infos:
+
+ # Combine file_infos and expanded_files_info
+ all_infos = (file_infos or []) + expanded_files_info
+ if all_infos:
original_filenames = {
- f["id"]: clean_connector_filename(f["name"], f.get("mimeType") or f.get("mimetype"))
- for f in file_infos
+ f["id"]: clean_connector_filename(
+ f["name"], f.get("mimeType") or f.get("mimetype", "")
+ )
+ for f in all_infos
if "id" in f and "name" in f
}
diff --git a/src/models/processors.py b/src/models/processors.py
index 05ffaf4a1..ba7ff205b 100644
--- a/src/models/processors.py
+++ b/src/models/processors.py
@@ -241,15 +241,16 @@ class TaskProcessor:
file_hash=file_hash,
)
slim_doc = process_text_file(file_path)
- # Override filename with original_filename if provided
- if original_filename:
- slim_doc["filename"] = original_filename
else:
full_doc = await self.docling_service.convert_file(
file_path, user_id=owner_user_id, auth_header=jwt_token
)
slim_doc = extract_relevant(full_doc)
+ # Override filename with original_filename if provided
+ if original_filename:
+ slim_doc["filename"] = original_filename
+
if chunk_size is not None:
try:
cs = int(chunk_size)
@@ -562,8 +563,11 @@ class ConnectorFileProcessor(TaskProcessor):
return
await self.delete_document_by_filename(document.filename, opensearch_client)
- # Create temporary file from document content
- suffix = get_file_extension(document.mimetype)
+ # Create temporary file from document content import os
+
+ suffix = os.path.splitext(document.filename)[1]
+ if not suffix:
+ suffix = get_file_extension(document.mimetype)
with auto_cleanup_tempfile(suffix=suffix) as tmp_path:
# Write content to temp file
with open(tmp_path, "wb") as f:
@@ -698,8 +702,11 @@ class LangflowConnectorFileProcessor(TaskProcessor):
return
await self.delete_document_by_filename(document.filename, opensearch_client)
- # Create temporary file and compute hash to check for duplicates
- suffix = get_file_extension(document.mimetype)
+ # Create temporary file and compute hash to check for duplicates import os
+
+ suffix = os.path.splitext(document.filename)[1]
+ if not suffix:
+ suffix = get_file_extension(document.mimetype)
with auto_cleanup_tempfile(suffix=suffix) as tmp_path:
# Write content to temp file
with open(tmp_path, "wb") as f:
@@ -774,7 +781,8 @@ class S3FileProcessor(TaskProcessor):
file_task.updated_at = time.time()
try:
- with auto_cleanup_tempfile() as tmp_path:
+ suffix = os.path.splitext(item)[1]
+ with auto_cleanup_tempfile(suffix=suffix) as tmp_path:
# Download object to temporary file
with open(tmp_path, "wb") as tmp_file:
self.s3_client.download_fileobj(self.bucket, item, tmp_file)
diff --git a/src/services/auth_service.py b/src/services/auth_service.py
index 6d0676ed7..c14e0a73c 100644
--- a/src/services/auth_service.py
+++ b/src/services/auth_service.py
@@ -306,7 +306,7 @@ class AuthService:
# Add expiry if provided
if token_data.get("expires_in"):
- expiry = datetime.now() + timedelta(seconds=int(token_data["expires_in"]))
+ expiry = datetime.utcnow() + timedelta(seconds=int(token_data["expires_in"]))
token_file_data["expiry"] = expiry.isoformat()
# Save tokens to file