Skip to content

LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows

fix: Handle existing files in GoogleDriveFS upload (#1146)

Ritwik G1640f4f51c4f
diff --git a/unstract/connectors/src/unstract/connectors/filesystems/google_drive/google_drive.py b/unstract/connectors/src/unstract/connectors/filesystems/google_drive/google_drive.py
index 514acdd67..1743246dd 100644
--- a/unstract/connectors/src/unstract/connectors/filesystems/google_drive/google_drive.py
+++ b/unstract/connectors/src/unstract/connectors/filesystems/google_drive/google_drive.py
@@ -122,3 +122,29 @@ class GoogleDriveFS(UnstractFileSystem):
raise ValueError("root_path is required to get root_dir for Google Drive")
input_dir = str(Path(root_path, input_dir.lstrip("/")))
return f"{input_dir.strip('/')}/"
+
+ # TODO: This should be removed later once the root casue is fixed.
+ # This is a bandaid fix to avoid duplicate file upload in google drive.
+ # GDrive allows multiple files with same name in a single folder which is
+ # causing file duplication.
+ # Below logic removes the file if already exists to avoid duplication.
+ # Since other conenctor behaviour is to replace file sif exists
+ # the deletion lgoci should be okay here.
+ def upload_file_to_storage(self, source_path: str, destination_path: str) -> None:
+ """Method to upload filepath from tool to destination connector directory.
+ If a file already exists at the destination path, it will be deleted first.
+
+ Args:
+ source_path (str): local path of file to be uploaded, coming from tool
+ destination_path (str): target path in the storage where the file will be
+ uploaded
+ """
+ normalized_path = os.path.normpath(destination_path)
+ destination_connector_fs = self.get_fsspec_fs()
+
+ # Check if file exists and delete it
+ if destination_connector_fs.exists(normalized_path):
+ destination_connector_fs.delete(normalized_path)
+
+ # Call parent class's upload method
+ super().upload_file_to_storage(source_path, destination_path)