Skip to content

LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows

Fix/enable-handle-fs-connector (#456)

Kirtiman Mishra4a677f0f36dc
diff --git a/backend/workflow_manager/endpoint/base_connector.py b/backend/workflow_manager/endpoint/base_connector.py
index e90ae03c2..6c35910e5 100644
--- a/backend/workflow_manager/endpoint/base_connector.py
+++ b/backend/workflow_manager/endpoint/base_connector.py
@@ -45,11 +45,26 @@ class BaseConnector(ExecutionFileHandler):
Raises:
KeyError: If the connector_id is not found in the connectors dictionary.
"""
+ return self.get_fs_connector(
+ settings=settings, connector_id=connector_id
+ ).get_fsspec_fs()
+
+ def get_fs_connector(
+ self, settings: dict[str, Any], connector_id: str
+ ) -> UnstractFileSystem:
+ """Get an fs connector based specified connector settings.
+
+ Parameters:
+ - settings (dict): Connector-specific settings.
+ - connector_id (str): Identifier for the desired connector.
+
+ Returns:
+ UnstractFileSystem: An unstract fs connector instance.
+ """
if connector_id not in connectors:
- raise ValueError(f"Invalid connector_id: {connector_id}")
+ raise ValueError(f"Connector '{connector_id}' is not supported.")
connector = connectors[connector_id][Common.METADATA][Common.CONNECTOR]
- connector_class: UnstractFileSystem = connector(settings)
- return connector_class.get_fsspec_fs()
+ return connector(settings)
@classmethod
def get_json_schema(cls, file_path: str) -> dict[str, Any]:
diff --git a/backend/workflow_manager/endpoint/destination.py b/backend/workflow_manager/endpoint/destination.py
index 6b88dd86c..bdb37385d 100644
--- a/backend/workflow_manager/endpoint/destination.py
+++ b/backend/workflow_manager/endpoint/destination.py
@@ -175,32 +175,29 @@ class DestinationConnector(BaseConnector):
connector: ConnectorInstance = self.endpoint.connector_instance
connector_settings: dict[str, Any] = connector.connector_metadata
destination_configurations: dict[str, Any] = self.endpoint.configuration
- root_path = str(connector_settings.get(DestinationKey.PATH))
- output_folder = str(
+ root_path = connector_settings.get(DestinationKey.PATH, "")
+
+ output_directory = str(
destination_configurations.get(DestinationKey.OUTPUT_FOLDER, "/")
)
- overwrite = bool(
- destination_configurations.get(
- DestinationKey.OVERWRITE_OUTPUT_DOCUMENT, False
- )
+ destination_fs = self.get_fs_connector(
+ settings=connector_settings, connector_id=connector.connector_id
)
- output_directory = os.path.join(root_path, output_folder)
-
+ output_directory = destination_fs.get_connector_root_dir(
+ input_dir=output_directory, root_path=root_path
+ )
+ logger.debug(f"destination output directory {output_directory}")
destination_volume_path = os.path.join(
self.execution_dir, ToolExecKey.OUTPUT_DIR
)
-
- connector_fs = self.get_fsspec(
- settings=connector_settings, connector_id=connector.connector_id
- )
- if not connector_fs.isdir(output_directory):
- connector_fs.mkdir(output_directory)
+ destination_fs.create_dir_if_not_exists(input_dir=output_directory)
+ destination_fsspec = destination_fs.get_fsspec_fs()
# Traverse local directory and create the same structure in the
# output_directory
for root, dirs, files in os.walk(destination_volume_path):
for dir_name in dirs:
- connector_fs.mkdir(
+ destination_fsspec.mkdir(
os.path.join(
output_directory,
os.path.relpath(root, destination_volume_path),
@@ -217,9 +214,7 @@ class DestinationConnector(BaseConnector):
)
normalized_path = os.path.normpath(destination_path)
with open(source_path, "rb") as source_file:
- connector_fs.write_bytes(
- normalized_path, source_file.read(), overwrite=overwrite
- )
+ destination_fsspec.write_bytes(normalized_path, source_file.read())
def insert_into_db(self, file_history: Optional[FileHistory]) -> None:
"""Insert data into the database."""
diff --git a/backend/workflow_manager/endpoint/source.py b/backend/workflow_manager/endpoint/source.py
index 9a4306e4c..33cb97484 100644
--- a/backend/workflow_manager/endpoint/source.py
+++ b/backend/workflow_manager/endpoint/source.py
@@ -4,7 +4,6 @@ import os
import shutil
from hashlib import md5, sha256
from io import BytesIO
-from pathlib import Path
from typing import Any, Optional
import fsspec
@@ -153,21 +152,27 @@ class SourceConnector(BaseConnector):
)
)
root_dir_path = connector_settings.get(ConnectorKeys.PATH, "")
+
input_directory = str(source_configurations.get(SourceKey.ROOT_FOLDER, ""))
- if root_dir_path: # user needs to manually type the optional file path
- input_directory = str(Path(root_dir_path, input_directory.lstrip("/")))
- if not isinstance(required_patterns, list):
- required_patterns = [required_patterns]
- source_fs = self.get_fsspec(
+ source_fs = self.get_fs_connector(
settings=connector_settings, connector_id=connector.connector_id
)
+ input_directory = source_fs.get_connector_root_dir(
+ input_dir=input_directory, root_path=root_dir_path
+ )
+ logger.debug(f"source input directory {input_directory}")
+ if not isinstance(required_patterns, list):
+ required_patterns = [required_patterns]
+
+ source_fs_fsspec = source_fs.get_fsspec_fs()
+
patterns = self.valid_file_patterns(required_patterns=required_patterns)
- is_directory = source_fs.isdir(input_directory)
+ is_directory = source_fs_fsspec.isdir(input_directory)
if not is_directory:
raise InvalidInputDirectory()
matched_files = self._get_matched_files(
- source_fs, input_directory, patterns, recursive, limit
+ source_fs_fsspec, input_directory, patterns, recursive, limit
)
self.publish_input_output_list_file_logs(input_directory, matched_files)
return matched_files
diff --git a/backend/workflow_manager/endpoint/static/dest/file.json b/backend/workflow_manager/endpoint/static/dest/file.json
index aa3a1263e..95111993f 100644
--- a/backend/workflow_manager/endpoint/static/dest/file.json
+++ b/backend/workflow_manager/endpoint/static/dest/file.json
@@ -14,12 +14,6 @@
"minLength": 1,
"maxLength": 100,
"format": "file-path"
- },
- "overwriteOutput": {
- "type": "boolean",
- "title": "Overwrite existing output",
- "default": true,
- "description": "Used to overwrite output document"
}
}
}
diff --git a/frontend/src/components/agency/ds-settings-card/DsSettingsCard.jsx b/frontend/src/components/agency/ds-settings-card/DsSettingsCard.jsx
index fdc752d03..8d5b193e9 100644
--- a/frontend/src/components/agency/ds-settings-card/DsSettingsCard.jsx
+++ b/frontend/src/components/agency/ds-settings-card/DsSettingsCard.jsx
@@ -36,8 +36,6 @@ const tooltip = {
const disabledIdsByType = {
FILE_SYSTEM: [
"box|4d94d237-ce4b-45d8-8f34-ddeefc37c0bf",
- "google_cloud_storage|109bbe7b-8861-45eb-8841-7244e833d97b",
- "azure_cloud_storage|1476a54a-ed17-4a01-9f8f-cb7e4cf91c8a",
"http|6fdea346-86e4-4383-9a21-132db7c9a576",
],
};
diff --git a/unstract/connectors/src/unstract/connectors/filesystems/google_drive/google_drive.py b/unstract/connectors/src/unstract/connectors/filesystems/google_drive/google_drive.py
index 1a2241b78..7beafa412 100644
--- a/unstract/connectors/src/unstract/connectors/filesystems/google_drive/google_drive.py
+++ b/unstract/connectors/src/unstract/connectors/filesystems/google_drive/google_drive.py
@@ -1,6 +1,7 @@
import json
import logging
import os
+from pathlib import Path
from typing import Any
from oauth2client.client import OAuth2Credentials
@@ -95,3 +96,12 @@ class GoogleDriveFS(UnstractFileSystem):
except Exception as e:
raise ConnectorError(str(e))
return True
+
+ @staticmethod
+ def get_connector_root_dir(input_dir: str, **kwargs: Any) -> str:
+ """Get roor dir of gdrive."""
+ root_path = kwargs.get("root_path")
+ if root_path is None:
+ raise ValueError("root_path is required to get root_dir for Google Drive")
+ input_dir = str(Path(root_path, input_dir.lstrip("/")))
+ return f"{input_dir.strip('/')}/"
diff --git a/unstract/connectors/src/unstract/connectors/filesystems/minio/minio.py b/unstract/connectors/src/unstract/connectors/filesystems/minio/minio.py
index f0cf798b1..4bdf2c110 100644
--- a/unstract/connectors/src/unstract/connectors/filesystems/minio/minio.py
+++ b/unstract/connectors/src/unstract/connectors/filesystems/minio/minio.py
@@ -17,7 +17,6 @@ class MinioFS(UnstractFileSystem):
secret = settings["secret"]
endpoint_url = settings["endpoint_url"]
self.bucket = settings["bucket"]
- self.path = settings["path"]
client_kwargs = {}
if "region_name" in settings and settings["region_name"] != "":
client_kwargs = {"region_name": settings["region_name"]}
diff --git a/unstract/connectors/src/unstract/connectors/filesystems/minio/static/json_schema.json b/unstract/connectors/src/unstract/connectors/filesystems/minio/static/json_schema.json
index b248cad7f..7f3d3997c 100644
--- a/unstract/connectors/src/unstract/connectors/filesystems/minio/static/json_schema.json
+++ b/unstract/connectors/src/unstract/connectors/filesystems/minio/static/json_schema.json
@@ -32,12 +32,6 @@
"title": "Bucket Name",
"description": "Name of the bucket to be restricted to."
},
- "path": {
- "type": "string",
- "title": "Path",
- "default": "",
- "description": "Path to restrict to. (example /path/to/restrict/to)"
- },
"endpoint_url": {
"type": "string",
"title": "Endpoint URL",
diff --git a/unstract/connectors/src/unstract/connectors/filesystems/unstract_file_system.py b/unstract/connectors/src/unstract/connectors/filesystems/unstract_file_system.py
index 304488e0d..34057315a 100644
--- a/unstract/connectors/src/unstract/connectors/filesystems/unstract_file_system.py
+++ b/unstract/connectors/src/unstract/connectors/filesystems/unstract_file_system.py
@@ -1,5 +1,6 @@
import logging
from abc import ABC, abstractmethod
+from typing import Any
from fsspec import AbstractFileSystem
@@ -69,3 +70,15 @@ class UnstractFileSystem(UnstractConnector, ABC):
def test_credentials(self) -> bool:
"""Override to test credentials for a connector."""
pass
+
+ @staticmethod
+ def get_connector_root_dir(input_dir: str, **kwargs: Any) -> str:
+ """Override to get root dir of a connector."""
+ return f"{input_dir.strip('/')}/"
+
+ def create_dir_if_not_exists(self, input_dir: str) -> None:
+ """Override to create dir of a connector if not exists."""
+ fs_fsspec = self.get_fsspec_fs()
+ is_dir = fs_fsspec.isdir(input_dir)
+ if not is_dir:
+ fs_fsspec.mkdir(input_dir)
diff --git a/unstract/connectors/src/unstract/connectors/filesystems/zs_dropbox/zs_dropbox.py b/unstract/connectors/src/unstract/connectors/filesystems/zs_dropbox/zs_dropbox.py
index 31b743cbb..797198caa 100644
--- a/unstract/connectors/src/unstract/connectors/filesystems/zs_dropbox/zs_dropbox.py
+++ b/unstract/connectors/src/unstract/connectors/filesystems/zs_dropbox/zs_dropbox.py
@@ -2,6 +2,7 @@ import logging
import os
from typing import Any
+from dropbox.exceptions import ApiError as DropBoxApiError
from dropbox.exceptions import DropboxException
from dropboxdrivefs import DropboxDriveFileSystem
@@ -74,3 +75,19 @@ class DropboxFS(UnstractFileSystem):
logger.error(f"Test creds failed: {e}")
raise ConnectorError(str(e))
return True
+
+ @staticmethod
+ def get_connector_root_dir(input_dir: str, **kwargs: Any) -> str:
+ """Get roor dir of zs dropbox."""
+ return f"/{input_dir.strip('/')}"
+
+ def create_dir_if_not_exists(self, input_dir: str) -> None:
+ """Create roor dir of zs dropbox if not exists."""
+ fs_fsspec = self.get_fsspec_fs()
+ try:
+ fs_fsspec.isdir(input_dir)
+ except (
+ DropBoxApiError
+ ) as e: # Dropbox returns this exception when directory is not present
+ logger.debug(f"Path not found in dropbox {e.error}")
+ fs_fsspec.mkdir(input_dir)