LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
Fix/enable-handle-fs-connector (#456)
Kirtiman Mishra4a677f0f36dc
diff --git a/backend/workflow_manager/endpoint/base_connector.py b/backend/workflow_manager/endpoint/base_connector.pyindex e90ae03c2..6c35910e5 100644--- a/backend/workflow_manager/endpoint/base_connector.py+++ b/backend/workflow_manager/endpoint/base_connector.py@@ -45,11 +45,26 @@ class BaseConnector(ExecutionFileHandler):Raises:KeyError: If the connector_id is not found in the connectors dictionary."""+ return self.get_fs_connector(+ settings=settings, connector_id=connector_id+ ).get_fsspec_fs()++ def get_fs_connector(+ self, settings: dict[str, Any], connector_id: str+ ) -> UnstractFileSystem:+ """Get an fs connector based specified connector settings.++ Parameters:+ - settings (dict): Connector-specific settings.+ - connector_id (str): Identifier for the desired connector.++ Returns:+ UnstractFileSystem: An unstract fs connector instance.+ """if connector_id not in connectors:- raise ValueError(f"Invalid connector_id: {connector_id}")+ raise ValueError(f"Connector '{connector_id}' is not supported.")connector = connectors[connector_id][Common.METADATA][Common.CONNECTOR]- connector_class: UnstractFileSystem = connector(settings)- return connector_class.get_fsspec_fs()+ return connector(settings)@classmethoddef get_json_schema(cls, file_path: str) -> dict[str, Any]:diff --git a/backend/workflow_manager/endpoint/destination.py b/backend/workflow_manager/endpoint/destination.pyindex 6b88dd86c..bdb37385d 100644--- a/backend/workflow_manager/endpoint/destination.py+++ b/backend/workflow_manager/endpoint/destination.py@@ -175,32 +175,29 @@ class DestinationConnector(BaseConnector):connector: ConnectorInstance = self.endpoint.connector_instanceconnector_settings: dict[str, Any] = connector.connector_metadatadestination_configurations: dict[str, Any] = self.endpoint.configuration- root_path = str(connector_settings.get(DestinationKey.PATH))- output_folder = str(+ root_path = connector_settings.get(DestinationKey.PATH, "")++ output_directory = str(destination_configurations.get(DestinationKey.OUTPUT_FOLDER, "/"))- overwrite = bool(- destination_configurations.get(- DestinationKey.OVERWRITE_OUTPUT_DOCUMENT, False- )+ destination_fs = self.get_fs_connector(+ settings=connector_settings, connector_id=connector.connector_id)- output_directory = os.path.join(root_path, output_folder)-+ output_directory = destination_fs.get_connector_root_dir(+ input_dir=output_directory, root_path=root_path+ )+ logger.debug(f"destination output directory {output_directory}")destination_volume_path = os.path.join(self.execution_dir, ToolExecKey.OUTPUT_DIR)-- connector_fs = self.get_fsspec(- settings=connector_settings, connector_id=connector.connector_id- )- if not connector_fs.isdir(output_directory):- connector_fs.mkdir(output_directory)+ destination_fs.create_dir_if_not_exists(input_dir=output_directory)+ destination_fsspec = destination_fs.get_fsspec_fs()# Traverse local directory and create the same structure in the# output_directoryfor root, dirs, files in os.walk(destination_volume_path):for dir_name in dirs:- connector_fs.mkdir(+ destination_fsspec.mkdir(os.path.join(output_directory,os.path.relpath(root, destination_volume_path),@@ -217,9 +214,7 @@ class DestinationConnector(BaseConnector):)normalized_path = os.path.normpath(destination_path)with open(source_path, "rb") as source_file:- connector_fs.write_bytes(- normalized_path, source_file.read(), overwrite=overwrite- )+ destination_fsspec.write_bytes(normalized_path, source_file.read())def insert_into_db(self, file_history: Optional[FileHistory]) -> None:"""Insert data into the database."""diff --git a/backend/workflow_manager/endpoint/source.py b/backend/workflow_manager/endpoint/source.pyindex 9a4306e4c..33cb97484 100644--- a/backend/workflow_manager/endpoint/source.py+++ b/backend/workflow_manager/endpoint/source.py@@ -4,7 +4,6 @@ import osimport shutilfrom hashlib import md5, sha256from io import BytesIO-from pathlib import Pathfrom typing import Any, Optionalimport fsspec@@ -153,21 +152,27 @@ class SourceConnector(BaseConnector):))root_dir_path = connector_settings.get(ConnectorKeys.PATH, "")+input_directory = str(source_configurations.get(SourceKey.ROOT_FOLDER, ""))- if root_dir_path: # user needs to manually type the optional file path- input_directory = str(Path(root_dir_path, input_directory.lstrip("/")))- if not isinstance(required_patterns, list):- required_patterns = [required_patterns]- source_fs = self.get_fsspec(+ source_fs = self.get_fs_connector(settings=connector_settings, connector_id=connector.connector_id)+ input_directory = source_fs.get_connector_root_dir(+ input_dir=input_directory, root_path=root_dir_path+ )+ logger.debug(f"source input directory {input_directory}")+ if not isinstance(required_patterns, list):+ required_patterns = [required_patterns]++ source_fs_fsspec = source_fs.get_fsspec_fs()+patterns = self.valid_file_patterns(required_patterns=required_patterns)- is_directory = source_fs.isdir(input_directory)+ is_directory = source_fs_fsspec.isdir(input_directory)if not is_directory:raise InvalidInputDirectory()matched_files = self._get_matched_files(- source_fs, input_directory, patterns, recursive, limit+ source_fs_fsspec, input_directory, patterns, recursive, limit)self.publish_input_output_list_file_logs(input_directory, matched_files)return matched_filesdiff --git a/backend/workflow_manager/endpoint/static/dest/file.json b/backend/workflow_manager/endpoint/static/dest/file.jsonindex aa3a1263e..95111993f 100644--- a/backend/workflow_manager/endpoint/static/dest/file.json+++ b/backend/workflow_manager/endpoint/static/dest/file.json@@ -14,12 +14,6 @@"minLength": 1,"maxLength": 100,"format": "file-path"- },- "overwriteOutput": {- "type": "boolean",- "title": "Overwrite existing output",- "default": true,- "description": "Used to overwrite output document"}}}diff --git a/frontend/src/components/agency/ds-settings-card/DsSettingsCard.jsx b/frontend/src/components/agency/ds-settings-card/DsSettingsCard.jsxindex fdc752d03..8d5b193e9 100644--- a/frontend/src/components/agency/ds-settings-card/DsSettingsCard.jsx+++ b/frontend/src/components/agency/ds-settings-card/DsSettingsCard.jsx@@ -36,8 +36,6 @@ const tooltip = {const disabledIdsByType = {FILE_SYSTEM: ["box|4d94d237-ce4b-45d8-8f34-ddeefc37c0bf",- "google_cloud_storage|109bbe7b-8861-45eb-8841-7244e833d97b",- "azure_cloud_storage|1476a54a-ed17-4a01-9f8f-cb7e4cf91c8a","http|6fdea346-86e4-4383-9a21-132db7c9a576",],};diff --git a/unstract/connectors/src/unstract/connectors/filesystems/google_drive/google_drive.py b/unstract/connectors/src/unstract/connectors/filesystems/google_drive/google_drive.pyindex 1a2241b78..7beafa412 100644--- a/unstract/connectors/src/unstract/connectors/filesystems/google_drive/google_drive.py+++ b/unstract/connectors/src/unstract/connectors/filesystems/google_drive/google_drive.py@@ -1,6 +1,7 @@import jsonimport loggingimport os+from pathlib import Pathfrom typing import Anyfrom oauth2client.client import OAuth2Credentials@@ -95,3 +96,12 @@ class GoogleDriveFS(UnstractFileSystem):except Exception as e:raise ConnectorError(str(e))return True++ @staticmethod+ def get_connector_root_dir(input_dir: str, **kwargs: Any) -> str:+ """Get roor dir of gdrive."""+ root_path = kwargs.get("root_path")+ if root_path is None:+ raise ValueError("root_path is required to get root_dir for Google Drive")+ input_dir = str(Path(root_path, input_dir.lstrip("/")))+ return f"{input_dir.strip('/')}/"diff --git a/unstract/connectors/src/unstract/connectors/filesystems/minio/minio.py b/unstract/connectors/src/unstract/connectors/filesystems/minio/minio.pyindex f0cf798b1..4bdf2c110 100644--- a/unstract/connectors/src/unstract/connectors/filesystems/minio/minio.py+++ b/unstract/connectors/src/unstract/connectors/filesystems/minio/minio.py@@ -17,7 +17,6 @@ class MinioFS(UnstractFileSystem):secret = settings["secret"]endpoint_url = settings["endpoint_url"]self.bucket = settings["bucket"]- self.path = settings["path"]client_kwargs = {}if "region_name" in settings and settings["region_name"] != "":client_kwargs = {"region_name": settings["region_name"]}diff --git a/unstract/connectors/src/unstract/connectors/filesystems/minio/static/json_schema.json b/unstract/connectors/src/unstract/connectors/filesystems/minio/static/json_schema.jsonindex b248cad7f..7f3d3997c 100644--- a/unstract/connectors/src/unstract/connectors/filesystems/minio/static/json_schema.json+++ b/unstract/connectors/src/unstract/connectors/filesystems/minio/static/json_schema.json@@ -32,12 +32,6 @@"title": "Bucket Name","description": "Name of the bucket to be restricted to."},- "path": {- "type": "string",- "title": "Path",- "default": "",- "description": "Path to restrict to. (example /path/to/restrict/to)"- },"endpoint_url": {"type": "string","title": "Endpoint URL",diff --git a/unstract/connectors/src/unstract/connectors/filesystems/unstract_file_system.py b/unstract/connectors/src/unstract/connectors/filesystems/unstract_file_system.pyindex 304488e0d..34057315a 100644--- a/unstract/connectors/src/unstract/connectors/filesystems/unstract_file_system.py+++ b/unstract/connectors/src/unstract/connectors/filesystems/unstract_file_system.py@@ -1,5 +1,6 @@import loggingfrom abc import ABC, abstractmethod+from typing import Anyfrom fsspec import AbstractFileSystem@@ -69,3 +70,15 @@ class UnstractFileSystem(UnstractConnector, ABC):def test_credentials(self) -> bool:"""Override to test credentials for a connector."""pass++ @staticmethod+ def get_connector_root_dir(input_dir: str, **kwargs: Any) -> str:+ """Override to get root dir of a connector."""+ return f"{input_dir.strip('/')}/"++ def create_dir_if_not_exists(self, input_dir: str) -> None:+ """Override to create dir of a connector if not exists."""+ fs_fsspec = self.get_fsspec_fs()+ is_dir = fs_fsspec.isdir(input_dir)+ if not is_dir:+ fs_fsspec.mkdir(input_dir)diff --git a/unstract/connectors/src/unstract/connectors/filesystems/zs_dropbox/zs_dropbox.py b/unstract/connectors/src/unstract/connectors/filesystems/zs_dropbox/zs_dropbox.pyindex 31b743cbb..797198caa 100644--- a/unstract/connectors/src/unstract/connectors/filesystems/zs_dropbox/zs_dropbox.py+++ b/unstract/connectors/src/unstract/connectors/filesystems/zs_dropbox/zs_dropbox.py@@ -2,6 +2,7 @@ import loggingimport osfrom typing import Any+from dropbox.exceptions import ApiError as DropBoxApiErrorfrom dropbox.exceptions import DropboxExceptionfrom dropboxdrivefs import DropboxDriveFileSystem@@ -74,3 +75,19 @@ class DropboxFS(UnstractFileSystem):logger.error(f"Test creds failed: {e}")raise ConnectorError(str(e))return True++ @staticmethod+ def get_connector_root_dir(input_dir: str, **kwargs: Any) -> str:+ """Get roor dir of zs dropbox."""+ return f"/{input_dir.strip('/')}"++ def create_dir_if_not_exists(self, input_dir: str) -> None:+ """Create roor dir of zs dropbox if not exists."""+ fs_fsspec = self.get_fsspec_fs()+ try:+ fs_fsspec.isdir(input_dir)+ except (+ DropBoxApiError+ ) as e: # Dropbox returns this exception when directory is not present+ logger.debug(f"Path not found in dropbox {e.error}")+ fs_fsspec.mkdir(input_dir)