LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
Feature/connector integration (#23)
Kirtiman Mishraafa50665be69
diff --git a/backend/pdm.lock b/backend/pdm.lockindex 77a03ccbb..1676964ad 100644--- a/backend/pdm.lock+++ b/backend/pdm.lock@@ -7,6 +7,25 @@ strategy = ["cross_platform", "inherit_metadata"]lock_version = "4.4.1"content_hash = "sha256:5dc2189aaa7017f49f8711bc05b6fc8a4f33ac93387b53bd3e9aff95a264c33f"+[[package]]+name = "adlfs"+version = "2023.8.0"+requires_python = ">=3.8"+summary = "Access Azure Datalake Gen1 with fsspec and dask"+groups = ["default"]+dependencies = [+ "aiohttp>=3.7.0",+ "azure-core<2.0.0,>=1.23.1",+ "azure-datalake-store<0.1,>=0.0.46",+ "azure-identity",+ "azure-storage-blob>=12.12.0",+ "fsspec>=2021.10.1",+]+files = [+ {file = "adlfs-2023.8.0-py3-none-any.whl", hash = "sha256:3eb248a3c2a30b419f1147bd7676d156b5219f96ef7f11d47166afd2a3bdb07e"},+ {file = "adlfs-2023.8.0.tar.gz", hash = "sha256:07e804f6df4593acfcaf01025b162e30ac13e523d3570279c98b2d91a18026d9"},+]+[[package]]name = "aiobotocore"version = "2.5.4"@@ -385,6 +404,71 @@ files = [{file = "Authlib-1.2.1.tar.gz", hash = "sha256:421f7c6b468d907ca2d9afede256f068f87e34d23dd221c07d13d4c234726afb"},]+[[package]]+name = "azure-core"+version = "1.30.0"+requires_python = ">=3.7"+summary = "Microsoft Azure Core Library for Python"+groups = ["default"]+dependencies = [+ "requests>=2.21.0",+ "six>=1.11.0",+ "typing-extensions>=4.6.0",+]+files = [+ {file = "azure-core-1.30.0.tar.gz", hash = "sha256:6f3a7883ef184722f6bd997262eddaf80cfe7e5b3e0caaaf8db1695695893d35"},+ {file = "azure_core-1.30.0-py3-none-any.whl", hash = "sha256:3dae7962aad109610e68c9a7abb31d79720e1d982ddf61363038d175a5025e89"},+]++[[package]]+name = "azure-datalake-store"+version = "0.0.53"+summary = "Azure Data Lake Store Filesystem Client Library for Python"+groups = ["default"]+dependencies = [+ "cffi",+ "msal<2,>=1.16.0",+ "requests>=2.20.0",+]+files = [+ {file = "azure-datalake-store-0.0.53.tar.gz", hash = "sha256:05b6de62ee3f2a0a6e6941e6933b792b800c3e7f6ffce2fc324bc19875757393"},+ {file = "azure_datalake_store-0.0.53-py2.py3-none-any.whl", hash = "sha256:a30c902a6e360aa47d7f69f086b426729784e71c536f330b691647a51dc42b2b"},+]++[[package]]+name = "azure-identity"+version = "1.15.0"+requires_python = ">=3.7"+summary = "Microsoft Azure Identity Library for Python"+groups = ["default"]+dependencies = [+ "azure-core<2.0.0,>=1.23.0",+ "cryptography>=2.5",+ "msal-extensions<2.0.0,>=0.3.0",+ "msal<2.0.0,>=1.24.0",+]+files = [+ {file = "azure-identity-1.15.0.tar.gz", hash = "sha256:4c28fc246b7f9265610eb5261d65931183d019a23d4b0e99357facb2e6c227c8"},+ {file = "azure_identity-1.15.0-py3-none-any.whl", hash = "sha256:a14b1f01c7036f11f148f22cd8c16e05035293d714458d6b44ddf534d93eb912"},+]++[[package]]+name = "azure-storage-blob"+version = "12.19.0"+requires_python = ">=3.7"+summary = "Microsoft Azure Blob Storage Client Library for Python"+groups = ["default"]+dependencies = [+ "azure-core<2.0.0,>=1.28.0",+ "cryptography>=2.1.4",+ "isodate>=0.6.1",+ "typing-extensions>=4.3.0",+]+files = [+ {file = "azure-storage-blob-12.19.0.tar.gz", hash = "sha256:26c0a4320a34a3c2a1b74528ba6812ebcb632a04cd67b1c7377232c4b01a5897"},+ {file = "azure_storage_blob-12.19.0-py3-none-any.whl", hash = "sha256:7bbc2c9c16678f7a420367fef6b172ba8730a7e66df7f4d7a55d5b3c8216615b"},+]+[[package]]name = "backoff"version = "2.2.1"@@ -798,6 +882,17 @@ files = [{file = "dataclasses_json-0.6.4.tar.gz", hash = "sha256:73696ebf24936560cca79a2430cbc4f3dd23ac7bf46ed17f38e5e5e7657a6377"},]+[[package]]+name = "decorator"+version = "5.1.1"+requires_python = ">=3.5"+summary = "Decorators for Humans"+groups = ["default"]+files = [+ {file = "decorator-5.1.1-py3-none-any.whl", hash = "sha256:b8c3f85900b9dc423225913c5aace94729fe1fa9763b38939a95226f02d37186"},+ {file = "decorator-5.1.1.tar.gz", hash = "sha256:637996211036b6385ef91435e4fae22989472f9d571faba8927ba8253acbc330"},+]+[[package]]name = "defusedxml"version = "0.8.0rc2"@@ -1229,6 +1324,26 @@ files = [{file = "funcy-2.0.tar.gz", hash = "sha256:3963315d59d41c6f30c04bc910e10ab50a3ac4a225868bfa96feed133df075cb"},]+[[package]]+name = "gcsfs"+version = "2023.6.0"+requires_python = ">=3.8"+summary = "Convenient Filesystem interface over GCS"+groups = ["default"]+dependencies = [+ "aiohttp!=4.0.0a0,!=4.0.0a1",+ "decorator>4.1.2",+ "fsspec==2023.6.0",+ "google-auth-oauthlib",+ "google-auth>=1.2",+ "google-cloud-storage",+ "requests",+]+files = [+ {file = "gcsfs-2023.6.0-py2.py3-none-any.whl", hash = "sha256:3b3c7d8eddd4ec1380f3b49fbb861ee1e974adb223564401f10884b6260d406f"},+ {file = "gcsfs-2023.6.0.tar.gz", hash = "sha256:30b14fccadb3b7f0d99b2cd03bd8507c40f3a9a7d05847edca571f642bedbdff"},+]+[[package]]name = "gitdb"version = "4.0.11"@@ -1359,6 +1474,21 @@ files = [{file = "google_auth_httplib2-0.2.0-py2.py3-none-any.whl", hash = "sha256:b65a0a2123300dd71281a7bf6e64d65a0759287df52729bdd1ae2e47dc311a3d"},]+[[package]]+name = "google-auth-oauthlib"+version = "1.2.0"+requires_python = ">=3.6"+summary = "Google Authentication Library"+groups = ["default"]+dependencies = [+ "google-auth>=2.15.0",+ "requests-oauthlib>=0.7.0",+]+files = [+ {file = "google-auth-oauthlib-1.2.0.tar.gz", hash = "sha256:292d2d3783349f2b0734a0a0207b1e1e322ac193c2c09d8f7c613fb7cc501ea8"},+ {file = "google_auth_oauthlib-1.2.0-py2.py3-none-any.whl", hash = "sha256:297c1ce4cb13a99b5834c74a1fe03252e1e499716718b190f56bcb9c4abc4faf"},+]+[[package]]name = "google-cloud-aiplatform"version = "1.40.0"@@ -1984,6 +2114,19 @@ files = [{file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"},]+[[package]]+name = "isodate"+version = "0.6.1"+summary = "An ISO 8601 date/time/duration parser and formatter"+groups = ["default"]+dependencies = [+ "six",+]+files = [+ {file = "isodate-0.6.1-py2.py3-none-any.whl", hash = "sha256:0751eece944162659049d35f4f549ed815792b38793f07cf73381c1c87cbed96"},+ {file = "isodate-0.6.1.tar.gz", hash = "sha256:48c5881de7e8b0a0d648cb024c8062dc84e7b840ed81e864c7614fd3c127bde9"},+]+[[package]]name = "jmespath"version = "1.0.1"@@ -2222,6 +2365,39 @@ files = [{file = "mpmath-1.3.0.tar.gz", hash = "sha256:7a28eb2a9774d00c7bc92411c19a89209d5da7c4c9a9e227be8330a23a25b91f"},]+[[package]]+name = "msal"+version = "1.27.0"+requires_python = ">=2.7"+summary = "The Microsoft Authentication Library (MSAL) for Python library enables your app to access the Microsoft Cloud by supporting authentication of users with Microsoft Azure Active Directory accounts (AAD) and Microsoft Accounts (MSA) using industry standard OAuth2 and OpenID Connect."+groups = ["default"]+dependencies = [+ "PyJWT[crypto]<3,>=1.0.0",+ "cryptography<45,>=0.6",+ "requests<3,>=2.0.0",+]+files = [+ {file = "msal-1.27.0-py2.py3-none-any.whl", hash = "sha256:572d07149b83e7343a85a3bcef8e581167b4ac76befcbbb6eef0c0e19643cdc0"},+ {file = "msal-1.27.0.tar.gz", hash = "sha256:3109503c038ba6b307152b0e8d34f98113f2e7a78986e28d0baf5b5303afda52"},+]++[[package]]+name = "msal-extensions"+version = "1.1.0"+requires_python = ">=3.7"+summary = "Microsoft Authentication Library extensions (MSAL EX) provides a persistence API that can save your data on disk, encrypted on Windows, macOS and Linux. Concurrent data access will be coordinated by a file lock mechanism."+groups = ["default"]+dependencies = [+ "msal<2.0.0,>=0.4.1",+ "packaging",+ "portalocker<3,>=1.0; platform_system != \"Windows\"",+ "portalocker<3,>=1.6; platform_system == \"Windows\"",+]+files = [+ {file = "msal-extensions-1.1.0.tar.gz", hash = "sha256:6ab357867062db7b253d0bd2df6d411c7891a0ee7308d54d1e4317c1d1c54252"},+ {file = "msal_extensions-1.1.0-py3-none-any.whl", hash = "sha256:01be9711b4c0b1a151450068eeb2c4f0997df3bba085ac299de3a66f585e382f"},+]+[[package]]name = "multidict"version = "6.0.5"@@ -3101,6 +3277,22 @@ files = [{file = "PyJWT-2.8.0.tar.gz", hash = "sha256:57e28d156e3d5c10088e0c68abb90bfac3df82b40a71bd0daa20c65ccd5c23de"},]+[[package]]+name = "pyjwt"+version = "2.8.0"+extras = ["crypto"]+requires_python = ">=3.7"+summary = "JSON Web Token implementation in Python"+groups = ["default"]+dependencies = [+ "PyJWT==2.8.0",+ "cryptography>=3.4.0",+]+files = [+ {file = "PyJWT-2.8.0-py3-none-any.whl", hash = "sha256:59127c392cc44c2da5bb3192169a91f429924e17aff6534d70fdc02ab3e04320"},+ {file = "PyJWT-2.8.0.tar.gz", hash = "sha256:57e28d156e3d5c10088e0c68abb90bfac3df82b40a71bd0daa20c65ccd5c23de"},+]+[[package]]name = "pymilvus"version = "2.3.4"@@ -4621,8 +4813,10 @@ groups = ["default"]dependencies = ["PyDrive2[fsspec]==1.15.4","PyMySQL==1.1.0",+ "adlfs==2023.8.0","boxfs==0.2.1","dropboxdrivefs==1.3.1",+ "gcsfs==2023.6.0","google-auth==2.20.0","google-cloud-bigquery==3.11.4","google-cloud-secret-manager==2.16.1",diff --git a/backend/workflow_manager/endpoint/constants.py b/backend/workflow_manager/endpoint/constants.pyindex c2ffe5072..061dc7a56 100644--- a/backend/workflow_manager/endpoint/constants.py+++ b/backend/workflow_manager/endpoint/constants.py@@ -5,6 +5,7 @@ class TableColumns:class DBConnectionClass:SNOWFLAKE = "SnowflakeConnection"+ BIGQUERY = "Client"class Snowflake:@@ -83,3 +84,13 @@ class SourceConstant:class ApiDeploymentResultStatus:SUCCESS = "Success"FAILED = "Failed"+++class BigQuery:+ """In big query, table name has to be in the format {db}.{schema}.{table}+ Throws error if any of the params not set.++ When converted to list table size should be 3+ """++ TABLE_NAME_SIZE = 3diff --git a/backend/workflow_manager/endpoint/database_utils.py b/backend/workflow_manager/endpoint/database_utils.pyindex 234493bda..ad59188c2 100644--- a/backend/workflow_manager/endpoint/database_utils.py+++ b/backend/workflow_manager/endpoint/database_utils.py@@ -3,16 +3,19 @@ import jsonimport loggingfrom typing import Any, Optional-from unstract.connectors.databases import connectors as db_connectors-from unstract.connectors.databases.unstract_db import UnstractDBfrom utils.constants import Commonfrom workflow_manager.endpoint.constants import (+ BigQuery,DBConnectionClass,Snowflake,TableColumns,)+from workflow_manager.endpoint.exceptions import BigQueryTableNotFoundfrom workflow_manager.workflow.enums import AgentName, ColumnModes+from unstract.connectors.databases import connectors as db_connectors+from unstract.connectors.databases.unstract_db import UnstractDB+logger = logging.getLogger(__name__)@@ -60,58 +63,84 @@ class DatabaseUtils:return sql_values@staticmethod- def get_column_types(- engine: Any, table_name: str, cls: Any = None- ) -> dict[str, str]:- """Retrieve column types for a specified table from a database engine.+ def get_column_types_util(columns_with_types: Any) -> dict[str, str]:+ """Converts db results columns_with_types to dict.Args:- engine (Any): The database engine used to execute queries.- table_name (str): The name of the table for which column types- are retrieved.- cls (Any, optional): The database connection class (e.g.,- DBConnectionClass.SNOWFLAKE) for handling database-specific- queries.- Defaults to None.+ columns_with_types (Any): _description_Returns:- dict: A dictionary mapping column names to their respective data- types.+ dict[str, str]: _description_+ """+ column_types: dict[str, str] = {}+ for column_name, data_type in columns_with_types:+ column_types[column_name] = data_type+ return column_types++ @staticmethod+ def get_column_types(+ cls: Any,+ table_name: str,+ connector_id: str,+ connector_settings: dict[str, Any],+ ) -> Any:+ """Get db column name and types.++ Args:+ cls (Any): _description_+ table_name (str): _description_+ connector_id (str): _description_+ connector_settings (dict[str, Any]): _description_Raises:- Exception: If there is an error while retrieving column types,- an exception is raised. Exit.+ ValueError: _description_+ e: _description_- Note:- - If `cls` is not provided or is None, the function assumes a- Default SQL database and queries column types accordingly.- - If `cls` is provided and matches DBConnectionClass.SNOWFLAKE,- the function queries column types using Snowflake-specific- syntax.+ Returns:+ Any: _description_"""+ column_types: dict[str, str] = {}try:- column_types: dict[str, str] = {}- with engine.cursor() as cursor:- if cls == DBConnectionClass.SNOWFLAKE:- results = cursor.execute(f"describe table {table_name}")- for column in results:- column_types[column[0].lower()] = column[1].split("(")[- 0- ]- else:- # Default to Other SQL DBs- # Postgresql treats the table names as in lowercase- # tested only with Postgresql- table_name = str.lower(table_name)- columns_with_types_query = (- "SELECT column_name, data_type FROM "- "information_schema.columns WHERE "- f"table_name = '{table_name}'"- )- cursor.execute(columns_with_types_query)- columns_with_types = cursor.fetchall()- for column_name, data_type in columns_with_types:- column_types[column_name] = data_type+ if cls == DBConnectionClass.SNOWFLAKE:+ query = f"describe table {table_name}"+ results = DatabaseUtils.execute_and_fetch_data(+ connector_id=connector_id,+ connector_settings=connector_settings,+ query=query,+ )+ for column in results:+ column_types[column[0].lower()] = column[1].split("(")[0]+ elif cls == DBConnectionClass.BIGQUERY:+ bigquery_table_name = str.lower(table_name).split(".")+ if len(bigquery_table_name) != BigQuery.TABLE_NAME_SIZE:+ raise BigQueryTableNotFound()+ database = bigquery_table_name[0]+ schema = bigquery_table_name[1]+ table = bigquery_table_name[2]+ query = (+ "SELECT column_name, data_type FROM "+ f"{database}.{schema}.INFORMATION_SCHEMA.COLUMNS WHERE "+ f"table_name = '{table}'"+ )+ results = DatabaseUtils.execute_and_fetch_data(+ connector_id=connector_id,+ connector_settings=connector_settings,+ query=query,+ )+ column_types = DatabaseUtils.get_column_types_util(results)+ else:+ table_name = str.lower(table_name)+ query = (+ "SELECT column_name, data_type FROM "+ "information_schema.columns WHERE "+ f"table_name = '{table_name}'"+ )+ results = DatabaseUtils.execute_and_fetch_data(+ connector_id=connector_id,+ connector_settings=connector_settings,+ query=query,+ )+ column_types = DatabaseUtils.get_column_types_util(results)except Exception as e:logger.error(f"Error getting column types for {table_name}: {str(e)}"@@ -180,13 +209,19 @@ class DatabaseUtils:@staticmethoddef get_sql_values_for_query(- engine: Any, table_name: str, values: dict[str, Any]+ engine: Any,+ connector_id: str,+ connector_settings: dict[str, Any],+ table_name: str,+ values: dict[str, Any],) -> list[str]:"""Generate SQL values for an insert query based on the provided valuesand table schema.Args:engine (Any): The database engine.+ connector_id: The connector id of the connector provided+ connector_settings: Connector settings provided by usertable_name (str): The name of the target table for the insert query.values (dict[str, Any]): A dictionary containing column-value pairsfor the insert query.@@ -202,24 +237,20 @@ class DatabaseUtils:- For other SQL databases, it uses default SQL generationbased on column types."""-- if engine.__class__.__name__ == DBConnectionClass.SNOWFLAKE:- # Handle Snowflake- column_types: dict[str, str] = DatabaseUtils.get_column_types(- engine=engine,- table_name=table_name,- cls=DBConnectionClass.SNOWFLAKE,- )+ class_name = engine.__class__.__name__+ column_types: dict[str, str] = DatabaseUtils.get_column_types(+ cls=class_name,+ table_name=table_name,+ connector_id=connector_id,+ connector_settings=connector_settings,+ )+ if class_name == DBConnectionClass.SNOWFLAKE:sql_values = DatabaseUtils.make_sql_values_for_query(values=values,column_types=column_types,cls=DBConnectionClass.SNOWFLAKE,)else:- # Default to Other SQL DBs- column_types = DatabaseUtils.get_column_types(- engine=engine, table_name=table_name- )sql_values = DatabaseUtils.make_sql_values_for_query(values=values, column_types=column_types)@@ -248,11 +279,13 @@ class DatabaseUtils:f"INSERT INTO {table_name} ({','.join(sql_keys)}) "f"SELECT {','.join(sql_values)}")-try:- with engine.cursor() as cursor:- cursor.execute(sql)- engine.commit()+ if hasattr(engine, "cursor"):+ with engine.cursor() as cursor:+ cursor.execute(sql)+ engine.commit()+ else:+ engine.query(sql)except Exception as e:logger.error(f"Error while writing data: {str(e)}")raise e@@ -266,3 +299,13 @@ class DatabaseUtils:]connector_class: UnstractDB = connector(connector_settings)return connector_class.get_engine()++ @staticmethod+ def execute_and_fetch_data(+ connector_id: str, connector_settings: dict[str, Any], query: str+ ) -> Any:+ connector = db_connectors[connector_id][Common.METADATA][+ Common.CONNECTOR+ ]+ connector_class: UnstractDB = connector(connector_settings)+ return connector_class.execute(query=query)diff --git a/backend/workflow_manager/endpoint/destination.py b/backend/workflow_manager/endpoint/destination.pyindex cdaae988b..dcbe52342 100644--- a/backend/workflow_manager/endpoint/destination.py+++ b/backend/workflow_manager/endpoint/destination.py@@ -223,7 +223,11 @@ class DestinationConnector(BaseConnector):connector_settings=connector_settings,)sql_values = DatabaseUtils.get_sql_values_for_query(- engine=engine, table_name=table_name, values=values+ engine=engine,+ connector_id=connector_instance.connector_id,+ connector_settings=connector_settings,+ table_name=table_name,+ values=values,)DatabaseUtils.execute_write_query(diff --git a/backend/workflow_manager/endpoint/exceptions.py b/backend/workflow_manager/endpoint/exceptions.pyindex 03e82ea06..b4590503b 100644--- a/backend/workflow_manager/endpoint/exceptions.py+++ b/backend/workflow_manager/endpoint/exceptions.py@@ -61,3 +61,11 @@ class ToolOutputTypeMismatch(APIException):default_detail = ("The data type of the tool's output does not match the expected type.")+++class BigQueryTableNotFound(APIException):+ status_code = 400+ default_detail = (+ "Please enter correct correct bigquery table in the form "+ "{table}.{schema}.{database}."+ )diff --git a/backend/workflow_manager/endpoint/source.py b/backend/workflow_manager/endpoint/source.pyindex 6dfe41e61..973655457 100644--- a/backend/workflow_manager/endpoint/source.py+++ b/backend/workflow_manager/endpoint/source.py@@ -162,7 +162,10 @@ class SourceConnector(BaseConnector):input_directory = str(source_configurations.get(SourceKey.ROOT_FOLDER, ""))- input_directory = str(Path(root_dir_path, input_directory.lstrip("/")))+ if root_dir_path: # user needs to manually type the optional file path+ input_directory = str(+ Path(root_dir_path, input_directory.lstrip("/"))+ )if not isinstance(required_patterns, list):required_patterns = [required_patterns]diff --git a/frontend/src/components/agency/configure-connector-modal/ConfigureConnectorModal.jsx b/frontend/src/components/agency/configure-connector-modal/ConfigureConnectorModal.jsxindex 614956d83..49e9cc959 100644--- a/frontend/src/components/agency/configure-connector-modal/ConfigureConnectorModal.jsx+++ b/frontend/src/components/agency/configure-connector-modal/ConfigureConnectorModal.jsx@@ -21,9 +21,9 @@ function ConfigureConnectorModal({formDataConfig,setFormDataConfig,isSpecConfigLoading,+ connType,}) {const [activeKey, setActiveKey] = useState("1");-const tabItems = [{key: "1",@@ -32,7 +32,7 @@ function ConfigureConnectorModal({{key: "2",label: "File System",- disabled: !connectorId,+ disabled: !connectorId || connType === "DATABASE",},];@@ -112,6 +112,7 @@ ConfigureConnectorModal.propTypes = {formDataConfig: PropTypes.object,setFormDataConfig: PropTypes.func.isRequired,isSpecConfigLoading: PropTypes.bool.isRequired,+ connType: PropTypes.string.isRequired,};export { ConfigureConnectorModal };diff --git a/frontend/src/components/agency/ds-settings-card/DsSettingsCard.jsx b/frontend/src/components/agency/ds-settings-card/DsSettingsCard.jsxindex fdce7fac5..5a24ed3e8 100644--- a/frontend/src/components/agency/ds-settings-card/DsSettingsCard.jsx+++ b/frontend/src/components/agency/ds-settings-card/DsSettingsCard.jsx@@ -330,7 +330,7 @@ function DsSettingsCard({ type, endpointDetails, message }) {</Tooltip></Space><div className="display-flex-align-center">- {connDetails?.icon ? (+ {connDetails?.connector_name ? (<Space><Imagesrc={connDetails?.icon}@@ -396,6 +396,7 @@ function DsSettingsCard({ type, endpointDetails, message }) {formDataConfig={formDataConfig}setFormDataConfig={setFormDataConfig}isSpecConfigLoading={isSpecConfigLoading}+ connType={connType}/></>);diff --git a/pdm.lock b/pdm.lockindex 1b75a9b64..c07caf241 100644--- a/pdm.lock+++ b/pdm.lock@@ -18,6 +18,25 @@ files = [{file = "absolufy_imports-0.3.1.tar.gz", hash = "sha256:c90638a6c0b66826d1fb4880ddc20ef7701af34192c94faf40b95d32b59f9793"},]+[[package]]+name = "adlfs"+version = "2023.8.0"+requires_python = ">=3.8"+summary = "Access Azure Datalake Gen1 with fsspec and dask"+groups = ["hook-check-django-migrations"]+dependencies = [+ "aiohttp>=3.7.0",+ "azure-core<2.0.0,>=1.23.1",+ "azure-datalake-store<0.1,>=0.0.46",+ "azure-identity",+ "azure-storage-blob>=12.12.0",+ "fsspec>=2021.10.1",+]+files = [+ {file = "adlfs-2023.8.0-py3-none-any.whl", hash = "sha256:3eb248a3c2a30b419f1147bd7676d156b5219f96ef7f11d47166afd2a3bdb07e"},+ {file = "adlfs-2023.8.0.tar.gz", hash = "sha256:07e804f6df4593acfcaf01025b162e30ac13e523d3570279c98b2d91a18026d9"},+]+[[package]]name = "aiobotocore"version = "2.5.4"@@ -412,6 +431,71 @@ files = [{file = "autopep8-2.0.4.tar.gz", hash = "sha256:2913064abd97b3419d1cc83ea71f042cb821f87e45b9c88cad5ad3c4ea87fe0c"},]+[[package]]+name = "azure-core"+version = "1.30.0"+requires_python = ">=3.7"+summary = "Microsoft Azure Core Library for Python"+groups = ["hook-check-django-migrations"]+dependencies = [+ "requests>=2.21.0",+ "six>=1.11.0",+ "typing-extensions>=4.6.0",+]+files = [+ {file = "azure-core-1.30.0.tar.gz", hash = "sha256:6f3a7883ef184722f6bd997262eddaf80cfe7e5b3e0caaaf8db1695695893d35"},+ {file = "azure_core-1.30.0-py3-none-any.whl", hash = "sha256:3dae7962aad109610e68c9a7abb31d79720e1d982ddf61363038d175a5025e89"},+]++[[package]]+name = "azure-datalake-store"+version = "0.0.53"+summary = "Azure Data Lake Store Filesystem Client Library for Python"+groups = ["hook-check-django-migrations"]+dependencies = [+ "cffi",+ "msal<2,>=1.16.0",+ "requests>=2.20.0",+]+files = [+ {file = "azure-datalake-store-0.0.53.tar.gz", hash = "sha256:05b6de62ee3f2a0a6e6941e6933b792b800c3e7f6ffce2fc324bc19875757393"},+ {file = "azure_datalake_store-0.0.53-py2.py3-none-any.whl", hash = "sha256:a30c902a6e360aa47d7f69f086b426729784e71c536f330b691647a51dc42b2b"},+]++[[package]]+name = "azure-identity"+version = "1.15.0"+requires_python = ">=3.7"+summary = "Microsoft Azure Identity Library for Python"+groups = ["hook-check-django-migrations"]+dependencies = [+ "azure-core<2.0.0,>=1.23.0",+ "cryptography>=2.5",+ "msal-extensions<2.0.0,>=0.3.0",+ "msal<2.0.0,>=1.24.0",+]+files = [+ {file = "azure-identity-1.15.0.tar.gz", hash = "sha256:4c28fc246b7f9265610eb5261d65931183d019a23d4b0e99357facb2e6c227c8"},+ {file = "azure_identity-1.15.0-py3-none-any.whl", hash = "sha256:a14b1f01c7036f11f148f22cd8c16e05035293d714458d6b44ddf534d93eb912"},+]++[[package]]+name = "azure-storage-blob"+version = "12.19.0"+requires_python = ">=3.7"+summary = "Microsoft Azure Blob Storage Client Library for Python"+groups = ["hook-check-django-migrations"]+dependencies = [+ "azure-core<2.0.0,>=1.28.0",+ "cryptography>=2.1.4",+ "isodate>=0.6.1",+ "typing-extensions>=4.3.0",+]+files = [+ {file = "azure-storage-blob-12.19.0.tar.gz", hash = "sha256:26c0a4320a34a3c2a1b74528ba6812ebcb632a04cd67b1c7377232c4b01a5897"},+ {file = "azure_storage_blob-12.19.0-py3-none-any.whl", hash = "sha256:7bbc2c9c16678f7a420367fef6b172ba8730a7e66df7f4d7a55d5b3c8216615b"},+]+[[package]]name = "backoff"version = "2.2.1"@@ -860,6 +944,17 @@ files = [{file = "dataclasses_json-0.6.4.tar.gz", hash = "sha256:73696ebf24936560cca79a2430cbc4f3dd23ac7bf46ed17f38e5e5e7657a6377"},]+[[package]]+name = "decorator"+version = "5.1.1"+requires_python = ">=3.5"+summary = "Decorators for Humans"+groups = ["hook-check-django-migrations"]+files = [+ {file = "decorator-5.1.1-py3-none-any.whl", hash = "sha256:b8c3f85900b9dc423225913c5aace94729fe1fa9763b38939a95226f02d37186"},+ {file = "decorator-5.1.1.tar.gz", hash = "sha256:637996211036b6385ef91435e4fae22989472f9d571faba8927ba8253acbc330"},+]+[[package]]name = "defusedxml"version = "0.8.0rc2"@@ -1309,6 +1404,26 @@ files = [{file = "funcy-2.0.tar.gz", hash = "sha256:3963315d59d41c6f30c04bc910e10ab50a3ac4a225868bfa96feed133df075cb"},]+[[package]]+name = "gcsfs"+version = "2023.6.0"+requires_python = ">=3.8"+summary = "Convenient Filesystem interface over GCS"+groups = ["hook-check-django-migrations"]+dependencies = [+ "aiohttp!=4.0.0a0,!=4.0.0a1",+ "decorator>4.1.2",+ "fsspec==2023.6.0",+ "google-auth-oauthlib",+ "google-auth>=1.2",+ "google-cloud-storage",+ "requests",+]+files = [+ {file = "gcsfs-2023.6.0-py2.py3-none-any.whl", hash = "sha256:3b3c7d8eddd4ec1380f3b49fbb861ee1e974adb223564401f10884b6260d406f"},+ {file = "gcsfs-2023.6.0.tar.gz", hash = "sha256:30b14fccadb3b7f0d99b2cd03bd8507c40f3a9a7d05847edca571f642bedbdff"},+]+[[package]]name = "gitdb"version = "4.0.11"@@ -1439,6 +1554,21 @@ files = [{file = "google_auth_httplib2-0.2.0-py2.py3-none-any.whl", hash = "sha256:b65a0a2123300dd71281a7bf6e64d65a0759287df52729bdd1ae2e47dc311a3d"},]+[[package]]+name = "google-auth-oauthlib"+version = "1.2.0"+requires_python = ">=3.6"+summary = "Google Authentication Library"+groups = ["hook-check-django-migrations"]+dependencies = [+ "google-auth>=2.15.0",+ "requests-oauthlib>=0.7.0",+]+files = [+ {file = "google-auth-oauthlib-1.2.0.tar.gz", hash = "sha256:292d2d3783349f2b0734a0a0207b1e1e322ac193c2c09d8f7c613fb7cc501ea8"},+ {file = "google_auth_oauthlib-1.2.0-py2.py3-none-any.whl", hash = "sha256:297c1ce4cb13a99b5834c74a1fe03252e1e499716718b190f56bcb9c4abc4faf"},+]+[[package]]name = "google-cloud-aiplatform"version = "1.40.0"@@ -2050,6 +2180,19 @@ files = [{file = "inflection-0.5.1.tar.gz", hash = "sha256:1a29730d366e996aaacffb2f1f1cb9593dc38e2ddd30c91250c6dde09ea9b417"},]+[[package]]+name = "isodate"+version = "0.6.1"+summary = "An ISO 8601 date/time/duration parser and formatter"+groups = ["hook-check-django-migrations"]+dependencies = [+ "six",+]+files = [+ {file = "isodate-0.6.1-py2.py3-none-any.whl", hash = "sha256:0751eece944162659049d35f4f549ed815792b38793f07cf73381c1c87cbed96"},+ {file = "isodate-0.6.1.tar.gz", hash = "sha256:48c5881de7e8b0a0d648cb024c8062dc84e7b840ed81e864c7614fd3c127bde9"},+]+[[package]]name = "isort"version = "5.12.0"@@ -2310,6 +2453,39 @@ files = [{file = "mpmath-1.3.0.tar.gz", hash = "sha256:7a28eb2a9774d00c7bc92411c19a89209d5da7c4c9a9e227be8330a23a25b91f"},]+[[package]]+name = "msal"+version = "1.27.0"+requires_python = ">=2.7"+summary = "The Microsoft Authentication Library (MSAL) for Python library enables your app to access the Microsoft Cloud by supporting authentication of users with Microsoft Azure Active Directory accounts (AAD) and Microsoft Accounts (MSA) using industry standard OAuth2 and OpenID Connect."+groups = ["hook-check-django-migrations"]+dependencies = [+ "PyJWT[crypto]<3,>=1.0.0",+ "cryptography<45,>=0.6",+ "requests<3,>=2.0.0",+]+files = [+ {file = "msal-1.27.0-py2.py3-none-any.whl", hash = "sha256:572d07149b83e7343a85a3bcef8e581167b4ac76befcbbb6eef0c0e19643cdc0"},+ {file = "msal-1.27.0.tar.gz", hash = "sha256:3109503c038ba6b307152b0e8d34f98113f2e7a78986e28d0baf5b5303afda52"},+]++[[package]]+name = "msal-extensions"+version = "1.1.0"+requires_python = ">=3.7"+summary = "Microsoft Authentication Library extensions (MSAL EX) provides a persistence API that can save your data on disk, encrypted on Windows, macOS and Linux. Concurrent data access will be coordinated by a file lock mechanism."+groups = ["hook-check-django-migrations"]+dependencies = [+ "msal<2.0.0,>=0.4.1",+ "packaging",+ "portalocker<3,>=1.0; platform_system != \"Windows\"",+ "portalocker<3,>=1.6; platform_system == \"Windows\"",+]+files = [+ {file = "msal-extensions-1.1.0.tar.gz", hash = "sha256:6ab357867062db7b253d0bd2df6d411c7891a0ee7308d54d1e4317c1d1c54252"},+ {file = "msal_extensions-1.1.0-py3-none-any.whl", hash = "sha256:01be9711b4c0b1a151450068eeb2c4f0997df3bba085ac299de3a66f585e382f"},+]+[[package]]name = "multidict"version = "6.0.5"@@ -3252,6 +3428,22 @@ files = [{file = "PyJWT-2.8.0.tar.gz", hash = "sha256:57e28d156e3d5c10088e0c68abb90bfac3df82b40a71bd0daa20c65ccd5c23de"},]+[[package]]+name = "pyjwt"+version = "2.8.0"+extras = ["crypto"]+requires_python = ">=3.7"+summary = "JSON Web Token implementation in Python"+groups = ["hook-check-django-migrations"]+dependencies = [+ "PyJWT==2.8.0",+ "cryptography>=3.4.0",+]+files = [+ {file = "PyJWT-2.8.0-py3-none-any.whl", hash = "sha256:59127c392cc44c2da5bb3192169a91f429924e17aff6534d70fdc02ab3e04320"},+ {file = "PyJWT-2.8.0.tar.gz", hash = "sha256:57e28d156e3d5c10088e0c68abb90bfac3df82b40a71bd0daa20c65ccd5c23de"},+]+[[package]]name = "pymilvus"version = "2.3.4"@@ -4788,8 +4980,10 @@ groups = ["hook-check-django-migrations"]dependencies = ["PyDrive2[fsspec]==1.15.4","PyMySQL==1.1.0",+ "adlfs==2023.8.0","boxfs==0.2.1","dropboxdrivefs==1.3.1",+ "gcsfs==2023.6.0","google-auth==2.20.0","google-cloud-bigquery==3.11.4","google-cloud-secret-manager==2.16.1",diff --git a/unstract/connectors/pyproject.toml b/unstract/connectors/pyproject.tomlindex 727300707..abf53c275 100644--- a/unstract/connectors/pyproject.toml+++ b/unstract/connectors/pyproject.toml@@ -21,6 +21,8 @@ dependencies = ["oauth2client==4.1.3", # For GDrive"dropboxdrivefs==1.3.1", # For Dropbox"boxfs==0.2.1", # For Box+ "gcsfs==2023.6.0", # For GoogleCloudStorage+ "adlfs==2023.8.0", # For AzureCloudStorage# Database connectors"psycopg2-binary==2.9.9", # For Postgres, Redshift"snowflake-connector-python[pandas]~=3.6.0", # For Snowflakediff --git a/unstract/connectors/src/unstract/connectors/databases/bigquery/bigquery.py b/unstract/connectors/src/unstract/connectors/databases/bigquery/bigquery.pyindex 9b265a624..c9cdfb982 100644--- a/unstract/connectors/src/unstract/connectors/databases/bigquery/bigquery.py+++ b/unstract/connectors/src/unstract/connectors/databases/bigquery/bigquery.py@@ -5,12 +5,15 @@ from typing import Anyfrom google.cloud import bigqueryfrom google.cloud.bigquery import Clientfrom unstract.connectors.databases.unstract_db import UnstractDB+from unstract.connectors.exceptions import ConnectorErrorclass BigQuery(UnstractDB):def __init__(self, settings: dict[str, Any]):super().__init__("BigQuery")- self.json_credentials = json.loads(settings.get("json_credentials", "{}"))+ self.json_credentials = json.loads(+ settings.get("json_credentials", "{}")+ )@staticmethoddef get_id() -> str:@@ -47,7 +50,13 @@ class BigQuery(UnstractDB):return Truedef get_engine(self) -> Client:- con: Client = bigquery.Client.from_service_account_info( # type: ignore+ return bigquery.Client.from_service_account_info( # type: ignoreinfo=self.json_credentials)- return con++ def execute(self, query: str) -> Any:+ try:+ query_job = self.get_engine().query(query)+ return query_job.result()+ except Exception as e:+ raise ConnectorError(str(e))diff --git a/unstract/connectors/src/unstract/connectors/databases/bigquery/static/json_schema.json b/unstract/connectors/src/unstract/connectors/databases/bigquery/static/json_schema.jsonindex e80977752..4148b9aa1 100644--- a/unstract/connectors/src/unstract/connectors/databases/bigquery/static/json_schema.json+++ b/unstract/connectors/src/unstract/connectors/databases/bigquery/static/json_schema.json@@ -14,6 +14,7 @@},"json_credentials": {"type": "string",+ "format": "password","title": "Credentials","default": "","description": "You can download credentials from Google Clouds's iam-admin/serviceaccounts/Keys. Click on Add keys."diff --git a/unstract/connectors/src/unstract/connectors/databases/postgresql/static/json_schema.json b/unstract/connectors/src/unstract/connectors/databases/postgresql/static/json_schema.jsonindex 7812bc459..764ca00aa 100644--- a/unstract/connectors/src/unstract/connectors/databases/postgresql/static/json_schema.json+++ b/unstract/connectors/src/unstract/connectors/databases/postgresql/static/json_schema.json@@ -2,59 +2,79 @@"title": "PostgreSQL database","description": "Designed with a patented new architecture to handle all aspects of data and analytics","type": "object",- "required": [- "connectorName",- "connection_url"- ],- "properties": {- "connectorName": {- "type": "string",- "title": "Name of the connector",- "default": "Unstract's PostgreSQL connector"- },- "connection_url": {- "type": "string",- "title": "ConnectionURL",- "description": "postgresql://username:password@localhost:5432/mydatabase",- "default": ""- },- "host": {- "type": "string",- "title": "Host"- }- },- "dependencies": {- "host":{- "properties":{- "port": {- "type": "string",- "title": "Port",- "default": "5432"- },- "database": {- "type": "string",- "title": "Database",- "default": "postgres"- },- "schema": {- "type": "string",- "title": "Schema",- "description": "Default schema: public (Optional)",- "placeholder": "Enter schema (optional)"- },- "user": {+ "allOf": [+ {+ "required": [+ "connectorName"+ ],+ "properties": {+ "connectorName": {"type": "string",- "title": "User",- "default": "postgres"+ "title": "Name of the connector",+ "default": "Unstract's PostgreSQL connector"+ }+ }+ },+ {+ "oneOf": [+ {+ "title": "Postgres URL",+ "properties": {+ "connection_url": {+ "type": "string",+ "title": "Connection URL",+ "description": "postgresql://username:password@localhost:5432/mydatabase",+ "default": ""+ }+ },+ "required": [+ "connection_url"+ ]},- "password": {- "type": "string",- "title": "Password",- "format": "password",- "default": ""+ {+ "title": "Postgres Params",+ "properties": {+ "host": {+ "type": "string",+ "title": "Host"+ },+ "port": {+ "type": "string",+ "title": "Port",+ "default": "5432"+ },+ "database": {+ "type": "string",+ "title": "Database",+ "default": "postgres"+ },+ "schema": {+ "type": "string",+ "title": "Schema",+ "description": "Default schema: public (optional)",+ "placeholder": "Enter schema (optional)"+ },+ "user": {+ "type": "string",+ "title": "User",+ "default": "postgres"+ },+ "password": {+ "type": "string",+ "title": "Password",+ "format": "password",+ "default": ""+ }+ },+ "required": [+ "host",+ "port",+ "database",+ "user",+ "password"+ ]}- },- "required": ["port", "database", "user", "password"]+ ]}- }+ ]}diff --git a/unstract/connectors/src/unstract/connectors/databases/snowflake/static/json_schema.json b/unstract/connectors/src/unstract/connectors/databases/snowflake/static/json_schema.jsonindex 53d677c5d..c3870457f 100644--- a/unstract/connectors/src/unstract/connectors/databases/snowflake/static/json_schema.json+++ b/unstract/connectors/src/unstract/connectors/databases/snowflake/static/json_schema.json@@ -9,7 +9,8 @@"account","database","schema",- "warehouse"+ "warehouse",+ "role"],"properties": {"connectorName": {diff --git a/unstract/connectors/src/unstract/connectors/databases/unstract_db.py b/unstract/connectors/src/unstract/connectors/databases/unstract_db.pyindex f35edd62e..af3435d8a 100644--- a/unstract/connectors/src/unstract/connectors/databases/unstract_db.py+++ b/unstract/connectors/src/unstract/connectors/databases/unstract_db.py@@ -69,3 +69,11 @@ class UnstractDB(UnstractConnector, ABC):except Exception as e:raise ConnectorError(str(e))return True++ def execute(self, query: str) -> Any:+ try:+ with self.get_engine().cursor() as cursor:+ cursor.execute(query)+ return cursor.fetchall()+ except Exception as e:+ raise ConnectorError(str(e))diff --git a/unstract/connectors/src/unstract/connectors/filesystems/azure_cloud_storage/__init__.py b/unstract/connectors/src/unstract/connectors/filesystems/azure_cloud_storage/__init__.pynew file mode 100644index 000000000..c1982a9ac--- /dev/null+++ b/unstract/connectors/src/unstract/connectors/filesystems/azure_cloud_storage/__init__.py@@ -0,0 +1,12 @@+from .azure_cloud_storage import AzureCloudStorageFS++__all__ = ["AzureCloudStorageFS"]+++metadata = {+ "name": AzureCloudStorageFS.__name__,+ "version": "1.0.0",+ "connector": AzureCloudStorageFS,+ "description": "AzureCloudStorageFS connector",+ "is_active": True,+}diff --git a/unstract/connectors/src/unstract/connectors/filesystems/azure_cloud_storage/azure_cloud_storage.py b/unstract/connectors/src/unstract/connectors/filesystems/azure_cloud_storage/azure_cloud_storage.pynew file mode 100644index 000000000..45a2ce1f7--- /dev/null+++ b/unstract/connectors/src/unstract/connectors/filesystems/azure_cloud_storage/azure_cloud_storage.py@@ -0,0 +1,76 @@+import logging+import os+from typing import Any++from adlfs import AzureBlobFileSystem++from unstract.connectors.exceptions import ConnectorError+from unstract.connectors.filesystems.unstract_file_system import (+ UnstractFileSystem,+)++logger = logging.getLogger(__name__)+++class AzureCloudStorageFS(UnstractFileSystem):+ def __init__(self, settings: dict[str, Any]):+ super().__init__("AzureCloudStorageFS")+ account_name = settings.get("account_name", "")+ access_key = settings.get("access_key", "")+ self.bucket = settings.get("bucket", "")+ self.azure_fs = AzureBlobFileSystem(+ account_name=account_name, credential=access_key+ )++ @staticmethod+ def get_id() -> str:+ return "azure_cloud_storage|1476a54a-ed17-4a01-9f8f-cb7e4cf91c8a"++ @staticmethod+ def get_name() -> str:+ return "Azure Cloud Storage"++ @staticmethod+ def get_description() -> str:+ return "Access files in your Azure Cloud Storage"++ @staticmethod+ def get_icon() -> str:+ return (+ "https://storage.googleapis.com/"+ "pandora-static/connector-icons/azure_blob_storage.png"+ )++ @staticmethod+ def get_json_schema() -> str:+ f = open(f"{os.path.dirname(__file__)}/static/json_schema.json")+ schema = f.read()+ f.close()+ return schema++ @staticmethod+ def requires_oauth() -> bool:+ return False++ @staticmethod+ def python_social_auth_backend() -> str:+ return ""++ @staticmethod+ def can_write() -> bool:+ return True++ @staticmethod+ def can_read() -> bool:+ return True++ def get_fsspec_fs(self) -> AzureBlobFileSystem:+ return self.azure_fs++ def test_credentials(self) -> bool:+ """To test credentials for Azure Cloud Storage."""+ try:+ self.get_fsspec_fs().ls(f"{self.bucket}")+ except Exception as e:+ raise ConnectorError(str(e))+ return Truediff --git a/unstract/connectors/src/unstract/connectors/filesystems/azure_cloud_storage/static/json_schema.json b/unstract/connectors/src/unstract/connectors/filesystems/azure_cloud_storage/static/json_schema.jsonnew file mode 100644index 000000000..4e3ec1fc8--- /dev/null+++ b/unstract/connectors/src/unstract/connectors/filesystems/azure_cloud_storage/static/json_schema.json@@ -0,0 +1,36 @@+{+ "title": "Azure Cloud Storage",+ "description": "Access files in your Azure Cloud Storage with this connector",+ "type": "object",+ "required": [+ "connectorName",+ "account_name",+ "access_key",+ "bucket"+ ],+ "properties": {+ "connectorName": {+ "type": "string",+ "title": "Name of the connector",+ "default": "Unstract's Azure cloud storage"+ },+ "account_name": {+ "type": "string",+ "title": "Account Name",+ "description": "Account name of azure cloud storage",+ "default":""+ },+ "access_key": {+ "type": "string",+ "title": "Access Key",+ "format": "password",+ "description": "Access key of the corresponding account",+ "default":""+ },+ "bucket": {+ "type": "string",+ "title": "Bucket Name",+ "description": "Name of the bucket to be restricted to."+ }+ }+}diff --git a/unstract/connectors/src/unstract/connectors/filesystems/google_cloud_storage/__init__.py b/unstract/connectors/src/unstract/connectors/filesystems/google_cloud_storage/__init__.pynew file mode 100644index 000000000..c1eaeb26b--- /dev/null+++ b/unstract/connectors/src/unstract/connectors/filesystems/google_cloud_storage/__init__.py@@ -0,0 +1,12 @@+from .google_cloud_storage import GoogleCloudStorageFS++__all__ = ["GoogleCloudStorageFS"]+++metadata = {+ "name": GoogleCloudStorageFS.__name__,+ "version": "1.0.0",+ "connector": GoogleCloudStorageFS,+ "description": "GoogleCloudStorageFS connector",+ "is_active": True,+}diff --git a/unstract/connectors/src/unstract/connectors/filesystems/google_cloud_storage/google_cloud_storage.py b/unstract/connectors/src/unstract/connectors/filesystems/google_cloud_storage/google_cloud_storage.pynew file mode 100644index 000000000..49f27e5c9--- /dev/null+++ b/unstract/connectors/src/unstract/connectors/filesystems/google_cloud_storage/google_cloud_storage.py@@ -0,0 +1,75 @@+import json+import logging+import os+from typing import Any++from gcsfs import GCSFileSystem++from unstract.connectors.exceptions import ConnectorError+from unstract.connectors.filesystems.unstract_file_system import (+ UnstractFileSystem,+)++logger = logging.getLogger(__name__)+++class GoogleCloudStorageFS(UnstractFileSystem):+ def __init__(self, settings: dict[str, Any]):+ super().__init__("GoogleCloudStorage")+ self.bucket = settings.get("bucket", "")+ project_id = settings.get("project_id", "")+ json_credentials = json.loads(settings.get("json_credentials", "{}"))+ self.gcs_fs = GCSFileSystem(token=json_credentials, project=project_id)++ @staticmethod+ def get_id() -> str:+ return "google_cloud_storage|109bbe7b-8861-45eb-8841-7244e833d97b"++ @staticmethod+ def get_name() -> str:+ return "Google Cloud Storage"++ @staticmethod+ def get_description() -> str:+ return "Access files in your Google Cloud Storage"++ @staticmethod+ def get_icon() -> str:+ return (+ "https://storage.googleapis.com/"+ "pandora-static/connector-icons/google_cloud_storage.png"+ )++ @staticmethod+ def get_json_schema() -> str:+ f = open(f"{os.path.dirname(__file__)}/static/json_schema.json")+ schema = f.read()+ f.close()+ return schema++ @staticmethod+ def requires_oauth() -> bool:+ return False++ @staticmethod+ def python_social_auth_backend() -> str:+ return ""++ @staticmethod+ def can_write() -> bool:+ return True++ @staticmethod+ def can_read() -> bool:+ return True++ def get_fsspec_fs(self) -> GCSFileSystem:+ return self.gcs_fs++ def test_credentials(self) -> bool:+ """To test credentials for Google Cloud Storage."""+ try:+ is_dir = bool(self.get_fsspec_fs().isdir(f"{self.bucket}"))+ return is_dir+ except Exception as e:+ raise ConnectorError(str(e))diff --git a/unstract/connectors/src/unstract/connectors/filesystems/google_cloud_storage/static/json_schema.json b/unstract/connectors/src/unstract/connectors/filesystems/google_cloud_storage/static/json_schema.jsonnew file mode 100644index 000000000..6f375f9e0--- /dev/null+++ b/unstract/connectors/src/unstract/connectors/filesystems/google_cloud_storage/static/json_schema.json@@ -0,0 +1,35 @@+{+ "title": "Google Cloud Storage",+ "description": "Access files in your Google Cloud Storage with this connector",+ "type": "object",+ "required": [+ "connectorName",+ "json_credentials",+ "project_id",+ "bucket"+ ],+ "properties": {+ "connectorName": {+ "type": "string",+ "title": "Name of the connector",+ "default": "Unstract's google cloud storage"+ },+ "json_credentials": {+ "type": "string",+ "title": "Credentials",+ "format": "password",+ "default": "",+ "description": "You can download credentials from Google Clouds's iam-admin/serviceaccounts/Keys. Click on Add keys."+ },+ "project_id": {+ "type": "string",+ "title": "Project ID",+ "description": "Name of the project ID."+ },+ "bucket": {+ "type": "string",+ "title": "Bucket Name",+ "description": "Name of the bucket to be restricted to."+ }+ }+}diff --git a/unstract/connectors/src/unstract/connectors/filesystems/google_drive/static/json_schema.json b/unstract/connectors/src/unstract/connectors/filesystems/google_drive/static/json_schema.jsonindex 7a4e8545e..79dc11fbd 100644--- a/unstract/connectors/src/unstract/connectors/filesystems/google_drive/static/json_schema.json+++ b/unstract/connectors/src/unstract/connectors/filesystems/google_drive/static/json_schema.json@@ -8,6 +8,7 @@"properties": {"connectorName": {"type": "string",+ "format": "password","title": "Name of the connector","default": "Unstract's google drive"}diff --git a/unstract/connectors/src/unstract/connectors/filesystems/zs_dropbox/exceptions.py b/unstract/connectors/src/unstract/connectors/filesystems/zs_dropbox/exceptions.pyindex b1d65a550..0c0d9cf2e 100644--- a/unstract/connectors/src/unstract/connectors/filesystems/zs_dropbox/exceptions.py+++ b/unstract/connectors/src/unstract/connectors/filesystems/zs_dropbox/exceptions.py@@ -6,8 +6,7 @@ from unstract.connectors.exceptions import ConnectorErrordef handle_dropbox_exception(e: DropboxException) -> ConnectorError:- message = f"Error with the Dropbox connector: {e}"- user_msg = None+ user_msg = ""if isinstance(e, ExcAuthError):if isinstance(e.error, AuthError):if e.error.is_expired_access_token():@@ -19,4 +18,4 @@ def handle_dropbox_exception(e: DropboxException) -> ConnectorError:elif isinstance(e, ApiError):if e.user_message_text is not None:user_msg = e.user_message_text- return ConnectorError(message=message, user_message=user_msg)+ return ConnectorError(message=user_msg, treat_as_user_message=True)diff --git a/unstract/connectors/src/unstract/connectors/filesystems/zs_dropbox/zs_dropbox.py b/unstract/connectors/src/unstract/connectors/filesystems/zs_dropbox/zs_dropbox.pyindex 53c321e37..36e1476c9 100644--- a/unstract/connectors/src/unstract/connectors/filesystems/zs_dropbox/zs_dropbox.py+++ b/unstract/connectors/src/unstract/connectors/filesystems/zs_dropbox/zs_dropbox.py@@ -5,7 +5,9 @@ from typing import Anyfrom dropbox.exceptions import DropboxExceptionfrom dropboxdrivefs import DropboxDriveFileSystemfrom unstract.connectors.exceptions import ConnectorError-from unstract.connectors.filesystems.unstract_file_system import UnstractFileSystem+from unstract.connectors.filesystems.unstract_file_system import (+ UnstractFileSystem,+)from .exceptions import handle_dropbox_exception@@ -34,7 +36,8 @@ class DropboxFS(UnstractFileSystem):def get_icon() -> str:# TODO: Add an icon to GCS and serve itreturn (- "https://storage.googleapis.com/pandora-static/connector-icons/Dropbox.png"+ "https://storage.googleapis.com"+ "/pandora-static/connector-icons/Dropbox.png")@staticmethod