Skip to content

LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows

Feature/connector integration (#23)

Kirtiman Mishraafa50665be69
diff --git a/backend/pdm.lock b/backend/pdm.lock
index 77a03ccbb..1676964ad 100644
--- a/backend/pdm.lock
+++ b/backend/pdm.lock
@@ -7,6 +7,25 @@ strategy = ["cross_platform", "inherit_metadata"]
lock_version = "4.4.1"
content_hash = "sha256:5dc2189aaa7017f49f8711bc05b6fc8a4f33ac93387b53bd3e9aff95a264c33f"
+[[package]]
+name = "adlfs"
+version = "2023.8.0"
+requires_python = ">=3.8"
+summary = "Access Azure Datalake Gen1 with fsspec and dask"
+groups = ["default"]
+dependencies = [
+ "aiohttp>=3.7.0",
+ "azure-core<2.0.0,>=1.23.1",
+ "azure-datalake-store<0.1,>=0.0.46",
+ "azure-identity",
+ "azure-storage-blob>=12.12.0",
+ "fsspec>=2021.10.1",
+]
+files = [
+ {file = "adlfs-2023.8.0-py3-none-any.whl", hash = "sha256:3eb248a3c2a30b419f1147bd7676d156b5219f96ef7f11d47166afd2a3bdb07e"},
+ {file = "adlfs-2023.8.0.tar.gz", hash = "sha256:07e804f6df4593acfcaf01025b162e30ac13e523d3570279c98b2d91a18026d9"},
+]
+
[[package]]
name = "aiobotocore"
version = "2.5.4"
@@ -385,6 +404,71 @@ files = [
{file = "Authlib-1.2.1.tar.gz", hash = "sha256:421f7c6b468d907ca2d9afede256f068f87e34d23dd221c07d13d4c234726afb"},
]
+[[package]]
+name = "azure-core"
+version = "1.30.0"
+requires_python = ">=3.7"
+summary = "Microsoft Azure Core Library for Python"
+groups = ["default"]
+dependencies = [
+ "requests>=2.21.0",
+ "six>=1.11.0",
+ "typing-extensions>=4.6.0",
+]
+files = [
+ {file = "azure-core-1.30.0.tar.gz", hash = "sha256:6f3a7883ef184722f6bd997262eddaf80cfe7e5b3e0caaaf8db1695695893d35"},
+ {file = "azure_core-1.30.0-py3-none-any.whl", hash = "sha256:3dae7962aad109610e68c9a7abb31d79720e1d982ddf61363038d175a5025e89"},
+]
+
+[[package]]
+name = "azure-datalake-store"
+version = "0.0.53"
+summary = "Azure Data Lake Store Filesystem Client Library for Python"
+groups = ["default"]
+dependencies = [
+ "cffi",
+ "msal<2,>=1.16.0",
+ "requests>=2.20.0",
+]
+files = [
+ {file = "azure-datalake-store-0.0.53.tar.gz", hash = "sha256:05b6de62ee3f2a0a6e6941e6933b792b800c3e7f6ffce2fc324bc19875757393"},
+ {file = "azure_datalake_store-0.0.53-py2.py3-none-any.whl", hash = "sha256:a30c902a6e360aa47d7f69f086b426729784e71c536f330b691647a51dc42b2b"},
+]
+
+[[package]]
+name = "azure-identity"
+version = "1.15.0"
+requires_python = ">=3.7"
+summary = "Microsoft Azure Identity Library for Python"
+groups = ["default"]
+dependencies = [
+ "azure-core<2.0.0,>=1.23.0",
+ "cryptography>=2.5",
+ "msal-extensions<2.0.0,>=0.3.0",
+ "msal<2.0.0,>=1.24.0",
+]
+files = [
+ {file = "azure-identity-1.15.0.tar.gz", hash = "sha256:4c28fc246b7f9265610eb5261d65931183d019a23d4b0e99357facb2e6c227c8"},
+ {file = "azure_identity-1.15.0-py3-none-any.whl", hash = "sha256:a14b1f01c7036f11f148f22cd8c16e05035293d714458d6b44ddf534d93eb912"},
+]
+
+[[package]]
+name = "azure-storage-blob"
+version = "12.19.0"
+requires_python = ">=3.7"
+summary = "Microsoft Azure Blob Storage Client Library for Python"
+groups = ["default"]
+dependencies = [
+ "azure-core<2.0.0,>=1.28.0",
+ "cryptography>=2.1.4",
+ "isodate>=0.6.1",
+ "typing-extensions>=4.3.0",
+]
+files = [
+ {file = "azure-storage-blob-12.19.0.tar.gz", hash = "sha256:26c0a4320a34a3c2a1b74528ba6812ebcb632a04cd67b1c7377232c4b01a5897"},
+ {file = "azure_storage_blob-12.19.0-py3-none-any.whl", hash = "sha256:7bbc2c9c16678f7a420367fef6b172ba8730a7e66df7f4d7a55d5b3c8216615b"},
+]
+
[[package]]
name = "backoff"
version = "2.2.1"
@@ -798,6 +882,17 @@ files = [
{file = "dataclasses_json-0.6.4.tar.gz", hash = "sha256:73696ebf24936560cca79a2430cbc4f3dd23ac7bf46ed17f38e5e5e7657a6377"},
]
+[[package]]
+name = "decorator"
+version = "5.1.1"
+requires_python = ">=3.5"
+summary = "Decorators for Humans"
+groups = ["default"]
+files = [
+ {file = "decorator-5.1.1-py3-none-any.whl", hash = "sha256:b8c3f85900b9dc423225913c5aace94729fe1fa9763b38939a95226f02d37186"},
+ {file = "decorator-5.1.1.tar.gz", hash = "sha256:637996211036b6385ef91435e4fae22989472f9d571faba8927ba8253acbc330"},
+]
+
[[package]]
name = "defusedxml"
version = "0.8.0rc2"
@@ -1229,6 +1324,26 @@ files = [
{file = "funcy-2.0.tar.gz", hash = "sha256:3963315d59d41c6f30c04bc910e10ab50a3ac4a225868bfa96feed133df075cb"},
]
+[[package]]
+name = "gcsfs"
+version = "2023.6.0"
+requires_python = ">=3.8"
+summary = "Convenient Filesystem interface over GCS"
+groups = ["default"]
+dependencies = [
+ "aiohttp!=4.0.0a0,!=4.0.0a1",
+ "decorator>4.1.2",
+ "fsspec==2023.6.0",
+ "google-auth-oauthlib",
+ "google-auth>=1.2",
+ "google-cloud-storage",
+ "requests",
+]
+files = [
+ {file = "gcsfs-2023.6.0-py2.py3-none-any.whl", hash = "sha256:3b3c7d8eddd4ec1380f3b49fbb861ee1e974adb223564401f10884b6260d406f"},
+ {file = "gcsfs-2023.6.0.tar.gz", hash = "sha256:30b14fccadb3b7f0d99b2cd03bd8507c40f3a9a7d05847edca571f642bedbdff"},
+]
+
[[package]]
name = "gitdb"
version = "4.0.11"
@@ -1359,6 +1474,21 @@ files = [
{file = "google_auth_httplib2-0.2.0-py2.py3-none-any.whl", hash = "sha256:b65a0a2123300dd71281a7bf6e64d65a0759287df52729bdd1ae2e47dc311a3d"},
]
+[[package]]
+name = "google-auth-oauthlib"
+version = "1.2.0"
+requires_python = ">=3.6"
+summary = "Google Authentication Library"
+groups = ["default"]
+dependencies = [
+ "google-auth>=2.15.0",
+ "requests-oauthlib>=0.7.0",
+]
+files = [
+ {file = "google-auth-oauthlib-1.2.0.tar.gz", hash = "sha256:292d2d3783349f2b0734a0a0207b1e1e322ac193c2c09d8f7c613fb7cc501ea8"},
+ {file = "google_auth_oauthlib-1.2.0-py2.py3-none-any.whl", hash = "sha256:297c1ce4cb13a99b5834c74a1fe03252e1e499716718b190f56bcb9c4abc4faf"},
+]
+
[[package]]
name = "google-cloud-aiplatform"
version = "1.40.0"
@@ -1984,6 +2114,19 @@ files = [
{file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"},
]
+[[package]]
+name = "isodate"
+version = "0.6.1"
+summary = "An ISO 8601 date/time/duration parser and formatter"
+groups = ["default"]
+dependencies = [
+ "six",
+]
+files = [
+ {file = "isodate-0.6.1-py2.py3-none-any.whl", hash = "sha256:0751eece944162659049d35f4f549ed815792b38793f07cf73381c1c87cbed96"},
+ {file = "isodate-0.6.1.tar.gz", hash = "sha256:48c5881de7e8b0a0d648cb024c8062dc84e7b840ed81e864c7614fd3c127bde9"},
+]
+
[[package]]
name = "jmespath"
version = "1.0.1"
@@ -2222,6 +2365,39 @@ files = [
{file = "mpmath-1.3.0.tar.gz", hash = "sha256:7a28eb2a9774d00c7bc92411c19a89209d5da7c4c9a9e227be8330a23a25b91f"},
]
+[[package]]
+name = "msal"
+version = "1.27.0"
+requires_python = ">=2.7"
+summary = "The Microsoft Authentication Library (MSAL) for Python library enables your app to access the Microsoft Cloud by supporting authentication of users with Microsoft Azure Active Directory accounts (AAD) and Microsoft Accounts (MSA) using industry standard OAuth2 and OpenID Connect."
+groups = ["default"]
+dependencies = [
+ "PyJWT[crypto]<3,>=1.0.0",
+ "cryptography<45,>=0.6",
+ "requests<3,>=2.0.0",
+]
+files = [
+ {file = "msal-1.27.0-py2.py3-none-any.whl", hash = "sha256:572d07149b83e7343a85a3bcef8e581167b4ac76befcbbb6eef0c0e19643cdc0"},
+ {file = "msal-1.27.0.tar.gz", hash = "sha256:3109503c038ba6b307152b0e8d34f98113f2e7a78986e28d0baf5b5303afda52"},
+]
+
+[[package]]
+name = "msal-extensions"
+version = "1.1.0"
+requires_python = ">=3.7"
+summary = "Microsoft Authentication Library extensions (MSAL EX) provides a persistence API that can save your data on disk, encrypted on Windows, macOS and Linux. Concurrent data access will be coordinated by a file lock mechanism."
+groups = ["default"]
+dependencies = [
+ "msal<2.0.0,>=0.4.1",
+ "packaging",
+ "portalocker<3,>=1.0; platform_system != \"Windows\"",
+ "portalocker<3,>=1.6; platform_system == \"Windows\"",
+]
+files = [
+ {file = "msal-extensions-1.1.0.tar.gz", hash = "sha256:6ab357867062db7b253d0bd2df6d411c7891a0ee7308d54d1e4317c1d1c54252"},
+ {file = "msal_extensions-1.1.0-py3-none-any.whl", hash = "sha256:01be9711b4c0b1a151450068eeb2c4f0997df3bba085ac299de3a66f585e382f"},
+]
+
[[package]]
name = "multidict"
version = "6.0.5"
@@ -3101,6 +3277,22 @@ files = [
{file = "PyJWT-2.8.0.tar.gz", hash = "sha256:57e28d156e3d5c10088e0c68abb90bfac3df82b40a71bd0daa20c65ccd5c23de"},
]
+[[package]]
+name = "pyjwt"
+version = "2.8.0"
+extras = ["crypto"]
+requires_python = ">=3.7"
+summary = "JSON Web Token implementation in Python"
+groups = ["default"]
+dependencies = [
+ "PyJWT==2.8.0",
+ "cryptography>=3.4.0",
+]
+files = [
+ {file = "PyJWT-2.8.0-py3-none-any.whl", hash = "sha256:59127c392cc44c2da5bb3192169a91f429924e17aff6534d70fdc02ab3e04320"},
+ {file = "PyJWT-2.8.0.tar.gz", hash = "sha256:57e28d156e3d5c10088e0c68abb90bfac3df82b40a71bd0daa20c65ccd5c23de"},
+]
+
[[package]]
name = "pymilvus"
version = "2.3.4"
@@ -4621,8 +4813,10 @@ groups = ["default"]
dependencies = [
"PyDrive2[fsspec]==1.15.4",
"PyMySQL==1.1.0",
+ "adlfs==2023.8.0",
"boxfs==0.2.1",
"dropboxdrivefs==1.3.1",
+ "gcsfs==2023.6.0",
"google-auth==2.20.0",
"google-cloud-bigquery==3.11.4",
"google-cloud-secret-manager==2.16.1",
diff --git a/backend/workflow_manager/endpoint/constants.py b/backend/workflow_manager/endpoint/constants.py
index c2ffe5072..061dc7a56 100644
--- a/backend/workflow_manager/endpoint/constants.py
+++ b/backend/workflow_manager/endpoint/constants.py
@@ -5,6 +5,7 @@ class TableColumns:
class DBConnectionClass:
SNOWFLAKE = "SnowflakeConnection"
+ BIGQUERY = "Client"
class Snowflake:
@@ -83,3 +84,13 @@ class SourceConstant:
class ApiDeploymentResultStatus:
SUCCESS = "Success"
FAILED = "Failed"
+
+
+class BigQuery:
+ """In big query, table name has to be in the format {db}.{schema}.{table}
+ Throws error if any of the params not set.
+
+ When converted to list table size should be 3
+ """
+
+ TABLE_NAME_SIZE = 3
diff --git a/backend/workflow_manager/endpoint/database_utils.py b/backend/workflow_manager/endpoint/database_utils.py
index 234493bda..ad59188c2 100644
--- a/backend/workflow_manager/endpoint/database_utils.py
+++ b/backend/workflow_manager/endpoint/database_utils.py
@@ -3,16 +3,19 @@ import json
import logging
from typing import Any, Optional
-from unstract.connectors.databases import connectors as db_connectors
-from unstract.connectors.databases.unstract_db import UnstractDB
from utils.constants import Common
from workflow_manager.endpoint.constants import (
+ BigQuery,
DBConnectionClass,
Snowflake,
TableColumns,
)
+from workflow_manager.endpoint.exceptions import BigQueryTableNotFound
from workflow_manager.workflow.enums import AgentName, ColumnModes
+from unstract.connectors.databases import connectors as db_connectors
+from unstract.connectors.databases.unstract_db import UnstractDB
+
logger = logging.getLogger(__name__)
@@ -60,58 +63,84 @@ class DatabaseUtils:
return sql_values
@staticmethod
- def get_column_types(
- engine: Any, table_name: str, cls: Any = None
- ) -> dict[str, str]:
- """Retrieve column types for a specified table from a database engine.
+ def get_column_types_util(columns_with_types: Any) -> dict[str, str]:
+ """Converts db results columns_with_types to dict.
Args:
- engine (Any): The database engine used to execute queries.
- table_name (str): The name of the table for which column types
- are retrieved.
- cls (Any, optional): The database connection class (e.g.,
- DBConnectionClass.SNOWFLAKE) for handling database-specific
- queries.
- Defaults to None.
+ columns_with_types (Any): _description_
Returns:
- dict: A dictionary mapping column names to their respective data
- types.
+ dict[str, str]: _description_
+ """
+ column_types: dict[str, str] = {}
+ for column_name, data_type in columns_with_types:
+ column_types[column_name] = data_type
+ return column_types
+
+ @staticmethod
+ def get_column_types(
+ cls: Any,
+ table_name: str,
+ connector_id: str,
+ connector_settings: dict[str, Any],
+ ) -> Any:
+ """Get db column name and types.
+
+ Args:
+ cls (Any): _description_
+ table_name (str): _description_
+ connector_id (str): _description_
+ connector_settings (dict[str, Any]): _description_
Raises:
- Exception: If there is an error while retrieving column types,
- an exception is raised. Exit.
+ ValueError: _description_
+ e: _description_
- Note:
- - If `cls` is not provided or is None, the function assumes a
- Default SQL database and queries column types accordingly.
- - If `cls` is provided and matches DBConnectionClass.SNOWFLAKE,
- the function queries column types using Snowflake-specific
- syntax.
+ Returns:
+ Any: _description_
"""
+ column_types: dict[str, str] = {}
try:
- column_types: dict[str, str] = {}
- with engine.cursor() as cursor:
- if cls == DBConnectionClass.SNOWFLAKE:
- results = cursor.execute(f"describe table {table_name}")
- for column in results:
- column_types[column[0].lower()] = column[1].split("(")[
- 0
- ]
- else:
- # Default to Other SQL DBs
- # Postgresql treats the table names as in lowercase
- # tested only with Postgresql
- table_name = str.lower(table_name)
- columns_with_types_query = (
- "SELECT column_name, data_type FROM "
- "information_schema.columns WHERE "
- f"table_name = '{table_name}'"
- )
- cursor.execute(columns_with_types_query)
- columns_with_types = cursor.fetchall()
- for column_name, data_type in columns_with_types:
- column_types[column_name] = data_type
+ if cls == DBConnectionClass.SNOWFLAKE:
+ query = f"describe table {table_name}"
+ results = DatabaseUtils.execute_and_fetch_data(
+ connector_id=connector_id,
+ connector_settings=connector_settings,
+ query=query,
+ )
+ for column in results:
+ column_types[column[0].lower()] = column[1].split("(")[0]
+ elif cls == DBConnectionClass.BIGQUERY:
+ bigquery_table_name = str.lower(table_name).split(".")
+ if len(bigquery_table_name) != BigQuery.TABLE_NAME_SIZE:
+ raise BigQueryTableNotFound()
+ database = bigquery_table_name[0]
+ schema = bigquery_table_name[1]
+ table = bigquery_table_name[2]
+ query = (
+ "SELECT column_name, data_type FROM "
+ f"{database}.{schema}.INFORMATION_SCHEMA.COLUMNS WHERE "
+ f"table_name = '{table}'"
+ )
+ results = DatabaseUtils.execute_and_fetch_data(
+ connector_id=connector_id,
+ connector_settings=connector_settings,
+ query=query,
+ )
+ column_types = DatabaseUtils.get_column_types_util(results)
+ else:
+ table_name = str.lower(table_name)
+ query = (
+ "SELECT column_name, data_type FROM "
+ "information_schema.columns WHERE "
+ f"table_name = '{table_name}'"
+ )
+ results = DatabaseUtils.execute_and_fetch_data(
+ connector_id=connector_id,
+ connector_settings=connector_settings,
+ query=query,
+ )
+ column_types = DatabaseUtils.get_column_types_util(results)
except Exception as e:
logger.error(
f"Error getting column types for {table_name}: {str(e)}"
@@ -180,13 +209,19 @@ class DatabaseUtils:
@staticmethod
def get_sql_values_for_query(
- engine: Any, table_name: str, values: dict[str, Any]
+ engine: Any,
+ connector_id: str,
+ connector_settings: dict[str, Any],
+ table_name: str,
+ values: dict[str, Any],
) -> list[str]:
"""Generate SQL values for an insert query based on the provided values
and table schema.
Args:
engine (Any): The database engine.
+ connector_id: The connector id of the connector provided
+ connector_settings: Connector settings provided by user
table_name (str): The name of the target table for the insert query.
values (dict[str, Any]): A dictionary containing column-value pairs
for the insert query.
@@ -202,24 +237,20 @@ class DatabaseUtils:
- For other SQL databases, it uses default SQL generation
based on column types.
"""
-
- if engine.__class__.__name__ == DBConnectionClass.SNOWFLAKE:
- # Handle Snowflake
- column_types: dict[str, str] = DatabaseUtils.get_column_types(
- engine=engine,
- table_name=table_name,
- cls=DBConnectionClass.SNOWFLAKE,
- )
+ class_name = engine.__class__.__name__
+ column_types: dict[str, str] = DatabaseUtils.get_column_types(
+ cls=class_name,
+ table_name=table_name,
+ connector_id=connector_id,
+ connector_settings=connector_settings,
+ )
+ if class_name == DBConnectionClass.SNOWFLAKE:
sql_values = DatabaseUtils.make_sql_values_for_query(
values=values,
column_types=column_types,
cls=DBConnectionClass.SNOWFLAKE,
)
else:
- # Default to Other SQL DBs
- column_types = DatabaseUtils.get_column_types(
- engine=engine, table_name=table_name
- )
sql_values = DatabaseUtils.make_sql_values_for_query(
values=values, column_types=column_types
)
@@ -248,11 +279,13 @@ class DatabaseUtils:
f"INSERT INTO {table_name} ({','.join(sql_keys)}) "
f"SELECT {','.join(sql_values)}"
)
-
try:
- with engine.cursor() as cursor:
- cursor.execute(sql)
- engine.commit()
+ if hasattr(engine, "cursor"):
+ with engine.cursor() as cursor:
+ cursor.execute(sql)
+ engine.commit()
+ else:
+ engine.query(sql)
except Exception as e:
logger.error(f"Error while writing data: {str(e)}")
raise e
@@ -266,3 +299,13 @@ class DatabaseUtils:
]
connector_class: UnstractDB = connector(connector_settings)
return connector_class.get_engine()
+
+ @staticmethod
+ def execute_and_fetch_data(
+ connector_id: str, connector_settings: dict[str, Any], query: str
+ ) -> Any:
+ connector = db_connectors[connector_id][Common.METADATA][
+ Common.CONNECTOR
+ ]
+ connector_class: UnstractDB = connector(connector_settings)
+ return connector_class.execute(query=query)
diff --git a/backend/workflow_manager/endpoint/destination.py b/backend/workflow_manager/endpoint/destination.py
index cdaae988b..dcbe52342 100644
--- a/backend/workflow_manager/endpoint/destination.py
+++ b/backend/workflow_manager/endpoint/destination.py
@@ -223,7 +223,11 @@ class DestinationConnector(BaseConnector):
connector_settings=connector_settings,
)
sql_values = DatabaseUtils.get_sql_values_for_query(
- engine=engine, table_name=table_name, values=values
+ engine=engine,
+ connector_id=connector_instance.connector_id,
+ connector_settings=connector_settings,
+ table_name=table_name,
+ values=values,
)
DatabaseUtils.execute_write_query(
diff --git a/backend/workflow_manager/endpoint/exceptions.py b/backend/workflow_manager/endpoint/exceptions.py
index 03e82ea06..b4590503b 100644
--- a/backend/workflow_manager/endpoint/exceptions.py
+++ b/backend/workflow_manager/endpoint/exceptions.py
@@ -61,3 +61,11 @@ class ToolOutputTypeMismatch(APIException):
default_detail = (
"The data type of the tool's output does not match the expected type."
)
+
+
+class BigQueryTableNotFound(APIException):
+ status_code = 400
+ default_detail = (
+ "Please enter correct correct bigquery table in the form "
+ "{table}.{schema}.{database}."
+ )
diff --git a/backend/workflow_manager/endpoint/source.py b/backend/workflow_manager/endpoint/source.py
index 6dfe41e61..973655457 100644
--- a/backend/workflow_manager/endpoint/source.py
+++ b/backend/workflow_manager/endpoint/source.py
@@ -162,7 +162,10 @@ class SourceConnector(BaseConnector):
input_directory = str(
source_configurations.get(SourceKey.ROOT_FOLDER, "")
)
- input_directory = str(Path(root_dir_path, input_directory.lstrip("/")))
+ if root_dir_path: # user needs to manually type the optional file path
+ input_directory = str(
+ Path(root_dir_path, input_directory.lstrip("/"))
+ )
if not isinstance(required_patterns, list):
required_patterns = [required_patterns]
diff --git a/frontend/src/components/agency/configure-connector-modal/ConfigureConnectorModal.jsx b/frontend/src/components/agency/configure-connector-modal/ConfigureConnectorModal.jsx
index 614956d83..49e9cc959 100644
--- a/frontend/src/components/agency/configure-connector-modal/ConfigureConnectorModal.jsx
+++ b/frontend/src/components/agency/configure-connector-modal/ConfigureConnectorModal.jsx
@@ -21,9 +21,9 @@ function ConfigureConnectorModal({
formDataConfig,
setFormDataConfig,
isSpecConfigLoading,
+ connType,
}) {
const [activeKey, setActiveKey] = useState("1");
-
const tabItems = [
{
key: "1",
@@ -32,7 +32,7 @@ function ConfigureConnectorModal({
{
key: "2",
label: "File System",
- disabled: !connectorId,
+ disabled: !connectorId || connType === "DATABASE",
},
];
@@ -112,6 +112,7 @@ ConfigureConnectorModal.propTypes = {
formDataConfig: PropTypes.object,
setFormDataConfig: PropTypes.func.isRequired,
isSpecConfigLoading: PropTypes.bool.isRequired,
+ connType: PropTypes.string.isRequired,
};
export { ConfigureConnectorModal };
diff --git a/frontend/src/components/agency/ds-settings-card/DsSettingsCard.jsx b/frontend/src/components/agency/ds-settings-card/DsSettingsCard.jsx
index fdce7fac5..5a24ed3e8 100644
--- a/frontend/src/components/agency/ds-settings-card/DsSettingsCard.jsx
+++ b/frontend/src/components/agency/ds-settings-card/DsSettingsCard.jsx
@@ -330,7 +330,7 @@ function DsSettingsCard({ type, endpointDetails, message }) {
</Tooltip>
</Space>
<div className="display-flex-align-center">
- {connDetails?.icon ? (
+ {connDetails?.connector_name ? (
<Space>
<Image
src={connDetails?.icon}
@@ -396,6 +396,7 @@ function DsSettingsCard({ type, endpointDetails, message }) {
formDataConfig={formDataConfig}
setFormDataConfig={setFormDataConfig}
isSpecConfigLoading={isSpecConfigLoading}
+ connType={connType}
/>
</>
);
diff --git a/pdm.lock b/pdm.lock
index 1b75a9b64..c07caf241 100644
--- a/pdm.lock
+++ b/pdm.lock
@@ -18,6 +18,25 @@ files = [
{file = "absolufy_imports-0.3.1.tar.gz", hash = "sha256:c90638a6c0b66826d1fb4880ddc20ef7701af34192c94faf40b95d32b59f9793"},
]
+[[package]]
+name = "adlfs"
+version = "2023.8.0"
+requires_python = ">=3.8"
+summary = "Access Azure Datalake Gen1 with fsspec and dask"
+groups = ["hook-check-django-migrations"]
+dependencies = [
+ "aiohttp>=3.7.0",
+ "azure-core<2.0.0,>=1.23.1",
+ "azure-datalake-store<0.1,>=0.0.46",
+ "azure-identity",
+ "azure-storage-blob>=12.12.0",
+ "fsspec>=2021.10.1",
+]
+files = [
+ {file = "adlfs-2023.8.0-py3-none-any.whl", hash = "sha256:3eb248a3c2a30b419f1147bd7676d156b5219f96ef7f11d47166afd2a3bdb07e"},
+ {file = "adlfs-2023.8.0.tar.gz", hash = "sha256:07e804f6df4593acfcaf01025b162e30ac13e523d3570279c98b2d91a18026d9"},
+]
+
[[package]]
name = "aiobotocore"
version = "2.5.4"
@@ -412,6 +431,71 @@ files = [
{file = "autopep8-2.0.4.tar.gz", hash = "sha256:2913064abd97b3419d1cc83ea71f042cb821f87e45b9c88cad5ad3c4ea87fe0c"},
]
+[[package]]
+name = "azure-core"
+version = "1.30.0"
+requires_python = ">=3.7"
+summary = "Microsoft Azure Core Library for Python"
+groups = ["hook-check-django-migrations"]
+dependencies = [
+ "requests>=2.21.0",
+ "six>=1.11.0",
+ "typing-extensions>=4.6.0",
+]
+files = [
+ {file = "azure-core-1.30.0.tar.gz", hash = "sha256:6f3a7883ef184722f6bd997262eddaf80cfe7e5b3e0caaaf8db1695695893d35"},
+ {file = "azure_core-1.30.0-py3-none-any.whl", hash = "sha256:3dae7962aad109610e68c9a7abb31d79720e1d982ddf61363038d175a5025e89"},
+]
+
+[[package]]
+name = "azure-datalake-store"
+version = "0.0.53"
+summary = "Azure Data Lake Store Filesystem Client Library for Python"
+groups = ["hook-check-django-migrations"]
+dependencies = [
+ "cffi",
+ "msal<2,>=1.16.0",
+ "requests>=2.20.0",
+]
+files = [
+ {file = "azure-datalake-store-0.0.53.tar.gz", hash = "sha256:05b6de62ee3f2a0a6e6941e6933b792b800c3e7f6ffce2fc324bc19875757393"},
+ {file = "azure_datalake_store-0.0.53-py2.py3-none-any.whl", hash = "sha256:a30c902a6e360aa47d7f69f086b426729784e71c536f330b691647a51dc42b2b"},
+]
+
+[[package]]
+name = "azure-identity"
+version = "1.15.0"
+requires_python = ">=3.7"
+summary = "Microsoft Azure Identity Library for Python"
+groups = ["hook-check-django-migrations"]
+dependencies = [
+ "azure-core<2.0.0,>=1.23.0",
+ "cryptography>=2.5",
+ "msal-extensions<2.0.0,>=0.3.0",
+ "msal<2.0.0,>=1.24.0",
+]
+files = [
+ {file = "azure-identity-1.15.0.tar.gz", hash = "sha256:4c28fc246b7f9265610eb5261d65931183d019a23d4b0e99357facb2e6c227c8"},
+ {file = "azure_identity-1.15.0-py3-none-any.whl", hash = "sha256:a14b1f01c7036f11f148f22cd8c16e05035293d714458d6b44ddf534d93eb912"},
+]
+
+[[package]]
+name = "azure-storage-blob"
+version = "12.19.0"
+requires_python = ">=3.7"
+summary = "Microsoft Azure Blob Storage Client Library for Python"
+groups = ["hook-check-django-migrations"]
+dependencies = [
+ "azure-core<2.0.0,>=1.28.0",
+ "cryptography>=2.1.4",
+ "isodate>=0.6.1",
+ "typing-extensions>=4.3.0",
+]
+files = [
+ {file = "azure-storage-blob-12.19.0.tar.gz", hash = "sha256:26c0a4320a34a3c2a1b74528ba6812ebcb632a04cd67b1c7377232c4b01a5897"},
+ {file = "azure_storage_blob-12.19.0-py3-none-any.whl", hash = "sha256:7bbc2c9c16678f7a420367fef6b172ba8730a7e66df7f4d7a55d5b3c8216615b"},
+]
+
[[package]]
name = "backoff"
version = "2.2.1"
@@ -860,6 +944,17 @@ files = [
{file = "dataclasses_json-0.6.4.tar.gz", hash = "sha256:73696ebf24936560cca79a2430cbc4f3dd23ac7bf46ed17f38e5e5e7657a6377"},
]
+[[package]]
+name = "decorator"
+version = "5.1.1"
+requires_python = ">=3.5"
+summary = "Decorators for Humans"
+groups = ["hook-check-django-migrations"]
+files = [
+ {file = "decorator-5.1.1-py3-none-any.whl", hash = "sha256:b8c3f85900b9dc423225913c5aace94729fe1fa9763b38939a95226f02d37186"},
+ {file = "decorator-5.1.1.tar.gz", hash = "sha256:637996211036b6385ef91435e4fae22989472f9d571faba8927ba8253acbc330"},
+]
+
[[package]]
name = "defusedxml"
version = "0.8.0rc2"
@@ -1309,6 +1404,26 @@ files = [
{file = "funcy-2.0.tar.gz", hash = "sha256:3963315d59d41c6f30c04bc910e10ab50a3ac4a225868bfa96feed133df075cb"},
]
+[[package]]
+name = "gcsfs"
+version = "2023.6.0"
+requires_python = ">=3.8"
+summary = "Convenient Filesystem interface over GCS"
+groups = ["hook-check-django-migrations"]
+dependencies = [
+ "aiohttp!=4.0.0a0,!=4.0.0a1",
+ "decorator>4.1.2",
+ "fsspec==2023.6.0",
+ "google-auth-oauthlib",
+ "google-auth>=1.2",
+ "google-cloud-storage",
+ "requests",
+]
+files = [
+ {file = "gcsfs-2023.6.0-py2.py3-none-any.whl", hash = "sha256:3b3c7d8eddd4ec1380f3b49fbb861ee1e974adb223564401f10884b6260d406f"},
+ {file = "gcsfs-2023.6.0.tar.gz", hash = "sha256:30b14fccadb3b7f0d99b2cd03bd8507c40f3a9a7d05847edca571f642bedbdff"},
+]
+
[[package]]
name = "gitdb"
version = "4.0.11"
@@ -1439,6 +1554,21 @@ files = [
{file = "google_auth_httplib2-0.2.0-py2.py3-none-any.whl", hash = "sha256:b65a0a2123300dd71281a7bf6e64d65a0759287df52729bdd1ae2e47dc311a3d"},
]
+[[package]]
+name = "google-auth-oauthlib"
+version = "1.2.0"
+requires_python = ">=3.6"
+summary = "Google Authentication Library"
+groups = ["hook-check-django-migrations"]
+dependencies = [
+ "google-auth>=2.15.0",
+ "requests-oauthlib>=0.7.0",
+]
+files = [
+ {file = "google-auth-oauthlib-1.2.0.tar.gz", hash = "sha256:292d2d3783349f2b0734a0a0207b1e1e322ac193c2c09d8f7c613fb7cc501ea8"},
+ {file = "google_auth_oauthlib-1.2.0-py2.py3-none-any.whl", hash = "sha256:297c1ce4cb13a99b5834c74a1fe03252e1e499716718b190f56bcb9c4abc4faf"},
+]
+
[[package]]
name = "google-cloud-aiplatform"
version = "1.40.0"
@@ -2050,6 +2180,19 @@ files = [
{file = "inflection-0.5.1.tar.gz", hash = "sha256:1a29730d366e996aaacffb2f1f1cb9593dc38e2ddd30c91250c6dde09ea9b417"},
]
+[[package]]
+name = "isodate"
+version = "0.6.1"
+summary = "An ISO 8601 date/time/duration parser and formatter"
+groups = ["hook-check-django-migrations"]
+dependencies = [
+ "six",
+]
+files = [
+ {file = "isodate-0.6.1-py2.py3-none-any.whl", hash = "sha256:0751eece944162659049d35f4f549ed815792b38793f07cf73381c1c87cbed96"},
+ {file = "isodate-0.6.1.tar.gz", hash = "sha256:48c5881de7e8b0a0d648cb024c8062dc84e7b840ed81e864c7614fd3c127bde9"},
+]
+
[[package]]
name = "isort"
version = "5.12.0"
@@ -2310,6 +2453,39 @@ files = [
{file = "mpmath-1.3.0.tar.gz", hash = "sha256:7a28eb2a9774d00c7bc92411c19a89209d5da7c4c9a9e227be8330a23a25b91f"},
]
+[[package]]
+name = "msal"
+version = "1.27.0"
+requires_python = ">=2.7"
+summary = "The Microsoft Authentication Library (MSAL) for Python library enables your app to access the Microsoft Cloud by supporting authentication of users with Microsoft Azure Active Directory accounts (AAD) and Microsoft Accounts (MSA) using industry standard OAuth2 and OpenID Connect."
+groups = ["hook-check-django-migrations"]
+dependencies = [
+ "PyJWT[crypto]<3,>=1.0.0",
+ "cryptography<45,>=0.6",
+ "requests<3,>=2.0.0",
+]
+files = [
+ {file = "msal-1.27.0-py2.py3-none-any.whl", hash = "sha256:572d07149b83e7343a85a3bcef8e581167b4ac76befcbbb6eef0c0e19643cdc0"},
+ {file = "msal-1.27.0.tar.gz", hash = "sha256:3109503c038ba6b307152b0e8d34f98113f2e7a78986e28d0baf5b5303afda52"},
+]
+
+[[package]]
+name = "msal-extensions"
+version = "1.1.0"
+requires_python = ">=3.7"
+summary = "Microsoft Authentication Library extensions (MSAL EX) provides a persistence API that can save your data on disk, encrypted on Windows, macOS and Linux. Concurrent data access will be coordinated by a file lock mechanism."
+groups = ["hook-check-django-migrations"]
+dependencies = [
+ "msal<2.0.0,>=0.4.1",
+ "packaging",
+ "portalocker<3,>=1.0; platform_system != \"Windows\"",
+ "portalocker<3,>=1.6; platform_system == \"Windows\"",
+]
+files = [
+ {file = "msal-extensions-1.1.0.tar.gz", hash = "sha256:6ab357867062db7b253d0bd2df6d411c7891a0ee7308d54d1e4317c1d1c54252"},
+ {file = "msal_extensions-1.1.0-py3-none-any.whl", hash = "sha256:01be9711b4c0b1a151450068eeb2c4f0997df3bba085ac299de3a66f585e382f"},
+]
+
[[package]]
name = "multidict"
version = "6.0.5"
@@ -3252,6 +3428,22 @@ files = [
{file = "PyJWT-2.8.0.tar.gz", hash = "sha256:57e28d156e3d5c10088e0c68abb90bfac3df82b40a71bd0daa20c65ccd5c23de"},
]
+[[package]]
+name = "pyjwt"
+version = "2.8.0"
+extras = ["crypto"]
+requires_python = ">=3.7"
+summary = "JSON Web Token implementation in Python"
+groups = ["hook-check-django-migrations"]
+dependencies = [
+ "PyJWT==2.8.0",
+ "cryptography>=3.4.0",
+]
+files = [
+ {file = "PyJWT-2.8.0-py3-none-any.whl", hash = "sha256:59127c392cc44c2da5bb3192169a91f429924e17aff6534d70fdc02ab3e04320"},
+ {file = "PyJWT-2.8.0.tar.gz", hash = "sha256:57e28d156e3d5c10088e0c68abb90bfac3df82b40a71bd0daa20c65ccd5c23de"},
+]
+
[[package]]
name = "pymilvus"
version = "2.3.4"
@@ -4788,8 +4980,10 @@ groups = ["hook-check-django-migrations"]
dependencies = [
"PyDrive2[fsspec]==1.15.4",
"PyMySQL==1.1.0",
+ "adlfs==2023.8.0",
"boxfs==0.2.1",
"dropboxdrivefs==1.3.1",
+ "gcsfs==2023.6.0",
"google-auth==2.20.0",
"google-cloud-bigquery==3.11.4",
"google-cloud-secret-manager==2.16.1",
diff --git a/unstract/connectors/pyproject.toml b/unstract/connectors/pyproject.toml
index 727300707..abf53c275 100644
--- a/unstract/connectors/pyproject.toml
+++ b/unstract/connectors/pyproject.toml
@@ -21,6 +21,8 @@ dependencies = [
"oauth2client==4.1.3", # For GDrive
"dropboxdrivefs==1.3.1", # For Dropbox
"boxfs==0.2.1", # For Box
+ "gcsfs==2023.6.0", # For GoogleCloudStorage
+ "adlfs==2023.8.0", # For AzureCloudStorage
# Database connectors
"psycopg2-binary==2.9.9", # For Postgres, Redshift
"snowflake-connector-python[pandas]~=3.6.0", # For Snowflake
diff --git a/unstract/connectors/src/unstract/connectors/databases/bigquery/bigquery.py b/unstract/connectors/src/unstract/connectors/databases/bigquery/bigquery.py
index 9b265a624..c9cdfb982 100644
--- a/unstract/connectors/src/unstract/connectors/databases/bigquery/bigquery.py
+++ b/unstract/connectors/src/unstract/connectors/databases/bigquery/bigquery.py
@@ -5,12 +5,15 @@ from typing import Any
from google.cloud import bigquery
from google.cloud.bigquery import Client
from unstract.connectors.databases.unstract_db import UnstractDB
+from unstract.connectors.exceptions import ConnectorError
class BigQuery(UnstractDB):
def __init__(self, settings: dict[str, Any]):
super().__init__("BigQuery")
- self.json_credentials = json.loads(settings.get("json_credentials", "{}"))
+ self.json_credentials = json.loads(
+ settings.get("json_credentials", "{}")
+ )
@staticmethod
def get_id() -> str:
@@ -47,7 +50,13 @@ class BigQuery(UnstractDB):
return True
def get_engine(self) -> Client:
- con: Client = bigquery.Client.from_service_account_info( # type: ignore
+ return bigquery.Client.from_service_account_info( # type: ignore
info=self.json_credentials
)
- return con
+
+ def execute(self, query: str) -> Any:
+ try:
+ query_job = self.get_engine().query(query)
+ return query_job.result()
+ except Exception as e:
+ raise ConnectorError(str(e))
diff --git a/unstract/connectors/src/unstract/connectors/databases/bigquery/static/json_schema.json b/unstract/connectors/src/unstract/connectors/databases/bigquery/static/json_schema.json
index e80977752..4148b9aa1 100644
--- a/unstract/connectors/src/unstract/connectors/databases/bigquery/static/json_schema.json
+++ b/unstract/connectors/src/unstract/connectors/databases/bigquery/static/json_schema.json
@@ -14,6 +14,7 @@
},
"json_credentials": {
"type": "string",
+ "format": "password",
"title": "Credentials",
"default": "",
"description": "You can download credentials from Google Clouds's iam-admin/serviceaccounts/Keys. Click on Add keys."
diff --git a/unstract/connectors/src/unstract/connectors/databases/postgresql/static/json_schema.json b/unstract/connectors/src/unstract/connectors/databases/postgresql/static/json_schema.json
index 7812bc459..764ca00aa 100644
--- a/unstract/connectors/src/unstract/connectors/databases/postgresql/static/json_schema.json
+++ b/unstract/connectors/src/unstract/connectors/databases/postgresql/static/json_schema.json
@@ -2,59 +2,79 @@
"title": "PostgreSQL database",
"description": "Designed with a patented new architecture to handle all aspects of data and analytics",
"type": "object",
- "required": [
- "connectorName",
- "connection_url"
- ],
- "properties": {
- "connectorName": {
- "type": "string",
- "title": "Name of the connector",
- "default": "Unstract's PostgreSQL connector"
- },
- "connection_url": {
- "type": "string",
- "title": "ConnectionURL",
- "description": "postgresql://username:password@localhost:5432/mydatabase",
- "default": ""
- },
- "host": {
- "type": "string",
- "title": "Host"
- }
- },
- "dependencies": {
- "host":{
- "properties":{
- "port": {
- "type": "string",
- "title": "Port",
- "default": "5432"
- },
- "database": {
- "type": "string",
- "title": "Database",
- "default": "postgres"
- },
- "schema": {
- "type": "string",
- "title": "Schema",
- "description": "Default schema: public (Optional)",
- "placeholder": "Enter schema (optional)"
- },
- "user": {
+ "allOf": [
+ {
+ "required": [
+ "connectorName"
+ ],
+ "properties": {
+ "connectorName": {
"type": "string",
- "title": "User",
- "default": "postgres"
+ "title": "Name of the connector",
+ "default": "Unstract's PostgreSQL connector"
+ }
+ }
+ },
+ {
+ "oneOf": [
+ {
+ "title": "Postgres URL",
+ "properties": {
+ "connection_url": {
+ "type": "string",
+ "title": "Connection URL",
+ "description": "postgresql://username:password@localhost:5432/mydatabase",
+ "default": ""
+ }
+ },
+ "required": [
+ "connection_url"
+ ]
},
- "password": {
- "type": "string",
- "title": "Password",
- "format": "password",
- "default": ""
+ {
+ "title": "Postgres Params",
+ "properties": {
+ "host": {
+ "type": "string",
+ "title": "Host"
+ },
+ "port": {
+ "type": "string",
+ "title": "Port",
+ "default": "5432"
+ },
+ "database": {
+ "type": "string",
+ "title": "Database",
+ "default": "postgres"
+ },
+ "schema": {
+ "type": "string",
+ "title": "Schema",
+ "description": "Default schema: public (optional)",
+ "placeholder": "Enter schema (optional)"
+ },
+ "user": {
+ "type": "string",
+ "title": "User",
+ "default": "postgres"
+ },
+ "password": {
+ "type": "string",
+ "title": "Password",
+ "format": "password",
+ "default": ""
+ }
+ },
+ "required": [
+ "host",
+ "port",
+ "database",
+ "user",
+ "password"
+ ]
}
- },
- "required": ["port", "database", "user", "password"]
+ ]
}
- }
+ ]
}
diff --git a/unstract/connectors/src/unstract/connectors/databases/snowflake/static/json_schema.json b/unstract/connectors/src/unstract/connectors/databases/snowflake/static/json_schema.json
index 53d677c5d..c3870457f 100644
--- a/unstract/connectors/src/unstract/connectors/databases/snowflake/static/json_schema.json
+++ b/unstract/connectors/src/unstract/connectors/databases/snowflake/static/json_schema.json
@@ -9,7 +9,8 @@
"account",
"database",
"schema",
- "warehouse"
+ "warehouse",
+ "role"
],
"properties": {
"connectorName": {
diff --git a/unstract/connectors/src/unstract/connectors/databases/unstract_db.py b/unstract/connectors/src/unstract/connectors/databases/unstract_db.py
index f35edd62e..af3435d8a 100644
--- a/unstract/connectors/src/unstract/connectors/databases/unstract_db.py
+++ b/unstract/connectors/src/unstract/connectors/databases/unstract_db.py
@@ -69,3 +69,11 @@ class UnstractDB(UnstractConnector, ABC):
except Exception as e:
raise ConnectorError(str(e))
return True
+
+ def execute(self, query: str) -> Any:
+ try:
+ with self.get_engine().cursor() as cursor:
+ cursor.execute(query)
+ return cursor.fetchall()
+ except Exception as e:
+ raise ConnectorError(str(e))
diff --git a/unstract/connectors/src/unstract/connectors/filesystems/azure_cloud_storage/__init__.py b/unstract/connectors/src/unstract/connectors/filesystems/azure_cloud_storage/__init__.py
new file mode 100644
index 000000000..c1982a9ac
--- /dev/null
+++ b/unstract/connectors/src/unstract/connectors/filesystems/azure_cloud_storage/__init__.py
@@ -0,0 +1,12 @@
+from .azure_cloud_storage import AzureCloudStorageFS
+
+__all__ = ["AzureCloudStorageFS"]
+
+
+metadata = {
+ "name": AzureCloudStorageFS.__name__,
+ "version": "1.0.0",
+ "connector": AzureCloudStorageFS,
+ "description": "AzureCloudStorageFS connector",
+ "is_active": True,
+}
diff --git a/unstract/connectors/src/unstract/connectors/filesystems/azure_cloud_storage/azure_cloud_storage.py b/unstract/connectors/src/unstract/connectors/filesystems/azure_cloud_storage/azure_cloud_storage.py
new file mode 100644
index 000000000..45a2ce1f7
--- /dev/null
+++ b/unstract/connectors/src/unstract/connectors/filesystems/azure_cloud_storage/azure_cloud_storage.py
@@ -0,0 +1,76 @@
+import logging
+import os
+from typing import Any
+
+from adlfs import AzureBlobFileSystem
+
+from unstract.connectors.exceptions import ConnectorError
+from unstract.connectors.filesystems.unstract_file_system import (
+ UnstractFileSystem,
+)
+
+logger = logging.getLogger(__name__)
+
+
+class AzureCloudStorageFS(UnstractFileSystem):
+ def __init__(self, settings: dict[str, Any]):
+ super().__init__("AzureCloudStorageFS")
+ account_name = settings.get("account_name", "")
+ access_key = settings.get("access_key", "")
+ self.bucket = settings.get("bucket", "")
+ self.azure_fs = AzureBlobFileSystem(
+ account_name=account_name, credential=access_key
+ )
+
+ @staticmethod
+ def get_id() -> str:
+ return "azure_cloud_storage|1476a54a-ed17-4a01-9f8f-cb7e4cf91c8a"
+
+ @staticmethod
+ def get_name() -> str:
+ return "Azure Cloud Storage"
+
+ @staticmethod
+ def get_description() -> str:
+ return "Access files in your Azure Cloud Storage"
+
+ @staticmethod
+ def get_icon() -> str:
+ return (
+ "https://storage.googleapis.com/"
+ "pandora-static/connector-icons/azure_blob_storage.png"
+ )
+
+ @staticmethod
+ def get_json_schema() -> str:
+ f = open(f"{os.path.dirname(__file__)}/static/json_schema.json")
+ schema = f.read()
+ f.close()
+ return schema
+
+ @staticmethod
+ def requires_oauth() -> bool:
+ return False
+
+ @staticmethod
+ def python_social_auth_backend() -> str:
+ return ""
+
+ @staticmethod
+ def can_write() -> bool:
+ return True
+
+ @staticmethod
+ def can_read() -> bool:
+ return True
+
+ def get_fsspec_fs(self) -> AzureBlobFileSystem:
+ return self.azure_fs
+
+ def test_credentials(self) -> bool:
+ """To test credentials for Azure Cloud Storage."""
+ try:
+ self.get_fsspec_fs().ls(f"{self.bucket}")
+ except Exception as e:
+ raise ConnectorError(str(e))
+ return True
diff --git a/unstract/connectors/src/unstract/connectors/filesystems/azure_cloud_storage/static/json_schema.json b/unstract/connectors/src/unstract/connectors/filesystems/azure_cloud_storage/static/json_schema.json
new file mode 100644
index 000000000..4e3ec1fc8
--- /dev/null
+++ b/unstract/connectors/src/unstract/connectors/filesystems/azure_cloud_storage/static/json_schema.json
@@ -0,0 +1,36 @@
+{
+ "title": "Azure Cloud Storage",
+ "description": "Access files in your Azure Cloud Storage with this connector",
+ "type": "object",
+ "required": [
+ "connectorName",
+ "account_name",
+ "access_key",
+ "bucket"
+ ],
+ "properties": {
+ "connectorName": {
+ "type": "string",
+ "title": "Name of the connector",
+ "default": "Unstract's Azure cloud storage"
+ },
+ "account_name": {
+ "type": "string",
+ "title": "Account Name",
+ "description": "Account name of azure cloud storage",
+ "default":""
+ },
+ "access_key": {
+ "type": "string",
+ "title": "Access Key",
+ "format": "password",
+ "description": "Access key of the corresponding account",
+ "default":""
+ },
+ "bucket": {
+ "type": "string",
+ "title": "Bucket Name",
+ "description": "Name of the bucket to be restricted to."
+ }
+ }
+}
diff --git a/unstract/connectors/src/unstract/connectors/filesystems/google_cloud_storage/__init__.py b/unstract/connectors/src/unstract/connectors/filesystems/google_cloud_storage/__init__.py
new file mode 100644
index 000000000..c1eaeb26b
--- /dev/null
+++ b/unstract/connectors/src/unstract/connectors/filesystems/google_cloud_storage/__init__.py
@@ -0,0 +1,12 @@
+from .google_cloud_storage import GoogleCloudStorageFS
+
+__all__ = ["GoogleCloudStorageFS"]
+
+
+metadata = {
+ "name": GoogleCloudStorageFS.__name__,
+ "version": "1.0.0",
+ "connector": GoogleCloudStorageFS,
+ "description": "GoogleCloudStorageFS connector",
+ "is_active": True,
+}
diff --git a/unstract/connectors/src/unstract/connectors/filesystems/google_cloud_storage/google_cloud_storage.py b/unstract/connectors/src/unstract/connectors/filesystems/google_cloud_storage/google_cloud_storage.py
new file mode 100644
index 000000000..49f27e5c9
--- /dev/null
+++ b/unstract/connectors/src/unstract/connectors/filesystems/google_cloud_storage/google_cloud_storage.py
@@ -0,0 +1,75 @@
+import json
+import logging
+import os
+from typing import Any
+
+from gcsfs import GCSFileSystem
+
+from unstract.connectors.exceptions import ConnectorError
+from unstract.connectors.filesystems.unstract_file_system import (
+ UnstractFileSystem,
+)
+
+logger = logging.getLogger(__name__)
+
+
+class GoogleCloudStorageFS(UnstractFileSystem):
+ def __init__(self, settings: dict[str, Any]):
+ super().__init__("GoogleCloudStorage")
+ self.bucket = settings.get("bucket", "")
+ project_id = settings.get("project_id", "")
+ json_credentials = json.loads(settings.get("json_credentials", "{}"))
+ self.gcs_fs = GCSFileSystem(token=json_credentials, project=project_id)
+
+ @staticmethod
+ def get_id() -> str:
+ return "google_cloud_storage|109bbe7b-8861-45eb-8841-7244e833d97b"
+
+ @staticmethod
+ def get_name() -> str:
+ return "Google Cloud Storage"
+
+ @staticmethod
+ def get_description() -> str:
+ return "Access files in your Google Cloud Storage"
+
+ @staticmethod
+ def get_icon() -> str:
+ return (
+ "https://storage.googleapis.com/"
+ "pandora-static/connector-icons/google_cloud_storage.png"
+ )
+
+ @staticmethod
+ def get_json_schema() -> str:
+ f = open(f"{os.path.dirname(__file__)}/static/json_schema.json")
+ schema = f.read()
+ f.close()
+ return schema
+
+ @staticmethod
+ def requires_oauth() -> bool:
+ return False
+
+ @staticmethod
+ def python_social_auth_backend() -> str:
+ return ""
+
+ @staticmethod
+ def can_write() -> bool:
+ return True
+
+ @staticmethod
+ def can_read() -> bool:
+ return True
+
+ def get_fsspec_fs(self) -> GCSFileSystem:
+ return self.gcs_fs
+
+ def test_credentials(self) -> bool:
+ """To test credentials for Google Cloud Storage."""
+ try:
+ is_dir = bool(self.get_fsspec_fs().isdir(f"{self.bucket}"))
+ return is_dir
+ except Exception as e:
+ raise ConnectorError(str(e))
diff --git a/unstract/connectors/src/unstract/connectors/filesystems/google_cloud_storage/static/json_schema.json b/unstract/connectors/src/unstract/connectors/filesystems/google_cloud_storage/static/json_schema.json
new file mode 100644
index 000000000..6f375f9e0
--- /dev/null
+++ b/unstract/connectors/src/unstract/connectors/filesystems/google_cloud_storage/static/json_schema.json
@@ -0,0 +1,35 @@
+{
+ "title": "Google Cloud Storage",
+ "description": "Access files in your Google Cloud Storage with this connector",
+ "type": "object",
+ "required": [
+ "connectorName",
+ "json_credentials",
+ "project_id",
+ "bucket"
+ ],
+ "properties": {
+ "connectorName": {
+ "type": "string",
+ "title": "Name of the connector",
+ "default": "Unstract's google cloud storage"
+ },
+ "json_credentials": {
+ "type": "string",
+ "title": "Credentials",
+ "format": "password",
+ "default": "",
+ "description": "You can download credentials from Google Clouds's iam-admin/serviceaccounts/Keys. Click on Add keys."
+ },
+ "project_id": {
+ "type": "string",
+ "title": "Project ID",
+ "description": "Name of the project ID."
+ },
+ "bucket": {
+ "type": "string",
+ "title": "Bucket Name",
+ "description": "Name of the bucket to be restricted to."
+ }
+ }
+}
diff --git a/unstract/connectors/src/unstract/connectors/filesystems/google_drive/static/json_schema.json b/unstract/connectors/src/unstract/connectors/filesystems/google_drive/static/json_schema.json
index 7a4e8545e..79dc11fbd 100644
--- a/unstract/connectors/src/unstract/connectors/filesystems/google_drive/static/json_schema.json
+++ b/unstract/connectors/src/unstract/connectors/filesystems/google_drive/static/json_schema.json
@@ -8,6 +8,7 @@
"properties": {
"connectorName": {
"type": "string",
+ "format": "password",
"title": "Name of the connector",
"default": "Unstract's google drive"
}
diff --git a/unstract/connectors/src/unstract/connectors/filesystems/zs_dropbox/exceptions.py b/unstract/connectors/src/unstract/connectors/filesystems/zs_dropbox/exceptions.py
index b1d65a550..0c0d9cf2e 100644
--- a/unstract/connectors/src/unstract/connectors/filesystems/zs_dropbox/exceptions.py
+++ b/unstract/connectors/src/unstract/connectors/filesystems/zs_dropbox/exceptions.py
@@ -6,8 +6,7 @@ from unstract.connectors.exceptions import ConnectorError
def handle_dropbox_exception(e: DropboxException) -> ConnectorError:
- message = f"Error with the Dropbox connector: {e}"
- user_msg = None
+ user_msg = ""
if isinstance(e, ExcAuthError):
if isinstance(e.error, AuthError):
if e.error.is_expired_access_token():
@@ -19,4 +18,4 @@ def handle_dropbox_exception(e: DropboxException) -> ConnectorError:
elif isinstance(e, ApiError):
if e.user_message_text is not None:
user_msg = e.user_message_text
- return ConnectorError(message=message, user_message=user_msg)
+ return ConnectorError(message=user_msg, treat_as_user_message=True)
diff --git a/unstract/connectors/src/unstract/connectors/filesystems/zs_dropbox/zs_dropbox.py b/unstract/connectors/src/unstract/connectors/filesystems/zs_dropbox/zs_dropbox.py
index 53c321e37..36e1476c9 100644
--- a/unstract/connectors/src/unstract/connectors/filesystems/zs_dropbox/zs_dropbox.py
+++ b/unstract/connectors/src/unstract/connectors/filesystems/zs_dropbox/zs_dropbox.py
@@ -5,7 +5,9 @@ from typing import Any
from dropbox.exceptions import DropboxException
from dropboxdrivefs import DropboxDriveFileSystem
from unstract.connectors.exceptions import ConnectorError
-from unstract.connectors.filesystems.unstract_file_system import UnstractFileSystem
+from unstract.connectors.filesystems.unstract_file_system import (
+ UnstractFileSystem,
+)
from .exceptions import handle_dropbox_exception
@@ -34,7 +36,8 @@ class DropboxFS(UnstractFileSystem):
def get_icon() -> str:
# TODO: Add an icon to GCS and serve it
return (
- "https://storage.googleapis.com/pandora-static/connector-icons/Dropbox.png"
+ "https://storage.googleapis.com"
+ "/pandora-static/connector-icons/Dropbox.png"
)
@staticmethod