Skip to content

LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows

fix: Removed unused bucket field from connectors (#596)

Chandrasekharan Mc8f7770d0e2d
diff --git a/backend/adapter_processor/adapter_processor.py b/backend/adapter_processor/adapter_processor.py
index 2cf188b98..cc4da9a9a 100644
--- a/backend/adapter_processor/adapter_processor.py
+++ b/backend/adapter_processor/adapter_processor.py
@@ -8,7 +8,6 @@ from adapter_processor.exceptions import (
InternalServiceError,
InValidAdapterId,
TestAdapterError,
- TestAdapterInputError,
)
from django.conf import settings
from django.core.exceptions import ObjectDoesNotExist
@@ -94,12 +93,6 @@ class AdapterProcessor:
test_result: bool = adapter_instance.test_connection()
logger.info(f"{adapter_id} test result: {test_result}")
return test_result
- # HACK: Remove after error is explicitly handled in VertexAI adapter
- except json.JSONDecodeError:
- raise TestAdapterInputError(
- "Credentials is not a valid service account JSON, "
- "please provide a valid JSON."
- )
except AdapterError as e:
raise TestAdapterError(str(e))
diff --git a/backend/connector/connector_instance_helper.py b/backend/connector/connector_instance_helper.py
index e9c7ec7c7..3af7eff41 100644
--- a/backend/connector/connector_instance_helper.py
+++ b/backend/connector/connector_instance_helper.py
@@ -44,7 +44,6 @@ class ConnectorInstanceHelper:
connector_metadata = {
UCSKey.KEY: settings.GOOGLE_STORAGE_ACCESS_KEY_ID,
UCSKey.SECRET: settings.GOOGLE_STORAGE_SECRET_ACCESS_KEY,
- UCSKey.BUCKET: bucket_name,
UCSKey.ENDPOINT_URL: settings.GOOGLE_STORAGE_BASE_URL,
}
connector_metadata__input = {
diff --git a/unstract/connectors/src/unstract/connectors/filesystems/azure_cloud_storage/azure_cloud_storage.py b/unstract/connectors/src/unstract/connectors/filesystems/azure_cloud_storage/azure_cloud_storage.py
index fe61fd6f2..c9a6311e3 100644
--- a/unstract/connectors/src/unstract/connectors/filesystems/azure_cloud_storage/azure_cloud_storage.py
+++ b/unstract/connectors/src/unstract/connectors/filesystems/azure_cloud_storage/azure_cloud_storage.py
@@ -15,7 +15,6 @@ class AzureCloudStorageFS(UnstractFileSystem):
super().__init__("AzureCloudStorageFS")
account_name = settings.get("account_name", "")
access_key = settings.get("access_key", "")
- self.bucket = settings.get("bucket", "")
self.azure_fs = AzureBlobFileSystem(
account_name=account_name, credential=access_key
)
@@ -65,9 +64,9 @@ class AzureCloudStorageFS(UnstractFileSystem):
def test_credentials(self) -> bool:
"""To test credentials for Azure Cloud Storage."""
try:
- is_dir = bool(self.get_fsspec_fs().isdir(self.bucket))
+ is_dir = bool(self.get_fsspec_fs().isdir(""))
if not is_dir:
- raise RuntimeError(f"'{self.bucket}' is not a valid bucket.")
+ raise RuntimeError("Could not access root directory.")
except Exception as e:
raise ConnectorError(
f"Error from Azure Cloud Storage while testing connection: {str(e)}"
diff --git a/unstract/connectors/src/unstract/connectors/filesystems/azure_cloud_storage/static/json_schema.json b/unstract/connectors/src/unstract/connectors/filesystems/azure_cloud_storage/static/json_schema.json
index 4e3ec1fc8..1d715cd27 100644
--- a/unstract/connectors/src/unstract/connectors/filesystems/azure_cloud_storage/static/json_schema.json
+++ b/unstract/connectors/src/unstract/connectors/filesystems/azure_cloud_storage/static/json_schema.json
@@ -5,8 +5,7 @@
"required": [
"connectorName",
"account_name",
- "access_key",
- "bucket"
+ "access_key"
],
"properties": {
"connectorName": {
@@ -26,11 +25,6 @@
"format": "password",
"description": "Access key of the corresponding account",
"default":""
- },
- "bucket": {
- "type": "string",
- "title": "Bucket Name",
- "description": "Name of the bucket to be restricted to."
}
}
}
diff --git a/unstract/connectors/src/unstract/connectors/filesystems/google_cloud_storage/google_cloud_storage.py b/unstract/connectors/src/unstract/connectors/filesystems/google_cloud_storage/google_cloud_storage.py
index b79bb8742..10433ca08 100644
--- a/unstract/connectors/src/unstract/connectors/filesystems/google_cloud_storage/google_cloud_storage.py
+++ b/unstract/connectors/src/unstract/connectors/filesystems/google_cloud_storage/google_cloud_storage.py
@@ -14,7 +14,6 @@ logger = logging.getLogger(__name__)
class GoogleCloudStorageFS(UnstractFileSystem):
def __init__(self, settings: dict[str, Any]):
super().__init__("GoogleCloudStorage")
- self.bucket = settings.get("bucket", "")
project_id = settings.get("project_id", "")
json_credentials = json.loads(settings.get("json_credentials", "{}"))
self.gcs_fs = GCSFileSystem(token=json_credentials, project=project_id)
@@ -64,9 +63,9 @@ class GoogleCloudStorageFS(UnstractFileSystem):
def test_credentials(self) -> bool:
"""To test credentials for Google Cloud Storage."""
try:
- is_dir = bool(self.get_fsspec_fs().isdir(self.bucket))
+ is_dir = bool(self.get_fsspec_fs().isdir(""))
if not is_dir:
- raise RuntimeError(f"'{self.bucket}' is not a valid bucket.")
+ raise RuntimeError("Could not access root directory.")
except Exception as e:
raise ConnectorError(
f"Error from Google Cloud Storage while testing connection: {str(e)}"
diff --git a/unstract/connectors/src/unstract/connectors/filesystems/google_cloud_storage/static/json_schema.json b/unstract/connectors/src/unstract/connectors/filesystems/google_cloud_storage/static/json_schema.json
index 6f375f9e0..cbf13ec61 100644
--- a/unstract/connectors/src/unstract/connectors/filesystems/google_cloud_storage/static/json_schema.json
+++ b/unstract/connectors/src/unstract/connectors/filesystems/google_cloud_storage/static/json_schema.json
@@ -5,8 +5,7 @@
"required": [
"connectorName",
"json_credentials",
- "project_id",
- "bucket"
+ "project_id"
],
"properties": {
"connectorName": {
@@ -25,11 +24,6 @@
"type": "string",
"title": "Project ID",
"description": "Name of the project ID."
- },
- "bucket": {
- "type": "string",
- "title": "Bucket Name",
- "description": "Name of the bucket to be restricted to."
}
}
}
diff --git a/unstract/connectors/src/unstract/connectors/filesystems/minio/minio.py b/unstract/connectors/src/unstract/connectors/filesystems/minio/minio.py
index b676cda42..7f99a6711 100644
--- a/unstract/connectors/src/unstract/connectors/filesystems/minio/minio.py
+++ b/unstract/connectors/src/unstract/connectors/filesystems/minio/minio.py
@@ -17,7 +17,6 @@ class MinioFS(UnstractFileSystem):
key = settings["key"]
secret = settings["secret"]
endpoint_url = settings["endpoint_url"]
- self.bucket = settings["bucket"]
client_kwargs = {}
if "region_name" in settings and settings["region_name"] != "":
client_kwargs = {"region_name": settings["region_name"]}
@@ -77,9 +76,9 @@ class MinioFS(UnstractFileSystem):
def test_credentials(self) -> bool:
"""To test credentials for Minio."""
try:
- is_dir = bool(self.get_fsspec_fs().isdir(self.bucket))
+ is_dir = bool(self.get_fsspec_fs().isdir(""))
if not is_dir:
- raise RuntimeError(f"'{self.bucket}' is not a valid bucket.")
+ raise RuntimeError("Could not access root directory.")
except Exception as e:
raise handle_s3fs_exception(e) from e
return True
diff --git a/unstract/connectors/src/unstract/connectors/filesystems/minio/static/json_schema.json b/unstract/connectors/src/unstract/connectors/filesystems/minio/static/json_schema.json
index 7f3d3997c..e3cd76b5e 100644
--- a/unstract/connectors/src/unstract/connectors/filesystems/minio/static/json_schema.json
+++ b/unstract/connectors/src/unstract/connectors/filesystems/minio/static/json_schema.json
@@ -6,7 +6,6 @@
"connectorName",
"key",
"secret",
- "bucket",
"endpoint_url"
],
"properties": {
@@ -27,11 +26,6 @@
"format": "password",
"description": "Secret Access Key"
},
- "bucket": {
- "type": "string",
- "title": "Bucket Name",
- "description": "Name of the bucket to be restricted to."
- },
"endpoint_url": {
"type": "string",
"title": "Endpoint URL",
diff --git a/unstract/connectors/src/unstract/connectors/filesystems/ucs/constants.py b/unstract/connectors/src/unstract/connectors/filesystems/ucs/constants.py
index 03c24bf06..048c92a6d 100644
--- a/unstract/connectors/src/unstract/connectors/filesystems/ucs/constants.py
+++ b/unstract/connectors/src/unstract/connectors/filesystems/ucs/constants.py
@@ -1,6 +1,5 @@
class UCSKey:
KEY = "key"
SECRET = "secret"
- BUCKET = "bucket"
ENDPOINT_URL = "endpoint_url"
PATH = "path"
diff --git a/unstract/connectors/src/unstract/connectors/filesystems/ucs/static/json_schema.json b/unstract/connectors/src/unstract/connectors/filesystems/ucs/static/json_schema.json
index be8522093..5b663577c 100644
--- a/unstract/connectors/src/unstract/connectors/filesystems/ucs/static/json_schema.json
+++ b/unstract/connectors/src/unstract/connectors/filesystems/ucs/static/json_schema.json
@@ -6,7 +6,6 @@
"connectorName",
"key",
"secret",
- "bucket",
"path"
],
"properties": {
@@ -25,11 +24,6 @@
"title": "Secret",
"format": "password"
},
- "bucket": {
- "type": "string",
- "title": "Bucket Name",
- "description": "Name of the bucket to be restricted to."
- },
"path": {
"type": "string",
"title": "Path",
diff --git a/unstract/connectors/tests/filesystems/test_miniofs.py b/unstract/connectors/tests/filesystems/test_miniofs.py
index 0443bbde1..649c577c0 100644
--- a/unstract/connectors/tests/filesystems/test_miniofs.py
+++ b/unstract/connectors/tests/filesystems/test_miniofs.py
@@ -10,14 +10,10 @@ class TestMinoFS(unittest.TestCase):
self.assertEqual(MinioFS.requires_oauth(), False)
access_key = os.environ.get("AWS_ACCESS_KEY_ID")
secret_key = os.environ.get("AWS_SECRET_ACCESS_KEY")
- bucket_name = os.environ.get(
- "FREE_STORAGE_AWS_BUCKET_NAME", "pandora-user-storage"
- )
s3 = MinioFS(
{
"key": access_key,
"secret": secret_key,
- "bucket": bucket_name,
"path": "/",
"endpoint_url": "https://s3.amazonaws.com",
}
@@ -31,12 +27,10 @@ class TestMinoFS(unittest.TestCase):
access_key = os.environ.get("MINIO_ACCESS_KEY_ID")
secret_key = os.environ.get("MINIO_SECRET_ACCESS_KEY")
print(access_key, secret_key)
- bucket_name = os.environ.get("FREE_STORAGE_AWS_BUCKET_NAME", "minio-test")
s3 = MinioFS(
{
"key": access_key,
"secret": secret_key,
- "bucket": bucket_name,
"endpoint_url": "http://localhost:9000",
"path": "/minio-test",
}
diff --git a/unstract/connectors/tests/filesystems/test_pcs.py b/unstract/connectors/tests/filesystems/test_pcs.py
index 645b78b89..b7f193788 100644
--- a/unstract/connectors/tests/filesystems/test_pcs.py
+++ b/unstract/connectors/tests/filesystems/test_pcs.py
@@ -9,12 +9,10 @@ class TestPCS_FS(unittest.TestCase):
self.assertEqual(UnstractCloudStorage.requires_oauth(), False)
access_key = os.environ.get("GOOGLE_STORAGE_ACCESS_KEY_ID")
secret_key = os.environ.get("GOOGLE_STORAGE_SECRET_ACCESS_KEY")
- bucket_name = os.environ.get("AWS_BUCKET_NAME", "pandora-user-storage")
gcs = UnstractCloudStorage(
{
"key": access_key,
"secret": secret_key,
- "bucket": bucket_name,
"path": "/",
"endpoint_url": "https://storage.googleapis.com",
}