LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
fix: Removed unused bucket field from connectors (#596)
Chandrasekharan Mc8f7770d0e2d
diff --git a/backend/adapter_processor/adapter_processor.py b/backend/adapter_processor/adapter_processor.pyindex 2cf188b98..cc4da9a9a 100644--- a/backend/adapter_processor/adapter_processor.py+++ b/backend/adapter_processor/adapter_processor.py@@ -8,7 +8,6 @@ from adapter_processor.exceptions import (InternalServiceError,InValidAdapterId,TestAdapterError,- TestAdapterInputError,)from django.conf import settingsfrom django.core.exceptions import ObjectDoesNotExist@@ -94,12 +93,6 @@ class AdapterProcessor:test_result: bool = adapter_instance.test_connection()logger.info(f"{adapter_id} test result: {test_result}")return test_result- # HACK: Remove after error is explicitly handled in VertexAI adapter- except json.JSONDecodeError:- raise TestAdapterInputError(- "Credentials is not a valid service account JSON, "- "please provide a valid JSON."- )except AdapterError as e:raise TestAdapterError(str(e))diff --git a/backend/connector/connector_instance_helper.py b/backend/connector/connector_instance_helper.pyindex e9c7ec7c7..3af7eff41 100644--- a/backend/connector/connector_instance_helper.py+++ b/backend/connector/connector_instance_helper.py@@ -44,7 +44,6 @@ class ConnectorInstanceHelper:connector_metadata = {UCSKey.KEY: settings.GOOGLE_STORAGE_ACCESS_KEY_ID,UCSKey.SECRET: settings.GOOGLE_STORAGE_SECRET_ACCESS_KEY,- UCSKey.BUCKET: bucket_name,UCSKey.ENDPOINT_URL: settings.GOOGLE_STORAGE_BASE_URL,}connector_metadata__input = {diff --git a/unstract/connectors/src/unstract/connectors/filesystems/azure_cloud_storage/azure_cloud_storage.py b/unstract/connectors/src/unstract/connectors/filesystems/azure_cloud_storage/azure_cloud_storage.pyindex fe61fd6f2..c9a6311e3 100644--- a/unstract/connectors/src/unstract/connectors/filesystems/azure_cloud_storage/azure_cloud_storage.py+++ b/unstract/connectors/src/unstract/connectors/filesystems/azure_cloud_storage/azure_cloud_storage.py@@ -15,7 +15,6 @@ class AzureCloudStorageFS(UnstractFileSystem):super().__init__("AzureCloudStorageFS")account_name = settings.get("account_name", "")access_key = settings.get("access_key", "")- self.bucket = settings.get("bucket", "")self.azure_fs = AzureBlobFileSystem(account_name=account_name, credential=access_key)@@ -65,9 +64,9 @@ class AzureCloudStorageFS(UnstractFileSystem):def test_credentials(self) -> bool:"""To test credentials for Azure Cloud Storage."""try:- is_dir = bool(self.get_fsspec_fs().isdir(self.bucket))+ is_dir = bool(self.get_fsspec_fs().isdir(""))if not is_dir:- raise RuntimeError(f"'{self.bucket}' is not a valid bucket.")+ raise RuntimeError("Could not access root directory.")except Exception as e:raise ConnectorError(f"Error from Azure Cloud Storage while testing connection: {str(e)}"diff --git a/unstract/connectors/src/unstract/connectors/filesystems/azure_cloud_storage/static/json_schema.json b/unstract/connectors/src/unstract/connectors/filesystems/azure_cloud_storage/static/json_schema.jsonindex 4e3ec1fc8..1d715cd27 100644--- a/unstract/connectors/src/unstract/connectors/filesystems/azure_cloud_storage/static/json_schema.json+++ b/unstract/connectors/src/unstract/connectors/filesystems/azure_cloud_storage/static/json_schema.json@@ -5,8 +5,7 @@"required": ["connectorName","account_name",- "access_key",- "bucket"+ "access_key"],"properties": {"connectorName": {@@ -26,11 +25,6 @@"format": "password","description": "Access key of the corresponding account","default":""- },- "bucket": {- "type": "string",- "title": "Bucket Name",- "description": "Name of the bucket to be restricted to."}}}diff --git a/unstract/connectors/src/unstract/connectors/filesystems/google_cloud_storage/google_cloud_storage.py b/unstract/connectors/src/unstract/connectors/filesystems/google_cloud_storage/google_cloud_storage.pyindex b79bb8742..10433ca08 100644--- a/unstract/connectors/src/unstract/connectors/filesystems/google_cloud_storage/google_cloud_storage.py+++ b/unstract/connectors/src/unstract/connectors/filesystems/google_cloud_storage/google_cloud_storage.py@@ -14,7 +14,6 @@ logger = logging.getLogger(__name__)class GoogleCloudStorageFS(UnstractFileSystem):def __init__(self, settings: dict[str, Any]):super().__init__("GoogleCloudStorage")- self.bucket = settings.get("bucket", "")project_id = settings.get("project_id", "")json_credentials = json.loads(settings.get("json_credentials", "{}"))self.gcs_fs = GCSFileSystem(token=json_credentials, project=project_id)@@ -64,9 +63,9 @@ class GoogleCloudStorageFS(UnstractFileSystem):def test_credentials(self) -> bool:"""To test credentials for Google Cloud Storage."""try:- is_dir = bool(self.get_fsspec_fs().isdir(self.bucket))+ is_dir = bool(self.get_fsspec_fs().isdir(""))if not is_dir:- raise RuntimeError(f"'{self.bucket}' is not a valid bucket.")+ raise RuntimeError("Could not access root directory.")except Exception as e:raise ConnectorError(f"Error from Google Cloud Storage while testing connection: {str(e)}"diff --git a/unstract/connectors/src/unstract/connectors/filesystems/google_cloud_storage/static/json_schema.json b/unstract/connectors/src/unstract/connectors/filesystems/google_cloud_storage/static/json_schema.jsonindex 6f375f9e0..cbf13ec61 100644--- a/unstract/connectors/src/unstract/connectors/filesystems/google_cloud_storage/static/json_schema.json+++ b/unstract/connectors/src/unstract/connectors/filesystems/google_cloud_storage/static/json_schema.json@@ -5,8 +5,7 @@"required": ["connectorName","json_credentials",- "project_id",- "bucket"+ "project_id"],"properties": {"connectorName": {@@ -25,11 +24,6 @@"type": "string","title": "Project ID","description": "Name of the project ID."- },- "bucket": {- "type": "string",- "title": "Bucket Name",- "description": "Name of the bucket to be restricted to."}}}diff --git a/unstract/connectors/src/unstract/connectors/filesystems/minio/minio.py b/unstract/connectors/src/unstract/connectors/filesystems/minio/minio.pyindex b676cda42..7f99a6711 100644--- a/unstract/connectors/src/unstract/connectors/filesystems/minio/minio.py+++ b/unstract/connectors/src/unstract/connectors/filesystems/minio/minio.py@@ -17,7 +17,6 @@ class MinioFS(UnstractFileSystem):key = settings["key"]secret = settings["secret"]endpoint_url = settings["endpoint_url"]- self.bucket = settings["bucket"]client_kwargs = {}if "region_name" in settings and settings["region_name"] != "":client_kwargs = {"region_name": settings["region_name"]}@@ -77,9 +76,9 @@ class MinioFS(UnstractFileSystem):def test_credentials(self) -> bool:"""To test credentials for Minio."""try:- is_dir = bool(self.get_fsspec_fs().isdir(self.bucket))+ is_dir = bool(self.get_fsspec_fs().isdir(""))if not is_dir:- raise RuntimeError(f"'{self.bucket}' is not a valid bucket.")+ raise RuntimeError("Could not access root directory.")except Exception as e:raise handle_s3fs_exception(e) from ereturn Truediff --git a/unstract/connectors/src/unstract/connectors/filesystems/minio/static/json_schema.json b/unstract/connectors/src/unstract/connectors/filesystems/minio/static/json_schema.jsonindex 7f3d3997c..e3cd76b5e 100644--- a/unstract/connectors/src/unstract/connectors/filesystems/minio/static/json_schema.json+++ b/unstract/connectors/src/unstract/connectors/filesystems/minio/static/json_schema.json@@ -6,7 +6,6 @@"connectorName","key","secret",- "bucket","endpoint_url"],"properties": {@@ -27,11 +26,6 @@"format": "password","description": "Secret Access Key"},- "bucket": {- "type": "string",- "title": "Bucket Name",- "description": "Name of the bucket to be restricted to."- },"endpoint_url": {"type": "string","title": "Endpoint URL",diff --git a/unstract/connectors/src/unstract/connectors/filesystems/ucs/constants.py b/unstract/connectors/src/unstract/connectors/filesystems/ucs/constants.pyindex 03c24bf06..048c92a6d 100644--- a/unstract/connectors/src/unstract/connectors/filesystems/ucs/constants.py+++ b/unstract/connectors/src/unstract/connectors/filesystems/ucs/constants.py@@ -1,6 +1,5 @@class UCSKey:KEY = "key"SECRET = "secret"- BUCKET = "bucket"ENDPOINT_URL = "endpoint_url"PATH = "path"diff --git a/unstract/connectors/src/unstract/connectors/filesystems/ucs/static/json_schema.json b/unstract/connectors/src/unstract/connectors/filesystems/ucs/static/json_schema.jsonindex be8522093..5b663577c 100644--- a/unstract/connectors/src/unstract/connectors/filesystems/ucs/static/json_schema.json+++ b/unstract/connectors/src/unstract/connectors/filesystems/ucs/static/json_schema.json@@ -6,7 +6,6 @@"connectorName","key","secret",- "bucket","path"],"properties": {@@ -25,11 +24,6 @@"title": "Secret","format": "password"},- "bucket": {- "type": "string",- "title": "Bucket Name",- "description": "Name of the bucket to be restricted to."- },"path": {"type": "string","title": "Path",diff --git a/unstract/connectors/tests/filesystems/test_miniofs.py b/unstract/connectors/tests/filesystems/test_miniofs.pyindex 0443bbde1..649c577c0 100644--- a/unstract/connectors/tests/filesystems/test_miniofs.py+++ b/unstract/connectors/tests/filesystems/test_miniofs.py@@ -10,14 +10,10 @@ class TestMinoFS(unittest.TestCase):self.assertEqual(MinioFS.requires_oauth(), False)access_key = os.environ.get("AWS_ACCESS_KEY_ID")secret_key = os.environ.get("AWS_SECRET_ACCESS_KEY")- bucket_name = os.environ.get(- "FREE_STORAGE_AWS_BUCKET_NAME", "pandora-user-storage"- )s3 = MinioFS({"key": access_key,"secret": secret_key,- "bucket": bucket_name,"path": "/","endpoint_url": "https://s3.amazonaws.com",}@@ -31,12 +27,10 @@ class TestMinoFS(unittest.TestCase):access_key = os.environ.get("MINIO_ACCESS_KEY_ID")secret_key = os.environ.get("MINIO_SECRET_ACCESS_KEY")print(access_key, secret_key)- bucket_name = os.environ.get("FREE_STORAGE_AWS_BUCKET_NAME", "minio-test")s3 = MinioFS({"key": access_key,"secret": secret_key,- "bucket": bucket_name,"endpoint_url": "http://localhost:9000","path": "/minio-test",}diff --git a/unstract/connectors/tests/filesystems/test_pcs.py b/unstract/connectors/tests/filesystems/test_pcs.pyindex 645b78b89..b7f193788 100644--- a/unstract/connectors/tests/filesystems/test_pcs.py+++ b/unstract/connectors/tests/filesystems/test_pcs.py@@ -9,12 +9,10 @@ class TestPCS_FS(unittest.TestCase):self.assertEqual(UnstractCloudStorage.requires_oauth(), False)access_key = os.environ.get("GOOGLE_STORAGE_ACCESS_KEY_ID")secret_key = os.environ.get("GOOGLE_STORAGE_SECRET_ACCESS_KEY")- bucket_name = os.environ.get("AWS_BUCKET_NAME", "pandora-user-storage")gcs = UnstractCloudStorage({"key": access_key,"secret": secret_key,- "bucket": bucket_name,"path": "/","endpoint_url": "https://storage.googleapis.com",}