LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
MT-v2: Adapter, connector minor changes related to decryption (#721)
Chandrasekharan M5533a313a961
diff --git a/backend/adapter_processor_v2/adapter_processor.py b/backend/adapter_processor_v2/adapter_processor.pyindex 0b95443d7..9057934f7 100644--- a/backend/adapter_processor_v2/adapter_processor.py+++ b/backend/adapter_processor_v2/adapter_processor.py@@ -16,8 +16,8 @@ from tenant_account_v2.organization_member_service import OrganizationMemberServfrom unstract.sdk.adapters.adapterkit import Adapterkitfrom unstract.sdk.adapters.base import Adapterfrom unstract.sdk.adapters.enums import AdapterTypes-from unstract.sdk.adapters.exceptions import AdapterErrorfrom unstract.sdk.adapters.x2text.constants import X2TextConstants+from unstract.sdk.exceptions import SdkErrorfrom .models import AdapterInstance, UserDefaultAdapter@@ -94,7 +94,7 @@ class AdapterProcessor:test_result: bool = adapter_instance.test_connection()logger.info(f"{adapter_id} test result: {test_result}")return test_result- except AdapterError as e:+ except SdkError as e:raise TestAdapterError(str(e))@staticmethoddiff --git a/backend/connector_v2/connector_instance_helper.py b/backend/connector_v2/connector_instance_helper.pyindex e6a7d4307..101cc4805 100644--- a/backend/connector_v2/connector_instance_helper.py+++ b/backend/connector_v2/connector_instance_helper.py@@ -44,7 +44,6 @@ class ConnectorInstanceHelper:connector_metadata = {UCSKey.KEY: settings.GOOGLE_STORAGE_ACCESS_KEY_ID,UCSKey.SECRET: settings.GOOGLE_STORAGE_SECRET_ACCESS_KEY,- UCSKey.BUCKET: bucket_name,UCSKey.ENDPOINT_URL: settings.GOOGLE_STORAGE_BASE_URL,}connector_metadata__input = {diff --git a/backend/connector_v2/models.py b/backend/connector_v2/models.pyindex f8db9915a..cb28b1332 100644--- a/backend/connector_v2/models.py+++ b/backend/connector_v2/models.py@@ -6,9 +6,10 @@ from account_v2.models import Userfrom connector_auth_v2.models import ConnectorAuthfrom connector_processor.connector_processor import ConnectorProcessorfrom connector_processor.constants import ConnectorKeys-from cryptography.fernet import Fernet+from cryptography.fernet import Fernet, InvalidTokenfrom django.conf import settingsfrom django.db import models+from utils.exceptions import InvalidEncryptionKeyfrom utils.models.base_model import BaseModelfrom utils.models.organization_mixin import (DefaultOrganizationManagerMixin,@@ -84,6 +85,7 @@ class ConnectorInstance(DefaultOrganizationMixin, BaseModel):# Managerobjects = ConnectorInstanceModelManager()+ # TODO: Remove if unuseddef get_connector_metadata(self) -> dict[str, str]:"""Gets connector metadata and refreshes the tokens if needed in caseof OAuth."""@@ -132,17 +134,19 @@ class ConnectorInstance(DefaultOrganizationMixin, BaseModel):Returns:dict: The decrypted connector metadata."""-- connector_metadata_bytes = self.get_connector_metadata_bytes()-- if connector_metadata_bytes is None:- return None-- if isinstance(connector_metadata_bytes, (dict)):- return connector_metadata_bytes- encryption_secret: str = settings.ENCRYPTION_KEY- cipher_suite: Fernet = Fernet(encryption_secret.encode("utf-8"))- decrypted_value = cipher_suite.decrypt(connector_metadata_bytes)+ try:+ connector_metadata_bytes = self.get_connector_metadata_bytes()++ if connector_metadata_bytes is None:+ return None++ if isinstance(connector_metadata_bytes, (dict)):+ return connector_metadata_bytes+ encryption_secret: str = settings.ENCRYPTION_KEY+ cipher_suite: Fernet = Fernet(encryption_secret.encode("utf-8"))+ decrypted_value = cipher_suite.decrypt(connector_metadata_bytes)+ except InvalidToken:+ raise InvalidEncryptionKey(entity=InvalidEncryptionKey.Entity.CONNECTOR)return json.loads(decrypted_value.decode("utf-8"))class Meta:diff --git a/backend/connector_v2/serializers.py b/backend/connector_v2/serializers.pyindex 4704635ba..3dd8f9419 100644--- a/backend/connector_v2/serializers.py+++ b/backend/connector_v2/serializers.py@@ -88,12 +88,7 @@ class ConnectorInstanceSerializer(AuditSerializer):rep[ConnectorKeys.ICON] = ConnectorProcessor.get_connector_data_with_key(instance.connector_id, ConnectorKeys.ICON)- encryption_secret: str = settings.ENCRYPTION_KEY- f: Fernet = Fernet(encryption_secret.encode("utf-8"))if instance.connector_metadata:- adapter_metadata = json.loads(- f.decrypt(bytes(instance.connector_metadata).decode("utf-8"))- )- rep[CIKey.CONNECTOR_METADATA] = adapter_metadata+ rep[CIKey.CONNECTOR_METADATA] = instance.metadatareturn repdiff --git a/backend/prompt_studio/prompt_studio_core_v2/constants.py b/backend/prompt_studio/prompt_studio_core_v2/constants.pyindex afba6153d..a583bd5d7 100644--- a/backend/prompt_studio/prompt_studio_core_v2/constants.py+++ b/backend/prompt_studio/prompt_studio_core_v2/constants.py@@ -90,7 +90,6 @@ class ToolStudioPromptKeys:INCLUDE_METADATA = "include_metadata"TXT_EXTENTION = ".txt"TABLE = "table"- EXTRACT = "extract"PLATFORM_POSTAMBLE = "platform_postamble"SUMMARIZE_AS_SOURCE = "summarize_as_source"VARIABLE_MAP = "variable_map"diff --git a/backend/prompt_studio/prompt_studio_core_v2/models.py b/backend/prompt_studio/prompt_studio_core_v2/models.pyindex 9cefe3fa2..87abd4766 100644--- a/backend/prompt_studio/prompt_studio_core_v2/models.py+++ b/backend/prompt_studio/prompt_studio_core_v2/models.py@@ -125,10 +125,6 @@ class CustomTool(DefaultOrganizationMixin, BaseModel):db_comment="Flag to enable or disable document highlighting", default=False)- enable_highlight = models.BooleanField(- db_comment="Flag to enable or disable document highlighting", default=False- )-# Introduced field to establish M2M relation between users and custom_tool.# This will introduce intermediary table which relates both the models.shared_users = models.ManyToManyField(User, related_name="shared_custom_tools")