LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
fix: Encryption key error handling fix (#567)
Chandrasekharan Mb3e3f4bf446d
diff --git a/backend/adapter_processor/exceptions.py b/backend/adapter_processor/exceptions.pyindex a0f18febc..876775ca5 100644--- a/backend/adapter_processor/exceptions.py+++ b/backend/adapter_processor/exceptions.py@@ -19,14 +19,6 @@ class InValidAdapterId(APIException):default_detail = "Adapter ID is not Valid."-class InvalidEncryptionKey(APIException):- status_code = 403- default_detail = (- "Platform encryption key for storing adapter credentials has changed! "- "Please inform the organization admin to contact support."- )--class InternalServiceError(APIException):status_code = 500default_detail = "Internal Service error"diff --git a/backend/adapter_processor/models.py b/backend/adapter_processor/models.pyindex f37198d1b..f856bcf6b 100644--- a/backend/adapter_processor/models.py+++ b/backend/adapter_processor/models.py@@ -4,13 +4,14 @@ import uuidfrom typing import Anyfrom account.models import User-from cryptography.fernet import Fernet+from cryptography.fernet import Fernet, InvalidTokenfrom django.conf import settingsfrom django.db import modelsfrom django.db.models import QuerySetfrom unstract.sdk.adapters.adapterkit import Adapterkitfrom unstract.sdk.adapters.enums import AdapterTypesfrom unstract.sdk.adapters.exceptions import AdapterError+from utils.exceptions import InvalidEncryptionKeyfrom utils.models.base_model import BaseModelADAPTER_NAME_SIZE = 128@@ -135,22 +136,24 @@ class AdapterInstance(BaseModel):self.save()- def get_adapter_meta_data(self) -> Any:- encryption_secret: str = settings.ENCRYPTION_KEY- f: Fernet = Fernet(encryption_secret.encode("utf-8"))+ @property+ def metadata(self) -> Any:+ try:+ encryption_secret: str = settings.ENCRYPTION_KEY+ f: Fernet = Fernet(encryption_secret.encode("utf-8"))- adapter_metadata = json.loads(- f.decrypt(bytes(self.adapter_metadata_b).decode("utf-8"))- )+ adapter_metadata = json.loads(+ f.decrypt(bytes(self.adapter_metadata_b).decode("utf-8"))+ )+ except InvalidToken:+ raise InvalidEncryptionKey(entity=InvalidEncryptionKey.Entity.ADAPTER)return adapter_metadatadef get_context_window_size(self) -> int:-- adapter_metadata = self.get_adapter_meta_data()# Get the adapter_instanceadapter_class = Adapterkit().get_adapter_class_by_adapter_id(self.adapter_id)try:- adapter_instance = adapter_class(adapter_metadata)+ adapter_instance = adapter_class(self.metadata)return adapter_instance.get_context_window_size()except AdapterError as e:logger.warning(f"Unable to retrieve context window size - {e}")diff --git a/backend/adapter_processor/serializers.py b/backend/adapter_processor/serializers.pyindex aecb16a76..1e74c19ad 100644--- a/backend/adapter_processor/serializers.py+++ b/backend/adapter_processor/serializers.py@@ -4,8 +4,7 @@ from typing import Anyfrom account.serializer import UserSerializerfrom adapter_processor.adapter_processor import AdapterProcessorfrom adapter_processor.constants import AdapterKeys-from adapter_processor.exceptions import InvalidEncryptionKey-from cryptography.fernet import Fernet, InvalidToken+from cryptography.fernet import Fernetfrom django.conf import settingsfrom rest_framework import serializersfrom rest_framework.serializers import ModelSerializer@@ -62,11 +61,7 @@ class AdapterInstanceSerializer(BaseAdapterSerializer):rep: dict[str, str] = super().to_representation(instance)rep.pop(AdapterKeys.ADAPTER_METADATA_B)-- try:- adapter_metadata = instance.get_adapter_meta_data()- except InvalidToken:- raise InvalidEncryptionKey+ adapter_metadata = instance.metadatarep[AdapterKeys.ADAPTER_METADATA] = adapter_metadata# Retrieve context window if adapter is a LLM# For other adapter types, context_window is not relevant.@@ -124,8 +119,7 @@ class AdapterListSerializer(BaseAdapterSerializer):rep[common.ICON] = AdapterProcessor.get_adapter_data_with_key(instance.adapter_id, common.ICON)- adapter_metadata = instance.get_adapter_meta_data()- model = adapter_metadata.get("model")+ model = instance.metadata.get("model")if model:rep["model"] = modeldiff --git a/backend/connector/models.py b/backend/connector/models.pyindex 55c228e47..958946efe 100644--- a/backend/connector/models.py+++ b/backend/connector/models.py@@ -7,10 +7,11 @@ from connector.fields import ConnectorAuthJSONFieldfrom connector_auth.models import ConnectorAuthfrom connector_processor.connector_processor import ConnectorProcessorfrom connector_processor.constants import ConnectorKeys-from cryptography.fernet import Fernet+from cryptography.fernet import Fernet, InvalidTokenfrom django.conf import settingsfrom django.db import modelsfrom project.models import Project+from utils.exceptions import InvalidEncryptionKeyfrom utils.models.base_model import BaseModelfrom workflow_manager.workflow.models import Workflow@@ -111,11 +112,14 @@ class ConnectorInstance(BaseModel):@propertydef metadata(self) -> Any:- encryption_secret: str = settings.ENCRYPTION_KEY- cipher_suite: Fernet = Fernet(encryption_secret.encode("utf-8"))- decrypted_value = cipher_suite.decrypt(- bytes(self.connector_metadata_b).decode("utf-8")- )+ try:+ encryption_secret: str = settings.ENCRYPTION_KEY+ cipher_suite: Fernet = Fernet(encryption_secret.encode("utf-8"))+ decrypted_value = cipher_suite.decrypt(+ bytes(self.connector_metadata_b).decode("utf-8")+ )+ except InvalidToken:+ raise InvalidEncryptionKey(entity=InvalidEncryptionKey.Entity.CONNECTOR)return json.loads(decrypted_value)class Meta:diff --git a/backend/connector/serializers.py b/backend/connector/serializers.pyindex 13fea7ae8..c5ce1054f 100644--- a/backend/connector/serializers.py+++ b/backend/connector/serializers.py@@ -75,13 +75,8 @@ class ConnectorInstanceSerializer(AuditSerializer):rep[ConnectorKeys.ICON] = ConnectorProcessor.get_connector_data_with_key(instance.connector_id, ConnectorKeys.ICON)- encryption_secret: str = settings.ENCRYPTION_KEY- f: Fernet = Fernet(encryption_secret.encode("utf-8"))rep.pop(CIKey.CONNECTOR_METADATA_B)if instance.connector_metadata_b:- adapter_metadata = json.loads(- f.decrypt(bytes(instance.connector_metadata_b).decode("utf-8"))- )- rep[CIKey.CONNECTOR_METADATA] = adapter_metadata+ rep[CIKey.CONNECTOR_METADATA] = instance.metadatareturn repdiff --git a/backend/prompt_studio/prompt_studio_core/prompt_studio_helper.py b/backend/prompt_studio/prompt_studio_core/prompt_studio_helper.pyindex 4ed32ca04..f17c4fa37 100644--- a/backend/prompt_studio/prompt_studio_core/prompt_studio_helper.py+++ b/backend/prompt_studio/prompt_studio_core/prompt_studio_helper.py@@ -1072,7 +1072,7 @@ class PromptStudioHelper:if answer["status"] == "ERROR":error_message = answer.get("error", None)raise AnswerFetchError(- f"Error while fetching response for prompt. {error_message}"+ f"Error while fetching response for prompt(s). {error_message}")output_response = json.loads(answer["structure_output"])return output_responsediff --git a/backend/utils/exceptions.py b/backend/utils/exceptions.pynew file mode 100644index 000000000..b2cd4fd74--- /dev/null+++ b/backend/utils/exceptions.py@@ -0,0 +1,34 @@+from enum import Enum+from typing import Optional++from rest_framework.exceptions import APIException+++class InvalidEncryptionKey(APIException):+ status_code = 403+ default_detail = (+ "Platform encryption key for storing sensitive credentials has changed! "+ "All encrypted entities are inaccessible. Please inform the "+ "platform admin immediately."+ )++ class Entity(Enum):+ ADAPTER = "adapter"+ CONNECTOR = "connector"++ def __init__(+ self,+ entity: Optional[Entity] = None,+ detail: Optional[str] = None,+ code: Optional[str] = None,+ ) -> None:+ if entity == self.Entity.ADAPTER:+ detail = self.default_detail.replace("sensitive", "adapter").replace(+ "encrypted entities", "adapters"+ )+ elif entity == self.Entity.CONNECTOR:+ detail = self.default_detail.replace("sensitive", "connector").replace(+ "encrypted entities", "connectors"+ )++ super().__init__(detail, code)diff --git a/frontend/src/hooks/useExceptionHandler.jsx b/frontend/src/hooks/useExceptionHandler.jsxindex 4f7835e54..e11f8f5e8 100644--- a/frontend/src/hooks/useExceptionHandler.jsx+++ b/frontend/src/hooks/useExceptionHandler.jsx@@ -40,7 +40,7 @@ const useExceptionHandler = () => {return {title: title,type: "error",- content: errors?.[0]?.detail ? errors[0].detail : errMessage,+ content: errors,duration: duration,};case "client_error":diff --git a/frontend/src/hooks/useSessionValid.js b/frontend/src/hooks/useSessionValid.jsindex fe34e91f2..30264b47c 100644--- a/frontend/src/hooks/useSessionValid.js+++ b/frontend/src/hooks/useSessionValid.js@@ -7,6 +7,7 @@ import { useExceptionHandler } from "../hooks/useExceptionHandler.jsx";import { useSessionStore } from "../store/session-store";import { useUserSession } from "./useUserSession.js";import { listFlags } from "../helpers/FeatureFlagsData.js";+import { useAlertStore } from "../store/alert-store";let getTrialDetails;let isPlatformAdmin;@@ -30,6 +31,7 @@ try {function useSessionValid() {const setSessionDetails = useSessionStore((state) => state.setSessionDetails);const handleException = useExceptionHandler();+ const { setAlertDetails } = useAlertStore();const navigate = useNavigate();const userSession = useUserSession();@@ -134,6 +136,7 @@ function useSessionValid() {setSessionDetails(getSessionData(userAndOrgDetails));} catch (err) {// TODO: Throw popup error message+ // REVIEW: Add condition to check for trial period statusif (err.response?.status === 402) {handleException(err);}@@ -145,6 +148,7 @@ function useSessionValid() {window.location.href = `/error?code=${code}&domain=${domainName}`;// May be need a logout button there or auto logout}+ setAlertDetails(handleException(err));}};}diff --git a/platform-service/src/unstract/platform_service/controller/__init__.py b/platform-service/src/unstract/platform_service/controller/__init__.pyindex adfbe6331..95e7074ff 100644--- a/platform-service/src/unstract/platform_service/controller/__init__.py+++ b/platform-service/src/unstract/platform_service/controller/__init__.py@@ -1,7 +1,10 @@-from typing import Any+import json+import traceback+from typing import Union-from flask import Blueprint, Response, jsonify-from unstract.platform_service.exceptions import CustomException+from flask import Blueprint, Response, jsonify, request+from unstract.platform_service.exceptions import APIError, ErrorResponse+from werkzeug.exceptions import HTTPExceptionfrom .health import health_bpfrom .platform import platform_bp@@ -11,8 +14,63 @@ api.register_blueprint(platform_bp)api.register_blueprint(health_bp)[email protected](CustomException)-def handle_custom_exception(error: Any) -> tuple[Response, Any]:- response = jsonify({"error": error.message})- response.status_code = error.code # You can customize the HTTP status code- return jsonify(response), error.code+def log_exceptions(e: HTTPException) -> None:+ """Helper method to log exceptions.++ Args:+ e (HTTPException): Exception to log+ """+ code = 500+ if hasattr(e, "code"):+ code = e.code or code++ if code >= 500:+ message = "{method} {url} {status}\n\n{error}\n\n````{tb}````".format(+ method=request.method,+ url=request.url,+ status=code,+ error=str(e),+ tb=traceback.format_exc(),+ )+ else:+ message = "{method} {url} {status} {error}".format(+ method=request.method,+ url=request.url,+ status=code,+ error=str(e),+ )++ # Avoids circular import errors while initializing app context+ from flask import current_app as app++ app.logger.error(message)++[email protected](HTTPException)+def handle_http_exception(e: HTTPException) -> Union[Response, tuple[Response, int]]:+ """Return JSON instead of HTML for HTTP errors."""+ log_exceptions(e)+ if isinstance(e, APIError):+ return jsonify(e.to_dict()), e.code+ else:+ response = e.get_response()+ response.data = json.dumps(+ ErrorResponse(error=e.description, name=e.name, code=e.code)+ )+ response.content_type = "application/json"+ return response++[email protected](Exception)+def handle_uncaught_exception(e: Exception) -> Union[Response, tuple[Response, int]]:+ """Handler for uncaught exceptions.++ Args:+ e (Exception): Any uncaught exception+ """+ # pass through HTTP errors+ if isinstance(e, HTTPException):+ return handle_http_exception(e)++ log_exceptions(e)+ return handle_http_exception(APIError())diff --git a/platform-service/src/unstract/platform_service/controller/platform.py b/platform-service/src/unstract/platform_service/controller/platform.pyindex 5cd3a4f4d..dbed388da 100644--- a/platform-service/src/unstract/platform_service/controller/platform.py+++ b/platform-service/src/unstract/platform_service/controller/platform.py@@ -1,15 +1,17 @@+import jsonimport uuidfrom datetime import datetimefrom typing import Any, Optionalimport peeweeimport redis-from cryptography.fernet import Fernet+from cryptography.fernet import Fernet, InvalidTokenfrom flask import Blueprint, Requestfrom flask import current_app as app-from flask import json, jsonify, make_response, request+from flask import jsonify, make_response, requestfrom unstract.platform_service.constants import DBTable, DBTableV2, FeatureFlagfrom unstract.platform_service.env import Env+from unstract.platform_service.exceptions import APIErrorfrom unstract.platform_service.helper.adapter_instance import (AdapterInstanceRequestHelper,)@@ -452,11 +454,21 @@ def adapter_instance() -> Any:)return jsonify(data_dict)+ except InvalidToken:+ msg = (+ "Platform encryption key for storing adapter credentials has "+ "changed! All adapters are inaccessible. Please inform "+ "the platform admin immediately."+ )+ app.logger.error(+ f"Error while getting db adapter settings for: "+ f"{adapter_instance_id}, Error: {msg}"+ )+ raise APIError(message=msg, code=403)except Exception as e:- print(e)app.logger.error(f"Error while getting db adapter settings for: "- f"{adapter_instance_id} Error: {str(e)}"+ f"{adapter_instance_id}, Error: {str(e)}")return "Internal Server Error", 500return "Method Not Allowed", 405@@ -492,10 +504,14 @@ def custom_tool_instance() -> Any:)return jsonify(data_dict)except Exception as e:- print(e)app.logger.error(f"Error while getting db adapter settings for: "f"{prompt_registry_id} Error: {str(e)}")return "Internal Server Error", 500return "Method Not Allowed", 405+++if __name__ == "__main__":+ # Start the server+ app.run(host="0.0.0.0", port="3001")diff --git a/platform-service/src/unstract/platform_service/exceptions.py b/platform-service/src/unstract/platform_service/exceptions.pyindex c9a0c89c7..97832719c 100644--- a/platform-service/src/unstract/platform_service/exceptions.py+++ b/platform-service/src/unstract/platform_service/exceptions.py@@ -1,5 +1,46 @@-class CustomException(Exception):- def __init__(self, message: str = "An error occurred", code: int = 500) -> None:- self.message = message- self.code = code- super().__init__(self.message)+from dataclasses import asdict, dataclass+from typing import Any, Optional++from werkzeug.exceptions import HTTPException++DEFAULT_ERR_MESSAGE = "Something went wrong"+++@dataclass+class ErrorResponse:+ """Represents error response from platform service."""++ error: str = DEFAULT_ERR_MESSAGE+ name: str = "PlatformServiceError"+ code: int = 500+ payload: Optional[Any] = None+++class APIError(HTTPException):+ code = 500+ message = DEFAULT_ERR_MESSAGE++ def __init__(+ self,+ message: Optional[str] = None,+ code: Optional[int] = None,+ payload: Any = None,+ ):+ if message:+ self.message = message+ if code:+ self.code = code+ self.payload = payload+ super().__init__(description=message)++ def to_dict(self) -> dict[str, Any]:+ err = ErrorResponse(+ error=self.message,+ code=self.code,+ payload=self.payload,+ name=self.__class__.__name__,+ )+ return asdict(err)++ def __str__(self) -> str:+ return str(self.message)diff --git a/platform-service/src/unstract/platform_service/helper/adapter_instance.py b/platform-service/src/unstract/platform_service/helper/adapter_instance.pyindex 3a4c6aa25..b8bf5a418 100644--- a/platform-service/src/unstract/platform_service/helper/adapter_instance.py+++ b/platform-service/src/unstract/platform_service/helper/adapter_instance.py@@ -2,7 +2,7 @@ from typing import Any, Optionalimport peeweefrom unstract.platform_service.constants import DBTableV2, FeatureFlag-from unstract.platform_service.exceptions import CustomException+from unstract.platform_service.exceptions import APIErrorfrom unstract.flags.feature_flag import check_feature_flag_status@@ -41,7 +41,7 @@ class AdapterInstanceRequestHelper:cursor = db_instance.execute_sql(query)result_row = cursor.fetchone()if not result_row:- raise CustomException(message="Adapter not found", code=404)+ raise APIError(message="Adapter not found", code=404)columns = [desc[0] for desc in cursor.description]data_dict: dict[str, Any] = dict(zip(columns, result_row))cursor.close()diff --git a/platform-service/src/unstract/platform_service/helper/prompt_studio.py b/platform-service/src/unstract/platform_service/helper/prompt_studio.pyindex 1a05f1468..5de4b2cae 100644--- a/platform-service/src/unstract/platform_service/helper/prompt_studio.py+++ b/platform-service/src/unstract/platform_service/helper/prompt_studio.py@@ -2,7 +2,7 @@ from typing import Anyimport peeweefrom unstract.platform_service.constants import DBTableV2, FeatureFlag-from unstract.platform_service.exceptions import CustomException+from unstract.platform_service.exceptions import APIErrorfrom unstract.flags.feature_flag import check_feature_flag_status@@ -41,7 +41,7 @@ class PromptStudioRequestHelper:cursor = db_instance.execute_sql(query)result_row = cursor.fetchone()if not result_row:- raise CustomException(message="Custom Tool not found", code=404)+ raise APIError(message="Custom Tool not found", code=404)columns = [desc[0] for desc in cursor.description]data_dict: dict[str, Any] = dict(zip(columns, result_row))cursor.close()diff --git a/prompt-service/src/unstract/prompt_service/helper.py b/prompt-service/src/unstract/prompt_service/helper.pyindex 764502e29..82385c791 100644--- a/prompt-service/src/unstract/prompt_service/helper.py+++ b/prompt-service/src/unstract/prompt_service/helper.py@@ -12,6 +12,7 @@ from unstract.prompt_service.config import dbfrom unstract.prompt_service.constants import PromptServiceContants as PSKeysfrom unstract.prompt_service.exceptions import APIError, RateLimitErrorfrom unstract.sdk.exceptions import RateLimitError as SdkRateLimitError+from unstract.sdk.exceptions import SdkErrorfrom unstract.sdk.llm import LLMload_dotenv()@@ -292,7 +293,7 @@ def run_completion(# TODO: Catch and handle specific exception hereexcept SdkRateLimitError as e:raise RateLimitError(f"Rate limit error. {str(e)}") from e- except Exception as e:+ except SdkError as e:logger.error(f"Error fetching response for prompt: {e}.")# TODO: Publish this error as a FE updateraise APIError(str(e)) from e