LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
UN-1736 [FIX] ETL workflow failure when renamed adapter is used in tool settings (#1451)
Kirtiman Mishrad25a4bcf3fab
diff --git a/backend/adapter_processor_v2/adapter_processor.py b/backend/adapter_processor_v2/adapter_processor.pyindex 19c36b78c..31ad4a383 100644--- a/backend/adapter_processor_v2/adapter_processor.py+++ b/backend/adapter_processor_v2/adapter_processor.py@@ -11,6 +11,7 @@ from tenant_account_v2.organization_member_service import OrganizationMemberServfrom adapter_processor_v2.constants import AdapterKeys, AllowedDomainsfrom adapter_processor_v2.exceptions import (+ AdapterNotFound,InternalServiceError,InValidAdapterId,TestAdapterError,@@ -227,7 +228,7 @@ class AdapterProcessor:def get_adapter_by_name_and_type(adapter_type: AdapterTypes,adapter_name: str | None = None,- ) -> AdapterInstance | None:+ ) -> AdapterInstance:"""Get the adapter instance by its name and type.Parameters:@@ -236,18 +237,27 @@ class AdapterProcessor:Returns:- AdapterInstance: The adapter with the specified name and type.++ Raises:+ - AdapterNotFound: If the adapter is not found."""- if adapter_name:- adapter: AdapterInstance = AdapterInstance.objects.get(- adapter_name=adapter_name, adapter_type=adapter_type.value- )- else:- try:+ try:+ if adapter_name:+ adapter: AdapterInstance = AdapterInstance.objects.get(+ adapter_name=adapter_name, adapter_type=adapter_type.value+ )+ else:adapter = AdapterInstance.objects.get(adapter_type=adapter_type.value, is_default=True)- except AdapterInstance.DoesNotExist:- return None+ except AdapterInstance.DoesNotExist:+ error_msg = (+ f"Couldn't find adapter with name '{adapter_name}' and type '{adapter_type.value}'"+ if adapter_name+ else f"Couldn't find default adapter with type '{adapter_type.value}'"+ )+ logger.error(error_msg)+ raise AdapterNotFound(error_msg)return adapter@staticmethoddiff --git a/backend/adapter_processor_v2/exceptions.py b/backend/adapter_processor_v2/exceptions.pyindex af5d03b1a..4e0b9bd8a 100644--- a/backend/adapter_processor_v2/exceptions.py+++ b/backend/adapter_processor_v2/exceptions.py@@ -88,3 +88,15 @@ class DeleteAdapterInUseError(APIException):"It is used in a workflow or a prompt studio project")super().__init__(detail, code)+++class AdapterNotFound(APIException):+ status_code = 404+ default_detail = "Adapter not found"++ def __init__(+ self,+ detail: str | None = None,+ code: str | None = None,+ ) -> None:+ super().__init__(detail, code)diff --git a/backend/tool_instance_v2/serializers.py b/backend/tool_instance_v2/serializers.pyindex 9d7a80669..e51b64010 100644--- a/backend/tool_instance_v2/serializers.py+++ b/backend/tool_instance_v2/serializers.py@@ -2,6 +2,8 @@ import loggingimport uuidfrom typing import Any+from adapter_processor_v2.adapter_processor import AdapterProcessor+from adapter_processor_v2.models import AdapterInstancefrom prompt_studio.prompt_studio_registry_v2.constants import PromptStudioRegistryKeysfrom rest_framework.serializers import ListField, Serializer, UUIDField, ValidationErrorfrom workflow_manager.workflow_v2.constants import WorkflowKey@@ -12,8 +14,11 @@ from tool_instance_v2.constants import ToolInstanceKey as TIKeyfrom tool_instance_v2.constants import ToolKeyfrom tool_instance_v2.exceptions import ToolDoesNotExistfrom tool_instance_v2.models import ToolInstance+from tool_instance_v2.tool_instance_helper import ToolInstanceHelperfrom tool_instance_v2.tool_processor import ToolProcessor+from unstract.sdk.adapters.enums import AdapterTypesfrom unstract.tool_registry.dto import Tool+from unstract.tool_registry.tool_utils import ToolUtilslogger = logging.getLogger(__name__)@@ -51,8 +56,99 @@ class ToolInstanceSerializer(AuditSerializer):return reprep[ToolKey.ICON] = tool.iconrep[ToolKey.NAME] = tool.properties.display_name++ # Transform adapter IDs back to names for UI display+ metadata = rep.get(TIKey.METADATA, {})+ if metadata:+ rep[TIKey.METADATA] = self._transform_adapter_ids_to_names_for_display(+ metadata, tool_function+ )+return rep+ def _transform_adapter_ids_to_names_for_display(+ self, metadata: dict[str, Any], tool_uid: str+ ) -> dict[str, Any]:+ """Transform adapter IDs back to names for UI display."""+ # Create a copy to avoid mutating the original metadata+ display_metadata = metadata.copy()++ try:+ tool: Tool = ToolProcessor.get_tool_by_uid(tool_uid=tool_uid)+ schema = ToolUtils.get_json_schema_for_tool(tool)++ # Create mapping of adapter keys to their types+ adapter_key_to_type = {}+ for key in schema.get_llm_adapter_properties().keys():+ adapter_key_to_type[key] = AdapterTypes.LLM+ for key in schema.get_embedding_adapter_properties().keys():+ adapter_key_to_type[key] = AdapterTypes.EMBEDDING+ for key in schema.get_vector_db_adapter_properties().keys():+ adapter_key_to_type[key] = AdapterTypes.VECTOR_DB+ for key in schema.get_text_extractor_adapter_properties().keys():+ adapter_key_to_type[key] = AdapterTypes.X2TEXT+ for key in schema.get_ocr_adapter_properties().keys():+ adapter_key_to_type[key] = AdapterTypes.OCR++ # Transform IDs back to names for display+ for adapter_key, adapter_type in adapter_key_to_type.items():+ adapter_value = display_metadata.get(adapter_key)+ if not adapter_value:+ continue++ # If it's a UUID (adapter ID), convert to name for display+ if ToolInstanceHelper.is_uuid_format(adapter_value):+ try:+ adapter_name = AdapterInstance.objects.get(+ id=adapter_value+ ).adapter_name+ if adapter_name:+ display_metadata[adapter_key] = adapter_name+ else:+ # Adapter ID exists but returns None - mark as orphaned+ display_metadata[adapter_key] = (+ f"[DELETED ADAPTER: {adapter_value}...]"+ )+ logger.warning(+ f"Adapter ID {adapter_value} references a deleted or inaccessible adapter"+ )+ except Exception as e:+ # If conversion fails, show user-friendly error+ display_metadata[adapter_key] = f"[{adapter_value}...] NOT FOUND"+ logger.error(+ f"Could not resolve adapter ID {adapter_value} to name: {e}"+ )+ # If it's not a UUID, validate if the adapter name still exists+ elif adapter_value and not ToolInstanceHelper.is_uuid_format(+ adapter_value+ ):+ try:+ # Validate if adapter name still exists+ adapter = AdapterProcessor.get_adapter_by_name_and_type(+ adapter_type=adapter_type, adapter_name=adapter_value+ )+ if adapter:+ # Adapter name is valid, display as-is+ display_metadata[adapter_key] = adapter_value+ else:+ # Adapter name no longer exists (likely renamed)+ display_metadata[adapter_key] = (+ f"{adapter_value} [NEEDS UPDATE]"+ )+ logger.warning(+ f"Adapter name '{adapter_value}' no longer exists for type {adapter_type} - may have been renamed"+ )+ except Exception as e:+ # If validation fails, show error+ display_metadata[adapter_key] = f"[{adapter_value} NOT FOUND]"+ logger.error(+ f"Could not validate adapter name '{adapter_value}': {e}"+ )++ except Exception as e:+ logger.error(f"Error transforming adapter IDs to names: {e}", exc_info=True)+ return display_metadata+def create(self, validated_data: dict[str, Any]) -> Any:workflow_id = validated_data.pop(WorkflowKey.WF_ID)try:diff --git a/backend/tool_instance_v2/tool_instance_helper.py b/backend/tool_instance_v2/tool_instance_helper.pyindex f8ffc4a93..48319341f 100644--- a/backend/tool_instance_v2/tool_instance_helper.py+++ b/backend/tool_instance_v2/tool_instance_helper.py@@ -69,6 +69,9 @@ class ToolInstanceHelper:) -> None:"""Update the metadata dictionary with adapter properties.+ Handles both legacy metadata (adapter names) and new metadata (adapter IDs).+ Automatically converts adapter names to IDs for consistency.+Parameters:metadata (dict[str, Any]):The metadata dictionary to be updated with adapter properties.@@ -81,17 +84,30 @@ class ToolInstanceHelper:Returns:None+"""if adapter_key in metadata:- adapter_name = metadata[adapter_key]- adapter = AdapterProcessor.get_adapter_by_name_and_type(- adapter_type=adapter_type, adapter_name=adapter_name- )- adapter_id = str(adapter.id) if adapter else None- metadata_key_for_id = adapter_property.get(- AdapterPropertyKey.ADAPTER_ID_KEY, AdapterPropertyKey.ADAPTER_ID- )- metadata[metadata_key_for_id] = adapter_id+ adapter_value = metadata[adapter_key]+ adapter_id = None+ if ToolInstanceHelper.is_uuid_format(adapter_value):+ logger.debug(f"Adapter value '{adapter_value}' is already in UUID format")+ adapter = AdapterInstance.objects.get(+ id=adapter_value, adapter_type=adapter_type.value+ )+ adapter_id = str(adapter.id)+ metadata_key_for_id = adapter_property.get(+ AdapterPropertyKey.ADAPTER_ID_KEY, AdapterPropertyKey.ADAPTER_ID+ )+ metadata[metadata_key_for_id] = adapter_id+ else:+ adapter = AdapterProcessor.get_adapter_by_name_and_type(+ adapter_type=adapter_type, adapter_name=adapter_value+ )+ adapter_id = str(adapter.id)+ metadata_key_for_id = adapter_property.get(+ AdapterPropertyKey.ADAPTER_ID_KEY, AdapterPropertyKey.ADAPTER_ID+ )+ metadata[metadata_key_for_id] = adapter_id@staticmethoddef update_metadata_with_adapter_instances(@@ -189,7 +205,7 @@ class ToolInstanceHelper:metadata_key_for_id = adapter_property.get(AdapterPropertyKey.ADAPTER_ID_KEY, AdapterPropertyKey.ADAPTER_ID)- metadata[adapter_key] = adapter.adapter_name+ metadata[adapter_key] = str(adapter.id)metadata[metadata_key_for_id] = str(adapter.id)@staticmethod@@ -251,6 +267,120 @@ class ToolInstanceHelper:tool_instance.step = step + 1tool_instance.save()+ @staticmethod+ def is_uuid_format(value: str) -> bool:+ """Check if a string is in UUID format."""+ if not value or not isinstance(value, str):+ return False+ try:+ uuid.UUID(value)+ return True+ except ValueError:+ return False++ @staticmethod+ def has_adapter_ids_in_metadata(metadata: dict[str, Any], tool_uid: str) -> bool:+ """Check if metadata already uses adapter IDs instead of names."""+ tool: Tool = ToolProcessor.get_tool_by_uid(tool_uid=tool_uid)+ schema: Spec = ToolUtils.get_json_schema_for_tool(tool)++ # Get all adapter keys for this tool+ adapter_keys = set()+ adapter_keys.update(schema.get_llm_adapter_properties().keys())+ adapter_keys.update(schema.get_embedding_adapter_properties().keys())+ adapter_keys.update(schema.get_vector_db_adapter_properties().keys())+ adapter_keys.update(schema.get_text_extractor_adapter_properties().keys())+ adapter_keys.update(schema.get_ocr_adapter_properties().keys())++ # Check if any adapter value is in UUID format (indicates already migrated)+ for key in adapter_keys:+ if (+ key in metadata+ and metadata[key]+ and ToolInstanceHelper.is_uuid_format(metadata[key])+ ):+ return True # Found at least one UUID, assume migrated++ return False # No UUIDs found, still using names++ @staticmethod+ def _migrate_adapter_keys_for_type(+ metadata: dict[str, Any],+ adapter_keys: dict[str, Any],+ adapter_type: AdapterTypes,+ ) -> tuple[dict[str, Any], bool]:+ """Helper method to migrate adapter names to IDs for a specific adapter type."""+ needs_update = False++ for adapter_key in adapter_keys.keys():+ if (+ adapter_key in metadata+ and metadata[adapter_key]+ and not ToolInstanceHelper.is_uuid_format(metadata[adapter_key])+ ):+ adapter_name = metadata[adapter_key]+ adapter = AdapterProcessor.get_adapter_by_name_and_type(+ adapter_type=adapter_type, adapter_name=adapter_name+ )+ logger.info(+ f"Migrating {adapter_type.value} adapter '{adapter_name}' to ID '{adapter.id}'"+ )+ metadata[adapter_key] = str(adapter.id)+ needs_update = True++ return metadata, needs_update++ @staticmethod+ def ensure_adapter_ids_in_metadata(+ tool_instance: ToolInstance, user: User = None+ ) -> dict[str, Any]:+ """Lazy migration: convert adapter names to IDs if needed and update database."""+ metadata = tool_instance.metadata.copy()++ # Check if already migrated+ if ToolInstanceHelper.has_adapter_ids_in_metadata(+ metadata, tool_instance.tool_id+ ):+ return metadata++ logger.info(f"Performing lazy migration for tool instance {tool_instance.id}")++ # Get tool schema and name for error messages+ tool: Tool = ToolProcessor.get_tool_by_uid(tool_uid=tool_instance.tool_id)+ schema: Spec = ToolUtils.get_json_schema_for_tool(tool)++ overall_needs_update = False++ # Migrate each adapter type+ adapter_types_to_migrate = [+ (schema.get_llm_adapter_properties(), AdapterTypes.LLM),+ (schema.get_embedding_adapter_properties(), AdapterTypes.EMBEDDING),+ (schema.get_vector_db_adapter_properties(), AdapterTypes.VECTOR_DB),+ (schema.get_text_extractor_adapter_properties(), AdapterTypes.X2TEXT),+ (schema.get_ocr_adapter_properties(), AdapterTypes.OCR),+ ]++ for adapter_properties, adapter_type in adapter_types_to_migrate:+ if adapter_properties: # Only process if this tool uses this adapter type+ metadata, needs_update = (+ ToolInstanceHelper._migrate_adapter_keys_for_type(+ metadata=metadata,+ adapter_keys=adapter_properties,+ adapter_type=adapter_type,+ )+ )+ overall_needs_update = overall_needs_update or needs_update++ # Update database if changes were made+ if overall_needs_update:+ tool_instance.metadata = metadata+ tool_instance.save()+ logger.info(+ f"Successfully migrated tool instance {tool_instance.id} to use adapter IDs"+ )++ return metadata+@staticmethoddef validate_tool_settings(user: User, tool_uid: str, tool_meta: dict[str, Any]diff --git a/backend/tool_instance_v2/tool_processor.py b/backend/tool_instance_v2/tool_processor.pyindex 41193e2a7..ac2a8de73 100644--- a/backend/tool_instance_v2/tool_processor.py+++ b/backend/tool_instance_v2/tool_processor.py@@ -62,6 +62,22 @@ class ToolProcessor:schema_json: dict[str, Any] = schema.to_dict()return schema_json+ @staticmethod+ def _update_schema_for_adapter_type(+ schema: Spec, keys: list[str], adapter_type: AdapterTypes, user: User+ ) -> None:+ """Helper method to update schema properties for a specific adapter type."""+ if not keys:+ return++ adapters = AdapterProcessor.get_adapters_by_type(adapter_type, user=user)+ adapter_ids = [str(adapter.id) for adapter in adapters]+ adapter_names = [adapter.adapter_name for adapter in adapters]++ for key in keys:+ schema.properties[key]["enum"] = adapter_ids+ schema.properties[key]["enumNames"] = adapter_names+@staticmethoddef update_schema_with_adapter_configurations(schema: Spec, user: User) -> None:"""Updates the JSON schema with the available adapter configurations@@ -79,41 +95,21 @@ class ToolProcessor:x2text_keys = schema.get_text_extractor_adapter_properties_keys()ocr_keys = schema.get_ocr_adapter_properties_keys()- if llm_keys:- adapters = AdapterProcessor.get_adapters_by_type(AdapterTypes.LLM, user=user)- for key in llm_keys:- adapter_names = map(lambda adapter: str(adapter.adapter_name), adapters)- schema.properties[key]["enum"] = list(adapter_names)-- if embedding_keys:- adapters = AdapterProcessor.get_adapters_by_type(- AdapterTypes.EMBEDDING, user=user- )- for key in embedding_keys:- adapter_names = map(lambda adapter: str(adapter.adapter_name), adapters)- schema.properties[key]["enum"] = list(adapter_names)-- if vector_db_keys:- adapters = AdapterProcessor.get_adapters_by_type(- AdapterTypes.VECTOR_DB, user=user- )- for key in vector_db_keys:- adapter_names = map(lambda adapter: str(adapter.adapter_name), adapters)- schema.properties[key]["enum"] = list(adapter_names)-- if x2text_keys:- adapters = AdapterProcessor.get_adapters_by_type(- AdapterTypes.X2TEXT, user=user- )- for key in x2text_keys:- adapter_names = map(lambda adapter: str(adapter.adapter_name), adapters)- schema.properties[key]["enum"] = list(adapter_names)-- if ocr_keys:- adapters = AdapterProcessor.get_adapters_by_type(AdapterTypes.OCR, user=user)- for key in ocr_keys:- adapter_names = map(lambda adapter: str(adapter.adapter_name), adapters)- schema.properties[key]["enum"] = list(adapter_names)+ ToolProcessor._update_schema_for_adapter_type(+ schema, llm_keys, AdapterTypes.LLM, user+ )+ ToolProcessor._update_schema_for_adapter_type(+ schema, embedding_keys, AdapterTypes.EMBEDDING, user+ )+ ToolProcessor._update_schema_for_adapter_type(+ schema, vector_db_keys, AdapterTypes.VECTOR_DB, user+ )+ ToolProcessor._update_schema_for_adapter_type(+ schema, x2text_keys, AdapterTypes.X2TEXT, user+ )+ ToolProcessor._update_schema_for_adapter_type(+ schema, ocr_keys, AdapterTypes.OCR, user+ )@staticmethoddef get_tool_list(user: User) -> list[dict[str, Any]]:diff --git a/backend/workflow_manager/workflow_v2/workflow_helper.py b/backend/workflow_manager/workflow_v2/workflow_helper.pyindex c42e06180..8969c5414 100644--- a/backend/workflow_manager/workflow_v2/workflow_helper.py+++ b/backend/workflow_manager/workflow_v2/workflow_helper.py@@ -28,9 +28,7 @@ from utils.user_context import UserContextfrom backend.celery_service import app as celery_appfrom unstract.workflow_execution.enums import LogStagefrom workflow_manager.endpoint_v2.destination import DestinationConnector-from workflow_manager.endpoint_v2.dto import (- FileHash,-)+from workflow_manager.endpoint_v2.dto import FileHashfrom workflow_manager.endpoint_v2.result_cache_utils import ResultCacheUtilsfrom workflow_manager.endpoint_v2.source import SourceConnectorfrom workflow_manager.execution.dto import ExecutionCache@@ -42,11 +40,7 @@ from workflow_manager.workflow_v2.constants import (WorkflowExecutionKey,WorkflowMessages,)-from workflow_manager.workflow_v2.dto import (- ExecutionResponse,- FileBatchData,- FileData,-)+from workflow_manager.workflow_v2.dto import ExecutionResponse, FileBatchData, FileDatafrom workflow_manager.workflow_v2.enums import (ExecutionStatus,SchemaEntity,@@ -238,10 +232,15 @@ class WorkflowHelper:tool_instances: list[ToolInstance],) -> None:for tool in tool_instances:+ # Ensure adapter IDs are resolved before validation+ user = tool.workflow.created_by+ migrated_metadata = ToolInstanceHelper.ensure_adapter_ids_in_metadata(+ tool, user=user+ )ToolInstanceHelper.validate_tool_settings(- user=tool.workflow.created_by,+ user=user,tool_uid=tool.tool_id,- tool_meta=tool.metadata,+ tool_meta=migrated_metadata,)@staticmethod