Skip to content

LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows

UN-1736 [FIX] ETL workflow failure when renamed adapter is used in tool settings (#1451)

Kirtiman Mishrad25a4bcf3fab
diff --git a/backend/adapter_processor_v2/adapter_processor.py b/backend/adapter_processor_v2/adapter_processor.py
index 19c36b78c..31ad4a383 100644
--- a/backend/adapter_processor_v2/adapter_processor.py
+++ b/backend/adapter_processor_v2/adapter_processor.py
@@ -11,6 +11,7 @@ from tenant_account_v2.organization_member_service import OrganizationMemberServ
from adapter_processor_v2.constants import AdapterKeys, AllowedDomains
from adapter_processor_v2.exceptions import (
+ AdapterNotFound,
InternalServiceError,
InValidAdapterId,
TestAdapterError,
@@ -227,7 +228,7 @@ class AdapterProcessor:
def get_adapter_by_name_and_type(
adapter_type: AdapterTypes,
adapter_name: str | None = None,
- ) -> AdapterInstance | None:
+ ) -> AdapterInstance:
"""Get the adapter instance by its name and type.
Parameters:
@@ -236,18 +237,27 @@ class AdapterProcessor:
Returns:
- AdapterInstance: The adapter with the specified name and type.
+
+ Raises:
+ - AdapterNotFound: If the adapter is not found.
"""
- if adapter_name:
- adapter: AdapterInstance = AdapterInstance.objects.get(
- adapter_name=adapter_name, adapter_type=adapter_type.value
- )
- else:
- try:
+ try:
+ if adapter_name:
+ adapter: AdapterInstance = AdapterInstance.objects.get(
+ adapter_name=adapter_name, adapter_type=adapter_type.value
+ )
+ else:
adapter = AdapterInstance.objects.get(
adapter_type=adapter_type.value, is_default=True
)
- except AdapterInstance.DoesNotExist:
- return None
+ except AdapterInstance.DoesNotExist:
+ error_msg = (
+ f"Couldn't find adapter with name '{adapter_name}' and type '{adapter_type.value}'"
+ if adapter_name
+ else f"Couldn't find default adapter with type '{adapter_type.value}'"
+ )
+ logger.error(error_msg)
+ raise AdapterNotFound(error_msg)
return adapter
@staticmethod
diff --git a/backend/adapter_processor_v2/exceptions.py b/backend/adapter_processor_v2/exceptions.py
index af5d03b1a..4e0b9bd8a 100644
--- a/backend/adapter_processor_v2/exceptions.py
+++ b/backend/adapter_processor_v2/exceptions.py
@@ -88,3 +88,15 @@ class DeleteAdapterInUseError(APIException):
"It is used in a workflow or a prompt studio project"
)
super().__init__(detail, code)
+
+
+class AdapterNotFound(APIException):
+ status_code = 404
+ default_detail = "Adapter not found"
+
+ def __init__(
+ self,
+ detail: str | None = None,
+ code: str | None = None,
+ ) -> None:
+ super().__init__(detail, code)
diff --git a/backend/tool_instance_v2/serializers.py b/backend/tool_instance_v2/serializers.py
index 9d7a80669..e51b64010 100644
--- a/backend/tool_instance_v2/serializers.py
+++ b/backend/tool_instance_v2/serializers.py
@@ -2,6 +2,8 @@ import logging
import uuid
from typing import Any
+from adapter_processor_v2.adapter_processor import AdapterProcessor
+from adapter_processor_v2.models import AdapterInstance
from prompt_studio.prompt_studio_registry_v2.constants import PromptStudioRegistryKeys
from rest_framework.serializers import ListField, Serializer, UUIDField, ValidationError
from workflow_manager.workflow_v2.constants import WorkflowKey
@@ -12,8 +14,11 @@ from tool_instance_v2.constants import ToolInstanceKey as TIKey
from tool_instance_v2.constants import ToolKey
from tool_instance_v2.exceptions import ToolDoesNotExist
from tool_instance_v2.models import ToolInstance
+from tool_instance_v2.tool_instance_helper import ToolInstanceHelper
from tool_instance_v2.tool_processor import ToolProcessor
+from unstract.sdk.adapters.enums import AdapterTypes
from unstract.tool_registry.dto import Tool
+from unstract.tool_registry.tool_utils import ToolUtils
logger = logging.getLogger(__name__)
@@ -51,8 +56,99 @@ class ToolInstanceSerializer(AuditSerializer):
return rep
rep[ToolKey.ICON] = tool.icon
rep[ToolKey.NAME] = tool.properties.display_name
+
+ # Transform adapter IDs back to names for UI display
+ metadata = rep.get(TIKey.METADATA, {})
+ if metadata:
+ rep[TIKey.METADATA] = self._transform_adapter_ids_to_names_for_display(
+ metadata, tool_function
+ )
+
return rep
+ def _transform_adapter_ids_to_names_for_display(
+ self, metadata: dict[str, Any], tool_uid: str
+ ) -> dict[str, Any]:
+ """Transform adapter IDs back to names for UI display."""
+ # Create a copy to avoid mutating the original metadata
+ display_metadata = metadata.copy()
+
+ try:
+ tool: Tool = ToolProcessor.get_tool_by_uid(tool_uid=tool_uid)
+ schema = ToolUtils.get_json_schema_for_tool(tool)
+
+ # Create mapping of adapter keys to their types
+ adapter_key_to_type = {}
+ for key in schema.get_llm_adapter_properties().keys():
+ adapter_key_to_type[key] = AdapterTypes.LLM
+ for key in schema.get_embedding_adapter_properties().keys():
+ adapter_key_to_type[key] = AdapterTypes.EMBEDDING
+ for key in schema.get_vector_db_adapter_properties().keys():
+ adapter_key_to_type[key] = AdapterTypes.VECTOR_DB
+ for key in schema.get_text_extractor_adapter_properties().keys():
+ adapter_key_to_type[key] = AdapterTypes.X2TEXT
+ for key in schema.get_ocr_adapter_properties().keys():
+ adapter_key_to_type[key] = AdapterTypes.OCR
+
+ # Transform IDs back to names for display
+ for adapter_key, adapter_type in adapter_key_to_type.items():
+ adapter_value = display_metadata.get(adapter_key)
+ if not adapter_value:
+ continue
+
+ # If it's a UUID (adapter ID), convert to name for display
+ if ToolInstanceHelper.is_uuid_format(adapter_value):
+ try:
+ adapter_name = AdapterInstance.objects.get(
+ id=adapter_value
+ ).adapter_name
+ if adapter_name:
+ display_metadata[adapter_key] = adapter_name
+ else:
+ # Adapter ID exists but returns None - mark as orphaned
+ display_metadata[adapter_key] = (
+ f"[DELETED ADAPTER: {adapter_value}...]"
+ )
+ logger.warning(
+ f"Adapter ID {adapter_value} references a deleted or inaccessible adapter"
+ )
+ except Exception as e:
+ # If conversion fails, show user-friendly error
+ display_metadata[adapter_key] = f"[{adapter_value}...] NOT FOUND"
+ logger.error(
+ f"Could not resolve adapter ID {adapter_value} to name: {e}"
+ )
+ # If it's not a UUID, validate if the adapter name still exists
+ elif adapter_value and not ToolInstanceHelper.is_uuid_format(
+ adapter_value
+ ):
+ try:
+ # Validate if adapter name still exists
+ adapter = AdapterProcessor.get_adapter_by_name_and_type(
+ adapter_type=adapter_type, adapter_name=adapter_value
+ )
+ if adapter:
+ # Adapter name is valid, display as-is
+ display_metadata[adapter_key] = adapter_value
+ else:
+ # Adapter name no longer exists (likely renamed)
+ display_metadata[adapter_key] = (
+ f"{adapter_value} [NEEDS UPDATE]"
+ )
+ logger.warning(
+ f"Adapter name '{adapter_value}' no longer exists for type {adapter_type} - may have been renamed"
+ )
+ except Exception as e:
+ # If validation fails, show error
+ display_metadata[adapter_key] = f"[{adapter_value} NOT FOUND]"
+ logger.error(
+ f"Could not validate adapter name '{adapter_value}': {e}"
+ )
+
+ except Exception as e:
+ logger.error(f"Error transforming adapter IDs to names: {e}", exc_info=True)
+ return display_metadata
+
def create(self, validated_data: dict[str, Any]) -> Any:
workflow_id = validated_data.pop(WorkflowKey.WF_ID)
try:
diff --git a/backend/tool_instance_v2/tool_instance_helper.py b/backend/tool_instance_v2/tool_instance_helper.py
index f8ffc4a93..48319341f 100644
--- a/backend/tool_instance_v2/tool_instance_helper.py
+++ b/backend/tool_instance_v2/tool_instance_helper.py
@@ -69,6 +69,9 @@ class ToolInstanceHelper:
) -> None:
"""Update the metadata dictionary with adapter properties.
+ Handles both legacy metadata (adapter names) and new metadata (adapter IDs).
+ Automatically converts adapter names to IDs for consistency.
+
Parameters:
metadata (dict[str, Any]):
The metadata dictionary to be updated with adapter properties.
@@ -81,17 +84,30 @@ class ToolInstanceHelper:
Returns:
None
+
"""
if adapter_key in metadata:
- adapter_name = metadata[adapter_key]
- adapter = AdapterProcessor.get_adapter_by_name_and_type(
- adapter_type=adapter_type, adapter_name=adapter_name
- )
- adapter_id = str(adapter.id) if adapter else None
- metadata_key_for_id = adapter_property.get(
- AdapterPropertyKey.ADAPTER_ID_KEY, AdapterPropertyKey.ADAPTER_ID
- )
- metadata[metadata_key_for_id] = adapter_id
+ adapter_value = metadata[adapter_key]
+ adapter_id = None
+ if ToolInstanceHelper.is_uuid_format(adapter_value):
+ logger.debug(f"Adapter value '{adapter_value}' is already in UUID format")
+ adapter = AdapterInstance.objects.get(
+ id=adapter_value, adapter_type=adapter_type.value
+ )
+ adapter_id = str(adapter.id)
+ metadata_key_for_id = adapter_property.get(
+ AdapterPropertyKey.ADAPTER_ID_KEY, AdapterPropertyKey.ADAPTER_ID
+ )
+ metadata[metadata_key_for_id] = adapter_id
+ else:
+ adapter = AdapterProcessor.get_adapter_by_name_and_type(
+ adapter_type=adapter_type, adapter_name=adapter_value
+ )
+ adapter_id = str(adapter.id)
+ metadata_key_for_id = adapter_property.get(
+ AdapterPropertyKey.ADAPTER_ID_KEY, AdapterPropertyKey.ADAPTER_ID
+ )
+ metadata[metadata_key_for_id] = adapter_id
@staticmethod
def update_metadata_with_adapter_instances(
@@ -189,7 +205,7 @@ class ToolInstanceHelper:
metadata_key_for_id = adapter_property.get(
AdapterPropertyKey.ADAPTER_ID_KEY, AdapterPropertyKey.ADAPTER_ID
)
- metadata[adapter_key] = adapter.adapter_name
+ metadata[adapter_key] = str(adapter.id)
metadata[metadata_key_for_id] = str(adapter.id)
@staticmethod
@@ -251,6 +267,120 @@ class ToolInstanceHelper:
tool_instance.step = step + 1
tool_instance.save()
+ @staticmethod
+ def is_uuid_format(value: str) -> bool:
+ """Check if a string is in UUID format."""
+ if not value or not isinstance(value, str):
+ return False
+ try:
+ uuid.UUID(value)
+ return True
+ except ValueError:
+ return False
+
+ @staticmethod
+ def has_adapter_ids_in_metadata(metadata: dict[str, Any], tool_uid: str) -> bool:
+ """Check if metadata already uses adapter IDs instead of names."""
+ tool: Tool = ToolProcessor.get_tool_by_uid(tool_uid=tool_uid)
+ schema: Spec = ToolUtils.get_json_schema_for_tool(tool)
+
+ # Get all adapter keys for this tool
+ adapter_keys = set()
+ adapter_keys.update(schema.get_llm_adapter_properties().keys())
+ adapter_keys.update(schema.get_embedding_adapter_properties().keys())
+ adapter_keys.update(schema.get_vector_db_adapter_properties().keys())
+ adapter_keys.update(schema.get_text_extractor_adapter_properties().keys())
+ adapter_keys.update(schema.get_ocr_adapter_properties().keys())
+
+ # Check if any adapter value is in UUID format (indicates already migrated)
+ for key in adapter_keys:
+ if (
+ key in metadata
+ and metadata[key]
+ and ToolInstanceHelper.is_uuid_format(metadata[key])
+ ):
+ return True # Found at least one UUID, assume migrated
+
+ return False # No UUIDs found, still using names
+
+ @staticmethod
+ def _migrate_adapter_keys_for_type(
+ metadata: dict[str, Any],
+ adapter_keys: dict[str, Any],
+ adapter_type: AdapterTypes,
+ ) -> tuple[dict[str, Any], bool]:
+ """Helper method to migrate adapter names to IDs for a specific adapter type."""
+ needs_update = False
+
+ for adapter_key in adapter_keys.keys():
+ if (
+ adapter_key in metadata
+ and metadata[adapter_key]
+ and not ToolInstanceHelper.is_uuid_format(metadata[adapter_key])
+ ):
+ adapter_name = metadata[adapter_key]
+ adapter = AdapterProcessor.get_adapter_by_name_and_type(
+ adapter_type=adapter_type, adapter_name=adapter_name
+ )
+ logger.info(
+ f"Migrating {adapter_type.value} adapter '{adapter_name}' to ID '{adapter.id}'"
+ )
+ metadata[adapter_key] = str(adapter.id)
+ needs_update = True
+
+ return metadata, needs_update
+
+ @staticmethod
+ def ensure_adapter_ids_in_metadata(
+ tool_instance: ToolInstance, user: User = None
+ ) -> dict[str, Any]:
+ """Lazy migration: convert adapter names to IDs if needed and update database."""
+ metadata = tool_instance.metadata.copy()
+
+ # Check if already migrated
+ if ToolInstanceHelper.has_adapter_ids_in_metadata(
+ metadata, tool_instance.tool_id
+ ):
+ return metadata
+
+ logger.info(f"Performing lazy migration for tool instance {tool_instance.id}")
+
+ # Get tool schema and name for error messages
+ tool: Tool = ToolProcessor.get_tool_by_uid(tool_uid=tool_instance.tool_id)
+ schema: Spec = ToolUtils.get_json_schema_for_tool(tool)
+
+ overall_needs_update = False
+
+ # Migrate each adapter type
+ adapter_types_to_migrate = [
+ (schema.get_llm_adapter_properties(), AdapterTypes.LLM),
+ (schema.get_embedding_adapter_properties(), AdapterTypes.EMBEDDING),
+ (schema.get_vector_db_adapter_properties(), AdapterTypes.VECTOR_DB),
+ (schema.get_text_extractor_adapter_properties(), AdapterTypes.X2TEXT),
+ (schema.get_ocr_adapter_properties(), AdapterTypes.OCR),
+ ]
+
+ for adapter_properties, adapter_type in adapter_types_to_migrate:
+ if adapter_properties: # Only process if this tool uses this adapter type
+ metadata, needs_update = (
+ ToolInstanceHelper._migrate_adapter_keys_for_type(
+ metadata=metadata,
+ adapter_keys=adapter_properties,
+ adapter_type=adapter_type,
+ )
+ )
+ overall_needs_update = overall_needs_update or needs_update
+
+ # Update database if changes were made
+ if overall_needs_update:
+ tool_instance.metadata = metadata
+ tool_instance.save()
+ logger.info(
+ f"Successfully migrated tool instance {tool_instance.id} to use adapter IDs"
+ )
+
+ return metadata
+
@staticmethod
def validate_tool_settings(
user: User, tool_uid: str, tool_meta: dict[str, Any]
diff --git a/backend/tool_instance_v2/tool_processor.py b/backend/tool_instance_v2/tool_processor.py
index 41193e2a7..ac2a8de73 100644
--- a/backend/tool_instance_v2/tool_processor.py
+++ b/backend/tool_instance_v2/tool_processor.py
@@ -62,6 +62,22 @@ class ToolProcessor:
schema_json: dict[str, Any] = schema.to_dict()
return schema_json
+ @staticmethod
+ def _update_schema_for_adapter_type(
+ schema: Spec, keys: list[str], adapter_type: AdapterTypes, user: User
+ ) -> None:
+ """Helper method to update schema properties for a specific adapter type."""
+ if not keys:
+ return
+
+ adapters = AdapterProcessor.get_adapters_by_type(adapter_type, user=user)
+ adapter_ids = [str(adapter.id) for adapter in adapters]
+ adapter_names = [adapter.adapter_name for adapter in adapters]
+
+ for key in keys:
+ schema.properties[key]["enum"] = adapter_ids
+ schema.properties[key]["enumNames"] = adapter_names
+
@staticmethod
def update_schema_with_adapter_configurations(schema: Spec, user: User) -> None:
"""Updates the JSON schema with the available adapter configurations
@@ -79,41 +95,21 @@ class ToolProcessor:
x2text_keys = schema.get_text_extractor_adapter_properties_keys()
ocr_keys = schema.get_ocr_adapter_properties_keys()
- if llm_keys:
- adapters = AdapterProcessor.get_adapters_by_type(AdapterTypes.LLM, user=user)
- for key in llm_keys:
- adapter_names = map(lambda adapter: str(adapter.adapter_name), adapters)
- schema.properties[key]["enum"] = list(adapter_names)
-
- if embedding_keys:
- adapters = AdapterProcessor.get_adapters_by_type(
- AdapterTypes.EMBEDDING, user=user
- )
- for key in embedding_keys:
- adapter_names = map(lambda adapter: str(adapter.adapter_name), adapters)
- schema.properties[key]["enum"] = list(adapter_names)
-
- if vector_db_keys:
- adapters = AdapterProcessor.get_adapters_by_type(
- AdapterTypes.VECTOR_DB, user=user
- )
- for key in vector_db_keys:
- adapter_names = map(lambda adapter: str(adapter.adapter_name), adapters)
- schema.properties[key]["enum"] = list(adapter_names)
-
- if x2text_keys:
- adapters = AdapterProcessor.get_adapters_by_type(
- AdapterTypes.X2TEXT, user=user
- )
- for key in x2text_keys:
- adapter_names = map(lambda adapter: str(adapter.adapter_name), adapters)
- schema.properties[key]["enum"] = list(adapter_names)
-
- if ocr_keys:
- adapters = AdapterProcessor.get_adapters_by_type(AdapterTypes.OCR, user=user)
- for key in ocr_keys:
- adapter_names = map(lambda adapter: str(adapter.adapter_name), adapters)
- schema.properties[key]["enum"] = list(adapter_names)
+ ToolProcessor._update_schema_for_adapter_type(
+ schema, llm_keys, AdapterTypes.LLM, user
+ )
+ ToolProcessor._update_schema_for_adapter_type(
+ schema, embedding_keys, AdapterTypes.EMBEDDING, user
+ )
+ ToolProcessor._update_schema_for_adapter_type(
+ schema, vector_db_keys, AdapterTypes.VECTOR_DB, user
+ )
+ ToolProcessor._update_schema_for_adapter_type(
+ schema, x2text_keys, AdapterTypes.X2TEXT, user
+ )
+ ToolProcessor._update_schema_for_adapter_type(
+ schema, ocr_keys, AdapterTypes.OCR, user
+ )
@staticmethod
def get_tool_list(user: User) -> list[dict[str, Any]]:
diff --git a/backend/workflow_manager/workflow_v2/workflow_helper.py b/backend/workflow_manager/workflow_v2/workflow_helper.py
index c42e06180..8969c5414 100644
--- a/backend/workflow_manager/workflow_v2/workflow_helper.py
+++ b/backend/workflow_manager/workflow_v2/workflow_helper.py
@@ -28,9 +28,7 @@ from utils.user_context import UserContext
from backend.celery_service import app as celery_app
from unstract.workflow_execution.enums import LogStage
from workflow_manager.endpoint_v2.destination import DestinationConnector
-from workflow_manager.endpoint_v2.dto import (
- FileHash,
-)
+from workflow_manager.endpoint_v2.dto import FileHash
from workflow_manager.endpoint_v2.result_cache_utils import ResultCacheUtils
from workflow_manager.endpoint_v2.source import SourceConnector
from workflow_manager.execution.dto import ExecutionCache
@@ -42,11 +40,7 @@ from workflow_manager.workflow_v2.constants import (
WorkflowExecutionKey,
WorkflowMessages,
)
-from workflow_manager.workflow_v2.dto import (
- ExecutionResponse,
- FileBatchData,
- FileData,
-)
+from workflow_manager.workflow_v2.dto import ExecutionResponse, FileBatchData, FileData
from workflow_manager.workflow_v2.enums import (
ExecutionStatus,
SchemaEntity,
@@ -238,10 +232,15 @@ class WorkflowHelper:
tool_instances: list[ToolInstance],
) -> None:
for tool in tool_instances:
+ # Ensure adapter IDs are resolved before validation
+ user = tool.workflow.created_by
+ migrated_metadata = ToolInstanceHelper.ensure_adapter_ids_in_metadata(
+ tool, user=user
+ )
ToolInstanceHelper.validate_tool_settings(
- user=tool.workflow.created_by,
+ user=user,
tool_uid=tool.tool_id,
- tool_meta=tool.metadata,
+ tool_meta=migrated_metadata,
)
@staticmethod