LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
"""Internal API Views for Tool Instance Operations
This module contains internal API endpoints used by workers for tool execution.
"""
import logging
from django.views.decorators.csrf import csrf_exempt
from rest_framework import status, viewsets
from rest_framework.decorators import api_view
from rest_framework.response import Response
from utils.organization_utils import filter_queryset_by_organization
from tool_instance_v2.models import ToolInstance
from tool_instance_v2.serializers import ToolInstanceSerializer
from tool_instance_v2.tool_instance_helper import ToolInstanceHelper
from tool_instance_v2.tool_processor import ToolProcessor
logger = logging.getLogger(__name__)
class ToolExecutionInternalViewSet(viewsets.ModelViewSet):
"""Internal API for tool execution operations used by lightweight workers."""
serializer_class = ToolInstanceSerializer
# OrganizationFilterBackend is off here; scoping runs through
# filter_queryset_by_organization, which fails closed, so a caller without
# X-Organization-ID gets zero rows.
skip_org_filter = True
def get_queryset(self):
# Filter by organization context set by internal API middleware
# Use relationship path: ToolInstance -> Workflow -> Organization
queryset = ToolInstance.objects.all()
return filter_queryset_by_organization(
queryset, self.request, "workflow__organization"
)
def execute_tool(self, request, pk=None):
"""Execute a specific tool with provided input data.
This replaces the direct tool execution that was previously done
in the heavy Django workers.
"""
try:
tool_instance = self.get_object()
# Extract execution parameters from request
input_data = request.data.get("input_data", {})
file_data = request.data.get("file_data", {})
execution_context = request.data.get("execution_context", {})
# Execute tool using existing tool processor
execution_result = ToolProcessor.execute_tool(
tool_instance=tool_instance,
input_data=input_data,
file_data=file_data,
context=execution_context,
user=request.user,
)
return Response(
{
"status": "success",
"tool_instance_id": str(tool_instance.id),
"execution_result": execution_result,
"tool_function": tool_instance.tool_function,
"step": tool_instance.step,
},
status=status.HTTP_200_OK,
)
except Exception as e:
logger.error(f"Tool execution failed for tool {pk}: {e}")
return Response(
{
"status": "error",
"error_message": str(e),
"tool_instance_id": str(pk) if pk else None,
},
status=status.HTTP_500_INTERNAL_SERVER_ERROR,
)
@csrf_exempt # Safe: Internal API with Bearer token auth, no session/cookies
@api_view(["GET"])
def tool_execution_status_internal(request, execution_id):
"""Get tool execution status for internal API calls."""
try:
# This would track tool execution status
# For now, return a basic status structure
return Response(
{
"execution_id": execution_id,
"status": "completed", # Could be: pending, running, completed, failed
"progress": 100,
"results": [],
"error_message": None,
},
status=status.HTTP_200_OK,
)
except Exception as e:
logger.error(f"Failed to get tool execution status for {execution_id}: {e}")
return Response({"error": str(e)}, status=status.HTTP_500_INTERNAL_SERVER_ERROR)
@csrf_exempt # Safe: Internal API with Bearer token auth, no session/cookies
@api_view(["GET"])
def tool_by_id_internal(request, tool_id):
"""Get tool information by tool ID for internal API calls."""
try:
logger.info(f"Getting tool information for tool ID: {tool_id}")
# Get tool from registry using ToolProcessor
try:
tool = ToolProcessor.get_tool_by_uid(tool_id)
logger.info(f"Successfully retrieved tool from ToolProcessor: {tool_id}")
except Exception as tool_fetch_error:
logger.error(
f"Failed to fetch tool {tool_id} from ToolProcessor: {tool_fetch_error}"
)
# Return fallback using Structure Tool image (which actually exists)
from django.conf import settings
return Response(
{
"tool": {
"tool_id": tool_id,
"properties": {
"displayName": f"Missing Tool ({tool_id[:8]}...)",
"functionName": tool_id,
"description": "Tool not found in registry or Prompt Studio",
"toolVersion": "unknown",
},
"image_name": settings.STRUCTURE_TOOL_IMAGE_NAME,
"image_tag": settings.STRUCTURE_TOOL_IMAGE_TAG,
"name": f"Missing Tool ({tool_id[:8]}...)",
"description": "Tool not found in registry or Prompt Studio",
"version": "unknown",
"note": "Fallback data for missing tool",
}
},
status=status.HTTP_200_OK,
)
# Convert Properties object to dict for JSON serialization
properties_dict = {}
try:
if hasattr(tool.properties, "to_dict"):
# Use the to_dict method if available (which handles Adapter serialization)
properties_dict = tool.properties.to_dict()
logger.info(f"Properties serialized using to_dict() for tool {tool_id}")
elif hasattr(tool.properties, "dict"):
properties_dict = tool.properties.dict()
logger.info(f"Properties serialized using dict() for tool {tool_id}")
elif hasattr(tool.properties, "__dict__"):
properties_dict = tool.properties.__dict__
logger.info(f"Properties serialized using __dict__ for tool {tool_id}")
else:
# Try to convert to dict if it's iterable
try:
properties_dict = dict(tool.properties)
logger.info(
f"Properties serialized using dict conversion for tool {tool_id}"
)
except (TypeError, ValueError):
properties_dict = {"default": "true"} # Fallback
logger.warning(f"Using fallback properties for tool {tool_id}")
except Exception as props_error:
logger.error(
f"Failed to serialize properties for tool {tool_id}: {props_error}"
)
properties_dict = {"error": "serialization_failed"}
# Handle spec serialization if needed
if hasattr(tool, "spec") and tool.spec:
if hasattr(tool.spec, "to_dict"):
tool.spec.to_dict()
elif hasattr(tool.spec, "__dict__"):
pass
# Return tool information with essential fields only to avoid serialization issues
return Response(
{
"tool": {
"tool_id": tool_id,
"properties": properties_dict,
"image_name": str(tool.image_name)
if tool.image_name
else "default-tool",
"image_tag": str(tool.image_tag) if tool.image_tag else "latest",
"name": getattr(tool, "name", tool_id),
"description": getattr(tool, "description", ""),
"version": getattr(tool, "version", "latest"),
}
},
status=status.HTTP_200_OK,
)
except Exception as e:
logger.error(f"Failed to get tool information for {tool_id}: {e}")
import traceback
logger.error(f"Full traceback: {traceback.format_exc()}")
# Always return fallback data instead of error to allow workflow to continue
from django.conf import settings
return Response(
{
"tool": {
"tool_id": tool_id,
"properties": {
"displayName": f"Error Tool ({tool_id[:8]}...)",
"functionName": tool_id,
"description": f"Error processing tool: {str(e)[:100]}",
"toolVersion": "error",
},
"image_name": settings.STRUCTURE_TOOL_IMAGE_NAME,
"image_tag": settings.STRUCTURE_TOOL_IMAGE_TAG,
"name": f"Error Tool ({tool_id[:8]}...)",
"description": f"Error: {str(e)[:100]}",
"version": "error",
"error": str(e),
"note": "Fallback data for tool processing error",
}
},
status=status.HTTP_200_OK, # Return 200 to allow workflow to continue
)
@csrf_exempt # Safe: Internal API with Bearer token auth, no session/cookies
@api_view(["GET"])
def tool_instances_by_workflow_internal(request, workflow_id):
"""Get tool instances for a workflow for internal API calls."""
try:
from workflow_manager.workflow_v2.models.workflow import Workflow
logger.info(f"Getting tool instances for workflow: {workflow_id}")
# Get workflow with organization filtering first (via DefaultOrganizationManagerMixin)
try:
workflow = Workflow.objects.get(id=workflow_id)
logger.info(f"Found workflow: {workflow.id}")
except Workflow.DoesNotExist:
logger.error(f"Workflow not found: {workflow_id}")
return Response(
{"error": "Workflow not found or access denied"},
status=status.HTTP_404_NOT_FOUND,
)
# Get tool instances for the workflow with organization filtering
# Filter through the relationship: ToolInstance -> Workflow -> Organization
tool_instances_queryset = ToolInstance.objects.filter(workflow=workflow)
tool_instances_queryset = filter_queryset_by_organization(
tool_instances_queryset, request, "workflow__organization"
)
tool_instances = tool_instances_queryset.order_by("step")
logger.info(f"Found {len(tool_instances)} tool instances")
# Serialize the tool instances
try:
logger.info("Starting serialization of tool instances")
serializer = ToolInstanceSerializer(tool_instances, many=True)
logger.info("Accessing serializer.data")
serializer_data = serializer.data
logger.info(f"Serialization completed, got {len(serializer_data)} items")
except Exception as serializer_error:
logger.error(f"Serialization error: {serializer_error}")
# Try to return basic data without enhanced tool information
basic_data = []
for instance in tool_instances:
basic_data.append(
{
"id": str(instance.id),
"tool_id": instance.tool_id,
"step": instance.step,
"metadata": instance.metadata,
}
)
logger.info(f"Returning {len(basic_data)} basic tool instances")
return Response(
{
"workflow_id": workflow_id,
"tool_instances": basic_data,
"total_count": len(tool_instances),
"note": "Basic data returned due to serialization error",
},
status=status.HTTP_200_OK,
)
return Response(
{
"workflow_id": workflow_id,
"tool_instances": serializer_data,
"total_count": len(tool_instances),
},
status=status.HTTP_200_OK,
)
except Exception as e:
logger.exception(f"Failed to get tool instances for workflow {workflow_id}: {e}")
return Response({"error": str(e)}, status=status.HTTP_500_INTERNAL_SERVER_ERROR)
@csrf_exempt # Safe: Internal API with Bearer token auth, no session/cookies
@api_view(["POST"])
def validate_tool_instances_internal(request):
"""Validate tool instances and ensure adapter IDs are migrated.
This internal endpoint validates tool instances for a workflow, ensuring:
1. Adapter names are migrated to IDs
2. User has permissions to access tools and adapters
3. Tool settings match JSON schema requirements
Used by workers to validate tools before execution.
Args:
request: Request containing:
- workflow_id: ID of the workflow
- tool_instances: List of tool instance IDs
Returns:
Response with validation results and migrated metadata
"""
workflow_id = request.data.get("workflow_id")
tool_instance_ids = request.data.get("tool_instances", [])
if not workflow_id:
return Response(
{"error": "workflow_id is required"}, status=status.HTTP_400_BAD_REQUEST
)
validated_instances = []
validation_errors = []
try:
# Get tool instances from database with organization filtering
tool_instances_queryset = ToolInstance.objects.filter(
workflow_id=workflow_id, id__in=tool_instance_ids
).select_related("workflow", "workflow__created_by")
# Apply organization filtering
tool_instances_queryset = filter_queryset_by_organization(
tool_instances_queryset, request, "workflow__organization"
)
tool_instances = list(tool_instances_queryset)
# Validate each tool instance
for tool in tool_instances:
try:
# Get the user who created the workflow
user = tool.workflow.created_by
# Ensure adapter IDs are migrated from names to IDs
migrated_metadata = ToolInstanceHelper.ensure_adapter_ids_in_metadata(
tool, user=user
)
# Validate tool settings
ToolInstanceHelper.validate_tool_settings(
user=user,
tool_uid=tool.tool_id,
tool_meta=migrated_metadata,
)
# Add to validated list with migrated metadata
validated_instances.append(
{
"id": str(tool.id),
"tool_id": tool.tool_id,
"metadata": migrated_metadata,
"step": tool.step,
"status": "valid",
}
)
except Exception as e:
validation_errors.append(
{
"tool_id": tool.tool_id,
"tool_instance_id": str(tool.id),
"error": str(e),
}
)
logger.error(f"Tool validation failed for {tool.tool_id}: {e}")
# Return validation results
response_data = {
"success": len(validation_errors) == 0,
"validated_instances": validated_instances,
"errors": validation_errors,
"workflow_id": workflow_id,
}
if validation_errors:
return Response(response_data, status=status.HTTP_422_UNPROCESSABLE_ENTITY)
return Response(response_data, status=status.HTTP_200_OK)
except Exception as e:
logger.error(f"Tool validation failed: {e}", exc_info=True)
return Response(
{"error": f"Tool validation failed: {str(e)}"},
status=status.HTTP_500_INTERNAL_SERVER_ERROR,
)