LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
"""Workflow Manager Internal API Views
Handles workflow execution related endpoints for internal services.
"""
import logging
import uuid
from django.db import transaction
from django.http import Http404
from django.shortcuts import get_object_or_404
from rest_framework import status, viewsets
from rest_framework.decorators import action
from rest_framework.response import Response
from rest_framework.views import APIView
from tool_instance_v2.models import ToolInstance
from utils.constants import Account
from utils.local_context import StateStore
from utils.organization_utils import filter_queryset_by_organization
# Import new dataclasses for WorkflowDefinitionAPIView
from unstract.core.data_models import (
ConnectionType,
ConnectorInstanceData,
WorkflowDefinitionResponseData,
WorkflowEndpointConfigData,
WorkflowEndpointConfigResponseData,
)
from workflow_manager.endpoint_v2.endpoint_utils import WorkflowEndpointUtils
from workflow_manager.endpoint_v2.models import WorkflowEndpoint
from workflow_manager.file_execution.models import WorkflowFileExecution
from workflow_manager.workflow_v2.models.execution import WorkflowExecution
from workflow_manager.workflow_v2.models.workflow import Workflow
from .internal_serializers import (
FileBatchCreateSerializer,
FileBatchResponseSerializer,
WorkflowExecutionContextSerializer,
WorkflowExecutionSerializer,
WorkflowExecutionStatusUpdateSerializer,
)
logger = logging.getLogger(__name__)
class WorkflowExecutionInternalViewSet(viewsets.ReadOnlyModelViewSet):
"""Internal API ViewSet for Workflow Execution operations.
Provides workflow execution CRUD operations for internal services.
"""
serializer_class = WorkflowExecutionSerializer
lookup_field = "id"
# OrganizationFilterBackend is off here; get_queryset() scopes instead, via
# filter_queryset_by_organization, which fails closed. X-Organization-ID is
# therefore required in practice: a worker that omits it gets zero rows.
skip_org_filter = True
def get_queryset(self):
"""Get workflow executions filtered by organization context."""
queryset = WorkflowExecution.objects.select_related("workflow").prefetch_related(
"tags"
)
return filter_queryset_by_organization(
queryset, self.request, "workflow__organization"
)
def list(self, request, *args, **kwargs):
"""List workflow executions with proper query parameter filtering."""
try:
# Start with organization-filtered queryset
queryset = self.get_queryset()
# Apply query parameter filters
workflow_id = request.query_params.get("workflow_id")
if workflow_id:
queryset = queryset.filter(workflow_id=workflow_id)
status_filter = request.query_params.get("status__in")
if status_filter:
# Handle comma-separated status values
statuses = [s.strip() for s in status_filter.split(",")]
queryset = queryset.filter(status__in=statuses)
# Apply any other filters
status = request.query_params.get("status")
if status:
queryset = queryset.filter(status=status)
# Order by creation time (newest first) for consistent results
queryset = queryset.order_by("-created_at")
# Serialize the filtered queryset
serializer = self.get_serializer(queryset, many=True)
logger.info(
f"WorkflowExecution list: returned {len(serializer.data)} executions"
)
logger.debug(
f"Applied filters - workflow_id: {workflow_id}, status__in: {status_filter}, status: {status}"
)
return Response(serializer.data)
except Exception as e:
logger.error(f"Error in WorkflowExecution list: {str(e)}")
return Response(
{"error": "Failed to list workflow executions", "detail": str(e)},
status=status.HTTP_500_INTERNAL_SERVER_ERROR,
)
def retrieve(self, request, *args, **kwargs):
"""Get specific workflow execution with context."""
try:
execution = self.get_object()
# Check if cost data is requested (expensive operation)
include_cost = request.GET.get("include_cost", "false").lower() == "true"
file_execution = request.GET.get("file_execution", "true").lower() == "true"
# Build comprehensive context
workflow_definition = {}
if execution.workflow:
workflow_definition = {
"workflow_id": str(execution.workflow.id),
"workflow_name": execution.workflow.workflow_name,
"workflow_type": execution.workflow.deployment_type,
"description": execution.workflow.description,
"source_settings": execution.workflow.source_settings or {},
"destination_settings": execution.workflow.destination_settings or {},
"is_active": execution.workflow.is_active,
"status": execution.workflow.status,
}
context_data = {
"execution": execution, # Pass model instance, not serialized data
"workflow_definition": workflow_definition,
"source_config": self._get_source_config(execution),
"destination_config": self._get_destination_config(execution),
"organization_context": self._get_organization_context(execution),
"file_executions": list(execution.file_executions.values())
if file_execution
else [],
}
# Only calculate cost if explicitly requested (expensive database operation)
if include_cost:
context_data["aggregated_usage_cost"] = execution.aggregated_usage_cost
serializer = WorkflowExecutionContextSerializer(context_data)
return Response(serializer.data)
except (Http404, WorkflowExecution.DoesNotExist):
# A deleted execution must return 404 (deterministic) so the reaper's
# orphan-claim sweep can GC it, instead of 500 (which it treats as
# transient → retry forever). BUT get_object() is org-scoped and this try
# wraps the whole method, so an Http404 here can also mean "exists but
# scoped out of this request" or a nested Http404 for a *different*
# resource — NOT "execution deleted". Since the reaper turns a genuine 404
# into an irreversible claim GC, confirm the row is truly absent (unscoped)
# before saying not-found; otherwise 500 so the reaper retains the claim.
execution_id = kwargs.get("pk") or kwargs.get("id")
# Own try/except: a DatabaseError from this unscoped exists-check is
# raised INSIDE this except block, so the sibling ``except Exception``
# below would NOT catch it (Python doesn't route between sibling
# handlers) — it would escape as Django's unstructured 500. On any
# exists-check failure we can't confirm absence, so fail closed to a
# structured 500 (retain the claim), never a 404 that would GC it.
try:
row_absent = not WorkflowExecution.objects.filter(
id=execution_id
).exists()
except Exception:
logger.exception(
f"WorkflowExecution {execution_id} existence check failed "
"(retrieve) — returning 500 (retain)"
)
return Response(
{"error": "Failed to retrieve workflow execution"},
status=status.HTTP_500_INTERNAL_SERVER_ERROR,
)
if row_absent:
logger.info(
f"WorkflowExecution {execution_id} not found (retrieve) — 404"
)
return Response(
{"error": "WorkflowExecution not found"},
status=status.HTTP_404_NOT_FOUND,
)
logger.warning(
f"WorkflowExecution {execution_id} exists but was not retrievable in "
"this context (org scope / nested lookup) — returning 500 (retain)"
)
return Response(
{"error": "Failed to retrieve workflow execution"},
status=status.HTTP_500_INTERNAL_SERVER_ERROR,
)
except Exception as e:
logger.error(
f"Failed to retrieve workflow execution {kwargs.get('id')}: {str(e)}"
)
return Response(
{"error": "Failed to retrieve workflow execution", "detail": str(e)},
status=status.HTTP_500_INTERNAL_SERVER_ERROR,
)
def _get_source_config(self, execution: WorkflowExecution) -> dict:
"""Get source configuration for execution with connector instance details."""
try:
workflow = execution.workflow
if not workflow:
logger.warning(f"No workflow found for execution {execution.id}")
return {}
# Get workflow-level source settings
source_settings = {}
workflow_type = "general_workflow"
is_api = False
if execution.pipeline_id:
# Check if pipeline_id references a Pipeline or APIDeployment (like serializer)
from api_v2.models import APIDeployment
from pipeline_v2.models import Pipeline
try:
# First check if it's a Pipeline
pipeline = Pipeline.objects.get(id=execution.pipeline_id)
source_settings = pipeline.workflow.source_settings or {}
workflow_type = "pipeline"
is_api = False
logger.debug(
f"Pipeline {execution.pipeline_id} source settings: {bool(source_settings)}"
)
except Pipeline.DoesNotExist:
# Check if it's an APIDeployment (like serializer does)
try:
api_deployment = APIDeployment.objects.get(
id=execution.pipeline_id
)
source_settings = workflow.source_settings or {}
workflow_type = "api_deployment"
is_api = True
logger.debug(
f"APIDeployment {execution.pipeline_id} found for execution {execution.id}"
)
except APIDeployment.DoesNotExist:
# Neither Pipeline nor APIDeployment exists
logger.warning(
f"Neither Pipeline nor APIDeployment found for ID {execution.pipeline_id} in execution {execution.id}"
)
source_settings = workflow.source_settings or {}
workflow_type = "pipeline_not_found"
else:
# API deployment or general workflow execution
source_settings = workflow.source_settings or {}
if (
workflow
and hasattr(workflow, "api_deployments")
and workflow.api_deployments.filter(is_active=True).exists()
):
workflow_type = "api_deployment"
is_api = True
logger.debug(
f"Workflow {workflow.id} source settings: {bool(source_settings)}"
)
# Get source connector instance from workflow endpoints
from workflow_manager.endpoint_v2.models import WorkflowEndpoint
source_connector_data = None
try:
# Look for source endpoint with connector instance
source_endpoint = (
WorkflowEndpoint.objects.select_related("connector_instance")
.filter(
workflow=workflow,
endpoint_type=WorkflowEndpoint.EndpointType.SOURCE,
)
.first()
)
if source_endpoint and source_endpoint.connector_instance:
source_connector_instance = source_endpoint.connector_instance
source_connector_data = {
"connector_id": source_connector_instance.connector_id,
"connector_settings": source_connector_instance.metadata or {},
"connector_name": getattr(
source_connector_instance, "connector_name", ""
),
}
logger.debug(
f"Found source connector instance: {source_connector_instance.connector_id}"
)
# Include endpoint configuration in source settings
if source_endpoint.configuration:
source_settings.update(source_endpoint.configuration)
else:
logger.debug("No source connector instance found for workflow")
except Exception as source_error:
logger.warning(
f"Failed to get source connector info for workflow {workflow.id}: {str(source_error)}"
)
# Build comprehensive source config
source_config = {
"type": workflow_type,
"source_settings": source_settings,
"is_api": is_api,
}
# Add pipeline/deployment specific info
if execution.pipeline_id and workflow_type != "pipeline_not_found":
source_config["pipeline_id"] = str(execution.pipeline_id)
elif workflow_type == "api_deployment":
api_deployment = workflow.api_deployments.first()
if api_deployment:
source_config["deployment_id"] = str(api_deployment.id)
# Add source connector instance data if available
if source_connector_data:
source_config.update(source_connector_data)
logger.debug("Added source connector instance data to source config")
return source_config
except Exception as e:
logger.warning(
f"Failed to get source config for execution {execution.id}: {str(e)}"
)
return {}
def _get_destination_config(self, execution: WorkflowExecution) -> dict:
"""Get destination configuration for execution with connector instance details."""
try:
workflow = execution.workflow
if not workflow:
logger.warning(f"No workflow found for execution {execution.id}")
return {}
# Get destination settings from workflow
destination_settings = {}
if execution.pipeline_id:
# ETL/Task pipeline execution - get settings from pipeline's workflow
from pipeline_v2.models import Pipeline
try:
pipeline = Pipeline.objects.get(id=execution.pipeline_id)
destination_settings = pipeline.workflow.destination_settings or {}
logger.debug(
f"Pipeline {execution.pipeline_id} destination settings: {bool(destination_settings)}"
)
except Pipeline.DoesNotExist:
logger.warning(
f"Pipeline {execution.pipeline_id} not found for execution {execution.id}"
)
destination_settings = workflow.destination_settings or {}
else:
# API deployment or general workflow execution
destination_settings = workflow.destination_settings or {}
logger.debug(
f"Workflow {workflow.id} destination settings: {bool(destination_settings)}"
)
# Get connection type and connector instance from workflow endpoints
from workflow_manager.endpoint_v2.models import WorkflowEndpoint
connection_type = "FILESYSTEM" # Default
is_api = False
connector_instance_data = None
try:
# Look for destination endpoint with connector instance
dest_endpoint = (
WorkflowEndpoint.objects.select_related("connector_instance")
.filter(
workflow=workflow,
endpoint_type=WorkflowEndpoint.EndpointType.DESTINATION,
)
.first()
)
if dest_endpoint:
connection_type = dest_endpoint.connection_type or "FILESYSTEM"
is_api = connection_type in ["API", "APPDEPLOYMENT"]
# Include connector instance details if available
if dest_endpoint.connector_instance:
connector_instance = dest_endpoint.connector_instance
connector_instance_data = {
"connector_id": connector_instance.connector_id,
"connector_settings": connector_instance.metadata or {},
"connector_name": getattr(
connector_instance, "connector_name", ""
),
}
logger.debug(
f"Found connector instance: {connector_instance.connector_id}"
)
# Include endpoint configuration
if dest_endpoint.configuration:
destination_settings.update(dest_endpoint.configuration)
logger.debug(
f"Found destination endpoint: {connection_type}, is_api: {is_api}"
)
else:
# Check if workflow has API deployments
if (
hasattr(workflow, "api_deployments")
and workflow.api_deployments.filter(is_active=True).exists()
):
connection_type = "API"
is_api = True
logger.debug(
"Workflow has active API deployments, treating as API destination"
)
except Exception as endpoint_error:
logger.warning(
f"Failed to get endpoint info for workflow {workflow.id}: {str(endpoint_error)}"
)
# Get source connector information for file reading in manual review
source_connector_data = None
try:
# Look for source endpoint with connector instance
source_endpoint = (
WorkflowEndpoint.objects.select_related("connector_instance")
.filter(
workflow=workflow,
endpoint_type=WorkflowEndpoint.EndpointType.SOURCE,
)
.first()
)
if source_endpoint and source_endpoint.connector_instance:
source_connector_instance = source_endpoint.connector_instance
source_connector_data = {
"source_connector_id": source_connector_instance.connector_id,
"source_connector_settings": source_connector_instance.metadata
or {},
}
logger.debug(
f"Found source connector instance: {source_connector_instance.connector_id}"
)
else:
logger.debug("No source connector instance found for workflow")
except Exception as source_error:
logger.warning(
f"Failed to get source connector info for workflow {workflow.id}: {str(source_error)}"
)
# Build comprehensive destination config
destination_config = {
"connection_type": connection_type,
"settings": destination_settings,
"is_api": is_api,
"use_file_history": True,
}
# Add connector instance data if available
if connector_instance_data:
destination_config.update(connector_instance_data)
logger.debug("Added connector instance data to destination config")
else:
logger.debug("No connector instance found for destination endpoint")
# Add source connector data for manual review file reading
if source_connector_data:
destination_config.update(source_connector_data)
logger.debug(
"Added source connector data to destination config for manual review"
)
return destination_config
except Exception as e:
logger.warning(
f"Failed to get destination config for execution {execution.id}: {str(e)}"
)
return {}
def _get_organization_context(self, execution: WorkflowExecution) -> dict:
"""Get organization context for execution."""
try:
# Get organization from the workflow, not directly from execution
if execution.workflow and hasattr(execution.workflow, "organization"):
org = execution.workflow.organization
return {
"organization_id": str(
org.organization_id
), # organization identifier
"organization_uuid": str(org.id), # organization uuid
"organization_name": org.display_name, # organization name
"settings": {}, # Add organization-specific settings if needed
}
else:
logger.warning(f"No organization found for execution {execution.id}")
return {
"organization_id": None,
"organization_uuid": None,
"organization_name": "Unknown",
"settings": {},
}
except Exception as e:
logger.warning(
f"Failed to get organization context for execution {execution.id}: {str(e)}"
)
return {
"organization_id": None,
"organization_uuid": None,
"organization_name": "Unknown",
"settings": {},
}
@action(detail=True, methods=["post"])
def update_status(self, request, id=None):
"""Update workflow execution status."""
try:
logger.info(f"Updating status for execution {id}")
execution = self.get_object()
serializer = WorkflowExecutionStatusUpdateSerializer(data=request.data)
if not serializer.is_valid():
return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST)
validated_data = serializer.validated_data
# Use update_execution() for proper wall-clock time calculation rather
# than manual field setting, which bypassed the execution time logic.
error_message = self._truncate_error_message(
validated_data.get("error_message"), id
)
increment_attempt = (
validated_data.get("attempts") is not None
and validated_data.get("attempts") > execution.attempts
)
from workflow_manager.workflow_v2.enums import ExecutionStatus
status_enum = ExecutionStatus(validated_data["status"])
logger.info(f"Updating status for execution {id} to {status_enum}")
# Status and file-count aggregates must commit together. Otherwise a
# failure between the two writes can leave the row at a terminal status
# with failed_files=None, which is_failure_run() reads as a success and
# silently bypasses notify_on_failures subscribers.
with transaction.atomic():
row_locked = False
# Concurrency guard for PG-transport executions, identified by the
# durable ``queue_message_id`` marker — NOT a re-evaluated Flipt flag.
# Keying on the row's own transport keeps the invariant correct even
# if the rollout flag is toggled off or Flipt is unreachable while a
# PG execution is still finalizing (per the "transport carried on the
# row, never re-read from Flipt" rule).
#
# Protect COMPLETED and STOPPED — the two statuses that assert a
# deliberate final outcome. COMPLETED is set exclusively by a
# successful callback (its redelivery is skipped, see
# _callback_already_ran); STOPPED records an explicit user/operator
# stop. Under the PG path's high concurrency ~15 call sites write
# status via unlocked read-modify-write, so a stale writer can commit
# AFTER one of these and clobber it (lost update → execution stranded
# in EXECUTING, a confusing success→failed flip, or a straggler
# callback erasing a user's stop). Lock the row and refuse to overwrite
# either with a different status.
#
# ERROR is deliberately NOT protected: other paths (upstream error,
# the reaper) can set it BEFORE the first real callback, which then
# legitimately corrects the row to COMPLETED when the files actually
# succeeded — blocking that would freeze a genuinely successful run at
# a premature ERROR. Celery executions (queue_message_id NULL) keep the
# legacy path.
#
# ONE lock per call: the row read here is handed to update_execution()
# with row_locked=True, so it does not lock the same row a second time. The
# Celery path takes no lock here and lets update_execution() take its
# own — still one. FOR NO KEY UPDATE (no_key=True) keeps writers
# mutually exclusive without blocking the FOR KEY SHARE that every
# child-row INSERT takes on this row for its foreign-key check.
if execution.queue_message_id is not None:
execution = WorkflowExecution.objects.select_for_update(
no_key=True
).get(pk=execution.pk)
row_locked = True
protected = {
ExecutionStatus.COMPLETED.value,
ExecutionStatus.STOPPED.value,
}
if (
execution.status in protected
and status_enum.value != execution.status
):
logger.warning(
f"update_status: refusing to overwrite protected status "
f"'{execution.status}' with '{status_enum.value}' "
f"for execution {id}"
)
return Response(
{
"status": "skipped",
"reason": "already_final",
"execution_id": str(execution.id),
"new_status": execution.status,
}
)
execution.update_execution(
status=status_enum,
error=error_message,
increment_attempt=increment_attempt,
row_locked=row_locked,
)
# File aggregates are not handled by update_execution; persist
# separately. The floor's file-row count runs here, AFTER the lock,
# on purpose: any status writer we waited behind (e.g. the reaper
# cascading files to ERROR) has committed by now, and a count taken
# before the wait would miss its rows and let failed_files be written
# too low. (A file row whose INSERT has not committed is not counted
# either way: Django's foreign keys are DEFERRABLE INITIALLY DEFERRED,
# so an open insert holds no lock on this row until it commits, and
# FOR UPDATE never waited for it.)
self._update_file_aggregates(execution, validated_data)
# Reaper recovery: cascade the terminal status to any non-terminal
# file executions so the execution and its files agree (atomic with
# the status write above — they can't diverge).
if validated_data.get("cascade_terminal_files"):
self._cascade_terminal_files(execution, status_enum, error_message)
logger.info(
f"Updated workflow execution {id} status to {validated_data['status']}"
)
return Response(
{
"status": "updated",
"execution_id": str(execution.id),
"new_status": execution.status,
}
)
except Exception as e:
logger.error(f"Failed to update workflow execution status {id}: {str(e)}")
return Response(
{"error": "Failed to update workflow execution status", "detail": str(e)},
status=status.HTTP_500_INTERNAL_SERVER_ERROR,
)
@action(detail=False, methods=["post"])
def recover_stuck_pg_executions(self, request):
"""Reaper safety-net: finalize PG executions stranded non-terminal after
all their files completed.
A lost-update race (unlocked concurrent status writes) or a swallowed
finalization write can leave a PG execution in EXECUTING/PENDING forever
even though every file execution is terminal — and because the barrier row
self-deletes at fan-in, the reaper's other (PG-table) sweeps can't see it.
This endpoint is the leader-gated reaper's trigger: it finds PG executions
stuck past ``stuck_seconds`` whose files are ALL terminal, recomputes the
correct terminal status from those files, and finalizes them.
**Covers BOTH transports (UN-3796).** It was scoped to PG by
``queue_message_id__isnull=False``, which made a Celery execution invisible —
so one stranded by a PG cutover (its workers removed while its chord callback
was still to fire) sat ``EXECUTING`` forever with nothing able to close it. The
finalization logic itself was always transport-agnostic: it reads file statuses
and recomputes the terminal status, never touching a queue. Only the selection
was narrow.
Now requires the row to have been dispatched on **some** transport —
``queue_message_id`` (PG) or ``task_id`` (Celery). That deliberately excludes
rows with BOTH handles NULL, which are never-dispatched and belong to
``workflow_v2/undispatched_sweep.py``; the two predicates are disjoint on
``task_id`` so they can never contend for the same row. (The
``total == 0 → skipped`` guard below would also catch those, but relying on a
downstream guard for correctness is how overlaps get reintroduced.)
The method and route keep the ``_pg_`` name despite no longer being PG-only:
the URL is an internal-API contract between backend and workers, and renaming
it would break during a rolling deploy where an older worker still calls the
old path. Misleading name, deliberate trade.
"""
from datetime import timedelta
from django.db.models import Exists, Max, OuterRef, Q
from django.utils import timezone
from workflow_manager.workflow_v2.enums import ExecutionStatus
# Validate/clamp inputs: a negative stuck_seconds would push the cutoff into
# the future and match LIVE executions (finalizing running work); an
# unbounded limit would let one call scan the whole table.
try:
stuck_seconds = int(request.data.get("stuck_seconds") or 9000)
limit = int(request.data.get("limit") or 100)
except (TypeError, ValueError):
return Response(
{"error": "stuck_seconds and limit must be integers"},
status=status.HTTP_400_BAD_REQUEST,
)
stuck_seconds = max(1, stuck_seconds)
limit = max(1, min(limit, 1000))
cutoff = timezone.now() - timedelta(seconds=stuck_seconds)
terminal = ExecutionStatus.terminal_values()
# Select ONLY rows the per-row guard below can actually finalize: at least one
# file execution, and none of them still non-terminal.
#
# Without this the window fills with rows _recover_one_stuck_pg_execution()
# always skips — and a skip leaves ``modified_at`` untouched, so the SAME rows
# are re-selected on every sweep, forever. A genuinely recoverable execution
# behind them is never reached: not scanned, not skipped, simply invisible.
# Measured on integration during the UN-3796 cutover rehearsal: 1964 candidates,
# of which 476 had no file rows and 646 had a non-terminal file — 57% permanent
# skip fodder — while the freshly stranded execution sat at rank 1964 and every
# sweep logged a healthy-looking "scanned 100, recovered 0".
#
# Filtering here rather than leaning on the downstream guard is also what makes
# the drain real: every selected row finalizes and drops out of the candidate
# set, so the backlog shrinks monotonically. That is precisely what keeps the
# oldest-first ordering below starvation-free — FIFO is only fair if the queue
# actually moves.
files = WorkflowFileExecution.objects.filter(workflow_execution=OuterRef("pk"))
stuck_ids = list(
WorkflowExecution.objects.filter(
# Dispatched by ANY evidence: the positive stamp, or either transport
# handle. dispatched_at alone is not enough — pre-0027 rows never got
# one — and the handles alone are not enough either, which is the gap
# that makes this three-way rather than two.
#
# Once undispatched_sweep also requires `dispatched_at IS NULL`, a row
# that WAS dispatched but whose handle write failed stops matching that
# sweep. Without dispatched_at here it would match neither, and an
# execution whose files all finished could sit non-terminal forever with
# nothing able to close it — trading "wrongly terminalised" for "never
# terminalised". The three-way test keeps the two sweeps disjoint AND
# jointly exhaustive: undispatched takes rows where all three are absent,
# this takes rows where any one is present.
Q(dispatched_at__isnull=False)
| Q(queue_message_id__isnull=False)
| Q(task_id__isnull=False),
status__in=[
ExecutionStatus.PENDING.value,
ExecutionStatus.EXECUTING.value,
],
modified_at__lt=cutoff,
)
.filter(Exists(files), ~Exists(files.exclude(status__in=terminal)))
# Staleness must be measured on the FILES, not on the execution row.
#
# ``workflow_execution.modified_at`` is effectively the START time: file
# completions write the file row, not the execution row, so it does not
# advance while work proceeds. ``modified_at < cutoff`` therefore reads as
# "started more than stuck_seconds ago", not "quiet for stuck_seconds" —
# and every run longer than the window is formally eligible for recovery
# WHILE IT IS STILL RUNNING. (Observed: a 14-minute ETL whose execution row
# still read its 12:01:44 start at 12:15, with a 600 s window.)
#
# The all-files-terminal filter above stops that being catastrophic, but it
# leaves a live race: between the last file going terminal and the callback
# finalizing, a healthy execution passes every check here. If a sweep lands
# in that window the reaper finalizes first, the terminal-one-way guard then
# refuses the callback's own write, and the notification never fires.
#
# Requiring the LAST FILE to also be older than the cutoff closes it: an
# execution whose files just finished is not yet stale, so the callback keeps
# its window and only genuinely abandoned work is recovered. Both conditions
# are kept — the indexed execution-row predicate cheaply narrows the scan,
# this one decides correctness.
.annotate(last_file_at=Max("file_executions__modified_at"))
.filter(last_file_at__lt=cutoff)
.order_by("modified_at")
.values_list("id", flat=True)[:limit]
)
outcomes = {"recovered": 0, "skipped": 0, "failed": 0}
for exec_id in stuck_ids:
outcomes[self._recover_one_stuck_pg_execution(exec_id, cutoff, terminal)] += 1
logger.info(
"recover_stuck_pg_executions: scanned=%s recovered=%s skipped=%s failed=%s",
len(stuck_ids),
outcomes["recovered"],
outcomes["skipped"],
outcomes["failed"],
)
return Response(
{
"recovered": outcomes["recovered"],
"skipped": outcomes["skipped"],
"scanned": len(stuck_ids),
"failed": outcomes["failed"],
}
)
def _recover_one_stuck_pg_execution(self, exec_id, cutoff, terminal) -> str:
"""Finalize one stuck PG execution under a row lock.
Returns the outcome — ``"recovered"`` | ``"skipped"`` | ``"failed"``.
Extracted from :meth:`recover_stuck_pg_executions` to keep that action's
cognitive complexity low; see it for the full rationale.
"""
from django.db.models import Count, Q
from workflow_manager.workflow_v2.enums import ExecutionStatus
try:
with transaction.atomic():
execution = WorkflowExecution.objects.select_for_update(no_key=True).get(
pk=exec_id
)
# Re-check under lock: skip if it reached terminal or was just
# touched (no longer stuck) — never race a live execution.
if execution.status in terminal or execution.modified_at >= cutoff:
return "skipped"
# One conditional aggregate instead of three COUNT round-trips.
counts = WorkflowFileExecution.objects.filter(
workflow_execution=execution
).aggregate(
total=Count("id"),
terminal=Count("id", filter=Q(status__in=terminal)),
errored=Count("id", filter=Q(status=ExecutionStatus.ERROR.value)),
stopped=Count("id", filter=Q(status=ExecutionStatus.STOPPED.value)),
completed=Count(
"id", filter=Q(status=ExecutionStatus.COMPLETED.value)
),
)
total = counts["total"]
if total == 0 or counts["terminal"] < total:
# total==0: a valid PG message may still be QUEUED (a
# backlog/outage can outlast stuck_seconds), so failing it would
# wrongly terminalize a live execution the one-way guard then
# blocks from recovery — file-less recovery needs a queue-state
# check (reaper-side, follow-up). terminal < total: a file is
# still non-terminal → genuinely processing. Skip either way.
return "skipped"
# Recompute the correct terminal status: ERROR if any file errored,
# else STOPPED if any was stopped (a cancelled run must not be turned
# into ERROR), else COMPLETED.
if counts["errored"]:
computed = ExecutionStatus.ERROR
elif counts["stopped"]:
computed = ExecutionStatus.STOPPED
else:
computed = ExecutionStatus.COMPLETED
# Already locked above; hand the locked read over so the row is
# not locked a second time.
execution.update_execution(status=computed, row_locked=True)
self._restamp_execution_time_from_files(execution)
self._update_file_aggregates(
execution,
{
"total_files": total,
"successful_files": counts["completed"],
"failed_files": counts["errored"],
},
)
logger.warning(
"Reaper safety-net: finalized stranded PG execution %s to %s "
"(files %s ok / %s errored / %s stopped)",
exec_id,
computed.value,
counts["completed"],
counts["errored"],
counts["stopped"],
)
return "recovered"
except Exception:
logger.exception("recover_stuck_pg_executions: failed to recover %s", exec_id)
return "failed"
@staticmethod
def _cascade_terminal_files(execution, status_enum, error_message) -> None:
"""Mark this execution's non-terminal (PENDING/EXECUTING) file executions to
*status_enum*, so a reaper-recovered strand doesn't leave the execution
terminal while its files stay EXECUTING (the b11ba2f3 inconsistency).
No-op unless *status_enum* is terminal. Runs in the caller's transaction, so
the execution status and the file cascade commit together. Idempotent — a
re-run finds those files already terminal and updates nothing.
"""
from workflow_manager.file_execution.models import WorkflowFileExecution
from workflow_manager.workflow_v2.enums import ExecutionStatus
terminal = ExecutionStatus.terminal_values() # canonical COMPLETED/ERROR/STOPPED
if status_enum.value not in terminal:
return
# Bulk .update() deliberately bypasses WorkflowFileExecution.update_status():
# this is a give-up recovery, so per-file execution_time is unknown (stays
# NULL) and the parent's failed/successful_files aggregates are NOT
# reconciled here — the execution's own terminal status (set above) is the
# source of truth (is_failure_run / notifications read it), and the
# serializer already derives failed counts for a terminal-failure run.
cascaded = (
WorkflowFileExecution.objects.filter(workflow_execution=execution)
.exclude(status__in=terminal)
.update(
status=status_enum.value,
execution_error=(
error_message
or "[reaper-recovery] Execution stranded; file marked terminal "
"to match the execution."
),
)
)
if cascaded:
logger.info(
f"Cascaded terminal status {status_enum.value} to {cascaded} "
f"non-terminal file execution(s) of execution {execution.id}"
)
@staticmethod
def _truncate_error_message(error_msg: str | None, execution_id) -> str | None:
"""Truncate an error message to the 256-char column limit (with ellipsis)."""
if not error_msg:
return None
if len(error_msg) > 256:
logger.warning(
f"Error message truncated for execution {execution_id} "
f"(original length: {len(error_msg)})"
)
return error_msg[:253] + "..."
return error_msg
@staticmethod
def _restamp_execution_time_from_files(execution) -> None:
"""Correct the runtime ``update_execution()`` just stamped from ``now()``.
Both transports' status writers set ``execution_time = now - created_at`` on
any terminal transition (``_apply_legacy_update`` / ``_apply_pg_guarded_update``
in the execution model). That is right for a live finalization, where "now" IS
when the work ended, and wrong here: this sweep finalizes executions whose files
finished long before anything noticed, so the stamp would record how late the
reaper was rather than how long the work took. On the integration backlog that
meant multi-day runtimes on executions that ran for minutes — every one of them
feeding whatever dashboards read this column.
Recompute from the last file to reach a terminal state, which is the closest
durable record of when the work actually ended. If no file carries a timestamp,
leave whatever is there alone rather than substituting a worse guess.
"""
from django.db.models import Max
last_file_at = WorkflowFileExecution.objects.filter(
workflow_execution=execution
).aggregate(last=Max("modified_at"))["last"]
if not last_file_at or not execution.created_at:
return
execution.execution_time = round(
(last_file_at - execution.created_at).total_seconds(), 3
)
execution.save(update_fields=["execution_time"])
@staticmethod
def _update_file_aggregates(execution, validated_data) -> None:
"""Persist total / failed / successful file counts when present.
Floored at what the file rows say, because the worker's aggregate only
counts files it was dispatched and misses any rejected before that.
"""
update_fields: list[str] = []
for field in ("total_files", "successful_files", "failed_files"):
if validated_data.get(field) is not None:
setattr(execution, field, validated_data[field])
update_fields.append(field)
if not update_fields:
return
WorkflowExecutionInternalViewSet._floor_counters_at_file_rows(
execution, update_fields
)
execution.save(update_fields=update_fields)
@staticmethod
def _count_file_rows(execution) -> dict:
"""Count this execution's file rows, total and ERROR, in one query.
Returns ``{"total": int, "errored": int}``. Errors are left to propagate:
the request then fails and the transaction rolls back, so the worker retries
instead of committing counters the floor never checked (an under-counted
failed_files would read as a success and skip failure notifications).
"""
from django.db.models import Count, Q
from workflow_manager.workflow_v2.enums import ExecutionStatus
return WorkflowFileExecution.objects.filter(
workflow_execution_id=execution.pk
).aggregate(
total=Count("id"),
errored=Count("id", filter=Q(status=ExecutionStatus.ERROR.value)),
)
@staticmethod
def _floor_counters_at_file_rows(execution, update_fields: "list[str]") -> None:
"""Raise the counters being written to match the file rows on disk.
Only ever corrects upward, and only to a count already true, so a run
with nothing rejected is left alone.
"""
wants_failed = "failed_files" in update_fields
wants_total = "total_files" in update_fields
if not (wants_failed or wants_total):
return
counts = WorkflowExecutionInternalViewSet._count_file_rows(execution)
if wants_failed:
execution.failed_files = max(execution.failed_files or 0, counts["errored"])
if wants_total:
execution.total_files = max(execution.total_files or 0, counts["total"])
class FileBatchCreateAPIView(APIView):
"""Internal API endpoint for creating file batches for workflow execution."""
def post(self, request):
"""Create file execution records in batches."""
try:
serializer = FileBatchCreateSerializer(data=request.data)
if not serializer.is_valid():
return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST)
validated_data = serializer.validated_data
workflow_execution_id = validated_data["workflow_execution_id"]
files = validated_data["files"]
is_api = validated_data.get("is_api", False)
# Get workflow execution
workflow_execution = get_object_or_404(
WorkflowExecution, id=workflow_execution_id
)
created_files = []
skipped_files = []
batch_id = uuid.uuid4()
with transaction.atomic():
for file_data in files:
try:
# Create file execution record
file_execution = WorkflowFileExecution.objects.create(
id=uuid.uuid4(),
workflow_execution=workflow_execution,
file_name=file_data.get("file_name", ""),
file_path=file_data.get("file_path", ""),
file_size=file_data.get("file_size", 0),
file_hash=file_data.get("file_hash", ""),
provider_file_uuid=file_data.get("provider_file_uuid", ""),
mime_type=file_data.get("mime_type", ""),
fs_metadata=file_data.get("fs_metadata", {}),
status="PENDING",
)
created_files.append(
{
"id": str(file_execution.id),
"file_name": file_execution.file_name,
"status": file_execution.status,
}
)
except Exception as file_error:
logger.warning(
f"Failed to create file execution for {file_data.get('file_name')}: {file_error}"
)
skipped_files.append(
{
"file_name": file_data.get("file_name", "unknown"),
"error": str(file_error),
}
)
response_data = {
"batch_id": batch_id,
"workflow_execution_id": workflow_execution_id,
"total_files": len(files),
"created_file_executions": created_files,
"skipped_files": skipped_files,
"is_api": is_api,
}
response_serializer = FileBatchResponseSerializer(response_data)
logger.info(
f"Created file batch {batch_id} with {len(created_files)} files for execution {workflow_execution_id}"
)
return Response(response_serializer.data, status=status.HTTP_201_CREATED)
except Exception as e:
logger.error(f"Failed to create file batch: {str(e)}")
return Response(
{"error": "Failed to create file batch", "detail": str(e)},
status=status.HTTP_500_INTERNAL_SERVER_ERROR,
)
class ToolExecutionInternalAPIView(APIView):
"""Internal API endpoint for tool execution operations."""
def get(self, request, workflow_id):
"""Get tool instances for a workflow."""
try:
# Get workflow with automatic organization filtering (via DefaultOrganizationManagerMixin)
try:
# This will automatically apply organization filtering via DefaultOrganizationManagerMixin
workflow = Workflow.objects.get(id=workflow_id)
logger.debug(f"Found workflow {workflow_id} for tool instances request")
except Workflow.DoesNotExist:
logger.error(f"Workflow {workflow_id} not found or not accessible")
return Response(
{"error": "Workflow not found or access denied"},
status=status.HTTP_404_NOT_FOUND,
)
# Get tool instances for the workflow with organization filtering
# Filter through the relationship: ToolInstance -> Workflow -> Organization
tool_instances_queryset = ToolInstance.objects.filter(workflow=workflow)
tool_instances_queryset = filter_queryset_by_organization(
tool_instances_queryset, request, "workflow__organization"
)
tool_instances = tool_instances_queryset.order_by("step")
instances_data = []
for tool_instance in tool_instances:
instances_data.append(
{
"id": str(tool_instance.id),
"tool_id": str(tool_instance.tool_id)
if tool_instance.tool_id
else None,
"step": tool_instance.step,
"tool_settings": tool_instance.metadata or {},
"created_at": tool_instance.created_at.isoformat()
if tool_instance.created_at
else None,
"modified_at": tool_instance.modified_at.isoformat()
if tool_instance.modified_at
else None,
}
)
response_data = {
"workflow_id": workflow_id,
"tool_instances": instances_data,
"total_instances": len(instances_data),
}
logger.info(
f"Retrieved {len(instances_data)} tool instances for workflow {workflow_id}"
)
return Response(response_data)
except Exception as e:
logger.error(
f"Failed to get tool instances for workflow {workflow_id}: {str(e)}"
)
return Response(
{"error": "Failed to get tool instances", "detail": str(e)},
status=status.HTTP_500_INTERNAL_SERVER_ERROR,
)
# ExecutionFinalizationAPIView class removed - it was unused dead code
# Workers now use simple update_workflow_execution_status instead of complex finalization
# This eliminates unnecessary API complexity and improves callback performance
class WorkflowFileExecutionCheckActiveAPIView(APIView):
"""Internal API for checking if files are actively being processed."""
def post(self, request):
"""Check if files are in PENDING or EXECUTING state in other workflow executions."""
try:
workflow_id = request.data.get("workflow_id")
# Support both legacy and new formats
provider_file_uuids = request.data.get(
"provider_file_uuids", []
) # Legacy format
files = request.data.get("files", []) # New format: [{uuid, path}]
current_execution_id = request.data.get("current_execution_id")
# Convert legacy format to new format for backward compatibility
if provider_file_uuids and not files:
files = [{"uuid": uuid, "path": None} for uuid in provider_file_uuids]
elif files:
# Ensure files have required fields
for file_data in files:
if "uuid" not in file_data:
return Response(
{"error": "Each file must have 'uuid' field"},
status=status.HTTP_400_BAD_REQUEST,
)
if not workflow_id or not files:
return Response(
{
"error": "workflow_id and files (or provider_file_uuids) are required"
},
status=status.HTTP_400_BAD_REQUEST,
)
logger.info(
f"Checking active files for workflow {workflow_id}, "
f"excluding execution {current_execution_id}, "
f"checking {len(files)} files"
)
# Check for files in PENDING or EXECUTING state in other workflow executions
active_files = {} # {uuid: [execution_data]} - legacy format
active_identifiers = set() # Composite identifiers for new format
db_queries = 0
# Prepare all files for database check
# Note: Workers already check cache before calling this API, so no need to check again
files_needing_db_check = []
for file_data in files:
provider_uuid = file_data["uuid"]
file_path = file_data.get("path")
composite_id = (
f"{provider_uuid}:{file_path}" if file_path else provider_uuid
)
# All files need database check
files_needing_db_check.append(
{
"uuid": provider_uuid,
"path": file_path,
"composite_id": composite_id,
}
)
# Bulk database query for all files
if files_needing_db_check:
logger.info(
f"[ActiveCheck] Performing bulk database check for {len(files_needing_db_check)} files"
)
self._bulk_database_check(
files_needing_db_check=files_needing_db_check,
workflow_id=workflow_id,
current_execution_id=current_execution_id,
active_files=active_files,
active_identifiers=active_identifiers,
)
db_queries = 2 # At most 2 bulk queries (path-aware + legacy)
logger.info(
f"[ActiveCheck] Active check complete: {len(active_files)}/{len(files)} files active "
f"(db_queries: {db_queries})"
)
# Log final active identifiers for debugging
if active_identifiers:
logger.debug(
f"[ActiveCheck] Active identifiers: {sorted(active_identifiers)}"
)
else:
logger.debug("[ActiveCheck] No files are currently active")
return Response(
{
"active_files": active_files, # Legacy format: {uuid: [execution_data]}
"active_uuids": list(
active_files.keys()
), # Legacy format: [uuid1, uuid2]
"active_identifiers": list(
active_identifiers
), # New format: ["uuid:path", "uuid2:path2"]
"total_checked": len(files),
"total_active": len(active_files),
"cache_stats": {
"db_queries": db_queries,
},
}
)
except Exception as e:
logger.error(f"Error checking active files: {str(e)}", exc_info=True)
return Response(
{"error": "Failed to check active files", "detail": str(e)},
status=status.HTTP_500_INTERNAL_SERVER_ERROR,
)
def _bulk_database_check(
self,
files_needing_db_check: list[dict],
workflow_id: str,
current_execution_id: str | None,
active_files: dict,
active_identifiers: set,
):
"""Perform bulk database queries instead of individual queries for each file."""
if not files_needing_db_check:
return
# Separate files by query type
path_aware_files = [f for f in files_needing_db_check if f["path"] is not None]
legacy_files = [f for f in files_needing_db_check if f["path"] is None]
logger.debug(
f"[ActiveCheck] Querying {len(path_aware_files)} path-aware, "
f"{len(legacy_files)} UUID-only files"
)
# Query 1: Bulk query for path-aware files
if path_aware_files:
self._bulk_query_path_aware(
path_aware_files,
workflow_id,
current_execution_id,
active_files,
active_identifiers,
)
# Query 2: Bulk query for UUID-only files
if legacy_files:
self._bulk_query_uuid_only(
legacy_files,
workflow_id,
current_execution_id,
active_files,
active_identifiers,
)
def _bulk_query_path_aware(
self,
path_aware_files: list[dict],
workflow_id: str,
current_execution_id: str | None,
active_files: dict,
active_identifiers: set,
):
"""Bulk query for files with specific paths using two-step workflow scoping."""
from django.db.models import Q
# Step 1: Get ACTIVE workflow executions for this workflow
active_workflow_executions = WorkflowExecution.objects.filter(
workflow_id=workflow_id, status__in=["PENDING", "EXECUTING"]
)
if current_execution_id:
active_workflow_executions = active_workflow_executions.exclude(
id=current_execution_id
)
active_execution_ids = list(
active_workflow_executions.values_list("id", flat=True)
)
if not active_execution_ids:
logger.debug(
"[ActiveCheck] No active workflow executions found, path-aware query returns 0 results"
)
return
# Step 2: Build OR conditions for file matching: (uuid1 AND path1) OR (uuid2 AND path2) OR ...
path_conditions = Q()
for file_info in path_aware_files:
path_conditions |= Q(
provider_file_uuid=file_info["uuid"], file_path=file_info["path"]
)
# Step 3: Execute bulk query on workflow_file_executions from active workflow executions only
query = WorkflowFileExecution.objects.filter(
workflow_execution_id__in=active_execution_ids, # Scoped to active workflow executions
status__in=["PENDING", "EXECUTING"], # File execution must also be active
).filter(path_conditions)
active_executions = query.values(
"id",
"workflow_execution_id",
"file_name",
"file_path",
"status",
"created_at",
"provider_file_uuid",
)
logger.info(
f"[ActiveCheck] Path-aware query found {active_executions.count()} active records"
)
# Map results back to files with validation
for record in active_executions:
provider_uuid = record["provider_file_uuid"]
file_path = record["file_path"]
composite_id = f"{provider_uuid}:{file_path}"
execution_id = record["workflow_execution_id"]
# Validation: Ensure this execution ID is in our expected active executions list
if execution_id not in active_execution_ids:
logger.error(
f"[ActiveCheck] VALIDATION ERROR: Found file execution {record['id']} "
f"with workflow_execution_id {execution_id} that's not in our active executions list!"
)
continue
logger.debug(
f"[ActiveCheck] Active record {record['id']}: "
f"uuid={provider_uuid[:8]}..., status={record['status']}, "
f"path={file_path}, workflow_execution={execution_id} ✓"
)
# Track in both formats
if provider_uuid not in active_files:
active_files[provider_uuid] = []
active_files[provider_uuid].append(dict(record))
active_identifiers.add(composite_id)
logger.debug(f"[ActiveCheck] File {composite_id} is actively being processed")
def _bulk_query_uuid_only(
self,
legacy_files: list[dict],
workflow_id: str,
current_execution_id: str | None,
active_files: dict,
active_identifiers: set,
):
"""Bulk query for UUID-only files (no path available) using two-step workflow scoping."""
# Step 1: Get ACTIVE workflow executions for this workflow
active_workflow_executions = WorkflowExecution.objects.filter(
workflow_id=workflow_id, status__in=["PENDING", "EXECUTING"]
)
if current_execution_id:
active_workflow_executions = active_workflow_executions.exclude(
id=current_execution_id
)
active_execution_ids = list(
active_workflow_executions.values_list("id", flat=True)
)
if not active_execution_ids:
logger.debug(
"[ActiveCheck] No active workflow executions found, UUID-only query returns 0 results"
)
return
# Step 2: Extract UUIDs for IN query
uuid_only_uuids = [f["uuid"] for f in legacy_files]
# Step 3: Execute bulk query on workflow_file_executions from active workflow executions only
query = WorkflowFileExecution.objects.filter(
workflow_execution_id__in=active_execution_ids, # Scoped to active workflow executions
provider_file_uuid__in=uuid_only_uuids,
status__in=["PENDING", "EXECUTING"], # File execution must also be active
)
logger.debug(f"[ActiveCheck] Legacy bulk SQL: {query.query}")
active_executions = query.values(
"id",
"workflow_execution_id",
"file_name",
"file_path",
"status",
"created_at",
"provider_file_uuid",
)
logger.info(
f"[ActiveCheck] UUID-only query found {active_executions.count()} active records"
)
# Map results back to files with validation
for record in active_executions:
provider_uuid = record["provider_file_uuid"]
composite_id = provider_uuid # Legacy: no path suffix
execution_id = record["workflow_execution_id"]
# Validation: Ensure this execution ID is in our expected active executions list
if execution_id not in active_execution_ids:
logger.error(
f"[ActiveCheck] VALIDATION ERROR: Found file execution {record['id']} "
f"with workflow_execution_id {execution_id} that's not in our active executions list!"
)
continue
logger.debug(
f"[ActiveCheck] Active record {record['id']}: "
f"uuid={provider_uuid[:8]}..., status={record['status']}, "
f"path={record['file_path']}, workflow_execution={execution_id} ✓"
)
# Track in both formats
if provider_uuid not in active_files:
active_files[provider_uuid] = []
active_files[provider_uuid].append(dict(record))
active_identifiers.add(composite_id)
logger.info(
f"[ActiveCheck] File {composite_id} is actively being processed (legacy)"
)
class WorkflowFileExecutionAPIView(APIView):
"""Internal API for workflow file execution operations."""
def post(self, request):
"""Get or create workflow file execution record."""
try:
execution_id = request.data.get("execution_id")
file_hash = request.data.get("file_hash", {})
workflow_id = request.data.get("workflow_id")
logger.info(
f"1Received file execution request for execution {execution_id} and workflow {workflow_id}"
)
if not execution_id or not workflow_id:
return Response(
{"error": "execution_id and workflow_id are required"},
status=status.HTTP_400_BAD_REQUEST,
)
logger.info(
f"2Received file execution request for execution {execution_id} and workflow {workflow_id}"
)
# Get workflow execution
try:
workflow_execution = WorkflowExecution.objects.get(id=execution_id)
except WorkflowExecution.DoesNotExist:
return Response(
{"error": f"Workflow execution {execution_id} not found"},
status=status.HTTP_404_NOT_FOUND,
)
logger.info(
f"3Received file execution request for execution {execution_id} and workflow {workflow_id}"
)
# Get or create workflow file execution
file_execution, created = WorkflowFileExecution.objects.get_or_create(
workflow_execution=workflow_execution,
file_hash=file_hash.get("file_hash", ""),
defaults={
"file_name": file_hash.get("file_name", ""),
"file_path": file_hash.get("file_path", ""),
"file_size": file_hash.get("file_size", 0),
"mime_type": file_hash.get("mime_type", ""),
"provider_file_uuid": file_hash.get("provider_file_uuid"),
"fs_metadata": file_hash.get("fs_metadata", {}),
"status": "PENDING",
},
)
logger.info(f"4Received file execution request for file_hash {file_hash}")
return Response(
{
"id": str(file_execution.id),
"file_name": file_execution.file_name,
"file_path": file_execution.file_path,
"status": file_execution.status,
"created": created,
}
)
except Exception as e:
logger.error(f"Failed to get/create workflow file execution: {str(e)}")
return Response(
{
"error": "Failed to get/create workflow file execution",
"detail": str(e),
},
status=status.HTTP_500_INTERNAL_SERVER_ERROR,
)
class WorkflowExecuteFileAPIView(APIView):
"""Internal API for executing workflow for a single file."""
def post(self, request):
"""Execute workflow for a single file."""
try:
workflow_id = request.data.get("workflow_id")
execution_id = request.data.get("execution_id")
file_data = request.data.get("file_data", {})
organization_id = request.data.get("organization_id")
if not all([workflow_id, execution_id, file_data, organization_id]):
return Response(
{
"error": "workflow_id, execution_id, file_data, and organization_id are required"
},
status=status.HTTP_400_BAD_REQUEST,
)
# Set organization context
StateStore.set(Account.ORGANIZATION_ID, organization_id)
# Get workflow and execution
try:
workflow = Workflow.objects.get(id=workflow_id)
workflow_execution = WorkflowExecution.objects.get(id=execution_id)
except (Workflow.DoesNotExist, WorkflowExecution.DoesNotExist) as e:
return Response(
{"error": f"Workflow or execution not found: {str(e)}"},
status=status.HTTP_404_NOT_FOUND,
)
# Get tool instances
tool_instances = ToolInstance.objects.filter(workflow=workflow).order_by(
"step"
)
# Execute workflow using WorkflowExecutionServiceHelper
try:
from workflow_manager.workflow_v2.execution import (
WorkflowExecutionServiceHelper,
)
execution_helper = WorkflowExecutionServiceHelper(
workflow=workflow,
tool_instances=list(tool_instances),
organization_id=organization_id,
workflow_execution=workflow_execution,
)
# Execute the workflow for this file
result = execution_helper.execute_single_file(
file_data=file_data,
file_name=file_data.get("name", ""),
file_path=file_data.get("file_path", ""),
)
return Response(
{
"status": "success",
"execution_id": execution_id,
"result": result,
"file_name": file_data.get("name"),
}
)
except Exception as exec_error:
logger.error(f"Workflow execution failed: {str(exec_error)}")
return Response(
{
"status": "error",
"execution_id": execution_id,
"error": str(exec_error),
"file_name": file_data.get("name"),
}
)
except Exception as e:
logger.error(f"Failed to execute workflow for file: {str(e)}")
return Response(
{"error": "Failed to execute workflow for file", "detail": str(e)},
status=status.HTTP_500_INTERNAL_SERVER_ERROR,
)
class WorkflowEndpointAPIView(APIView):
"""Internal API for getting workflow endpoints.
Used by workers to determine if a workflow is API-based or filesystem-based.
"""
def get(self, request, workflow_id):
"""Get workflow endpoints for connection type detection."""
try:
from utils.user_context import UserContext
from workflow_manager.endpoint_v2.models import WorkflowEndpoint
# Enhanced debug logging for organization context
organization_id = getattr(request, "organization_id", None)
organization_from_context = UserContext.get_organization()
statestore_org_id = StateStore.get(Account.ORGANIZATION_ID)
request_debug = {
"workflow_id": str(workflow_id),
"request_organization_id": organization_id,
"statestore_org_id": statestore_org_id,
"usercontext_organization": str(organization_from_context.id)
if organization_from_context
else None,
"usercontext_org_name": organization_from_context.display_name
if organization_from_context
else None,
"internal_service": getattr(request, "internal_service", False),
"authenticated_via": getattr(request, "authenticated_via", None),
"path": request.path,
}
# Headers omitted: they carry the internal service bearer key.
logger.debug("WorkflowEndpointAPIView debug - %s", request_debug)
# Get workflow using the DefaultOrganizationManagerMixin which automatically filters by organization
try:
# This will automatically apply organization filtering via DefaultOrganizationManagerMixin
workflow = Workflow.objects.get(id=workflow_id)
logger.info(
f"Found workflow {workflow_id}: organization={workflow.organization_id}, name={getattr(workflow, 'workflow_name', 'Unknown')}"
)
except Workflow.DoesNotExist:
logger.error(
f"Workflow {workflow_id} not found or not accessible by organization {organization_id}"
)
return Response(
{"error": "Workflow not found or access denied"},
status=status.HTTP_404_NOT_FOUND,
)
# Get workflow endpoints with connector instance data
workflow_endpoints = WorkflowEndpoint.objects.select_related(
"connector_instance"
).filter(workflow=workflow)
source_endpoint = None
destination_endpoint = None
has_api_endpoints = False
for endpoint in workflow_endpoints:
endpoint_data = WorkflowEndpointConfigData(
endpoint_id=endpoint.id,
endpoint_type=endpoint.endpoint_type,
connection_type=endpoint.connection_type,
configuration=endpoint.configuration,
)
# Include connector instance information if available
if endpoint.connector_instance:
connector_instance_data = ConnectorInstanceData(
connector_id=endpoint.connector_instance.connector_id,
connector_name=endpoint.connector_instance.connector_name,
connector_metadata=endpoint.connector_instance.metadata or {},
)
endpoint_data.connector_instance = connector_instance_data
# endpoint_data["connector_instance"] = connector_instance_data
logger.debug(
f"Added connector instance data for endpoint {endpoint.id}: {endpoint.connector_instance.connector_id}"
)
else:
endpoint_data.connector_instance = None
# endpoint_data["connector_instance"] = None
logger.debug(
f"No connector instance found for endpoint {endpoint.id}"
)
if endpoint.endpoint_type == WorkflowEndpoint.EndpointType.SOURCE:
source_endpoint = endpoint_data
elif endpoint.endpoint_type == WorkflowEndpoint.EndpointType.DESTINATION:
destination_endpoint = endpoint_data
if endpoint.connection_type == ConnectionType.API.value:
has_api_endpoints = True
endpoint_config = WorkflowEndpointConfigResponseData(
workflow_id=str(workflow_id),
has_api_endpoints=has_api_endpoints,
source_endpoint=source_endpoint,
destination_endpoint=destination_endpoint,
)
response_data = endpoint_config.to_dict()
logger.info(
f"Retrieved endpoints for workflow {workflow_id}, API endpoints: {has_api_endpoints}"
)
return Response(response_data, status=status.HTTP_200_OK)
except Exception as e:
logger.error(f"Failed to get workflow endpoints for {workflow_id}: {str(e)}")
return Response(
{"error": "Failed to get workflow endpoints", "detail": str(e)},
status=status.HTTP_500_INTERNAL_SERVER_ERROR,
)
class WorkflowSourceFilesAPIView(APIView):
"""Internal API for getting workflow source files.
Used by workers to get source files for processing.
"""
def post(self, request, workflow_id):
"""Get source files for a workflow execution."""
try:
from utils.user_context import UserContext
from unstract.workflow_execution.enums import LogStage
from workflow_manager.endpoint_v2.source import SourceConnector
from workflow_manager.utils.workflow_log import WorkflowLog
# Get request data
execution_id = request.data.get("execution_id")
pipeline_id = request.data.get("pipeline_id")
use_file_history = request.data.get("use_file_history", True)
if not execution_id:
return Response(
{"error": "execution_id is required"},
status=status.HTTP_400_BAD_REQUEST,
)
# Enhanced debug logging for organization context
organization_id = getattr(request, "organization_id", None)
organization_from_context = UserContext.get_organization()
statestore_org_id = StateStore.get(Account.ORGANIZATION_ID)
request_debug = {
"workflow_id": str(workflow_id),
"execution_id": str(execution_id),
"pipeline_id": str(pipeline_id) if pipeline_id else None,
"request_organization_id": organization_id,
"statestore_org_id": statestore_org_id,
"usercontext_organization": str(organization_from_context.id)
if organization_from_context
else None,
"use_file_history": use_file_history,
}
logger.info(f"WorkflowSourceFilesAPIView debug - {request_debug}")
# Get workflow using the DefaultOrganizationManagerMixin which automatically filters by organization
try:
workflow = Workflow.objects.get(id=workflow_id)
logger.info(f"Found workflow {workflow_id} for source files request")
except Workflow.DoesNotExist:
logger.error(f"Workflow {workflow_id} not found or not accessible")
return Response(
{"error": "Workflow not found or access denied"},
status=status.HTTP_404_NOT_FOUND,
)
# Get workflow execution
try:
WorkflowExecution.objects.get(id=execution_id)
logger.info(f"Found workflow execution {execution_id}")
except WorkflowExecution.DoesNotExist:
logger.error(f"Workflow execution {execution_id} not found")
return Response(
{"error": "Workflow execution not found"},
status=status.HTTP_404_NOT_FOUND,
)
# Create workflow log
workflow_log = WorkflowLog(
execution_id=execution_id,
organization_id=organization_id,
log_stage=LogStage.INITIALIZE,
pipeline_id=pipeline_id,
)
# Create source connector
source = SourceConnector(
workflow=workflow,
execution_id=str(execution_id),
workflow_log=workflow_log,
use_file_history=use_file_history,
organization_id=organization_id,
)
# Validate and get source files
source.validate()
# Get input files from source (this includes file listing and processing)
input_files, total_files = source.list_files_from_source({})
# Convert input_files to serializable format and include connector context
serializable_files = {}
connector_metadata = None
connector_id = None
# Get connector metadata from the workflow endpoint for FILESYSTEM access
if source.endpoint and source.endpoint.connector_instance:
connector_metadata = source.endpoint.connector_instance.connector_metadata
connector_id = source.endpoint.connector_instance.connector_id
logger.info(f"Including connector context: connector_id={connector_id}")
for file_name, file_hash in input_files.items():
if hasattr(file_hash, "to_json"):
file_data = file_hash.to_json()
else:
file_data = file_hash
# Add connector context to each file for worker access
if connector_metadata and connector_id:
file_data["connector_metadata"] = connector_metadata
file_data["connector_id"] = connector_id
serializable_files[file_name] = file_data
logger.info(
f"Retrieved {total_files} source files for workflow {workflow_id}, execution {execution_id}"
)
return Response(
{
"files": serializable_files,
"total_files": total_files,
"workflow_id": str(workflow_id),
"execution_id": str(execution_id),
"pipeline_id": str(pipeline_id) if pipeline_id else None,
}
)
except Exception as e:
logger.error(
f"Failed to get source files for workflow {workflow_id}: {str(e)}",
exc_info=True,
)
return Response(
{"error": "Failed to get source files", "detail": str(e)},
status=status.HTTP_500_INTERNAL_SERVER_ERROR,
)
class FileCountIncrementAPIView(APIView):
"""Internal API for incrementing file counts during execution.
Replicates Django ExecutionCacheUtils functionality for workers.
"""
def post(self, request):
"""Increment file counts for execution."""
try:
workflow_id = request.data.get("workflow_id")
execution_id = request.data.get("execution_id")
increment_type = request.data.get("increment_type") # 'completed' or 'failed'
if not all([workflow_id, execution_id, increment_type]):
return Response(
{
"error": "workflow_id, execution_id, and increment_type are required"
},
status=status.HTTP_400_BAD_REQUEST,
)
# Get workflow execution
try:
WorkflowExecution.objects.get(id=execution_id)
except WorkflowExecution.DoesNotExist:
return Response(
{"error": "Workflow execution not found"},
status=status.HTTP_404_NOT_FOUND,
)
# Use Django backend's ExecutionCacheUtils to increment counts
from workflow_manager.execution.execution_cache_utils import (
ExecutionCacheUtils,
)
if increment_type == "completed":
ExecutionCacheUtils.increment_completed_files(
workflow_id=workflow_id, execution_id=execution_id
)
logger.info(f"Incremented completed files for execution {execution_id}")
elif increment_type == "failed":
ExecutionCacheUtils.increment_failed_files(
workflow_id=workflow_id, execution_id=execution_id
)
logger.info(f"Incremented failed files for execution {execution_id}")
else:
return Response(
{
"error": f"Invalid increment_type: {increment_type}. Must be 'completed' or 'failed'"
},
status=status.HTTP_400_BAD_REQUEST,
)
return Response(
{
"success": True,
"workflow_id": workflow_id,
"execution_id": execution_id,
"increment_type": increment_type,
}
)
except Exception as e:
logger.error(f"Failed to increment file count: {str(e)}")
return Response(
{"error": "Failed to increment file count", "detail": str(e)},
status=status.HTTP_500_INTERNAL_SERVER_ERROR,
)
class PipelineStatusUpdateAPIView(APIView):
"""Internal API for updating pipeline status.
Used by workers to update pipeline execution status.
"""
def post(self, request, pipeline_id):
"""Update pipeline status."""
try:
from pipeline_v2.models import Pipeline
from workflow_manager.utils.pipeline_utils import PipelineUtils
# Get request data
execution_id = request.data.get("execution_id")
status_value = request.data.get("status")
if not execution_id or not status_value:
return Response(
{"error": "execution_id and status are required"},
status=status.HTTP_400_BAD_REQUEST,
)
# Get pipeline with organization filtering
try:
# Apply organization filtering to pipeline query
pipeline_queryset = Pipeline.objects.filter(id=pipeline_id)
pipeline_queryset = filter_queryset_by_organization(
pipeline_queryset, request, "organization"
)
pipeline_queryset.get()
logger.info(
f"Found pipeline {pipeline_id} for status update (org: {getattr(request, 'organization_id', 'unknown')})"
)
except Pipeline.DoesNotExist:
org_id = getattr(request, "organization_id", "unknown")
logger.error(
f"Pipeline {pipeline_id} not found or not accessible by organization {org_id}"
)
return Response(
{"error": "Pipeline not found"}, status=status.HTTP_404_NOT_FOUND
)
# Get workflow execution with organization filtering
try:
# Apply organization filtering to workflow execution query
execution_queryset = WorkflowExecution.objects.filter(id=execution_id)
execution_queryset = filter_queryset_by_organization(
execution_queryset, request, "workflow__organization"
)
workflow_execution = execution_queryset.get()
logger.info(
f"Found workflow execution {execution_id} (org: {getattr(request, 'organization_id', 'unknown')})"
)
except WorkflowExecution.DoesNotExist:
org_id = getattr(request, "organization_id", "unknown")
logger.error(
f"Workflow execution {execution_id} not found or not accessible by organization {org_id}"
)
return Response(
{"error": "Workflow execution not found"},
status=status.HTTP_404_NOT_FOUND,
)
# Update pipeline status using the utility method
PipelineUtils.update_pipeline_status(
pipeline_id=str(pipeline_id), workflow_execution=workflow_execution
)
logger.info(
f"Updated pipeline {pipeline_id} status for execution {execution_id}"
)
return Response(
{
"status": "updated",
"pipeline_id": str(pipeline_id),
"execution_id": str(execution_id),
"new_status": status_value,
}
)
except Exception as e:
logger.error(
f"Failed to update pipeline {pipeline_id} status: {str(e)}", exc_info=True
)
return Response(
{"error": "Failed to update pipeline status", "detail": str(e)},
status=status.HTTP_500_INTERNAL_SERVER_ERROR,
)
class WorkflowDefinitionAPIView(APIView):
"""Internal API endpoint for getting workflow definitions.
Fixed to handle missing endpoints gracefully and use correct workflow type detection.
"""
def get(self, request, workflow_id):
"""Get workflow definition with proper type detection and endpoint handling."""
try:
from workflow_manager.workflow_v2.models.workflow import Workflow
# Get workflow with organization filtering
try:
workflow = Workflow.objects.get(id=workflow_id)
# Verify organization access
filter_queryset_by_organization(
Workflow.objects.filter(id=workflow_id), request, "organization"
).get()
except Workflow.DoesNotExist:
return Response(
{"error": "Workflow not found or access denied"},
status=status.HTTP_404_NOT_FOUND,
)
# Step 1: Get source configuration with graceful error handling
source_config = self._get_source_endpoint_config(workflow_id, workflow)
# Step 2: Get destination configuration with graceful error handling
destination_config = self._get_destination_endpoint_config(
workflow_id, workflow
)
# Step 3: Build comprehensive workflow definition using dataclasses
workflow_definition = WorkflowDefinitionResponseData(
workflow_id=str(workflow.id),
workflow_name=workflow.workflow_name,
source_config=source_config,
destination_config=destination_config,
organization_id=str(workflow.organization.organization_id),
created_at=workflow.created_at.isoformat(),
modified_at=workflow.modified_at.isoformat(),
is_active=workflow.is_active,
)
response_data = workflow_definition.to_dict()
logger.info(
f"Retrieved workflow definition for {workflow_id}: {workflow_definition.workflow_type} (source: {workflow_definition.source_config.connection_type})"
)
return Response(response_data, status=status.HTTP_200_OK)
except Exception as e:
logger.error(
f"Failed to get workflow definition for {workflow_id}: {str(e)}",
exc_info=True,
)
return Response(
{"error": "Failed to get workflow definition", "detail": str(e)},
status=status.HTTP_500_INTERNAL_SERVER_ERROR,
)
def _get_source_endpoint_config(
self, workflow_id: str, workflow
) -> WorkflowEndpointConfigData:
"""Get source endpoint configuration with credential resolution."""
try:
source_endpoint = WorkflowEndpointUtils.get_endpoint_for_workflow_by_type(
workflow_id, WorkflowEndpoint.EndpointType.SOURCE
)
# Start with folder/path configuration from endpoint
merged_configuration = source_endpoint.configuration or {}
# Create connector instance data and resolve credentials if available
connector_instance_data = None
if source_endpoint.connector_instance:
connector_instance = source_endpoint.connector_instance
# Use exact same pattern as backend source.py
# Get connector metadata (which contains decrypted credentials)
connector_credentials = {}
try:
# Follow backend pattern: use connector.metadata for credentials
# This contains the actual decrypted credentials (json_credentials, project_id, etc.)
connector_credentials = connector_instance.metadata or {}
# Optionally refresh OAuth tokens if needed (like backend does)
if connector_instance.connector_auth:
try:
# This refreshes tokens and updates metadata if needed
connector_instance.get_connector_metadata()
# Use the updated metadata
connector_credentials = connector_instance.metadata or {}
logger.debug(
f"Refreshed connector metadata for {connector_instance.connector_id}"
)
except Exception as refresh_error:
logger.warning(
f"Failed to refresh connector metadata for {connector_instance.id}: {str(refresh_error)}"
)
# Continue with existing metadata
logger.debug(
f"Retrieved connector settings for {connector_instance.connector_id}"
)
except Exception as cred_error:
logger.warning(
f"Failed to retrieve connector settings for {connector_instance.id}: {str(cred_error)}"
)
# Continue without credentials - let connector handle the error
# Merge folder configuration with connector credentials
# Folder settings take precedence over connector defaults
merged_configuration = {**connector_credentials, **merged_configuration}
connector_instance_data = ConnectorInstanceData(
connector_id=connector_instance.connector_id,
connector_name=getattr(connector_instance, "connector_name", ""),
connector_metadata=connector_instance.metadata or {},
)
logger.debug(
f"Found source endpoint for workflow {workflow_id}: {source_endpoint.connection_type} with {len(merged_configuration)} config keys"
)
return WorkflowEndpointConfigData(
endpoint_id=str(source_endpoint.id),
endpoint_type=source_endpoint.endpoint_type,
connection_type=source_endpoint.connection_type,
configuration=merged_configuration,
connector_instance=connector_instance_data,
)
except WorkflowEndpoint.DoesNotExist:
logger.info(
f"No source endpoint found for workflow {workflow_id}, returning empty config"
)
return WorkflowEndpointConfigData(
endpoint_id="",
endpoint_type=WorkflowEndpoint.EndpointType.SOURCE,
connection_type="NONE",
)
except Exception as e:
logger.warning(
f"Error getting source endpoint for workflow {workflow_id}: {str(e)}"
)
return WorkflowEndpointConfigData(
endpoint_id="",
endpoint_type=WorkflowEndpoint.EndpointType.SOURCE,
connection_type="NONE",
)
def _get_destination_endpoint_config(
self, workflow_id: str, workflow
) -> WorkflowEndpointConfigData:
"""Get destination endpoint configuration with credential resolution."""
try:
destination_endpoint = (
WorkflowEndpointUtils.get_endpoint_for_workflow_by_type(
workflow_id, WorkflowEndpoint.EndpointType.DESTINATION
)
)
# Start with configuration from endpoint
merged_configuration = destination_endpoint.configuration or {}
# Create connector instance data and resolve credentials if available
connector_instance_data = None
if destination_endpoint.connector_instance:
connector_instance = destination_endpoint.connector_instance
# Use exact same pattern as backend source.py
# Get connector metadata (which contains decrypted credentials)
connector_credentials = {}
try:
# Follow backend pattern: use connector.metadata for credentials
# This contains the actual decrypted credentials (host, database, username, password, etc.)
connector_credentials = connector_instance.metadata or {}
# Optionally refresh OAuth tokens if needed (like backend does)
if connector_instance.connector_auth:
try:
# This refreshes tokens and updates metadata if needed
connector_instance.get_connector_metadata()
# Use the updated metadata
connector_credentials = connector_instance.metadata or {}
logger.debug(
f"Refreshed destination connector metadata for {connector_instance.connector_id}"
)
except Exception as refresh_error:
logger.warning(
f"Failed to refresh destination connector metadata for {connector_instance.id}: {str(refresh_error)}"
)
# Continue with existing metadata
logger.debug(
f"Retrieved destination connector settings for {connector_instance.connector_id}"
)
except Exception as cred_error:
logger.warning(
f"Failed to retrieve destination connector settings for {connector_instance.id}: {str(cred_error)}"
)
# Continue without credentials - let connector handle the error
# Merge configuration with connector credentials
# Endpoint settings take precedence over connector defaults
merged_configuration = {**connector_credentials, **merged_configuration}
connector_instance_data = ConnectorInstanceData(
connector_id=connector_instance.connector_id,
connector_name=connector_instance.connector_name,
connector_metadata=connector_instance.metadata or {},
)
logger.debug(
f"Found destination endpoint for workflow {workflow_id}: {destination_endpoint.connection_type} with {len(merged_configuration)} config keys"
)
return WorkflowEndpointConfigData(
endpoint_id=str(destination_endpoint.id),
endpoint_type=destination_endpoint.endpoint_type,
connection_type=destination_endpoint.connection_type,
configuration=merged_configuration,
connector_instance=connector_instance_data,
)
except WorkflowEndpoint.DoesNotExist:
logger.info(
f"No destination endpoint found for workflow {workflow_id}, returning empty config"
)
return WorkflowEndpointConfigData(
endpoint_id="",
endpoint_type=WorkflowEndpoint.EndpointType.DESTINATION,
connection_type="NONE",
)
except Exception as e:
logger.warning(
f"Error getting destination endpoint for workflow {workflow_id}: {str(e)}"
)
return WorkflowEndpointConfigData(
endpoint_id="",
endpoint_type=WorkflowEndpoint.EndpointType.DESTINATION,
connection_type="NONE",
)
class PipelineTypeAPIView(APIView):
"""Internal API endpoint for determining pipeline type.
Checks APIDeployment first, then Pipeline model to determine if pipeline is:
- API (if found in APIDeployment model)
- ETL/TASK/APP (if found in Pipeline model with pipeline_type field)
"""
def get(self, request, pipeline_id):
"""Determine pipeline type from APIDeployment or Pipeline models."""
try:
from api_v2.models import APIDeployment
from pipeline_v2.models import Pipeline
organization_id = getattr(request, "organization_id", None)
# First check if this is an API deployment
try:
api_deployment = APIDeployment.objects.get(id=pipeline_id)
# Verify organization access
if (
organization_id
and str(api_deployment.organization.organization_id)
!= organization_id
):
return Response(
{"error": "API deployment not found in organization"},
status=status.HTTP_404_NOT_FOUND,
)
logger.info(f"Pipeline {pipeline_id} identified as API deployment")
return Response(
{
"pipeline_id": str(pipeline_id),
"pipeline_type": "API",
"source": "api_deployment",
"workflow_id": str(api_deployment.workflow_id),
"display_name": api_deployment.display_name,
"is_active": api_deployment.is_active,
},
status=status.HTTP_200_OK,
)
except APIDeployment.DoesNotExist:
# Not an API deployment, check Pipeline model
pass
# Check if this is a regular pipeline (ETL/TASK/APP)
try:
pipeline = Pipeline.objects.get(id=pipeline_id)
# Verify organization access
if (
organization_id
and str(pipeline.organization.organization_id) != organization_id
):
return Response(
{"error": "Pipeline not found in organization"},
status=status.HTTP_404_NOT_FOUND,
)
# Map Pipeline.PipelineType to expected values
pipeline_type = pipeline.pipeline_type
if pipeline_type == Pipeline.PipelineType.ETL:
resolved_type = "ETL"
elif pipeline_type == Pipeline.PipelineType.TASK:
resolved_type = "TASK"
elif pipeline_type == Pipeline.PipelineType.APP:
resolved_type = "APP"
else:
resolved_type = "ETL" # Default fallback
logger.info(
f"Pipeline {pipeline_id} identified as {resolved_type} pipeline"
)
return Response(
{
"pipeline_id": str(pipeline_id),
"pipeline_type": resolved_type,
"source": "pipeline",
"workflow_id": str(pipeline.workflow_id),
"pipeline_name": pipeline.pipeline_name,
"active": pipeline.active,
"scheduled": pipeline.scheduled,
},
status=status.HTTP_200_OK,
)
except Pipeline.DoesNotExist:
# Pipeline not found in either model
logger.warning(
f"Pipeline {pipeline_id} not found in APIDeployment or Pipeline models"
)
return Response(
{
"error": "Pipeline not found",
"detail": f"Pipeline {pipeline_id} not found in APIDeployment or Pipeline models",
},
status=status.HTTP_404_NOT_FOUND,
)
except Exception as e:
logger.error(f"Failed to determine pipeline type for {pipeline_id}: {str(e)}")
return Response(
{"error": "Failed to determine pipeline type", "detail": str(e)},
status=status.HTTP_500_INTERNAL_SERVER_ERROR,
)
class BatchStatusUpdateAPIView(APIView):
"""Internal API endpoint for batch status updates.
Allows updating multiple workflow executions in a single request.
"""
def post(self, request):
"""Update multiple workflow execution statuses."""
try:
updates = request.data.get("updates", [])
if not updates:
return Response(
{"error": "updates list is required"},
status=status.HTTP_400_BAD_REQUEST,
)
successful_updates = []
failed_updates = []
with transaction.atomic():
for update in updates:
try:
execution_id = update.get("execution_id")
status_value = update.get("status")
if not execution_id or not status_value:
failed_updates.append(
{
"execution_id": execution_id,
"error": "execution_id and status are required",
}
)
continue
# Get workflow execution with organization filtering
execution_queryset = WorkflowExecution.objects.filter(
id=execution_id
)
execution_queryset = filter_queryset_by_organization(
execution_queryset, request, "workflow__organization"
)
execution = execution_queryset.get()
# Update status
execution.status = status_value
# Update optional fields
if update.get("error_message"):
execution.error_message = update["error_message"][
:256
] # Truncate to fit constraint
if update.get("total_files") is not None:
execution.total_files = update["total_files"]
if update.get("execution_time") is not None:
execution.execution_time = update["execution_time"]
execution.save()
# This write bypasses update_execution, so it registers
# the inbox hook itself.
from inbox.events import notify_on_terminal_status
notify_on_terminal_status(execution.pk, status_value)
successful_updates.append(
{
"execution_id": str(execution.id),
"status": execution.status,
}
)
except WorkflowExecution.DoesNotExist:
failed_updates.append(
{
"execution_id": execution_id,
"error": "Workflow execution not found",
}
)
except Exception as e:
failed_updates.append(
{"execution_id": execution_id, "error": str(e)}
)
logger.info(
f"Batch status update completed: {len(successful_updates)} successful, {len(failed_updates)} failed"
)
return Response(
{
"successful_updates": successful_updates,
"failed_updates": failed_updates,
"total_processed": len(updates),
}
)
except Exception as e:
logger.error(f"Failed to process batch status update: {str(e)}")
return Response(
{"error": "Failed to process batch status update", "detail": str(e)},
status=status.HTTP_500_INTERNAL_SERVER_ERROR,
)
class WorkflowExecutionCleanupAPIView(APIView):
"""Internal API endpoint for cleaning up workflow execution resources."""
def post(self, request):
"""Cleanup resources for multiple workflow executions."""
try:
execution_ids = request.data.get("execution_ids", [])
cleanup_types = request.data.get("cleanup_types", ["cache", "temp_files"])
if not execution_ids:
return Response(
{"error": "execution_ids list is required"},
status=status.HTTP_400_BAD_REQUEST,
)
cleaned_executions = []
failed_cleanups = []
for execution_id in execution_ids:
try:
# Get workflow execution with organization filtering
execution_queryset = WorkflowExecution.objects.filter(id=execution_id)
execution_queryset = filter_queryset_by_organization(
execution_queryset, request, "workflow__organization"
)
execution = execution_queryset.get()
# Perform cleanup based on cleanup_types
cleanup_results = {}
if "cache" in cleanup_types:
# Clean execution cache
try:
from workflow_manager.execution.execution_cache_utils import (
ExecutionCacheUtils,
)
ExecutionCacheUtils.cleanup_execution_cache(str(execution.id))
cleanup_results["cache"] = "cleaned"
except Exception as cache_error:
cleanup_results["cache"] = f"error: {str(cache_error)}"
if "temp_files" in cleanup_types:
# Clean temporary files
try:
# Import filesystem utilities
from unstract.filesystem import FileStorageType, FileSystem
# Clean execution directory
file_system = FileSystem(FileStorageType.WORKFLOW_EXECUTION)
file_storage = file_system.get_file_storage()
org_id = (
execution.workflow.organization_id
if execution.workflow
else "default"
)
execution_dir = f"unstract/execution/{org_id}/{execution.workflow_id}/{execution.id}"
if file_storage.exists(execution_dir):
file_storage.delete(execution_dir)
cleanup_results["temp_files"] = "cleaned"
else:
cleanup_results["temp_files"] = "not_found"
except Exception as file_error:
cleanup_results["temp_files"] = f"error: {str(file_error)}"
cleaned_executions.append(
{
"execution_id": str(execution.id),
"cleanup_results": cleanup_results,
}
)
except WorkflowExecution.DoesNotExist:
failed_cleanups.append(
{
"execution_id": execution_id,
"error": "Workflow execution not found",
}
)
except Exception as e:
failed_cleanups.append(
{"execution_id": execution_id, "error": str(e)}
)
logger.info(
f"Cleanup completed: {len(cleaned_executions)} successful, {len(failed_cleanups)} failed"
)
return Response(
{
"cleaned_executions": cleaned_executions,
"failed_cleanups": failed_cleanups,
"total_processed": len(execution_ids),
}
)
except Exception as e:
logger.error(f"Failed to process cleanup request: {str(e)}")
return Response(
{"error": "Failed to process cleanup request", "detail": str(e)},
status=status.HTTP_500_INTERNAL_SERVER_ERROR,
)
class WorkflowExecutionMetricsAPIView(APIView):
"""Internal API endpoint for getting workflow execution metrics."""
def get(self, request):
"""Get execution metrics with optional filtering."""
try:
# Get query parameters
start_date = request.query_params.get("start_date")
end_date = request.query_params.get("end_date")
workflow_id = request.query_params.get("workflow_id")
status = request.query_params.get("status")
# Build base queryset with organization filtering
executions = WorkflowExecution.objects.all()
executions = filter_queryset_by_organization(
executions, request, "workflow__organization"
)
# Apply filters
if start_date:
from datetime import datetime
executions = executions.filter(
created_at__gte=datetime.fromisoformat(start_date)
)
if end_date:
from datetime import datetime
executions = executions.filter(
created_at__lte=datetime.fromisoformat(end_date)
)
if workflow_id:
executions = executions.filter(workflow_id=workflow_id)
if status:
executions = executions.filter(status=status)
# Calculate metrics
from django.db.models import Avg, Count, Sum
total_executions = executions.count()
# Status breakdown
status_counts = executions.values("status").annotate(count=Count("id"))
status_breakdown = {item["status"]: item["count"] for item in status_counts}
# Success rate
completed_count = status_breakdown.get("COMPLETED", 0)
success_rate = (
(completed_count / total_executions) if total_executions > 0 else 0
)
# Average execution time
avg_execution_time = (
executions.aggregate(avg_time=Avg("execution_time"))["avg_time"] or 0
)
# Total files processed
total_files_processed = (
executions.aggregate(total_files=Sum("total_files"))["total_files"] or 0
)
metrics = {
"total_executions": total_executions,
"status_breakdown": status_breakdown,
"success_rate": success_rate,
"average_execution_time": avg_execution_time,
"total_files_processed": total_files_processed,
"filters_applied": {
"start_date": start_date,
"end_date": end_date,
"workflow_id": workflow_id,
"status": status,
},
}
logger.info(
f"Generated execution metrics: {total_executions} executions, {success_rate:.2%} success rate"
)
return Response(metrics)
except Exception as e:
logger.error(f"Failed to get execution metrics: {str(e)}")
return Response(
{"error": "Failed to get execution metrics", "detail": str(e)},
status=status.HTTP_500_INTERNAL_SERVER_ERROR,
)
class FileHistoryBatchCheckView(APIView):
"""Internal API view to check file history in batch for workers.
This enables file deduplication by checking which files have already been processed.
"""
def post(self, request):
"""Check file history for a batch of file hashes.
POST /internal/workflows/{workflow_id}/file-history/batch-check/
Request body:
{
"workflow_id": "uuid",
"file_hashes": ["hash1", "hash2", ...],
"organization_id": "uuid"
}
Response:
{
"processed_file_hashes": ["hash1", "hash3", ...]
}
"""
try:
workflow_id = request.data.get("workflow_id")
file_hashes = request.data.get("file_hashes", [])
organization_id = request.data.get("organization_id")
if not workflow_id or not file_hashes:
return Response(
{"error": "workflow_id and file_hashes are required"},
status=status.HTTP_400_BAD_REQUEST,
)
# Set organization context if provided
if organization_id:
StateStore.set(Account.ORGANIZATION_ID, organization_id)
# Get workflow
try:
workflow = filter_queryset_by_organization(
Workflow.objects.all(), request, "organization"
).get(id=workflow_id)
except Workflow.DoesNotExist:
return Response(
{"error": "Workflow not found or access denied"},
status=status.HTTP_404_NOT_FOUND,
)
# Check file history for the provided hashes
from workflow_manager.workflow_v2.models.file_history import FileHistory
# Apply organization filtering to FileHistory query
file_history_queryset = FileHistory.objects.filter(
workflow=workflow,
cache_key__in=file_hashes,
status="COMPLETED", # Only consider successfully completed files
)
# Apply organization filtering through workflow relationship
file_history_queryset = filter_queryset_by_organization(
file_history_queryset, request, "workflow__organization"
)
# Get max execution count for this workflow (for worker decision making)
max_execution_count = workflow.get_max_execution_count()
# Get full file history details for cached results
file_histories = file_history_queryset.values(
"cache_key",
"result",
"metadata",
"error",
"file_path",
"provider_file_uuid",
"execution_count",
"status",
)
# Build response with both processed hashes (for compatibility) and full details
processed_file_hashes = []
file_history_details = {}
for fh in file_histories:
cache_key = fh["cache_key"]
processed_file_hashes.append(cache_key)
file_history_details[cache_key] = {
"result": fh["result"],
"metadata": fh["metadata"],
"error": fh["error"],
"file_path": fh["file_path"],
"provider_file_uuid": fh["provider_file_uuid"],
"execution_count": fh["execution_count"],
"status": fh["status"],
"max_execution_count": max_execution_count, # Include for worker logic
}
logger.info(
f"File history batch check: {len(processed_file_hashes)}/{len(file_hashes)} files already processed "
f"(max_execution_count: {max_execution_count})"
)
return Response(
{
"processed_file_hashes": processed_file_hashes, # For backward compatibility
"file_history_details": file_history_details, # Full details for cached results
"max_execution_count": max_execution_count, # Global max for this workflow
}
)
except Exception as e:
logger.error(f"File history batch check failed: {str(e)}")
return Response(
{"error": "File history batch check failed", "detail": str(e)},
status=status.HTTP_500_INTERNAL_SERVER_ERROR,
)
class FileHistoryCreateView(APIView):
"""Internal API view to create file history entries for workers.
This enables workers to create file history entries after successful processing.
"""
def post(self, request):
"""Create a file history entry.
POST /internal/workflow-manager/file-history/create/
Request body:
{
"workflow_id": "uuid",
"cache_key": "file_hash",
"provider_file_uuid": "uuid_or_null",
"file_path": "path/to/file",
"file_name": "filename.ext",
"status": "COMPLETED",
"result": "execution_result",
"error": "error_message_or_empty",
"metadata": {},
"organization_id": "uuid"
}
Response:
{
"created": true,
"file_history_id": "uuid"
}
"""
try:
workflow_id = request.data.get("workflow_id")
cache_key = request.data.get("cache_key")
organization_id = request.data.get("organization_id")
provider_file_uuid = request.data.get("provider_file_uuid")
file_path = request.data.get("file_path")
file_name = request.data.get("file_name")
file_size = request.data.get("file_size")
file_hash = request.data.get("file_hash")
mime_type = request.data.get("mime_type")
is_api = request.data.get("is_api")
status = request.data.get("status")
result = request.data.get("result")
error = request.data.get("error")
metadata = request.data.get("metadata")
logger.info(
f"File history create: workflow_id={workflow_id}, cache_key={cache_key}, organization_id={organization_id}"
)
if not workflow_id or not cache_key:
return Response(
{"error": "workflow_id and cache_key are required"},
status=status.HTTP_400_BAD_REQUEST,
)
# Set organization context if provided
if organization_id:
StateStore.set(Account.ORGANIZATION_ID, organization_id)
# Get workflow
try:
workflow = filter_queryset_by_organization(
Workflow.objects.all(), request, "organization"
).get(id=workflow_id)
except Workflow.DoesNotExist:
return Response(
{"error": "Workflow not found or access denied"},
status=status.HTTP_404_NOT_FOUND,
)
# Create file history entry using the FileHistoryHelper
from unstract.core.data_models import FileHashData
from workflow_manager.workflow_v2.enums import ExecutionStatus
from workflow_manager.workflow_v2.file_history_helper import FileHistoryHelper
# Create FileHashData object from request data using shared class
file_hash = FileHashData(
file_name=file_name,
file_path=file_path,
file_hash=cache_key,
file_size=file_size,
mime_type=mime_type,
provider_file_uuid=provider_file_uuid,
fs_metadata={},
source_connection_type="",
file_destination="",
is_executed=False,
file_number=None,
connector_metadata={},
connector_id=None,
use_file_history=True,
)
# Check if file history should be created based on use_file_history flag
# if not file_hash.use_file_history:
# logger.info(
# f"Skipping file history creation for {file_hash.file_name} - use_file_history=False"
# )
# return Response({"created": False, "reason": "use_file_history=False"})
# Map string status to ExecutionStatus enum
status_str = request.data.get("status", "COMPLETED")
try:
execution_status = ExecutionStatus[status_str]
except KeyError:
execution_status = ExecutionStatus.COMPLETED
# Create file history entry
file_history = FileHistoryHelper.create_file_history(
file_hash=file_hash,
workflow=workflow,
status=execution_status,
result=result,
metadata=metadata,
error=error,
is_api=is_api,
)
logger.info(
f"Created/updated file history entry {file_history.id} for file {file_name} "
f"(execution_count: {file_history.execution_count})"
)
return Response(
{
"created": True,
"file_history_id": str(file_history.id),
"execution_count": file_history.execution_count,
}
)
except Exception as e:
logger.error(f"File history creation failed: {str(e)}")
return Response(
{"error": "File history creation failed", "detail": str(e)},
status=status.HTTP_500_INTERNAL_SERVER_ERROR,
)
class PipelineNameAPIView(APIView):
"""Internal API endpoint for fetching pipeline names from models.
This endpoint fetches the actual pipeline name from Pipeline.pipeline_name
or APIDeployment.api_name based on the pipeline ID.
Used by callback workers to get correct pipeline names for notifications.
"""
def get(self, request, pipeline_id):
"""Fetch pipeline name from Pipeline or APIDeployment model."""
try:
from api_v2.models import APIDeployment
from pipeline_v2.models import Pipeline
organization_id = getattr(request, "organization_id", None)
if organization_id:
StateStore.set(Account.ORGANIZATION_ID, organization_id)
# First check if this is an API deployment
try:
api_deployment = APIDeployment.objects.get(id=pipeline_id)
logger.info(
f"Found API deployment {pipeline_id}: name='{api_deployment.api_name}'"
)
# Verify organization access
if (
organization_id
and str(api_deployment.organization.organization_id)
!= organization_id
):
logger.warning(
f"API deployment {pipeline_id} not found in organization {organization_id}"
)
return Response(
{"error": "API deployment not found in organization"},
status=status.HTTP_404_NOT_FOUND,
)
logger.info(
f"Found API deployment {pipeline_id}: name='{api_deployment.api_name}'"
)
return Response(
{
"pipeline_id": str(pipeline_id),
"pipeline_name": api_deployment.api_name,
"pipeline_type": "API",
"source": "api_deployment",
"display_name": api_deployment.display_name,
}
)
except APIDeployment.DoesNotExist:
pass
# Check Pipeline model
try:
pipeline = Pipeline.objects.get(id=pipeline_id)
# Verify organization access
if (
organization_id
and str(pipeline.organization.organization_id) != organization_id
):
logger.warning(
f"Pipeline {pipeline_id} not found in organization {organization_id}"
)
return Response(
{"error": "Pipeline not found in organization"},
status=status.HTTP_404_NOT_FOUND,
)
logger.info(
f"Found Pipeline {pipeline_id}: name='{pipeline.pipeline_name}', type='{pipeline.pipeline_type}'"
)
return Response(
{
"pipeline_id": str(pipeline_id),
"pipeline_name": pipeline.pipeline_name,
"pipeline_type": pipeline.pipeline_type,
"source": "pipeline",
"workflow_id": str(pipeline.workflow_id)
if pipeline.workflow
else None,
}
)
except Pipeline.DoesNotExist:
logger.warning(
f"Pipeline {pipeline_id} not found in Pipeline model either"
)
pass
# Not found in either model
return Response(
{
"error": "Pipeline not found",
"detail": f"Pipeline {pipeline_id} not found in APIDeployment or Pipeline models",
"pipeline_id": str(pipeline_id),
},
status=status.HTTP_404_NOT_FOUND,
)
except Exception as e:
logger.error(f"Error fetching pipeline name for {pipeline_id}: {str(e)}")
return Response(
{
"error": "Failed to fetch pipeline name",
"detail": str(e),
"pipeline_id": str(pipeline_id),
},
status=status.HTTP_500_INTERNAL_SERVER_ERROR,
)