LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
fix: Added workflow as fk in execution table, filtered by users (#1226)
Chandrasekharan M0d3e93a78dda
diff --git a/backend/pipeline_v2/views.py b/backend/pipeline_v2/views.pyindex 1d13e4381..21fa37e53 100644--- a/backend/pipeline_v2/views.py+++ b/backend/pipeline_v2/views.py@@ -39,16 +39,14 @@ class PipelineViewSet(viewsets.ModelViewSet):permission_classes = [IsOwner]serializer_class = PipelineSerializer- def get_queryset(self) -> Optional[QuerySet]:- type = self.request.query_params.get(PipelineConstants.TYPE)- if type is not None:- queryset = Pipeline.objects.filter(- created_by=self.request.user, pipeline_type=type- )- return queryset- elif type is None:- queryset = Pipeline.objects.filter(created_by=self.request.user)- return queryset+ def get_queryset(self) -> QuerySet:+ queryset = Pipeline.objects.filter(created_by=self.request.user)++ # Apply type filter if specified+ pipeline_type = self.request.query_params.get(PipelineConstants.TYPE)+ if pipeline_type is not None:+ queryset = queryset.filter(pipeline_type=pipeline_type)+ return querysetdef get_serializer_class(self) -> serializers.Serializer:if self.action == "execute":diff --git a/backend/workflow_manager/execution/views/execution.py b/backend/workflow_manager/execution/views/execution.pyindex 8cede247d..61f94e8a2 100644--- a/backend/workflow_manager/execution/views/execution.py+++ b/backend/workflow_manager/execution/views/execution.py@@ -3,7 +3,7 @@ import loggingfrom django_filters.rest_framework import DjangoFilterBackendfrom rest_framework import viewsetsfrom rest_framework.filters import OrderingFilter-from rest_framework.permissions import IsAuthenticated+from rest_framework.permissions import BasePermission, IsAuthenticatedfrom utils.pagination import CustomPaginationfrom workflow_manager.execution.filter import ExecutionFilterfrom workflow_manager.execution.serializer import ExecutionSerializer@@ -12,8 +12,28 @@ from workflow_manager.workflow_v2.models import WorkflowExecutionlogger = logging.getLogger(__name__)+class UserWorkflowExecutionPermission(BasePermission):+ """+ Permission class that only allows users to see executions of workflows they created.++ This class controls access at the object level, determining if a user can view+ a specific execution based on workflow ownership.+ """++ def has_permission(self, request, view):+ # Allow access to the view itself+ # object-level permissions will filter individual items+ return True++ def has_object_permission(self, request, view, obj):+ # Check if this specific execution belongs to a workflow created by the user+ return (+ WorkflowExecution.objects.for_user(request.user).filter(id=obj.id).exists()+ )++class ExecutionViewSet(viewsets.ReadOnlyModelViewSet):- permission_classes = [IsAuthenticated]+ permission_classes = [IsAuthenticated, UserWorkflowExecutionPermission]serializer_class = ExecutionSerializerpagination_class = CustomPaginationfilter_backends = [DjangoFilterBackend, OrderingFilter]@@ -21,3 +41,7 @@ class ExecutionViewSet(viewsets.ReadOnlyModelViewSet):ordering = ["-created_at"]filterset_class = ExecutionFilterqueryset = WorkflowExecution.objects.all()++ def get_queryset(self):+ # Use the custom manager method to filter executions for the current user+ return WorkflowExecution.objects.for_user(self.request.user)diff --git a/backend/workflow_manager/workflow_v2/execution.py b/backend/workflow_manager/workflow_v2/execution.pyindex ba5f3e363..dff30fdde 100644--- a/backend/workflow_manager/workflow_v2/execution.py+++ b/backend/workflow_manager/workflow_v2/execution.py@@ -1,5 +1,6 @@import loggingimport time+import uuidfrom typing import Optionalfrom account_v2.constants import Common@@ -140,10 +141,13 @@ class WorkflowExecutionServiceHelper(WorkflowExecutionService):else WorkflowExecution.Type.COMPLETE)execution_log_id = log_events_id if log_events_id else pipeline_id- # TODO: Using objects.create() instead- workflow_execution = WorkflowExecution(++ # Create the workflow execution+ workflow_execution = WorkflowExecution.objects.create(+ id=execution_id if execution_id else uuid.uuid4(),pipeline_id=pipeline_id,workflow_id=workflow_id,+ related_workflow=Workflow.objects.get(id=workflow_id),execution_mode=mode,execution_method=execution_method,execution_type=execution_type,@@ -151,11 +155,9 @@ class WorkflowExecutionServiceHelper(WorkflowExecutionService):execution_log_id=execution_log_id,total_files=total_files,)- if execution_id:- workflow_execution.id = execution_id- workflow_execution.save()- if tags:- workflow_execution.tags.set(tags)++ # Set tags if provided (many-to-many relationships must be set after creation)+ workflow_execution.tags.set(tags or [])return workflow_executiondef update_execution(diff --git a/backend/workflow_manager/workflow_v2/migrations/0012_remove_workflowexecution_workflow_id_and_more.py b/backend/workflow_manager/workflow_v2/migrations/0012_remove_workflowexecution_workflow_id_and_more.pynew file mode 100644index 000000000..abd0459fe--- /dev/null+++ b/backend/workflow_manager/workflow_v2/migrations/0012_remove_workflowexecution_workflow_id_and_more.py@@ -0,0 +1,71 @@+# Generated by Django 4.2.1 on 2025-04-07 09:25++import logging++import django.db.models.deletion+from django.db import connection, migrations, models++logger = logging.getLogger(__name__)+++def remove_execution_of_deleted_wfs(apps, schema_editor):+ """Removes records from workflow_execution which do not have an associated workflow."""+ with connection.cursor() as cursor:+ # Count records with invalid workflow_id+ cursor.execute(+ """+ SELECT COUNT(*) FROM workflow_execution+ WHERE workflow_id IS NOT NULL AND workflow_id::text NOT IN+ (SELECT id::text FROM workflow)+ """+ )+ invalid_count = cursor.fetchone()[0]++ if invalid_count > 0:+ logger.warning(+ f"Deleting '{invalid_count}' workflow executions with "+ "invalid workflow references"+ )++ # Delete records with invalid workflow_id+ cursor.execute(+ """+ DELETE FROM workflow_execution+ WHERE workflow_id IS NOT NULL AND workflow_id::text NOT IN+ (SELECT id::text FROM workflow)+ """+ )+++class Migration(migrations.Migration):++ dependencies = [+ ("workflow_v2", "0011_remove_filehistory_unique_workflow_cachekey_and_more"),+ ]++ operations = [+ # Run custom function to handle workflow_id conversion+ migrations.RunPython(+ remove_execution_of_deleted_wfs, reverse_code=migrations.RunPython.noop+ ),+ # Alter the field to be a foreign key while keeping the same column name+ migrations.AlterField(+ model_name="workflowexecution",+ name="workflow_id",+ field=models.ForeignKey(+ db_column="workflow_id",+ db_comment="Workflow to be executed",+ editable=False,+ null=True,+ on_delete=django.db.models.deletion.CASCADE,+ related_name="workflow_executions",+ to="workflow_v2.workflow",+ ),+ ),+ # Rename the field in Django's ORM (doesn't affect the database column name)+ migrations.RenameField(+ model_name="workflowexecution",+ old_name="workflow_id",+ new_name="workflow",+ ),+ ]diff --git a/backend/workflow_manager/workflow_v2/models/execution.py b/backend/workflow_manager/workflow_v2/models/execution.pyindex 67a099b1e..6cf404352 100644--- a/backend/workflow_manager/workflow_v2/models/execution.py+++ b/backend/workflow_manager/workflow_v2/models/execution.py@@ -6,7 +6,7 @@ from typing import Optionalfrom api_v2.models import APIDeploymentfrom django.core.exceptions import ObjectDoesNotExistfrom django.db import models-from django.db.models import Sum+from django.db.models import QuerySet, Sumfrom pipeline_v2.models import Pipelinefrom tags.models import Tagfrom usage_v2.constants import UsageKeys@@ -22,7 +22,48 @@ logger = logging.getLogger(__name__)EXECUTION_ERROR_LENGTH = 256+class WorkflowExecutionManager(models.Manager):+ """Custom manager for WorkflowExecution model to handle user-specific filtering."""++ def for_user(self, user) -> QuerySet:+ """Filter user's workflow executions.+ Show those belonging to workflows created by the specified user.++ Args:+ user: The user to filter executions for++ Returns:+ QuerySet of executions that the user has permission to access+ """+ # Return executions where the workflow's created_by matches the user+ return self.filter(workflow__created_by=user)++ def clean_invalid_workflows(self):+ """Remove execution records with invalid workflow references.++ This is a utility method to clean up data when converting from workflow_id to+ a proper foreign key relationship. It deletes any execution records where the+ workflow reference doesn't exist in the database.++ Returns:+ int: Number of deleted records+ """+ # Find executions with no valid workflow reference+ invalid_executions = self.filter(workflow__isnull=True)++ count = invalid_executions.count()+ if count > 0:+ logger.info(+ f"Deleting {count} execution records with invalid workflow references"+ )+ invalid_executions.delete()+ return count++class WorkflowExecution(BaseModel):+ # Use the custom manager+ objects = WorkflowExecutionManager()+class Mode(models.TextChoices):INSTANT = "INSTANT", "will be executed immediately"QUEUE = "QUEUE", "will be placed in a queue"@@ -47,10 +88,16 @@ class WorkflowExecution(BaseModel):null=True,db_comment="task id of asynchronous execution",)- # TODO: Make as foreign key to access the instance directly- workflow_id = models.UUIDField(- editable=False, db_comment="Id of workflow to be executed"+ workflow = models.ForeignKey(+ Workflow,+ on_delete=models.CASCADE,+ editable=False,+ db_comment="Workflow to be executed",+ related_name="workflow_executions",+ null=True,+ db_column="workflow_id", # Reuse the existing column name)+execution_mode = models.CharField(choices=Mode.choices, db_comment="Mode of execution")@@ -106,24 +153,9 @@ class WorkflowExecution(BaseModel):@propertydef workflow_name(self) -> Optional[str]:"""Obtains the workflow's name associated to this execution."""- try:- return Workflow.objects.get(id=self.workflow_id).workflow_name- except ObjectDoesNotExist:- logger.warning(- f"Expected workflow ID '{self.workflow_id}' to exist but missing"- )- return None-- @property- def workflow(self) -> Optional[Workflow]:- """Obtains the workflow associated to this execution."""- try:- return Workflow.objects.get(id=self.workflow_id)- except ObjectDoesNotExist:- logger.warning(- f"Expected workflow ID '{self.workflow_id}' to exist but missing"- )- return None+ if self.workflow:+ return self.workflow.workflow_name+ return None@propertydef pipeline_name(self) -> Optional[str]:@@ -195,7 +227,7 @@ class WorkflowExecution(BaseModel):return (f"Workflow execution: {self.id} ("f"pipeline ID: {self.pipeline_id}, "- f"workflow iD: {self.workflow_id}, "+ f"workflow: {self.workflow}, "f"status: {self.status}, "f"files: {self.total_files}, "f"error message: {self.error_message})"