Skip to content

LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows

fix: Added workflow as fk in execution table, filtered by users (#1226)

Chandrasekharan M0d3e93a78dda
diff --git a/backend/pipeline_v2/views.py b/backend/pipeline_v2/views.py
index 1d13e4381..21fa37e53 100644
--- a/backend/pipeline_v2/views.py
+++ b/backend/pipeline_v2/views.py
@@ -39,16 +39,14 @@ class PipelineViewSet(viewsets.ModelViewSet):
permission_classes = [IsOwner]
serializer_class = PipelineSerializer
- def get_queryset(self) -> Optional[QuerySet]:
- type = self.request.query_params.get(PipelineConstants.TYPE)
- if type is not None:
- queryset = Pipeline.objects.filter(
- created_by=self.request.user, pipeline_type=type
- )
- return queryset
- elif type is None:
- queryset = Pipeline.objects.filter(created_by=self.request.user)
- return queryset
+ def get_queryset(self) -> QuerySet:
+ queryset = Pipeline.objects.filter(created_by=self.request.user)
+
+ # Apply type filter if specified
+ pipeline_type = self.request.query_params.get(PipelineConstants.TYPE)
+ if pipeline_type is not None:
+ queryset = queryset.filter(pipeline_type=pipeline_type)
+ return queryset
def get_serializer_class(self) -> serializers.Serializer:
if self.action == "execute":
diff --git a/backend/workflow_manager/execution/views/execution.py b/backend/workflow_manager/execution/views/execution.py
index 8cede247d..61f94e8a2 100644
--- a/backend/workflow_manager/execution/views/execution.py
+++ b/backend/workflow_manager/execution/views/execution.py
@@ -3,7 +3,7 @@ import logging
from django_filters.rest_framework import DjangoFilterBackend
from rest_framework import viewsets
from rest_framework.filters import OrderingFilter
-from rest_framework.permissions import IsAuthenticated
+from rest_framework.permissions import BasePermission, IsAuthenticated
from utils.pagination import CustomPagination
from workflow_manager.execution.filter import ExecutionFilter
from workflow_manager.execution.serializer import ExecutionSerializer
@@ -12,8 +12,28 @@ from workflow_manager.workflow_v2.models import WorkflowExecution
logger = logging.getLogger(__name__)
+class UserWorkflowExecutionPermission(BasePermission):
+ """
+ Permission class that only allows users to see executions of workflows they created.
+
+ This class controls access at the object level, determining if a user can view
+ a specific execution based on workflow ownership.
+ """
+
+ def has_permission(self, request, view):
+ # Allow access to the view itself
+ # object-level permissions will filter individual items
+ return True
+
+ def has_object_permission(self, request, view, obj):
+ # Check if this specific execution belongs to a workflow created by the user
+ return (
+ WorkflowExecution.objects.for_user(request.user).filter(id=obj.id).exists()
+ )
+
+
class ExecutionViewSet(viewsets.ReadOnlyModelViewSet):
- permission_classes = [IsAuthenticated]
+ permission_classes = [IsAuthenticated, UserWorkflowExecutionPermission]
serializer_class = ExecutionSerializer
pagination_class = CustomPagination
filter_backends = [DjangoFilterBackend, OrderingFilter]
@@ -21,3 +41,7 @@ class ExecutionViewSet(viewsets.ReadOnlyModelViewSet):
ordering = ["-created_at"]
filterset_class = ExecutionFilter
queryset = WorkflowExecution.objects.all()
+
+ def get_queryset(self):
+ # Use the custom manager method to filter executions for the current user
+ return WorkflowExecution.objects.for_user(self.request.user)
diff --git a/backend/workflow_manager/workflow_v2/execution.py b/backend/workflow_manager/workflow_v2/execution.py
index ba5f3e363..dff30fdde 100644
--- a/backend/workflow_manager/workflow_v2/execution.py
+++ b/backend/workflow_manager/workflow_v2/execution.py
@@ -1,5 +1,6 @@
import logging
import time
+import uuid
from typing import Optional
from account_v2.constants import Common
@@ -140,10 +141,13 @@ class WorkflowExecutionServiceHelper(WorkflowExecutionService):
else WorkflowExecution.Type.COMPLETE
)
execution_log_id = log_events_id if log_events_id else pipeline_id
- # TODO: Using objects.create() instead
- workflow_execution = WorkflowExecution(
+
+ # Create the workflow execution
+ workflow_execution = WorkflowExecution.objects.create(
+ id=execution_id if execution_id else uuid.uuid4(),
pipeline_id=pipeline_id,
workflow_id=workflow_id,
+ related_workflow=Workflow.objects.get(id=workflow_id),
execution_mode=mode,
execution_method=execution_method,
execution_type=execution_type,
@@ -151,11 +155,9 @@ class WorkflowExecutionServiceHelper(WorkflowExecutionService):
execution_log_id=execution_log_id,
total_files=total_files,
)
- if execution_id:
- workflow_execution.id = execution_id
- workflow_execution.save()
- if tags:
- workflow_execution.tags.set(tags)
+
+ # Set tags if provided (many-to-many relationships must be set after creation)
+ workflow_execution.tags.set(tags or [])
return workflow_execution
def update_execution(
diff --git a/backend/workflow_manager/workflow_v2/migrations/0012_remove_workflowexecution_workflow_id_and_more.py b/backend/workflow_manager/workflow_v2/migrations/0012_remove_workflowexecution_workflow_id_and_more.py
new file mode 100644
index 000000000..abd0459fe
--- /dev/null
+++ b/backend/workflow_manager/workflow_v2/migrations/0012_remove_workflowexecution_workflow_id_and_more.py
@@ -0,0 +1,71 @@
+# Generated by Django 4.2.1 on 2025-04-07 09:25
+
+import logging
+
+import django.db.models.deletion
+from django.db import connection, migrations, models
+
+logger = logging.getLogger(__name__)
+
+
+def remove_execution_of_deleted_wfs(apps, schema_editor):
+ """Removes records from workflow_execution which do not have an associated workflow."""
+ with connection.cursor() as cursor:
+ # Count records with invalid workflow_id
+ cursor.execute(
+ """
+ SELECT COUNT(*) FROM workflow_execution
+ WHERE workflow_id IS NOT NULL AND workflow_id::text NOT IN
+ (SELECT id::text FROM workflow)
+ """
+ )
+ invalid_count = cursor.fetchone()[0]
+
+ if invalid_count > 0:
+ logger.warning(
+ f"Deleting '{invalid_count}' workflow executions with "
+ "invalid workflow references"
+ )
+
+ # Delete records with invalid workflow_id
+ cursor.execute(
+ """
+ DELETE FROM workflow_execution
+ WHERE workflow_id IS NOT NULL AND workflow_id::text NOT IN
+ (SELECT id::text FROM workflow)
+ """
+ )
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ("workflow_v2", "0011_remove_filehistory_unique_workflow_cachekey_and_more"),
+ ]
+
+ operations = [
+ # Run custom function to handle workflow_id conversion
+ migrations.RunPython(
+ remove_execution_of_deleted_wfs, reverse_code=migrations.RunPython.noop
+ ),
+ # Alter the field to be a foreign key while keeping the same column name
+ migrations.AlterField(
+ model_name="workflowexecution",
+ name="workflow_id",
+ field=models.ForeignKey(
+ db_column="workflow_id",
+ db_comment="Workflow to be executed",
+ editable=False,
+ null=True,
+ on_delete=django.db.models.deletion.CASCADE,
+ related_name="workflow_executions",
+ to="workflow_v2.workflow",
+ ),
+ ),
+ # Rename the field in Django's ORM (doesn't affect the database column name)
+ migrations.RenameField(
+ model_name="workflowexecution",
+ old_name="workflow_id",
+ new_name="workflow",
+ ),
+ ]
diff --git a/backend/workflow_manager/workflow_v2/models/execution.py b/backend/workflow_manager/workflow_v2/models/execution.py
index 67a099b1e..6cf404352 100644
--- a/backend/workflow_manager/workflow_v2/models/execution.py
+++ b/backend/workflow_manager/workflow_v2/models/execution.py
@@ -6,7 +6,7 @@ from typing import Optional
from api_v2.models import APIDeployment
from django.core.exceptions import ObjectDoesNotExist
from django.db import models
-from django.db.models import Sum
+from django.db.models import QuerySet, Sum
from pipeline_v2.models import Pipeline
from tags.models import Tag
from usage_v2.constants import UsageKeys
@@ -22,7 +22,48 @@ logger = logging.getLogger(__name__)
EXECUTION_ERROR_LENGTH = 256
+class WorkflowExecutionManager(models.Manager):
+ """Custom manager for WorkflowExecution model to handle user-specific filtering."""
+
+ def for_user(self, user) -> QuerySet:
+ """Filter user's workflow executions.
+ Show those belonging to workflows created by the specified user.
+
+ Args:
+ user: The user to filter executions for
+
+ Returns:
+ QuerySet of executions that the user has permission to access
+ """
+ # Return executions where the workflow's created_by matches the user
+ return self.filter(workflow__created_by=user)
+
+ def clean_invalid_workflows(self):
+ """Remove execution records with invalid workflow references.
+
+ This is a utility method to clean up data when converting from workflow_id to
+ a proper foreign key relationship. It deletes any execution records where the
+ workflow reference doesn't exist in the database.
+
+ Returns:
+ int: Number of deleted records
+ """
+ # Find executions with no valid workflow reference
+ invalid_executions = self.filter(workflow__isnull=True)
+
+ count = invalid_executions.count()
+ if count > 0:
+ logger.info(
+ f"Deleting {count} execution records with invalid workflow references"
+ )
+ invalid_executions.delete()
+ return count
+
+
class WorkflowExecution(BaseModel):
+ # Use the custom manager
+ objects = WorkflowExecutionManager()
+
class Mode(models.TextChoices):
INSTANT = "INSTANT", "will be executed immediately"
QUEUE = "QUEUE", "will be placed in a queue"
@@ -47,10 +88,16 @@ class WorkflowExecution(BaseModel):
null=True,
db_comment="task id of asynchronous execution",
)
- # TODO: Make as foreign key to access the instance directly
- workflow_id = models.UUIDField(
- editable=False, db_comment="Id of workflow to be executed"
+ workflow = models.ForeignKey(
+ Workflow,
+ on_delete=models.CASCADE,
+ editable=False,
+ db_comment="Workflow to be executed",
+ related_name="workflow_executions",
+ null=True,
+ db_column="workflow_id", # Reuse the existing column name
)
+
execution_mode = models.CharField(
choices=Mode.choices, db_comment="Mode of execution"
)
@@ -106,24 +153,9 @@ class WorkflowExecution(BaseModel):
@property
def workflow_name(self) -> Optional[str]:
"""Obtains the workflow's name associated to this execution."""
- try:
- return Workflow.objects.get(id=self.workflow_id).workflow_name
- except ObjectDoesNotExist:
- logger.warning(
- f"Expected workflow ID '{self.workflow_id}' to exist but missing"
- )
- return None
-
- @property
- def workflow(self) -> Optional[Workflow]:
- """Obtains the workflow associated to this execution."""
- try:
- return Workflow.objects.get(id=self.workflow_id)
- except ObjectDoesNotExist:
- logger.warning(
- f"Expected workflow ID '{self.workflow_id}' to exist but missing"
- )
- return None
+ if self.workflow:
+ return self.workflow.workflow_name
+ return None
@property
def pipeline_name(self) -> Optional[str]:
@@ -195,7 +227,7 @@ class WorkflowExecution(BaseModel):
return (
f"Workflow execution: {self.id} ("
f"pipeline ID: {self.pipeline_id}, "
- f"workflow iD: {self.workflow_id}, "
+ f"workflow: {self.workflow}, "
f"status: {self.status}, "
f"files: {self.total_files}, "
f"error message: {self.error_message})"