LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
refactor: Moved file centric view and serializer to file execution app (#1171)
Chandrasekharan M1ad144e09e6c
diff --git a/backend/backend/urls_v2.py b/backend/backend/urls_v2.pyindex 84db146a7..cc5c25920 100644--- a/backend/backend/urls_v2.py+++ b/backend/backend/urls_v2.py@@ -61,4 +61,5 @@ urlpatterns = [),path("tags/", include("tags.urls")),path("execution/", include("workflow_manager.execution.urls")),+ path("execution/", include("workflow_manager.file_execution.urls")),]diff --git a/backend/workflow_manager/execution/serializer/__init__.py b/backend/workflow_manager/execution/serializer/__init__.pyindex 9b06d6d5b..39cf5cf71 100644--- a/backend/workflow_manager/execution/serializer/__init__.py+++ b/backend/workflow_manager/execution/serializer/__init__.py@@ -1,2 +1 @@from .execution import ExecutionSerializer # noqa: F401-from .file_centric import FileCentricExecutionSerializer # noqa: F401diff --git a/backend/workflow_manager/execution/serializer/file_centric.py b/backend/workflow_manager/execution/serializer/file_centric.pydeleted file mode 100644index e3cf446e7..000000000--- a/backend/workflow_manager/execution/serializer/file_centric.py+++ /dev/null@@ -1,40 +0,0 @@-from typing import Optional--from rest_framework import serializers-from workflow_manager.file_execution.models import (- WorkflowFileExecution as FileExecution,-)-from workflow_manager.workflow_v2.enums import ExecutionStatus--INIT_STATUS_MSG = "Waiting for a worker to pick up file's execution..."--DEFAULT_STATUS_MSG = (- "No status message available, please check again after a few minutes."-)---class FileCentricExecutionSerializer(serializers.ModelSerializer):- status_msg = serializers.SerializerMethodField()- file_size = serializers.ReadOnlyField(source="pretty_file_size")- execution_time = serializers.ReadOnlyField(source="pretty_execution_time")-- class Meta:- model = FileExecution- exclude = ["file_hash"]-- def get_status_msg(self, obj: FileExecution) -> Optional[dict[str, any]]:- if obj.status in [ExecutionStatus.PENDING, ExecutionStatus.QUEUED]:- return INIT_STATUS_MSG- elif obj.status == ExecutionStatus.ERROR:- return obj.execution_error-- latest_log = (- obj.execution_logs.exclude(data__level__in=["DEBUG", "WARN"])- .order_by("-event_time")- .first()- )- return (- latest_log.data["log"]- if latest_log and "log" in latest_log.data- else DEFAULT_STATUS_MSG- )diff --git a/backend/workflow_manager/execution/urls.py b/backend/workflow_manager/execution/urls.pyindex 0872aad04..d75795cdd 100644--- a/backend/workflow_manager/execution/urls.py+++ b/backend/workflow_manager/execution/urls.py@@ -1,9 +1,6 @@from django.urls import pathfrom rest_framework.urlpatterns import format_suffix_patterns-from workflow_manager.execution.views import (- ExecutionViewSet,- FileCentricExecutionViewSet,-)+from workflow_manager.execution.views import ExecutionViewSetfrom workflow_manager.workflow_v2.execution_log_view import (WorkflowExecutionLogViewSet as ExecutionLogViewSet,)@@ -14,14 +11,12 @@ execution_list = ExecutionViewSet.as_view(})execution_detail = ExecutionViewSet.as_view({"get": "retrieve"})-file_centric_list = FileCentricExecutionViewSet.as_view({"get": "list"})execution_log_list = ExecutionLogViewSet.as_view({"get": "list"})urlpatterns = format_suffix_patterns([path("", execution_list, name="execution-list"),path("<uuid:pk>/", execution_detail, name="execution-detail"),- path("<uuid:pk>/files/", file_centric_list, name="file-centric-execution-list"),path("<uuid:pk>/logs/", execution_log_list, name="execution-log"),])diff --git a/backend/workflow_manager/execution/views/__init__.py b/backend/workflow_manager/execution/views/__init__.pyindex 7b694c9bf..90a34404a 100644--- a/backend/workflow_manager/execution/views/__init__.py+++ b/backend/workflow_manager/execution/views/__init__.py@@ -1,2 +1 @@from .execution import ExecutionViewSet # noqa: F401-from .file_centric import FileCentricExecutionViewSet # noqa: F401diff --git a/backend/workflow_manager/execution/views/file_centric.py b/backend/workflow_manager/execution/views/file_centric.pydeleted file mode 100644index dc9e029eb..000000000--- a/backend/workflow_manager/execution/views/file_centric.py+++ /dev/null@@ -1,27 +0,0 @@-import logging--from django_filters.rest_framework import DjangoFilterBackend-from rest_framework import viewsets-from rest_framework.filters import OrderingFilter-from rest_framework.permissions import IsAuthenticated-from utils.pagination import CustomPagination-from workflow_manager.execution.serializer import FileCentricExecutionSerializer-from workflow_manager.file_execution.models import (- WorkflowFileExecution as FileExecution,-)--logger = logging.getLogger(__name__)---class FileCentricExecutionViewSet(viewsets.ReadOnlyModelViewSet):- permission_classes = [IsAuthenticated]- serializer_class = FileCentricExecutionSerializer- pagination_class = CustomPagination- filter_backends = [DjangoFilterBackend, OrderingFilter]- ordering_fields = ["created_at"]- ordering = ["created_at"]- filterset_fields = ["status"]-- def get_queryset(self):- execution_id = self.kwargs.get("pk")- return FileExecution.objects.filter(workflow_execution_id=execution_id)diff --git a/backend/workflow_manager/file_execution/serializers.py b/backend/workflow_manager/file_execution/serializers.pyindex 9232d7f25..68b73725f 100644--- a/backend/workflow_manager/file_execution/serializers.py+++ b/backend/workflow_manager/file_execution/serializers.py@@ -1,4 +1,10 @@+from typing import Optional+from rest_framework import serializers+from workflow_manager.file_execution.models import (+ WorkflowFileExecution as FileExecution,+)+from workflow_manager.workflow_v2.enums import ExecutionStatusfrom .models import WorkflowFileExecution@@ -7,3 +13,36 @@ class WorkflowFileExecutionSerializer(serializers.ModelSerializer):class Meta:model = WorkflowFileExecutionfields = "__all__"+++class FileCentricExecutionSerializer(serializers.ModelSerializer):+ INIT_STATUS_MSG = "Waiting for a worker to pick up file's execution..."++ DEFAULT_STATUS_MSG = (+ "No status message available, please check again after a few minutes."+ )++ status_msg = serializers.SerializerMethodField()+ file_size = serializers.ReadOnlyField(source="pretty_file_size")+ execution_time = serializers.ReadOnlyField(source="pretty_execution_time")++ class Meta:+ model = FileExecution+ exclude = ["file_hash"]++ def get_status_msg(self, obj: FileExecution) -> Optional[dict[str, any]]:+ if obj.status in [ExecutionStatus.PENDING, ExecutionStatus.QUEUED]:+ return self.INIT_STATUS_MSG+ elif obj.status == ExecutionStatus.ERROR:+ return obj.execution_error++ latest_log = (+ obj.execution_logs.exclude(data__level__in=["DEBUG", "WARN"])+ .order_by("-event_time")+ .first()+ )+ return (+ latest_log.data["log"]+ if latest_log and "log" in latest_log.data+ else self.DEFAULT_STATUS_MSG+ )diff --git a/backend/workflow_manager/file_execution/urls.py b/backend/workflow_manager/file_execution/urls.pynew file mode 100644index 000000000..bdf4b5cf4--- /dev/null+++ b/backend/workflow_manager/file_execution/urls.py@@ -0,0 +1,11 @@+from django.urls import path+from rest_framework.urlpatterns import format_suffix_patterns+from workflow_manager.file_execution.views import FileCentricExecutionViewSet++file_centric_list = FileCentricExecutionViewSet.as_view({"get": "list"})++urlpatterns = format_suffix_patterns(+ [+ path("<uuid:pk>/files/", file_centric_list, name="file-centric-execution-list"),+ ]+)diff --git a/backend/workflow_manager/file_execution/views.py b/backend/workflow_manager/file_execution/views.pyindex 60f00ef0e..cf304a5b1 100644--- a/backend/workflow_manager/file_execution/views.py+++ b/backend/workflow_manager/file_execution/views.py@@ -1 +1,23 @@-# Create your views here.+from django_filters.rest_framework import DjangoFilterBackend+from rest_framework import viewsets+from rest_framework.filters import OrderingFilter+from rest_framework.permissions import IsAuthenticated+from utils.pagination import CustomPagination+from workflow_manager.file_execution.models import (+ WorkflowFileExecution as FileExecution,+)+from workflow_manager.file_execution.serializers import FileCentricExecutionSerializer+++class FileCentricExecutionViewSet(viewsets.ReadOnlyModelViewSet):+ permission_classes = [IsAuthenticated]+ serializer_class = FileCentricExecutionSerializer+ pagination_class = CustomPagination+ filter_backends = [DjangoFilterBackend, OrderingFilter]+ ordering_fields = ["created_at"]+ ordering = ["created_at"]+ filterset_fields = ["status"]++ def get_queryset(self):+ execution_id = self.kwargs.get("pk")+ return FileExecution.objects.filter(workflow_execution_id=execution_id)