LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
feat: UN-1983 BE Enhancements for file centric logging (#1157)
Chandrasekharan Mde9257db795e
diff --git a/backend/utils/common_utils.py b/backend/utils/common_utils.pyindex 8e2c2973c..40ebaf93e 100644--- a/backend/utils/common_utils.py+++ b/backend/utils/common_utils.py@@ -1,6 +1,9 @@import json+from datetime import datetimefrom enum import Enum+from django.utils import timezone+class CommonUtils:@staticmethod@@ -31,6 +34,31 @@ class CommonUtils:return Falsereturn True+ # TODO: Use from SDK+ @staticmethod+ def pretty_file_size(size_in_bytes: float):+ """Convert bytes to human-readable format (KB, MB, GB, etc.)."""+ units = ["B", "KB", "MB", "GB", "TB"]+ size = float(size_in_bytes)+ for unit in units:+ if size < 1024:+ return f"{size:.2f} {unit}"+ size /= 1024+ return f"{size:.2f} PB"++ @staticmethod+ def time_since(start_time: datetime, precision: int = 3) -> float:+ """Compute the elapsed time given a starting time time in seconds++ Args:+ start_time (datetime): Time in seconds to compute from+ precision (int, optional): Floating point precision. Defaults to 3.++ Returns:+ float: Time elapsed since start_time+ """+ return round((timezone.now() - start_time).total_seconds(), precision)+class ModelEnum(Enum):@classmethoddiff --git a/backend/utils/constants.py b/backend/utils/constants.pyindex 377c21da6..bc5517d7a 100644--- a/backend/utils/constants.py+++ b/backend/utils/constants.py@@ -68,7 +68,5 @@ class ExecutionLogConstants:LOGS_BATCH_LIMIT: int = settings.LOGS_BATCH_LIMITLOG_QUEUE_NAME: str = "log_history_queue"CELERY_QUEUE_NAME = "celery_periodic_logs"- PERIODIC_TASK_NAME = "workflow_log_history"PERIODIC_TASK_NAME_V2 = "workflow_log_history_v2"- TASK = "workflow_manager.workflow.execution_log_utils.consume_log_history"TASK_V2 = "consume_log_history"diff --git a/backend/utils/dto.py b/backend/utils/dto.pyindex 8c06e3f91..674989f36 100644--- a/backend/utils/dto.py+++ b/backend/utils/dto.py@@ -1,8 +1,9 @@import jsonimport logging-from datetime import datetime+from datetime import datetime, timezonefrom typing import Any, Optional+from django.utils import timezone as dj_timezonefrom unstract.workflow_execution.enums import LogTypefrom unstract.core.constants import LogFieldName@@ -35,7 +36,9 @@ class LogDataDTO:self.file_execution_id: Optional[str] = file_execution_idself.organization_id: str = organization_idself.timestamp: int = timestamp- self.event_time: datetime = datetime.fromtimestamp(timestamp)+ self.event_time: datetime = dj_timezone.make_aware(+ datetime.fromtimestamp(timestamp), timezone.utc+ )self.log_type: LogType = log_typeself.data: dict[str, Any] = datadiff --git a/backend/workflow_manager/execution/serializer/execution.py b/backend/workflow_manager/execution/serializer/execution.pyindex 7fa9c0068..56d312979 100644--- a/backend/workflow_manager/execution/serializer/execution.py+++ b/backend/workflow_manager/execution/serializer/execution.py@@ -11,6 +11,7 @@ class ExecutionSerializer(serializers.ModelSerializer):pipeline_name = serializers.SerializerMethodField()successful_files = serializers.SerializerMethodField()failed_files = serializers.SerializerMethodField()+ execution_time = serializers.ReadOnlyField(source="pretty_execution_time")class Meta:model = WorkflowExecutiondiff --git a/backend/workflow_manager/execution/serializer/file_centric.py b/backend/workflow_manager/execution/serializer/file_centric.pyindex 2ce66d018..e3cf446e7 100644--- a/backend/workflow_manager/execution/serializer/file_centric.py+++ b/backend/workflow_manager/execution/serializer/file_centric.py@@ -4,21 +4,37 @@ from rest_framework import serializersfrom workflow_manager.file_execution.models import (WorkflowFileExecution as FileExecution,)+from workflow_manager.workflow_v2.enums import ExecutionStatus++INIT_STATUS_MSG = "Waiting for a worker to pick up file's execution..."++DEFAULT_STATUS_MSG = (+ "No status message available, please check again after a few minutes."+)class FileCentricExecutionSerializer(serializers.ModelSerializer):- latest_log = serializers.SerializerMethodField()+ status_msg = serializers.SerializerMethodField()+ file_size = serializers.ReadOnlyField(source="pretty_file_size")+ execution_time = serializers.ReadOnlyField(source="pretty_execution_time")class Meta:model = FileExecutionexclude = ["file_hash"]- def get_latest_log(self, obj: FileExecution) -> Optional[dict[str, any]]:+ def get_status_msg(self, obj: FileExecution) -> Optional[dict[str, any]]:+ if obj.status in [ExecutionStatus.PENDING, ExecutionStatus.QUEUED]:+ return INIT_STATUS_MSG+ elif obj.status == ExecutionStatus.ERROR:+ return obj.execution_error+latest_log = (obj.execution_logs.exclude(data__level__in=["DEBUG", "WARN"]).order_by("-event_time").first())return (- latest_log.data["log"] if latest_log and "log" in latest_log.data else None+ latest_log.data["log"]+ if latest_log and "log" in latest_log.data+ else DEFAULT_STATUS_MSG)diff --git a/backend/workflow_manager/execution/views/execution.py b/backend/workflow_manager/execution/views/execution.pyindex 9ee97794a..c5fe3ae60 100644--- a/backend/workflow_manager/execution/views/execution.py+++ b/backend/workflow_manager/execution/views/execution.py@@ -30,23 +30,23 @@ class ExecutionViewSet(viewsets.ReadOnlyModelViewSet):queryset = WorkflowExecution.objects.all()# Filter based on execution entity- if execution_entity == ExecutionEntity.API:+ if execution_entity == ExecutionEntity.API.value:queryset = queryset.filter(pipeline_id__in=APIDeployment.objects.values_list("id", flat=True))- elif execution_entity == ExecutionEntity.ETL:+ elif execution_entity == ExecutionEntity.ETL.value:queryset = queryset.filter(pipeline_id__in=Pipeline.objects.filter(pipeline_type=Pipeline.PipelineType.ETL).values_list("id", flat=True))- elif execution_entity == ExecutionEntity.TASK:+ elif execution_entity == ExecutionEntity.TASK.value:queryset = queryset.filter(pipeline_id__in=Pipeline.objects.filter(pipeline_type=Pipeline.PipelineType.TASK).values_list("id", flat=True))- elif execution_entity == ExecutionEntity.WORKFLOW:+ elif execution_entity == ExecutionEntity.WORKFLOW.value:queryset = queryset.filter(pipeline_id=None,workflow_id__in=Workflow.objects.values_list("id", flat=True),diff --git a/backend/workflow_manager/execution/views/file_centric.py b/backend/workflow_manager/execution/views/file_centric.pyindex d961ef069..dc9e029eb 100644--- a/backend/workflow_manager/execution/views/file_centric.py+++ b/backend/workflow_manager/execution/views/file_centric.py@@ -1,5 +1,6 @@import logging+from django_filters.rest_framework import DjangoFilterBackendfrom rest_framework import viewsetsfrom rest_framework.filters import OrderingFilterfrom rest_framework.permissions import IsAuthenticated@@ -16,9 +17,10 @@ class FileCentricExecutionViewSet(viewsets.ReadOnlyModelViewSet):permission_classes = [IsAuthenticated]serializer_class = FileCentricExecutionSerializerpagination_class = CustomPagination- filter_backends = [OrderingFilter]+ filter_backends = [DjangoFilterBackend, OrderingFilter]ordering_fields = ["created_at"]ordering = ["created_at"]+ filterset_fields = ["status"]def get_queryset(self):execution_id = self.kwargs.get("pk")diff --git a/backend/workflow_manager/file_execution/models.py b/backend/workflow_manager/file_execution/models.pyindex 51da76af7..1d052db02 100644--- a/backend/workflow_manager/file_execution/models.py+++ b/backend/workflow_manager/file_execution/models.py@@ -1,8 +1,9 @@import uuid+from datetime import timedeltafrom typing import Optionalfrom django.db import models-from django.utils import timezone+from utils.common_utils import CommonUtilsfrom utils.models.base_model import BaseModelfrom workflow_manager.workflow_v2.enums import ExecutionStatusfrom workflow_manager.workflow_v2.models.execution import WorkflowExecution@@ -128,13 +129,35 @@ class WorkflowFileExecution(BaseModel):]and not self.execution_time):- self.execution_time = round(- (timezone.now() - self.created_at).total_seconds(), 3- )+ self.execution_time = CommonUtils.time_since(self.created_at)self.execution_error = execution_errorself.save()+ @property+ def pretty_file_size(self) -> str:+ """Convert file_size from bytes to human-readable format++ Returns:+ str: File size with a precision of 2 decimals+ """+ return CommonUtils.pretty_file_size(self.file_size)++ @property+ def pretty_execution_time(self) -> str:+ """Convert execution_time from seconds to HH:MM:SS format++ Returns:+ str: Time in HH:MM:SS format+ """+ # Compute execution time for a run that's in progress+ time_in_secs = (+ self.execution_time+ if self.execution_time+ else CommonUtils.time_since(self.created_at)+ )+ return str(timedelta(seconds=time_in_secs)).split(".")[0]+class Meta:verbose_name = "Workflow File Execution"verbose_name_plural = "Workflow File Executions"diff --git a/backend/workflow_manager/workflow_v2/execution.py b/backend/workflow_manager/workflow_v2/execution.pyindex c03ffda81..31f2cb90f 100644--- a/backend/workflow_manager/workflow_v2/execution.py+++ b/backend/workflow_manager/workflow_v2/execution.py@@ -296,6 +296,8 @@ class WorkflowExecutionServiceHelper(WorkflowExecutionService):Returns:None"""+ # To not associate final logs with a file execution+ self.file_execution_id = Noneself.publish_update_log(LogState.END_WORKFLOW, "1", LogComponent.STATUS_BAR)self.publish_update_log(LogState.SUCCESS, "Executed successfully", LogComponent.WORKFLOW@@ -318,12 +320,13 @@ class WorkflowExecutionServiceHelper(WorkflowExecutionService):Returns:None"""+ msg = f"Processing file '{file_name}' ({current_file_idx}/{total_files})"self.publish_update_log(component=LogComponent.STATUS_BAR,state=LogState.MESSAGE,- message=f"Processing file {file_name} {current_file_idx}/{total_files}",+ message=msg,)- self.publish_log(f"Processing file {file_name}")+ self.publish_log(msg)def execute_input_file(self,diff --git a/backend/workflow_manager/workflow_v2/models/execution.py b/backend/workflow_manager/workflow_v2/models/execution.pyindex cacea1e49..12503024e 100644--- a/backend/workflow_manager/workflow_v2/models/execution.py+++ b/backend/workflow_manager/workflow_v2/models/execution.py@@ -1,5 +1,6 @@import loggingimport uuid+from datetime import timedeltafrom typing import Optionalfrom api_v2.models import APIDeployment@@ -7,6 +8,7 @@ from django.core.exceptions import ObjectDoesNotExistfrom django.db import modelsfrom pipeline_v2.models import Pipelinefrom tags.models import Tag+from utils.common_utils import CommonUtilsfrom utils.models.base_model import BaseModelfrom workflow_manager.workflow_v2.enums import ExecutionStatusfrom workflow_manager.workflow_v2.models import Workflow@@ -129,6 +131,21 @@ class WorkflowExecution(BaseModel):return None+ @property+ def pretty_execution_time(self) -> str:+ """Convert execution_time from seconds to HH:MM:SS format++ Returns:+ str: Time in HH:MM:SS format+ """+ # Compute execution time for a run that's in progress+ time_in_secs = (+ self.execution_time+ if self.execution_time+ else CommonUtils.time_since(self.created_at)+ )+ return str(timedelta(seconds=time_in_secs)).split(".")[0]+def __str__(self) -> str:return (f"Workflow execution: {self.id} ("diff --git a/unstract/workflow-execution/src/unstract/workflow_execution/workflow_execution.py b/unstract/workflow-execution/src/unstract/workflow_execution/workflow_execution.pyindex a11971c65..fec86c2b2 100644--- a/unstract/workflow-execution/src/unstract/workflow_execution/workflow_execution.py+++ b/unstract/workflow-execution/src/unstract/workflow_execution/workflow_execution.py@@ -182,8 +182,6 @@ class WorkflowExecutionService:Args:step (int): workflow stepsandbox (ToolSandbox): instance of tool sandbox- execution_type (ExecutionType): step or complete- last_step_output (list[Any]): output of previous stepRaises:error: _description_