Skip to content

LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows

feat: UN-1983 BE Enhancements for file centric logging (#1157)

Chandrasekharan Mde9257db795e
diff --git a/backend/utils/common_utils.py b/backend/utils/common_utils.py
index 8e2c2973c..40ebaf93e 100644
--- a/backend/utils/common_utils.py
+++ b/backend/utils/common_utils.py
@@ -1,6 +1,9 @@
import json
+from datetime import datetime
from enum import Enum
+from django.utils import timezone
+
class CommonUtils:
@staticmethod
@@ -31,6 +34,31 @@ class CommonUtils:
return False
return True
+ # TODO: Use from SDK
+ @staticmethod
+ def pretty_file_size(size_in_bytes: float):
+ """Convert bytes to human-readable format (KB, MB, GB, etc.)."""
+ units = ["B", "KB", "MB", "GB", "TB"]
+ size = float(size_in_bytes)
+ for unit in units:
+ if size < 1024:
+ return f"{size:.2f} {unit}"
+ size /= 1024
+ return f"{size:.2f} PB"
+
+ @staticmethod
+ def time_since(start_time: datetime, precision: int = 3) -> float:
+ """Compute the elapsed time given a starting time time in seconds
+
+ Args:
+ start_time (datetime): Time in seconds to compute from
+ precision (int, optional): Floating point precision. Defaults to 3.
+
+ Returns:
+ float: Time elapsed since start_time
+ """
+ return round((timezone.now() - start_time).total_seconds(), precision)
+
class ModelEnum(Enum):
@classmethod
diff --git a/backend/utils/constants.py b/backend/utils/constants.py
index 377c21da6..bc5517d7a 100644
--- a/backend/utils/constants.py
+++ b/backend/utils/constants.py
@@ -68,7 +68,5 @@ class ExecutionLogConstants:
LOGS_BATCH_LIMIT: int = settings.LOGS_BATCH_LIMIT
LOG_QUEUE_NAME: str = "log_history_queue"
CELERY_QUEUE_NAME = "celery_periodic_logs"
- PERIODIC_TASK_NAME = "workflow_log_history"
PERIODIC_TASK_NAME_V2 = "workflow_log_history_v2"
- TASK = "workflow_manager.workflow.execution_log_utils.consume_log_history"
TASK_V2 = "consume_log_history"
diff --git a/backend/utils/dto.py b/backend/utils/dto.py
index 8c06e3f91..674989f36 100644
--- a/backend/utils/dto.py
+++ b/backend/utils/dto.py
@@ -1,8 +1,9 @@
import json
import logging
-from datetime import datetime
+from datetime import datetime, timezone
from typing import Any, Optional
+from django.utils import timezone as dj_timezone
from unstract.workflow_execution.enums import LogType
from unstract.core.constants import LogFieldName
@@ -35,7 +36,9 @@ class LogDataDTO:
self.file_execution_id: Optional[str] = file_execution_id
self.organization_id: str = organization_id
self.timestamp: int = timestamp
- self.event_time: datetime = datetime.fromtimestamp(timestamp)
+ self.event_time: datetime = dj_timezone.make_aware(
+ datetime.fromtimestamp(timestamp), timezone.utc
+ )
self.log_type: LogType = log_type
self.data: dict[str, Any] = data
diff --git a/backend/workflow_manager/execution/serializer/execution.py b/backend/workflow_manager/execution/serializer/execution.py
index 7fa9c0068..56d312979 100644
--- a/backend/workflow_manager/execution/serializer/execution.py
+++ b/backend/workflow_manager/execution/serializer/execution.py
@@ -11,6 +11,7 @@ class ExecutionSerializer(serializers.ModelSerializer):
pipeline_name = serializers.SerializerMethodField()
successful_files = serializers.SerializerMethodField()
failed_files = serializers.SerializerMethodField()
+ execution_time = serializers.ReadOnlyField(source="pretty_execution_time")
class Meta:
model = WorkflowExecution
diff --git a/backend/workflow_manager/execution/serializer/file_centric.py b/backend/workflow_manager/execution/serializer/file_centric.py
index 2ce66d018..e3cf446e7 100644
--- a/backend/workflow_manager/execution/serializer/file_centric.py
+++ b/backend/workflow_manager/execution/serializer/file_centric.py
@@ -4,21 +4,37 @@ from rest_framework import serializers
from workflow_manager.file_execution.models import (
WorkflowFileExecution as FileExecution,
)
+from workflow_manager.workflow_v2.enums import ExecutionStatus
+
+INIT_STATUS_MSG = "Waiting for a worker to pick up file's execution..."
+
+DEFAULT_STATUS_MSG = (
+ "No status message available, please check again after a few minutes."
+)
class FileCentricExecutionSerializer(serializers.ModelSerializer):
- latest_log = serializers.SerializerMethodField()
+ status_msg = serializers.SerializerMethodField()
+ file_size = serializers.ReadOnlyField(source="pretty_file_size")
+ execution_time = serializers.ReadOnlyField(source="pretty_execution_time")
class Meta:
model = FileExecution
exclude = ["file_hash"]
- def get_latest_log(self, obj: FileExecution) -> Optional[dict[str, any]]:
+ def get_status_msg(self, obj: FileExecution) -> Optional[dict[str, any]]:
+ if obj.status in [ExecutionStatus.PENDING, ExecutionStatus.QUEUED]:
+ return INIT_STATUS_MSG
+ elif obj.status == ExecutionStatus.ERROR:
+ return obj.execution_error
+
latest_log = (
obj.execution_logs.exclude(data__level__in=["DEBUG", "WARN"])
.order_by("-event_time")
.first()
)
return (
- latest_log.data["log"] if latest_log and "log" in latest_log.data else None
+ latest_log.data["log"]
+ if latest_log and "log" in latest_log.data
+ else DEFAULT_STATUS_MSG
)
diff --git a/backend/workflow_manager/execution/views/execution.py b/backend/workflow_manager/execution/views/execution.py
index 9ee97794a..c5fe3ae60 100644
--- a/backend/workflow_manager/execution/views/execution.py
+++ b/backend/workflow_manager/execution/views/execution.py
@@ -30,23 +30,23 @@ class ExecutionViewSet(viewsets.ReadOnlyModelViewSet):
queryset = WorkflowExecution.objects.all()
# Filter based on execution entity
- if execution_entity == ExecutionEntity.API:
+ if execution_entity == ExecutionEntity.API.value:
queryset = queryset.filter(
pipeline_id__in=APIDeployment.objects.values_list("id", flat=True)
)
- elif execution_entity == ExecutionEntity.ETL:
+ elif execution_entity == ExecutionEntity.ETL.value:
queryset = queryset.filter(
pipeline_id__in=Pipeline.objects.filter(
pipeline_type=Pipeline.PipelineType.ETL
).values_list("id", flat=True)
)
- elif execution_entity == ExecutionEntity.TASK:
+ elif execution_entity == ExecutionEntity.TASK.value:
queryset = queryset.filter(
pipeline_id__in=Pipeline.objects.filter(
pipeline_type=Pipeline.PipelineType.TASK
).values_list("id", flat=True)
)
- elif execution_entity == ExecutionEntity.WORKFLOW:
+ elif execution_entity == ExecutionEntity.WORKFLOW.value:
queryset = queryset.filter(
pipeline_id=None,
workflow_id__in=Workflow.objects.values_list("id", flat=True),
diff --git a/backend/workflow_manager/execution/views/file_centric.py b/backend/workflow_manager/execution/views/file_centric.py
index d961ef069..dc9e029eb 100644
--- a/backend/workflow_manager/execution/views/file_centric.py
+++ b/backend/workflow_manager/execution/views/file_centric.py
@@ -1,5 +1,6 @@
import logging
+from django_filters.rest_framework import DjangoFilterBackend
from rest_framework import viewsets
from rest_framework.filters import OrderingFilter
from rest_framework.permissions import IsAuthenticated
@@ -16,9 +17,10 @@ class FileCentricExecutionViewSet(viewsets.ReadOnlyModelViewSet):
permission_classes = [IsAuthenticated]
serializer_class = FileCentricExecutionSerializer
pagination_class = CustomPagination
- filter_backends = [OrderingFilter]
+ filter_backends = [DjangoFilterBackend, OrderingFilter]
ordering_fields = ["created_at"]
ordering = ["created_at"]
+ filterset_fields = ["status"]
def get_queryset(self):
execution_id = self.kwargs.get("pk")
diff --git a/backend/workflow_manager/file_execution/models.py b/backend/workflow_manager/file_execution/models.py
index 51da76af7..1d052db02 100644
--- a/backend/workflow_manager/file_execution/models.py
+++ b/backend/workflow_manager/file_execution/models.py
@@ -1,8 +1,9 @@
import uuid
+from datetime import timedelta
from typing import Optional
from django.db import models
-from django.utils import timezone
+from utils.common_utils import CommonUtils
from utils.models.base_model import BaseModel
from workflow_manager.workflow_v2.enums import ExecutionStatus
from workflow_manager.workflow_v2.models.execution import WorkflowExecution
@@ -128,13 +129,35 @@ class WorkflowFileExecution(BaseModel):
]
and not self.execution_time
):
- self.execution_time = round(
- (timezone.now() - self.created_at).total_seconds(), 3
- )
+ self.execution_time = CommonUtils.time_since(self.created_at)
self.execution_error = execution_error
self.save()
+ @property
+ def pretty_file_size(self) -> str:
+ """Convert file_size from bytes to human-readable format
+
+ Returns:
+ str: File size with a precision of 2 decimals
+ """
+ return CommonUtils.pretty_file_size(self.file_size)
+
+ @property
+ def pretty_execution_time(self) -> str:
+ """Convert execution_time from seconds to HH:MM:SS format
+
+ Returns:
+ str: Time in HH:MM:SS format
+ """
+ # Compute execution time for a run that's in progress
+ time_in_secs = (
+ self.execution_time
+ if self.execution_time
+ else CommonUtils.time_since(self.created_at)
+ )
+ return str(timedelta(seconds=time_in_secs)).split(".")[0]
+
class Meta:
verbose_name = "Workflow File Execution"
verbose_name_plural = "Workflow File Executions"
diff --git a/backend/workflow_manager/workflow_v2/execution.py b/backend/workflow_manager/workflow_v2/execution.py
index c03ffda81..31f2cb90f 100644
--- a/backend/workflow_manager/workflow_v2/execution.py
+++ b/backend/workflow_manager/workflow_v2/execution.py
@@ -296,6 +296,8 @@ class WorkflowExecutionServiceHelper(WorkflowExecutionService):
Returns:
None
"""
+ # To not associate final logs with a file execution
+ self.file_execution_id = None
self.publish_update_log(LogState.END_WORKFLOW, "1", LogComponent.STATUS_BAR)
self.publish_update_log(
LogState.SUCCESS, "Executed successfully", LogComponent.WORKFLOW
@@ -318,12 +320,13 @@ class WorkflowExecutionServiceHelper(WorkflowExecutionService):
Returns:
None
"""
+ msg = f"Processing file '{file_name}' ({current_file_idx}/{total_files})"
self.publish_update_log(
component=LogComponent.STATUS_BAR,
state=LogState.MESSAGE,
- message=f"Processing file {file_name} {current_file_idx}/{total_files}",
+ message=msg,
)
- self.publish_log(f"Processing file {file_name}")
+ self.publish_log(msg)
def execute_input_file(
self,
diff --git a/backend/workflow_manager/workflow_v2/models/execution.py b/backend/workflow_manager/workflow_v2/models/execution.py
index cacea1e49..12503024e 100644
--- a/backend/workflow_manager/workflow_v2/models/execution.py
+++ b/backend/workflow_manager/workflow_v2/models/execution.py
@@ -1,5 +1,6 @@
import logging
import uuid
+from datetime import timedelta
from typing import Optional
from api_v2.models import APIDeployment
@@ -7,6 +8,7 @@ from django.core.exceptions import ObjectDoesNotExist
from django.db import models
from pipeline_v2.models import Pipeline
from tags.models import Tag
+from utils.common_utils import CommonUtils
from utils.models.base_model import BaseModel
from workflow_manager.workflow_v2.enums import ExecutionStatus
from workflow_manager.workflow_v2.models import Workflow
@@ -129,6 +131,21 @@ class WorkflowExecution(BaseModel):
return None
+ @property
+ def pretty_execution_time(self) -> str:
+ """Convert execution_time from seconds to HH:MM:SS format
+
+ Returns:
+ str: Time in HH:MM:SS format
+ """
+ # Compute execution time for a run that's in progress
+ time_in_secs = (
+ self.execution_time
+ if self.execution_time
+ else CommonUtils.time_since(self.created_at)
+ )
+ return str(timedelta(seconds=time_in_secs)).split(".")[0]
+
def __str__(self) -> str:
return (
f"Workflow execution: {self.id} ("
diff --git a/unstract/workflow-execution/src/unstract/workflow_execution/workflow_execution.py b/unstract/workflow-execution/src/unstract/workflow_execution/workflow_execution.py
index a11971c65..fec86c2b2 100644
--- a/unstract/workflow-execution/src/unstract/workflow_execution/workflow_execution.py
+++ b/unstract/workflow-execution/src/unstract/workflow_execution/workflow_execution.py
@@ -182,8 +182,6 @@ class WorkflowExecutionService:
Args:
step (int): workflow step
sandbox (ToolSandbox): instance of tool sandbox
- execution_type (ExecutionType): step or complete
- last_step_output (list[Any]): output of previous step
Raises:
error: _description_