Skip to content

LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows

persisting execution logs (#276)

ali640118a03678
diff --git a/backend/README.md b/backend/README.md
index 1185bf14a..40ebdf467 100644
--- a/backend/README.md
+++ b/backend/README.md
@@ -65,8 +65,10 @@ To update the username or password after it's been set:
Run the following command to start the worker:
```bash
-celery -A backend worker --loglevel=info
+celery -A backend worker --loglevel=info -Q celery,celery_periodic_logs
```
+- The `celery` queue is used for default Celery tasks.
+- The `celery_periodic_logs` queue is utilized for logging history tasks.
### Worker Dashboard
diff --git a/backend/backend/celery.py b/backend/backend/celery.py
index 0ff550bb0..57f387391 100644
--- a/backend/backend/celery.py
+++ b/backend/backend/celery.py
@@ -4,6 +4,7 @@ import os
from celery import Celery
from django.conf import settings
+from utils.constants import ExecutionLogConstants
# Set the default Django settings module for the 'celery' program.
os.environ.setdefault(
@@ -24,5 +25,13 @@ app.config_from_object("django.conf:settings", namespace="CELERY")
# Autodiscover tasks in all installed apps.
app.autodiscover_tasks()
+# Define the queues to purge when the Celery broker is restarted.
+queues_to_purge = [ExecutionLogConstants.CELERY_QUEUE_NAME]
+with app.connection() as connection:
+ channel = connection.channel()
+
+ for queue_name in queues_to_purge:
+ channel.queue_purge(queue_name)
+
# Use the Django-Celery-Beat scheduler.
app.conf.beat_scheduler = "django_celery_beat.schedulers:DatabaseScheduler"
diff --git a/backend/backend/settings/base.py b/backend/backend/settings/base.py
index 4b5bb2990..ab8d0dfce 100644
--- a/backend/backend/settings/base.py
+++ b/backend/backend/settings/base.py
@@ -155,6 +155,11 @@ SYSTEM_ADMIN_USERNAME = get_required_setting("SYSTEM_ADMIN_USERNAME")
SYSTEM_ADMIN_PASSWORD = get_required_setting("SYSTEM_ADMIN_PASSWORD")
SYSTEM_ADMIN_EMAIL = get_required_setting("SYSTEM_ADMIN_EMAIL")
SESSION_COOKIE_AGE = int(get_required_setting("SESSION_COOKIE_AGE", "86400"))
+ENABLE_LOG_HISTORY = get_required_setting("ENABLE_LOG_HISTORY")
+LOG_HISTORY_CONSUMER_INTERVAL = int(
+ get_required_setting("LOG_HISTORY_CONSUMER_INTERVAL", "60")
+)
+LOGS_BATCH_LIMIT = int(get_required_setting("LOGS_BATCH_LIMIT", "30"))
# Flag to Enable django admin
ADMIN_ENABLED = False
@@ -317,7 +322,7 @@ CACHES = {
"USERNAME": REDIS_USER,
"PASSWORD": REDIS_PASSWORD,
},
- "KEY_FUNCTION": "utils.cache_service.custom_key_function",
+ "KEY_FUNCTION": "utils.redis_cache.custom_key_function",
}
}
diff --git a/backend/pipeline/constants.py b/backend/pipeline/constants.py
index 970a3f277..cbb78c79a 100644
--- a/backend/pipeline/constants.py
+++ b/backend/pipeline/constants.py
@@ -61,5 +61,6 @@ class PipelineURL:
"""Constants for URL names."""
DETAIL = "pipeline-detail"
+ EXECUTIONS = "pipeline-executions"
LIST = "pipeline-list"
EXECUTE = "pipeline-execute"
diff --git a/backend/pipeline/execution_view.py b/backend/pipeline/execution_view.py
new file mode 100644
index 000000000..45f83f438
--- /dev/null
+++ b/backend/pipeline/execution_view.py
@@ -0,0 +1,35 @@
+from permissions.permission import IsOwner
+from pipeline.serializers.execute import DateRangeSerializer
+from rest_framework import viewsets
+from rest_framework.versioning import URLPathVersioning
+from utils.pagination import CustomPagination
+from workflow_manager.workflow.models.execution import WorkflowExecution
+from workflow_manager.workflow.serializers import WorkflowExecutionSerializer
+
+
+class PipelineExecutionViewSet(viewsets.ModelViewSet):
+ versioning_class = URLPathVersioning
+ permission_classes = [IsOwner]
+ serializer_class = WorkflowExecutionSerializer
+ pagination_class = CustomPagination
+
+ CREATED_AT_FIELD_DESC = "-created_at"
+ START_DATE_FIELD = "start_date"
+ END_DATE_FIELD = "end_date"
+
+ def get_queryset(self):
+ # Get the pipeline_id from the URL path
+ pipeline_id = self.kwargs.get("pk")
+ queryset = WorkflowExecution.objects.filter(pipeline_id=pipeline_id)
+
+ # Validate start_date and end_date parameters using DateRangeSerializer
+ date_range_serializer = DateRangeSerializer(data=self.request.query_params)
+ date_range_serializer.is_valid(raise_exception=True)
+ start_date = date_range_serializer.validated_data.get(self.START_DATE_FIELD)
+ end_date = date_range_serializer.validated_data.get(self.END_DATE_FIELD)
+
+ if start_date and end_date:
+ queryset = queryset.filter(created_at__range=(start_date, end_date))
+
+ queryset = queryset.order_by(self.CREATED_AT_FIELD_DESC)
+ return queryset
diff --git a/backend/pipeline/serializers/execute.py b/backend/pipeline/serializers/execute.py
index 88b57e199..cc52e3cfd 100644
--- a/backend/pipeline/serializers/execute.py
+++ b/backend/pipeline/serializers/execute.py
@@ -17,3 +17,8 @@ class PipelineExecuteSerializer(serializers.Serializer):
except Pipeline.DoesNotExist:
raise serializers.ValidationError("Invalid pipeline ID")
return value
+
+
+class DateRangeSerializer(serializers.Serializer):
+ start_date = serializers.DateTimeField(required=False)
+ end_date = serializers.DateTimeField(required=False)
diff --git a/backend/pipeline/urls.py b/backend/pipeline/urls.py
index 7af3dfcac..d03ed2587 100644
--- a/backend/pipeline/urls.py
+++ b/backend/pipeline/urls.py
@@ -1,5 +1,6 @@
from django.urls import path
from pipeline.constants import PipelineURL
+from pipeline.execution_view import PipelineExecutionViewSet
from pipeline.views import PipelineViewSet
from rest_framework.urlpatterns import format_suffix_patterns
@@ -9,6 +10,11 @@ pipeline_list = PipelineViewSet.as_view(
"post": "create",
}
)
+execution_list = PipelineExecutionViewSet.as_view(
+ {
+ "get": "list",
+ }
+)
pipeline_detail = PipelineViewSet.as_view(
{
"get": "retrieve",
@@ -25,6 +31,11 @@ urlpatterns = format_suffix_patterns(
[
path("pipeline/", pipeline_list, name=PipelineURL.LIST),
path("pipeline/<uuid:pk>/", pipeline_detail, name=PipelineURL.DETAIL),
+ path(
+ "pipeline/<uuid:pk>/executions/",
+ execution_list,
+ name=PipelineURL.EXECUTIONS,
+ ),
path("pipeline/execute/", pipeline_execute, name=PipelineURL.EXECUTE),
]
)
diff --git a/backend/pyproject.toml b/backend/pyproject.toml
index be3c1d781..f7349c512 100644
--- a/backend/pyproject.toml
+++ b/backend/pyproject.toml
@@ -78,7 +78,7 @@ migrate_db.cmd = "python manage.py migrate"
migrate_db.env_file = ".env"
migrate_db.help = "Performs DB migrations for Unstract backend"
# Commands for backend execution
-consumer.cmd = "celery -A backend worker --loglevel=info"
+consumer.cmd = "celery -A backend worker --loglevel=info -Q celery,celery_periodic_logs"
consumer.env_file = ".env"
consumer.help = "Runs the Unstract consumer"
# Celery Flower
diff --git a/backend/sample.env b/backend/sample.env
index c0f08baf9..9f3441568 100644
--- a/backend/sample.env
+++ b/backend/sample.env
@@ -130,3 +130,10 @@ ADAPTER_LLMW_POLL_INTERVAL=30
# Total number of times to poll the status API.
# 500 mins to allow 1500 (max pages limit) * 20 (approx time in sec to process a page)
ADAPTER_LLMW_MAX_POLLS=1000
+
+# Enable logging of workflow history.
+ENABLE_LOG_HISTORY=True
+# Interval in seconds for periodic consumer operations.
+LOG_HISTORY_CONSUMER_INTERVAL=30
+# Maximum number of logs to insert in a single batch.
+LOGS_BATCH_LIMIT=30
diff --git a/backend/utils/cache_service.py b/backend/utils/cache_service.py
index 83084e505..691b8ee2a 100644
--- a/backend/utils/cache_service.py
+++ b/backend/utils/cache_service.py
@@ -1,8 +1,10 @@
from typing import Any, Optional
-from account.custom_cache import CustomCache
from django.conf import settings
from django.core.cache import cache
+from django_redis import get_redis_connection
+
+redis_cache = get_redis_connection("default")
class CacheService:
@@ -56,16 +58,20 @@ class CacheService:
return cache.delete(key)
@staticmethod
- def add_cookie_id_to_user(user_id: str, cookie_id: str) -> None:
- custom_cache = CustomCache()
- key: str = f"{user_id}|cookies"
- custom_cache.rpush(key, cookie_id)
+ def rpush(key: str, value: str) -> None:
+ redis_cache.rpush(key, value)
+
+ @staticmethod
+ def lpop(key: str) -> Any:
+ return redis_cache.lpop(key)
+
+ @staticmethod
+ def lrem(key: str, value: str) -> None:
+ redis_cache.lrem(key, value)
@staticmethod
- def remove_cookie_id_from_user(user_id: str, cookie_id: str) -> None:
- custom_cache = CustomCache()
- key: str = f"{user_id}|cookies"
- custom_cache.lrem(key, cookie_id)
+ def lrange(key, start_index, end_index) -> list[Any]:
+ return redis_cache.lrange(key, start_index, end_index)
@staticmethod
def remove_all_session_keys(
diff --git a/backend/utils/constants.py b/backend/utils/constants.py
index d12cbf6d8..0ca37d3c4 100644
--- a/backend/utils/constants.py
+++ b/backend/utils/constants.py
@@ -1,3 +1,14 @@
+import os
+
+from django.conf import settings
+from utils.common_utils import CommonUtils
+
+os.environ.setdefault(
+ "DJANGO_SETTINGS_MODULE",
+ os.environ.get("DJANGO_SETTINGS_MODULE", "backend.settings.dev"),
+)
+
+
class Account:
CREATED_BY = "created_by"
MODIFIED_BY = "modified_by"
@@ -7,3 +18,43 @@ class Common:
METADATA = "metadata"
MODULE = "module"
CONNECTOR = "connector"
+
+
+class Pagination:
+ """Constants for Pagination.
+
+ Attributes:
+ PAGE_SIZE (int): The default number of items per page.
+ PAGE_SIZE_QUERY_PARAM (str): The name of the query parameter used to
+ specify page size in requests. Ex: ?page=2&<PAGE_SIZE_QUERY_PARAM>=3
+ MAX_PAGE_SIZE (int): The maximum allowed number of items per page.
+ """
+
+ PAGE_SIZE = 50
+ PAGE_SIZE_QUERY_PARAM = "page_size"
+ MAX_PAGE_SIZE = 1000
+
+
+class ExecutionLogConstants:
+ """Constants for ExecutionLog.
+
+ Attributes:
+ IS_ENABLED (bool): Whether to enable log history.
+ CONSUMER_INTERVAL (int): The interval (in seconds) between log history
+ consumers.
+ LOG_QUEUE_NAME (str): The name of the queue to store log history.
+ LOGS_BATCH_LIMIT (str): The maximum number of logs to store in a batch.
+ CELERY_QUEUE_NAME (str): The name of the Celery queue to schedule log
+ history consumers.
+ PERIODIC_TASK_NAME (str): The name of the Celery periodic task to schedule
+ log history consumers.
+ TASK (str): The name of the Celery task to schedule log history consumers.
+ """
+
+ IS_ENABLED: bool = CommonUtils.str_to_bool(settings.ENABLE_LOG_HISTORY)
+ CONSUMER_INTERVAL: int = settings.LOG_HISTORY_CONSUMER_INTERVAL
+ LOGS_BATCH_LIMIT: int = settings.LOGS_BATCH_LIMIT
+ LOG_QUEUE_NAME: str = "log_history_queue"
+ CELERY_QUEUE_NAME = "celery_periodic_logs"
+ PERIODIC_TASK_NAME = "workflow_log_history"
+ TASK = "workflow_manager.workflow.execution_log_utils.consume_log_history"
diff --git a/backend/utils/dto.py b/backend/utils/dto.py
new file mode 100644
index 000000000..445982d4c
--- /dev/null
+++ b/backend/utils/dto.py
@@ -0,0 +1,64 @@
+import json
+import logging
+from datetime import datetime
+from typing import Any, Optional
+
+from unstract.workflow_execution.enums import LogType
+
+from unstract.core.constants import LogFieldName
+
+logger = logging.getLogger(__name__)
+
+
+class LogDataDTO:
+ """Log data DTO
+ DTO for log data to store to QUEUE
+ Attributes:
+ execution_id: execution id
+ organization_id: organization id
+ timestamp: timestamp
+ log_type: log type
+ data: log data
+ """
+
+ def __init__(
+ self,
+ execution_id: str,
+ organization_id: str,
+ timestamp: int,
+ log_type: str,
+ data: dict[str, Any],
+ ):
+ self.execution_id: str = execution_id
+ self.organization_id: str = organization_id
+ self.timestamp: int = timestamp
+ self.event_time: datetime = datetime.fromtimestamp(timestamp)
+ self.log_type: LogType = log_type
+ self.data: dict[str, Any] = data
+
+ @classmethod
+ def from_json(cls, json_data: str) -> Optional["LogDataDTO"]:
+ try:
+ json_data = json.loads(json_data)
+ execution_id = json_data.get(LogFieldName.EXECUTION_ID)
+ organization_id = json_data.get(LogFieldName.ORGANIZATION_ID)
+ timestamp = json_data.get(LogFieldName.TIMESTAMP)
+ log_type = json_data.get(LogFieldName.TYPE)
+ data = json_data.get(LogFieldName.DATA)
+ if all((execution_id, organization_id, timestamp, log_type, data)):
+ return cls(execution_id, organization_id, timestamp, log_type, data)
+ except (json.JSONDecodeError, AttributeError):
+ logger.warning("Invalid log data: %s", json_data)
+ return None
+
+ def to_json(self) -> str:
+ return json.dumps(
+ {
+ LogFieldName.EXECUTION_ID: self.execution_id,
+ LogFieldName.ORGANIZATION_ID: self.organization_id,
+ LogFieldName.TIMESTAMP: self.timestamp,
+ LogFieldName.EVENT_TIME: self.event_time.isoformat(),
+ LogFieldName.TYPE: self.log_type,
+ LogFieldName.DATA: self.data,
+ }
+ )
diff --git a/backend/utils/log_events.py b/backend/utils/log_events.py
index d83cc03ee..c8ecfb989 100644
--- a/backend/utils/log_events.py
+++ b/backend/utils/log_events.py
@@ -1,13 +1,19 @@
+import json
import logging
import os
import threading
import time
-from typing import Any
+from typing import Any, Optional
import redis
import socketio
from django.conf import settings
from django.core.wsgi import WSGIHandler
+from unstract.workflow_execution.enums import LogType
+from utils.constants import ExecutionLogConstants
+from utils.dto import LogDataDTO
+
+from unstract.core.constants import LogFieldName
logger = logging.getLogger(__name__)
@@ -38,13 +44,105 @@ def disconnect(sid: str) -> None:
logger.info(f"[{os.getpid()}] Client with SID:{sid} disconnected")
-def _handle_pubsub_messages(message: Any) -> None:
- channel = message["channel"].decode("utf-8")
- data = message["data"]
+def _get_validated_log_data(json_data: Any) -> Optional[LogDataDTO]:
+ """Validate log data to persist history. This function takes log data in
+ JSON format, validates it, and returns a `LogDataDTO` object if the data is
+ valid. The validation process includes decoding bytes to string, parsing
+ the string as JSON, and checking for required fields and log type.
+
+ Args:
+ json_data (Any): Log data in JSON format
+ Returns:
+ Optional[LogDataDTO]: Log data DTO object
+ """
+ if isinstance(json_data, bytes):
+ json_data = json_data.decode("utf-8")
+
+ if isinstance(json_data, str):
+ try:
+ # Parse the string as JSON
+ json_data = json.loads(json_data)
+ except json.JSONDecodeError:
+ logger.error(f"Error decoding JSON data while validating {json_data}")
+ return
+
+ if not isinstance(json_data, dict):
+ logger.warning(f"Getting invalid data type while validating {json_data}")
+ return
+
+ # Extract required fields from the JSON data
+ execution_id = json_data.get(LogFieldName.EXECUTION_ID)
+ organization_id = json_data.get(LogFieldName.ORGANIZATION_ID)
+ timestamp = json_data.get(LogFieldName.TIMESTAMP)
+ log_type = json_data.get(LogFieldName.TYPE)
+
+ # Ensure the log type is LogType.LOG
+ if log_type != LogType.LOG.value:
+ return
+
+ # Check if all required fields are present
+ if not all((execution_id, organization_id, timestamp)):
+ logger.warning(f"Missing required fields while validating {json_data}")
+ return
+
+ return LogDataDTO(
+ execution_id=execution_id,
+ organization_id=organization_id,
+ timestamp=timestamp,
+ log_type=log_type,
+ data=json_data,
+ )
+
+
+def _store_execution_log(data: bytes) -> None:
+ """Store execution log in database
+ Args:
+ data (bytes): Execution log data in bytes format
+ """
+ if not ExecutionLogConstants.IS_ENABLED:
+ return
+ try:
+ log_data = _get_validated_log_data(json_data=data)
+ if log_data:
+ redis_conn.rpush(ExecutionLogConstants.LOG_QUEUE_NAME, log_data.to_json())
+ except Exception as e:
+ logger.error(f"Error storing execution log: {e}")
+
+
+def _emit_websocket_event(channel: str, data: bytes) -> None:
+ """Emit websocket event
+ Args:
+ channel (str): WebSocket channel
+ data (bytes): Execution log data in bytes format
+ """
payload = {"data": data}
+ try:
+ logger.debug(f"[{os.getpid()}] Push websocket event: {channel}, {payload}")
+ sio.emit(channel, payload)
+ except Exception as e:
+ logger.error(f"Error emitting WebSocket event: {e}")
+
+
+def _handle_pubsub_messages(message: dict[str, Any]) -> None:
+ """Handle pubsub messages
+ Args:
+ message (dict[str, Any]): Pub sub message
+ """
+ channel = message.get("channel")
+ data = message.get("data")
+
+ if not channel or not data:
+ logger.warning(f"Invalid message received: {message}")
+ return
+
+ try:
+ channel_str = channel.decode("utf-8")
+ except UnicodeDecodeError as e:
+ logger.error(f"Error decoding channel: {e}")
+ return
- logger.debug(f"[{os.getpid()}] Push websocket event: {channel}, {payload}")
- sio.emit(channel, {"data": data})
+ _store_execution_log(data)
+ _emit_websocket_event(channel_str, data)
def _pubsub_listen_forever() -> None:
diff --git a/backend/utils/pagination.py b/backend/utils/pagination.py
new file mode 100644
index 000000000..c4a30b60e
--- /dev/null
+++ b/backend/utils/pagination.py
@@ -0,0 +1,8 @@
+from rest_framework.pagination import PageNumberPagination
+from utils.constants import Pagination
+
+
+class CustomPagination(PageNumberPagination):
+ page_size = Pagination.PAGE_SIZE
+ page_size_query_param = Pagination.PAGE_SIZE_QUERY_PARAM
+ max_page_size = Pagination.MAX_PAGE_SIZE
diff --git a/backend/utils/redis_cache.py b/backend/utils/redis_cache.py
new file mode 100644
index 000000000..513d08edb
--- /dev/null
+++ b/backend/utils/redis_cache.py
@@ -0,0 +1,9 @@
+# KEY_FUNCTION for cache settings
+def custom_key_function(key: str, key_prefix: str, version: int) -> str:
+ version = int(version)
+ if version > 1:
+ return f"{key_prefix}:{version}:{key}"
+ if key_prefix:
+ return f"{key_prefix}:{key}"
+ else:
+ return key
diff --git a/backend/workflow_manager/workflow/apps.py b/backend/workflow_manager/workflow/apps.py
index 71e894363..3c0d821d7 100644
--- a/backend/workflow_manager/workflow/apps.py
+++ b/backend/workflow_manager/workflow/apps.py
@@ -4,3 +4,10 @@ from django.apps import AppConfig
class WorkflowConfig(AppConfig):
default_auto_field = "django.db.models.BigAutoField"
name = "workflow_manager.workflow"
+
+ def ready(self):
+ from workflow_manager.workflow.execution_log_utils import (
+ create_log_consumer_scheduler_if_not_exists,
+ )
+
+ create_log_consumer_scheduler_if_not_exists()
diff --git a/backend/workflow_manager/workflow/execution.py b/backend/workflow_manager/workflow/execution.py
index acfff7fee..b332c9ecb 100644
--- a/backend/workflow_manager/workflow/execution.py
+++ b/backend/workflow_manager/workflow/execution.py
@@ -145,7 +145,7 @@ class WorkflowExecutionServiceHelper(WorkflowExecutionService):
workflow_execution.save()
return workflow_execution
- def __update_execution(
+ def update_execution(
self,
status: Optional[ExecutionStatus] = None,
execution_time: Optional[float] = None,
@@ -171,11 +171,6 @@ class WorkflowExecutionServiceHelper(WorkflowExecutionService):
detailed_message: str,
execution_time: Optional[float] = None,
) -> WorkflowExecutionError:
- self.__update_execution(
- status=ExecutionStatus.ERROR,
- error=detailed_message,
- execution_time=execution_time,
- )
return WorkflowExecutionError(detail=error_message)
def has_successful_compilation(self) -> bool:
@@ -190,12 +185,16 @@ class WorkflowExecutionServiceHelper(WorkflowExecutionService):
def build(self) -> None:
if self.compilation_result["success"] is True:
self.build_workflow()
- self.__update_execution(status=ExecutionStatus.READY)
+ self.update_execution(status=ExecutionStatus.READY)
else:
logger.error(
"Errors while compiling workflow "
f"{self.compilation_result['problems']}"
)
+ self.update_execution(
+ status=ExecutionStatus.ERROR,
+ error=self.compilation_result["problems"][0],
+ )
raise self.__execution_error(
error_message=self.compilation_result["problems"][0],
detailed_message=self.compilation_result["problems"][0],
@@ -207,29 +206,18 @@ class WorkflowExecutionServiceHelper(WorkflowExecutionService):
execution_type = ExecutionType.STEP
if self.compilation_result["success"] is True:
- self.__update_execution(status=ExecutionStatus.READY)
if (
self.execution_mode == WorkflowExecution.Mode.INSTANT
or self.execution_mode == WorkflowExecution.Mode.QUEUE
):
start_time = time.time()
try:
- self.__update_execution(increment_attempt=True)
self.execute_workflow(execution_type=execution_type)
end_time = time.time()
execution_time = end_time - start_time
- self.__update_execution(
- status=ExecutionStatus.COMPLETED,
- execution_time=execution_time,
- )
except StopExecution as exception:
end_time = time.time()
execution_time = end_time - start_time
- self.__update_execution(
- status=ExecutionStatus.STOPPED,
- error=str(exception),
- execution_time=execution_time,
- )
logger.info(f"Execution {self.execution_id} stopped")
raise exception
except Exception as exception:
@@ -269,7 +257,9 @@ class WorkflowExecutionServiceHelper(WorkflowExecutionService):
LogState.RUNNING, "Ready for execution", LogComponent.WORKFLOW
)
- def publish_final_workflow_logs(self) -> None:
+ def publish_final_workflow_logs(
+ self, total_files: int, processed_files: int
+ ) -> None:
"""Publishes the final logs for the workflow.
Returns:
@@ -279,6 +269,10 @@ class WorkflowExecutionServiceHelper(WorkflowExecutionService):
self.publish_update_log(
LogState.SUCCESS, "Executed successfully", LogComponent.WORKFLOW
)
+ self.publish_log(
+ f"Execution completed successfully for the files {processed_files} "
+ f"out of {total_files}"
+ )
def publish_initial_tool_execution_logs(
self, current_step: int, total_step: int, file_name: str
@@ -305,7 +299,7 @@ class WorkflowExecutionServiceHelper(WorkflowExecutionService):
file_name: str,
single_step: bool,
file_history: Optional[FileHistory] = None,
- ) -> None:
+ ) -> bool:
"""Executes the input file.
Args:
@@ -314,22 +308,24 @@ class WorkflowExecutionServiceHelper(WorkflowExecutionService):
single step mode.
file_history (Optional[FileHistory], optional):
The file history object. Defaults to None.
-
Returns:
- None
+ bool: Flag indicating whether the file was executed.
"""
execution_type = ExecutionType.COMPLETE
+ is_executed = False
if single_step:
execution_type = ExecutionType.STEP
if not (file_history and file_history.is_completed()):
self.execute_uncached_input(file_name=file_name, single_step=single_step)
+ self.publish_log(f"Tool executed successfully for {file_name}")
+ is_executed = True
else:
self.publish_log(
f"Skipping file {file_name} as it is already processed."
"Clear the cache to process it again"
)
- self.publish_log(f"Tool executed successfully for {file_name}")
self._handle_execution_type(execution_type)
+ return is_executed
def execute_uncached_input(self, file_name: str, single_step: bool) -> None:
"""Executes the uncached input file.
diff --git a/backend/workflow_manager/workflow/execution_log_utils.py b/backend/workflow_manager/workflow/execution_log_utils.py
new file mode 100644
index 000000000..6c2b7bb14
--- /dev/null
+++ b/backend/workflow_manager/workflow/execution_log_utils.py
@@ -0,0 +1,106 @@
+import logging
+from collections import defaultdict
+
+from account.models import Organization
+from celery import shared_task
+from django.db import IntegrityError
+from django_celery_beat.models import IntervalSchedule, PeriodicTask
+from django_tenants.utils import get_tenant_model, tenant_context
+from utils.cache_service import CacheService
+from utils.constants import ExecutionLogConstants
+from utils.dto import LogDataDTO
+from workflow_manager.workflow.models.execution_log import ExecutionLog
+
+logger = logging.getLogger(__name__)
+
+
+@shared_task(bind=True)
+def consume_log_history(self):
+ organization_logs = defaultdict(list)
+ logs_count = 0
+
+ while logs_count < ExecutionLogConstants.LOGS_BATCH_LIMIT:
+ log = CacheService.lpop(ExecutionLogConstants.LOG_QUEUE_NAME)
+ if not log:
+ break
+
+ log_data = LogDataDTO.from_json(log)
+ if not log_data:
+ continue
+
+ organization_id = log_data.organization_id
+ organization_logs[organization_id].append(
+ ExecutionLog(
+ execution_id=log_data.execution_id,
+ data=log_data.data,
+ event_time=log_data.event_time,
+ )
+ )
+ logs_count += 1
+ logger.info(f"Logs count: {logs_count}")
+ for organization_id, logs in organization_logs.items():
+ store_to_db(organization_id, logs)
+
+
+def create_log_consumer_scheduler_if_not_exists():
+ try:
+ interval, _ = IntervalSchedule.objects.get_or_create(
+ every=ExecutionLogConstants.CONSUMER_INTERVAL,
+ period=IntervalSchedule.SECONDS,
+ )
+ except IntervalSchedule.MultipleObjectsReturned as error:
+ logger.error(f"Error occurred while getting interval schedule: {error}")
+ interval = IntervalSchedule.objects.filter(
+ every=ExecutionLogConstants.CONSUMER_INTERVAL,
+ period=IntervalSchedule.SECONDS,
+ ).first()
+ try:
+ # Create the scheduler
+ task, created = PeriodicTask.objects.get_or_create(
+ name=ExecutionLogConstants.PERIODIC_TASK_NAME,
+ task=ExecutionLogConstants.TASK,
+ defaults={
+ "interval": interval,
+ "queue": ExecutionLogConstants.CELERY_QUEUE_NAME,
+ "enabled": ExecutionLogConstants.IS_ENABLED,
+ },
+ )
+ if not created:
+ task.enabled = ExecutionLogConstants.IS_ENABLED
+ task.interval = interval
+ task.queue = ExecutionLogConstants.CELERY_QUEUE_NAME
+ task.save()
+ logger.info("Log consumer scheduler updated successfully.")
+ else:
+ logger.info("Log consumer scheduler created successfully.")
+ except IntegrityError as error:
+ logger.error(f"Error occurred while creating log consumer scheduler: {error}")
+
+
+def store_to_db(organization_id: str, execution_logs: list[ExecutionLog]) -> None:
+ try:
+ tenant: Organization = get_tenant_model().objects.get(
+ schema_name=organization_id
+ )
+ except Organization.DoesNotExist:
+ logger.error(f"Organization with ID {organization_id} does not exist.")
+ return
+
+ # Store the log data in the database within tenant context
+ with tenant_context(tenant):
+ ExecutionLog.objects.bulk_create(objs=execution_logs, ignore_conflicts=True)
+
+
+class ExecutionLogUtils:
+
+ @staticmethod
+ def get_execution_logs_by_execution_id(execution_id) -> list[ExecutionLog]:
+ """Get all execution logs for a given execution ID.
+
+ Args:
+ execution_id (int): The ID of the execution.
+
+ Returns:
+ list[ExecutionLog]: A list of ExecutionLog objects.
+ """
+ return ExecutionLog.objects.filter(execution_id=execution_id)
diff --git a/backend/workflow_manager/workflow/execution_log_view.py b/backend/workflow_manager/workflow/execution_log_view.py
new file mode 100644
index 000000000..137349e72
--- /dev/null
+++ b/backend/workflow_manager/workflow/execution_log_view.py
@@ -0,0 +1,27 @@
+import logging
+
+from permissions.permission import IsOwner
+from rest_framework import viewsets
+from rest_framework.versioning import URLPathVersioning
+from utils.pagination import CustomPagination
+from workflow_manager.workflow.models.execution_log import ExecutionLog
+from workflow_manager.workflow.serializers import WorkflowExecutionLogSerializer
+
+logger = logging.getLogger(__name__)
+
+
+class WorkflowExecutionLogViewSet(viewsets.ModelViewSet):
+ versioning_class = URLPathVersioning
+ permission_classes = [IsOwner]
+ serializer_class = WorkflowExecutionLogSerializer
+ pagination_class = CustomPagination
+
+ EVENT_TIME_FELID_ASC = "event_time"
+
+ def get_queryset(self):
+ # Get the execution_id:pk from the URL path
+ execution_id = self.kwargs.get("pk")
+ queryset = ExecutionLog.objects.filter(execution_id=execution_id).order_by(
+ self.EVENT_TIME_FELID_ASC
+ )
+ return queryset
diff --git a/backend/workflow_manager/workflow/execution_view.py b/backend/workflow_manager/workflow/execution_view.py
new file mode 100644
index 000000000..7c5866d0c
--- /dev/null
+++ b/backend/workflow_manager/workflow/execution_view.py
@@ -0,0 +1,25 @@
+import logging
+
+from permissions.permission import IsOwner
+from rest_framework import viewsets
+from rest_framework.versioning import URLPathVersioning
+from workflow_manager.workflow.models.execution import WorkflowExecution
+from workflow_manager.workflow.serializers import WorkflowExecutionSerializer
+
+logger = logging.getLogger(__name__)
+
+
+class WorkflowExecutionViewSet(viewsets.ModelViewSet):
+ versioning_class = URLPathVersioning
+ permission_classes = [IsOwner]
+ serializer_class = WorkflowExecutionSerializer
+
+ CREATED_AT_FIELD_DESC = "-created_at"
+
+ def get_queryset(self):
+ # Get the uuid:pk from the URL path
+ workflow_id = self.kwargs.get("pk")
+ queryset = WorkflowExecution.objects.filter(workflow_id=workflow_id).order_by(
+ self.CREATED_AT_FIELD_DESC
+ )
+ return queryset
diff --git a/backend/workflow_manager/workflow/migrations/0004_executionlog.py b/backend/workflow_manager/workflow/migrations/0004_executionlog.py
new file mode 100644
index 000000000..63e25da3c
--- /dev/null
+++ b/backend/workflow_manager/workflow/migrations/0004_executionlog.py
@@ -0,0 +1,40 @@
+# Generated by Django 4.2.1 on 2024-04-22 06:10
+
+import uuid
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ("workflow", "0003_workflowexecution_execution_log_id_and_more"),
+ ]
+
+ operations = [
+ migrations.CreateModel(
+ name="ExecutionLog",
+ fields=[
+ ("created_at", models.DateTimeField(auto_now_add=True)),
+ ("modified_at", models.DateTimeField(auto_now=True)),
+ (
+ "id",
+ models.UUIDField(
+ default=uuid.uuid4,
+ editable=False,
+ primary_key=True,
+ serialize=False,
+ ),
+ ),
+ (
+ "execution_id",
+ models.UUIDField(db_comment="Execution ID", editable=False),
+ ),
+ ("data", models.JSONField()),
+ ("event_time", models.DateTimeField()),
+ ],
+ options={
+ "abstract": False,
+ },
+ ),
+ ]
diff --git a/backend/workflow_manager/workflow/models/__init__.py b/backend/workflow_manager/workflow/models/__init__.py
index 57b3df1a1..938907762 100644
--- a/backend/workflow_manager/workflow/models/__init__.py
+++ b/backend/workflow_manager/workflow/models/__init__.py
@@ -1,3 +1,4 @@
from .execution import WorkflowExecution # noqa: F401
+from .execution_log import ExecutionLog # noqa: F401
from .file_history import FileHistory # noqa: F401
from .workflow import Workflow # noqa: F401
diff --git a/backend/workflow_manager/workflow/models/execution_log.py b/backend/workflow_manager/workflow/models/execution_log.py
new file mode 100644
index 000000000..d3b2c5ae7
--- /dev/null
+++ b/backend/workflow_manager/workflow/models/execution_log.py
@@ -0,0 +1,17 @@
+import uuid
+
+from django.db import models
+from utils.models.base_model import BaseModel
+
+
+class ExecutionLog(BaseModel):
+ id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
+ execution_id = models.UUIDField(
+ editable=False,
+ db_comment="Execution ID",
+ )
+ data = models.JSONField(db_comment="Execution log data")
+ event_time = models.DateTimeField(db_comment="Execution log event time")
+
+ def __str__(self):
+ return f"Execution ID: {self.execution_id}, Message: {self.data}"
diff --git a/backend/workflow_manager/workflow/serializers.py b/backend/workflow_manager/workflow/serializers.py
index 180884887..04dfcdac4 100644
--- a/backend/workflow_manager/workflow/serializers.py
+++ b/backend/workflow_manager/workflow/serializers.py
@@ -17,6 +17,8 @@ from workflow_manager.endpoint.models import WorkflowEndpoint
from workflow_manager.workflow.constants import WorkflowExecutionKey, WorkflowKey
from workflow_manager.workflow.exceptions import WorkflowGenerationError
from workflow_manager.workflow.generator import WorkflowGenerator
+from workflow_manager.workflow.models.execution import WorkflowExecution
+from workflow_manager.workflow.models.execution_log import ExecutionLog
from workflow_manager.workflow.models.workflow import Workflow
from backend.constants import RequestKey
@@ -149,3 +151,15 @@ class WorkflowEndpointSerializer(ModelSerializer):
class Meta:
model = WorkflowEndpoint
fields = "__all__"
+
+
+class WorkflowExecutionSerializer(ModelSerializer):
+ class Meta:
+ model = WorkflowExecution
+ fields = "__all__"
+
+
+class WorkflowExecutionLogSerializer(ModelSerializer):
+ class Meta:
+ model = ExecutionLog
+ fields = "__all__"
diff --git a/backend/workflow_manager/workflow/urls.py b/backend/workflow_manager/workflow/urls.py
index 144ad09b7..d75b5fc43 100644
--- a/backend/workflow_manager/workflow/urls.py
+++ b/backend/workflow_manager/workflow/urls.py
@@ -1,5 +1,7 @@
from django.urls import path
from rest_framework.urlpatterns import format_suffix_patterns
+from workflow_manager.workflow.execution_log_view import WorkflowExecutionLogViewSet
+from workflow_manager.workflow.execution_view import WorkflowExecutionViewSet
from workflow_manager.workflow.views import WorkflowViewSet
workflow_list = WorkflowViewSet.as_view(
@@ -19,7 +21,9 @@ workflow_detail = WorkflowViewSet.as_view(
# fmt: on
)
workflow_execute = WorkflowViewSet.as_view({"post": "execute", "put": "activate"})
-execution_entity = WorkflowViewSet.as_view({"get": "get_execution"})
+execution_entity = WorkflowExecutionViewSet.as_view({"get": "retrieve"})
+execution_list = WorkflowExecutionViewSet.as_view({"get": "list"})
+execution_log_list = WorkflowExecutionLogViewSet.as_view({"get": "list"})
workflow_clear_cache = WorkflowViewSet.as_view({"get": "clear_cache"})
workflow_clear_file_marker = WorkflowViewSet.as_view({"get": "clear_file_marker"})
workflow_schema = WorkflowViewSet.as_view({"get": "get_schema"})
@@ -49,11 +53,21 @@ urlpatterns = format_suffix_patterns(
workflow_execute,
name="active-workflow",
),
+ path(
+ "<uuid:pk>/execution/",
+ execution_list,
+ name="execution-list",
+ ),
path(
"execution/<uuid:pk>/",
execution_entity,
name="workflow-detail",
),
+ path(
+ "execution/<uuid:pk>/logs/",
+ execution_log_list,
+ name="execution-log",
+ ),
path(
"schema/",
workflow_schema,
diff --git a/backend/workflow_manager/workflow/workflow_helper.py b/backend/workflow_manager/workflow/workflow_helper.py
index db8a73d35..c74f2ccb2 100644
--- a/backend/workflow_manager/workflow/workflow_helper.py
+++ b/backend/workflow_manager/workflow/workflow_helper.py
@@ -18,7 +18,7 @@ from rest_framework import serializers
from tool_instance.constants import ToolInstanceKey
from tool_instance.models import ToolInstance
from tool_instance.tool_instance_helper import ToolInstanceHelper
-from unstract.workflow_execution.enums import LogComponent, LogState
+from unstract.workflow_execution.enums import LogComponent, LogLevel, LogState
from unstract.workflow_execution.exceptions import StopExecution
from utils.cache_service import CacheService
from workflow_manager.endpoint.destination import DestinationConnector
@@ -112,11 +112,16 @@ class WorkflowHelper:
) -> WorkflowExecution:
input_files = source.list_files_from_source()
total_files = len(input_files)
+ processed_files = 0
+ error_raised = 0
execution_service.publish_initial_workflow_logs(total_files)
+ execution_service.update_execution(
+ ExecutionStatus.EXECUTING, increment_attempt=True
+ )
for index, input_file in enumerate(input_files):
file_number = index + 1
try:
- WorkflowHelper.process_file(
+ is_executed, error = WorkflowHelper.process_file(
current_step=file_number,
total_step=total_files,
input_file=input_file,
@@ -127,9 +132,23 @@ class WorkflowHelper:
single_step=single_step,
hash_values_of_files=hash_values_of_files,
)
- except StopExecution:
+ if is_executed:
+ processed_files += 1
+ if error:
+ error_raised += 1
+ except StopExecution as exception:
+ execution_service.update_execution(
+ ExecutionStatus.STOPPED, error=str(exception)
+ )
break
- execution_service.publish_final_workflow_logs()
+ if error_raised and error_raised == total_files:
+ execution_service.update_execution(ExecutionStatus.ERROR)
+ else:
+ execution_service.update_execution(ExecutionStatus.COMPLETED)
+
+ execution_service.publish_final_workflow_logs(
+ total_files=total_files, processed_files=processed_files
+ )
return execution_service.get_execution_instance()
@staticmethod
@@ -143,9 +162,10 @@ class WorkflowHelper:
execution_service: WorkflowExecutionServiceHelper,
single_step: bool,
hash_values_of_files: dict[str, str],
- ) -> None:
+ ) -> tuple[bool, Optional[str]]:
file_history = None
error = None
+ is_executed = False
file_name, file_hash = source.add_file_to_volume(
input_file_path=input_file,
hash_values_of_files=hash_values_of_files,
@@ -157,7 +177,7 @@ class WorkflowHelper:
file_history = FileHistoryHelper.get_file_history(
workflow=workflow, cache_key=file_hash
)
- execution_service.execute_input_file(
+ is_executed = execution_service.execute_input_file(
file_name=file_name,
single_step=single_step,
file_history=file_history,
@@ -166,7 +186,8 @@ class WorkflowHelper:
raise
except Exception as e:
execution_service.publish_log(
- f"Error processing file {input_file}: {str(e)}"
+ f"Error processing file {input_file}: {str(e)}",
+ level=LogLevel.ERROR,
)
error = str(e)
execution_service.publish_update_log(
@@ -186,6 +207,7 @@ class WorkflowHelper:
f"{file_name} is Processed successfully",
LogComponent.DESTINATION,
)
+ return is_executed, error
@staticmethod
def validate_tool_instances_meta(
diff --git a/frontend/src/components/pipelines-or-deployments/log-modal/LogsModal.jsx b/frontend/src/components/pipelines-or-deployments/log-modal/LogsModal.jsx
new file mode 100644
index 000000000..f8d520fbb
--- /dev/null
+++ b/frontend/src/components/pipelines-or-deployments/log-modal/LogsModal.jsx
@@ -0,0 +1,175 @@
+import { Table, Modal, Button } from "antd";
+import PropTypes from "prop-types";
+import { useState } from "react";
+
+import { useSessionStore } from "../../../store/session-store.js";
+import { useAxiosPrivate } from "../../../hooks/useAxiosPrivate.js";
+import { useAlertStore } from "../../../store/alert-store.js";
+import { useExceptionHandler } from "../../../hooks/useExceptionHandler.jsx";
+import "./LogsModel.css";
+
+const LogsModal = ({
+ open,
+ setOpen,
+ logRecord,
+ totalLogs,
+ fetchExecutionLogs,
+}) => {
+ const [selectedLogId, setSelectedLogId] = useState(null);
+ const [logDescModalOpen, setLogDescModalOpen] = useState(false);
+ const [logDetails, setLogDetails] = useState([]);
+ const { sessionDetails } = useSessionStore();
+ const { setAlertDetails } = useAlertStore();
+ const axiosPrivate = useAxiosPrivate();
+ const handleException = useExceptionHandler();
+ const [totalCount, setTotalCount] = useState(0);
+
+ const fetchLogDetails = async (logId, page = 1, pageSize = 10) => {
+ const requestOptions = {
+ method: "GET",
+ url: `/api/v1/unstract/${sessionDetails?.orgId}/workflow/execution/${logId}/logs/`,
+ headers: {
+ "X-CSRFToken": sessionDetails?.csrfToken,
+ },
+ params: {
+ page: page,
+ page_size: pageSize,
+ },
+ };
+ axiosPrivate(requestOptions)
+ .then((res) => {
+ const logDetails = res.data.results.map((item) => ({
+ id: item.id,
+ log: item.data?.log,
+ type: item.data?.type,
+ stage: item.data?.stage,
+ level: item.data?.level,
+ event_time: item.event_time,
+ }));
+ setLogDetails(logDetails);
+ setTotalCount(res.data.count);
+ })
+ .catch((err) => {
+ setAlertDetails(handleException(err));
+ });
+ };
+
+ const handleLogIdClick = (logId) => {
+ setSelectedLogId(logId);
+ fetchLogDetails(logId);
+ setLogDescModalOpen(true);
+ };
+
+ const logColumns = [
+ {
+ title: "Executed At",
+ dataIndex: "created_at",
+ key: "created_at",
+ },
+ {
+ title: "Execution ID",
+ dataIndex: "execution_id",
+ key: "execution_id",
+ render: (text, record) => {
+ return (
+ <Button
+ type="link"
+ onClick={() => handleLogIdClick(record.execution_id)}
+ >
+ {text}
+ </Button>
+ );
+ },
+ },
+ {
+ title: "Status",
+ dataIndex: "status",
+ key: "status",
+ render: (level) => <span className={level.toLowerCase()}>{level}</span>,
+ },
+ ];
+
+ const logDetailsColumns = [
+ {
+ title: "Event Time",
+ dataIndex: "event_time",
+ key: "event_time",
+ },
+ {
+ title: "Event Stage",
+ dataIndex: "stage",
+ key: "stage",
+ },
+ {
+ title: "Log Level",
+ dataIndex: "level",
+ key: "level",
+ render: (level) => <span className={level.toLowerCase()}>{level}</span>,
+ },
+ {
+ title: "Log",
+ dataIndex: "log",
+ key: "log",
+ },
+ ];
+ return (
+ <>
+ <Modal
+ title="Execution Logs"
+ centered
+ open={open}
+ onCancel={() => setOpen(false)}
+ footer={null}
+ width="80%"
+ >
+ <Table
+ columns={logColumns}
+ dataSource={logRecord}
+ rowKey="id"
+ align="left"
+ pagination={{
+ pageSize: 10,
+ total: totalLogs,
+ onChange: (page, pageSize) => {
+ fetchExecutionLogs(page, pageSize);
+ },
+ }}
+ />
+ </Modal>
+
+ <Modal
+ title={`Execution Log Details - ${selectedLogId}`}
+ centered
+ open={logDescModalOpen}
+ onCancel={() => setLogDescModalOpen(false)}
+ footer={null}
+ width="80%"
+ wrapClassName="modal-body"
+ >
+ <Table
+ columns={logDetailsColumns}
+ dataSource={logDetails}
+ rowKey="id"
+ align="left"
+ pagination={{
+ pageSize: 10,
+ total: totalCount,
+ onChange: (page, pageSize) => {
+ fetchLogDetails(selectedLogId, page, pageSize);
+ },
+ }}
+ />
+ </Modal>
+ </>
+ );
+};
+
+LogsModal.propTypes = {
+ open: PropTypes.bool.isRequired,
+ setOpen: PropTypes.func.isRequired,
+ logRecord: PropTypes.array.isRequired,
+ totalLogs: PropTypes.number.isRequired,
+ fetchExecutionLogs: PropTypes.func.isRequired,
+};
+
+export { LogsModal };
diff --git a/frontend/src/components/pipelines-or-deployments/log-modal/LogsModel.css b/frontend/src/components/pipelines-or-deployments/log-modal/LogsModel.css
new file mode 100644
index 000000000..704a775a2
--- /dev/null
+++ b/frontend/src/components/pipelines-or-deployments/log-modal/LogsModel.css
@@ -0,0 +1,15 @@
+.error {
+ color: #FF0000;
+}
+
+.completed {
+ color: #034e03;
+}
+
+.warning {
+ color: #af6714;
+}
+
+table .ant-btn {
+ padding: 0;
+}
diff --git a/frontend/src/components/pipelines-or-deployments/pipelines/Pipelines.jsx b/frontend/src/components/pipelines-or-deployments/pipelines/Pipelines.jsx
index 49fed9536..c9ff548f0 100644
--- a/frontend/src/components/pipelines-or-deployments/pipelines/Pipelines.jsx
+++ b/frontend/src/components/pipelines-or-deployments/pipelines/Pipelines.jsx
@@ -4,6 +4,7 @@ import {
EllipsisOutlined,
SyncOutlined,
HighlightOutlined,
+ FileSearchOutlined,
} from "@ant-design/icons";
import { Dropdown, Image, Space, Switch, Typography } from "antd";
import PropTypes from "prop-types";
@@ -17,6 +18,7 @@ import { useSessionStore } from "../../../store/session-store.js";
import { Layout } from "../../deployments/layout/Layout.jsx";
import { SpinnerLoader } from "../../widgets/spinner-loader/SpinnerLoader.jsx";
import { DeleteModal } from "../delete-modal/DeleteModal.jsx";
+import { LogsModal } from "../log-modal/LogsModal.jsx";
import { EtlTaskDeploy } from "../etl-task-deploy/EtlTaskDeploy.jsx";
import "./Pipelines.css";
import { useExceptionHandler } from "../../../hooks/useExceptionHandler.jsx";
@@ -32,6 +34,9 @@ function Pipelines({ type }) {
const axiosPrivate = useAxiosPrivate();
const handleException = useExceptionHandler();
const [isEdit, setIsEdit] = useState(false);
+ const [openLogsModal, setOpenLogsModal] = useState(false);
+ const [executionLogs, setExecutionLogs] = useState([]);
+ const [executionLogsTotalCount, setExecutionLogsTotalCount] = useState(0);
useEffect(() => {
getPipelineList();
@@ -175,6 +180,34 @@ function Pipelines({ type }) {
});
};
+ const fetchExecutionLogs = (page = 1, pageSize = 10) => {
+ const requestOptions = {
+ method: "GET",
+ url: `/api/v1/unstract/${sessionDetails?.orgId}/pipeline/${selectedPorD.id}/executions/`,
+ headers: {
+ "X-CSRFToken": sessionDetails?.csrfToken,
+ },
+ params: {
+ page: page,
+ page_size: pageSize,
+ },
+ };
+ axiosPrivate(requestOptions)
+ .then((res) => {
+ const logs = res?.data?.results?.map((result) => ({
+ created_at: result.created_at,
+ execution_id: result.id,
+ status: result.status,
+ execution_time: result.execution_time,
+ }));
+ setExecutionLogs(logs);
+ setExecutionLogsTotalCount(res.data.count);
+ })
+ .catch((err) => {
+ setAlertDetails(handleException(err));
+ });
+ };
+
const clearCache = () => {
const requestOptions = {
method: "GET",
@@ -237,6 +270,26 @@ function Pipelines({ type }) {
},
{
key: "2",
+ label: (
+ <Space
+ direction="horizontal"
+ className="action-items"
+ onClick={() => {
+ setOpenLogsModal(true);
+ fetchExecutionLogs();
+ }}
+ >
+ <div>
+ <FileSearchOutlined />
+ </div>
+ <div>
+ <Typography.Text>View Logs</Typography.Text>
+ </div>
+ </Space>
+ ),
+ },
+ {
+ key: "3",
label: (
<Space
direction="horizontal"
@@ -253,7 +306,7 @@ function Pipelines({ type }) {
),
},
{
- key: "3",
+ key: "4",
label: (
<Space
direction="horizontal"
@@ -270,7 +323,7 @@ function Pipelines({ type }) {
),
},
{
- key: "4",
+ key: "5",
label: (
<Space
direction="horizontal"
@@ -431,6 +484,13 @@ function Pipelines({ type }) {
setSelectedRow={setSelectedPorD}
/>
)}
+ <LogsModal
+ open={openLogsModal}
+ setOpen={setOpenLogsModal}
+ logRecord={executionLogs}
+ totalLogs={executionLogsTotalCount}
+ fetchExecutionLogs={fetchExecutionLogs}
+ />
<DeleteModal
open={openDeleteModal}
setOpen={setOpenDeleteModal}
diff --git a/unstract/core/src/unstract/core/constants.py b/unstract/core/src/unstract/core/constants.py
new file mode 100644
index 000000000..5f397e7f6
--- /dev/null
+++ b/unstract/core/src/unstract/core/constants.py
@@ -0,0 +1,7 @@
+class LogFieldName:
+ EXECUTION_ID = "execution_id"
+ ORGANIZATION_ID = "organization_id"
+ TIMESTAMP = "timestamp"
+ TYPE = "type"
+ DATA = "data"
+ EVENT_TIME = "event_time"
diff --git a/unstract/core/src/unstract/core/pubsub_helper.py b/unstract/core/src/unstract/core/pubsub_helper.py
index 855a4d3f6..5658c0076 100644
--- a/unstract/core/src/unstract/core/pubsub_helper.py
+++ b/unstract/core/src/unstract/core/pubsub_helper.py
@@ -43,6 +43,8 @@ class LogPublisher:
step: Optional[int] = None,
iteration: Optional[int] = None,
iteration_total: Optional[int] = None,
+ execution_id: Optional[str] = None,
+ organization_id: Optional[str] = None,
) -> dict[str, Any]:
return {
"timestamp": datetime.now(timezone.utc).timestamp(),
@@ -56,6 +58,8 @@ class LogPublisher:
"step": step,
"iteration": iteration,
"iteration_total": iteration_total,
+ "execution_id": execution_id,
+ "organization_id": organization_id,
}
@staticmethod
diff --git a/unstract/workflow-execution/src/unstract/workflow_execution/tools_utils.py b/unstract/workflow-execution/src/unstract/workflow_execution/tools_utils.py
index b8a08d0a4..c915078ab 100644
--- a/unstract/workflow-execution/src/unstract/workflow_execution/tools_utils.py
+++ b/unstract/workflow-execution/src/unstract/workflow_execution/tools_utils.py
@@ -106,6 +106,8 @@ class ToolsUtils:
LogPublisher.log_workflow(
"BUILD",
f"------ Building step {tool_instance.step}/{tool_uid}",
+ execution_id=execution_id,
+ organization_id=self.organization_id,
),
)
@@ -117,7 +119,10 @@ class ToolsUtils:
LogPublisher.publish(
self.messaging_channel,
LogPublisher.log_workflow(
- "BUILD", f"Building the tool {tool_uid} now..."
+ "BUILD",
+ f"Building the tool {tool_uid} now...",
+ execution_id=execution_id,
+ organization_id=self.organization_id,
),
)
tool_sandbox = ToolSandbox(
diff --git a/unstract/workflow-execution/src/unstract/workflow_execution/workflow_execution.py b/unstract/workflow-execution/src/unstract/workflow_execution/workflow_execution.py
index 2c3448dd9..7e3d73273 100644
--- a/unstract/workflow-execution/src/unstract/workflow_execution/workflow_execution.py
+++ b/unstract/workflow-execution/src/unstract/workflow_execution/workflow_execution.py
@@ -375,6 +375,8 @@ class WorkflowExecutionService:
step=step,
iteration=iteration,
iteration_total=iteration_total,
+ execution_id=self.execution_id,
+ organization_id=self.organization_id,
)
LogPublisher.publish(self.messaging_channel, log_details)
diff --git a/worker/src/unstract/worker/worker.py b/worker/src/unstract/worker/worker.py
index 70ef1c183..b79f95913 100644
--- a/worker/src/unstract/worker/worker.py
+++ b/worker/src/unstract/worker/worker.py
@@ -2,6 +2,7 @@ import ast
import json
import os
import uuid
+from datetime import datetime, timezone
from typing import Any, Optional
from docker.errors import APIError
@@ -12,6 +13,7 @@ from unstract.worker.utils import Utils
import docker
from docker import DockerClient # type: ignore[attr-defined]
+from unstract.core.constants import LogFieldName
from unstract.core.pubsub_helper import LogPublisher
load_dotenv()
@@ -137,6 +139,8 @@ class UnstractWorker:
self,
container: Any,
tool_instance_id: str,
+ execution_id: str,
+ organization_id: str,
channel: Optional[str] = None,
) -> None:
for line in container.logs(stream=True, follow=True):
@@ -146,6 +150,8 @@ class UnstractWorker:
log_message=log_message,
tool_instance_id=tool_instance_id,
channel=channel,
+ execution_id=execution_id,
+ organization_id=organization_id,
)
def get_valid_log_message(self, log_message: str) -> Optional[dict[str, Any]]:
@@ -168,6 +174,8 @@ class UnstractWorker:
self,
log_message: str,
tool_instance_id: str,
+ execution_id: str,
+ organization_id: str,
channel: Optional[str] = None,
) -> Optional[dict[str, Any]]:
log_dict = self.get_valid_log_message(log_message)
@@ -176,7 +184,7 @@ class UnstractWorker:
log_type = self.get_log_type(log_dict)
if not self.is_valid_log_type(log_type):
self.logger.warning(
- f"Received invalid logType: {log_type} with log message: " f"{log_dict}"
+ f"Received invalid logType: {log_type} with log message: {log_dict}"
)
return None
if log_type == LogType.RESULT:
@@ -184,6 +192,10 @@ class UnstractWorker:
if log_type == LogType.UPDATE:
log_dict["component"] = tool_instance_id
if channel:
+ log_dict[LogFieldName.EXECUTION_ID] = execution_id
+ log_dict[LogFieldName.ORGANIZATION_ID] = organization_id
+ log_dict[LogFieldName.TIMESTAMP] = datetime.now(timezone.utc).timestamp()
+
# Publish to channel of socket io
LogPublisher.publish(channel, log_dict)
return None
@@ -366,6 +378,8 @@ class UnstractWorker:
container=container,
tool_instance_id=tool_instance_id,
channel=self.messaging_channel,
+ execution_id=execution_id,
+ organization_id=organization_id,
)
except Exception as e:
self.logger.error(f"Failed to run docker container: {e}")