LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
persisting execution logs (#276)
ali640118a03678
diff --git a/backend/README.md b/backend/README.mdindex 1185bf14a..40ebdf467 100644--- a/backend/README.md+++ b/backend/README.md@@ -65,8 +65,10 @@ To update the username or password after it's been set:Run the following command to start the worker:```bash-celery -A backend worker --loglevel=info+celery -A backend worker --loglevel=info -Q celery,celery_periodic_logs```+- The `celery` queue is used for default Celery tasks.+- The `celery_periodic_logs` queue is utilized for logging history tasks.### Worker Dashboarddiff --git a/backend/backend/celery.py b/backend/backend/celery.pyindex 0ff550bb0..57f387391 100644--- a/backend/backend/celery.py+++ b/backend/backend/celery.py@@ -4,6 +4,7 @@ import osfrom celery import Celeryfrom django.conf import settings+from utils.constants import ExecutionLogConstants# Set the default Django settings module for the 'celery' program.os.environ.setdefault(@@ -24,5 +25,13 @@ app.config_from_object("django.conf:settings", namespace="CELERY")# Autodiscover tasks in all installed apps.app.autodiscover_tasks()+# Define the queues to purge when the Celery broker is restarted.+queues_to_purge = [ExecutionLogConstants.CELERY_QUEUE_NAME]+with app.connection() as connection:+ channel = connection.channel()++ for queue_name in queues_to_purge:+ channel.queue_purge(queue_name)+# Use the Django-Celery-Beat scheduler.app.conf.beat_scheduler = "django_celery_beat.schedulers:DatabaseScheduler"diff --git a/backend/backend/settings/base.py b/backend/backend/settings/base.pyindex 4b5bb2990..ab8d0dfce 100644--- a/backend/backend/settings/base.py+++ b/backend/backend/settings/base.py@@ -155,6 +155,11 @@ SYSTEM_ADMIN_USERNAME = get_required_setting("SYSTEM_ADMIN_USERNAME")SYSTEM_ADMIN_PASSWORD = get_required_setting("SYSTEM_ADMIN_PASSWORD")SYSTEM_ADMIN_EMAIL = get_required_setting("SYSTEM_ADMIN_EMAIL")SESSION_COOKIE_AGE = int(get_required_setting("SESSION_COOKIE_AGE", "86400"))+ENABLE_LOG_HISTORY = get_required_setting("ENABLE_LOG_HISTORY")+LOG_HISTORY_CONSUMER_INTERVAL = int(+ get_required_setting("LOG_HISTORY_CONSUMER_INTERVAL", "60")+)+LOGS_BATCH_LIMIT = int(get_required_setting("LOGS_BATCH_LIMIT", "30"))# Flag to Enable django adminADMIN_ENABLED = False@@ -317,7 +322,7 @@ CACHES = {"USERNAME": REDIS_USER,"PASSWORD": REDIS_PASSWORD,},- "KEY_FUNCTION": "utils.cache_service.custom_key_function",+ "KEY_FUNCTION": "utils.redis_cache.custom_key_function",}}diff --git a/backend/pipeline/constants.py b/backend/pipeline/constants.pyindex 970a3f277..cbb78c79a 100644--- a/backend/pipeline/constants.py+++ b/backend/pipeline/constants.py@@ -61,5 +61,6 @@ class PipelineURL:"""Constants for URL names."""DETAIL = "pipeline-detail"+ EXECUTIONS = "pipeline-executions"LIST = "pipeline-list"EXECUTE = "pipeline-execute"diff --git a/backend/pipeline/execution_view.py b/backend/pipeline/execution_view.pynew file mode 100644index 000000000..45f83f438--- /dev/null+++ b/backend/pipeline/execution_view.py@@ -0,0 +1,35 @@+from permissions.permission import IsOwner+from pipeline.serializers.execute import DateRangeSerializer+from rest_framework import viewsets+from rest_framework.versioning import URLPathVersioning+from utils.pagination import CustomPagination+from workflow_manager.workflow.models.execution import WorkflowExecution+from workflow_manager.workflow.serializers import WorkflowExecutionSerializer+++class PipelineExecutionViewSet(viewsets.ModelViewSet):+ versioning_class = URLPathVersioning+ permission_classes = [IsOwner]+ serializer_class = WorkflowExecutionSerializer+ pagination_class = CustomPagination++ CREATED_AT_FIELD_DESC = "-created_at"+ START_DATE_FIELD = "start_date"+ END_DATE_FIELD = "end_date"++ def get_queryset(self):+ # Get the pipeline_id from the URL path+ pipeline_id = self.kwargs.get("pk")+ queryset = WorkflowExecution.objects.filter(pipeline_id=pipeline_id)++ # Validate start_date and end_date parameters using DateRangeSerializer+ date_range_serializer = DateRangeSerializer(data=self.request.query_params)+ date_range_serializer.is_valid(raise_exception=True)+ start_date = date_range_serializer.validated_data.get(self.START_DATE_FIELD)+ end_date = date_range_serializer.validated_data.get(self.END_DATE_FIELD)++ if start_date and end_date:+ queryset = queryset.filter(created_at__range=(start_date, end_date))++ queryset = queryset.order_by(self.CREATED_AT_FIELD_DESC)+ return querysetdiff --git a/backend/pipeline/serializers/execute.py b/backend/pipeline/serializers/execute.pyindex 88b57e199..cc52e3cfd 100644--- a/backend/pipeline/serializers/execute.py+++ b/backend/pipeline/serializers/execute.py@@ -17,3 +17,8 @@ class PipelineExecuteSerializer(serializers.Serializer):except Pipeline.DoesNotExist:raise serializers.ValidationError("Invalid pipeline ID")return value+++class DateRangeSerializer(serializers.Serializer):+ start_date = serializers.DateTimeField(required=False)+ end_date = serializers.DateTimeField(required=False)diff --git a/backend/pipeline/urls.py b/backend/pipeline/urls.pyindex 7af3dfcac..d03ed2587 100644--- a/backend/pipeline/urls.py+++ b/backend/pipeline/urls.py@@ -1,5 +1,6 @@from django.urls import pathfrom pipeline.constants import PipelineURL+from pipeline.execution_view import PipelineExecutionViewSetfrom pipeline.views import PipelineViewSetfrom rest_framework.urlpatterns import format_suffix_patterns@@ -9,6 +10,11 @@ pipeline_list = PipelineViewSet.as_view("post": "create",})+execution_list = PipelineExecutionViewSet.as_view(+ {+ "get": "list",+ }+)pipeline_detail = PipelineViewSet.as_view({"get": "retrieve",@@ -25,6 +31,11 @@ urlpatterns = format_suffix_patterns([path("pipeline/", pipeline_list, name=PipelineURL.LIST),path("pipeline/<uuid:pk>/", pipeline_detail, name=PipelineURL.DETAIL),+ path(+ "pipeline/<uuid:pk>/executions/",+ execution_list,+ name=PipelineURL.EXECUTIONS,+ ),path("pipeline/execute/", pipeline_execute, name=PipelineURL.EXECUTE),])diff --git a/backend/pyproject.toml b/backend/pyproject.tomlindex be3c1d781..f7349c512 100644--- a/backend/pyproject.toml+++ b/backend/pyproject.toml@@ -78,7 +78,7 @@ migrate_db.cmd = "python manage.py migrate"migrate_db.env_file = ".env"migrate_db.help = "Performs DB migrations for Unstract backend"# Commands for backend execution-consumer.cmd = "celery -A backend worker --loglevel=info"+consumer.cmd = "celery -A backend worker --loglevel=info -Q celery,celery_periodic_logs"consumer.env_file = ".env"consumer.help = "Runs the Unstract consumer"# Celery Flowerdiff --git a/backend/sample.env b/backend/sample.envindex c0f08baf9..9f3441568 100644--- a/backend/sample.env+++ b/backend/sample.env@@ -130,3 +130,10 @@ ADAPTER_LLMW_POLL_INTERVAL=30# Total number of times to poll the status API.# 500 mins to allow 1500 (max pages limit) * 20 (approx time in sec to process a page)ADAPTER_LLMW_MAX_POLLS=1000++# Enable logging of workflow history.+ENABLE_LOG_HISTORY=True+# Interval in seconds for periodic consumer operations.+LOG_HISTORY_CONSUMER_INTERVAL=30+# Maximum number of logs to insert in a single batch.+LOGS_BATCH_LIMIT=30diff --git a/backend/utils/cache_service.py b/backend/utils/cache_service.pyindex 83084e505..691b8ee2a 100644--- a/backend/utils/cache_service.py+++ b/backend/utils/cache_service.py@@ -1,8 +1,10 @@from typing import Any, Optional-from account.custom_cache import CustomCachefrom django.conf import settingsfrom django.core.cache import cache+from django_redis import get_redis_connection++redis_cache = get_redis_connection("default")class CacheService:@@ -56,16 +58,20 @@ class CacheService:return cache.delete(key)@staticmethod- def add_cookie_id_to_user(user_id: str, cookie_id: str) -> None:- custom_cache = CustomCache()- key: str = f"{user_id}|cookies"- custom_cache.rpush(key, cookie_id)+ def rpush(key: str, value: str) -> None:+ redis_cache.rpush(key, value)++ @staticmethod+ def lpop(key: str) -> Any:+ return redis_cache.lpop(key)++ @staticmethod+ def lrem(key: str, value: str) -> None:+ redis_cache.lrem(key, value)@staticmethod- def remove_cookie_id_from_user(user_id: str, cookie_id: str) -> None:- custom_cache = CustomCache()- key: str = f"{user_id}|cookies"- custom_cache.lrem(key, cookie_id)+ def lrange(key, start_index, end_index) -> list[Any]:+ return redis_cache.lrange(key, start_index, end_index)@staticmethoddef remove_all_session_keys(diff --git a/backend/utils/constants.py b/backend/utils/constants.pyindex d12cbf6d8..0ca37d3c4 100644--- a/backend/utils/constants.py+++ b/backend/utils/constants.py@@ -1,3 +1,14 @@+import os++from django.conf import settings+from utils.common_utils import CommonUtils++os.environ.setdefault(+ "DJANGO_SETTINGS_MODULE",+ os.environ.get("DJANGO_SETTINGS_MODULE", "backend.settings.dev"),+)++class Account:CREATED_BY = "created_by"MODIFIED_BY = "modified_by"@@ -7,3 +18,43 @@ class Common:METADATA = "metadata"MODULE = "module"CONNECTOR = "connector"+++class Pagination:+ """Constants for Pagination.++ Attributes:+ PAGE_SIZE (int): The default number of items per page.+ PAGE_SIZE_QUERY_PARAM (str): The name of the query parameter used to+ specify page size in requests. Ex: ?page=2&<PAGE_SIZE_QUERY_PARAM>=3+ MAX_PAGE_SIZE (int): The maximum allowed number of items per page.+ """++ PAGE_SIZE = 50+ PAGE_SIZE_QUERY_PARAM = "page_size"+ MAX_PAGE_SIZE = 1000+++class ExecutionLogConstants:+ """Constants for ExecutionLog.++ Attributes:+ IS_ENABLED (bool): Whether to enable log history.+ CONSUMER_INTERVAL (int): The interval (in seconds) between log history+ consumers.+ LOG_QUEUE_NAME (str): The name of the queue to store log history.+ LOGS_BATCH_LIMIT (str): The maximum number of logs to store in a batch.+ CELERY_QUEUE_NAME (str): The name of the Celery queue to schedule log+ history consumers.+ PERIODIC_TASK_NAME (str): The name of the Celery periodic task to schedule+ log history consumers.+ TASK (str): The name of the Celery task to schedule log history consumers.+ """++ IS_ENABLED: bool = CommonUtils.str_to_bool(settings.ENABLE_LOG_HISTORY)+ CONSUMER_INTERVAL: int = settings.LOG_HISTORY_CONSUMER_INTERVAL+ LOGS_BATCH_LIMIT: int = settings.LOGS_BATCH_LIMIT+ LOG_QUEUE_NAME: str = "log_history_queue"+ CELERY_QUEUE_NAME = "celery_periodic_logs"+ PERIODIC_TASK_NAME = "workflow_log_history"+ TASK = "workflow_manager.workflow.execution_log_utils.consume_log_history"diff --git a/backend/utils/dto.py b/backend/utils/dto.pynew file mode 100644index 000000000..445982d4c--- /dev/null+++ b/backend/utils/dto.py@@ -0,0 +1,64 @@+import json+import logging+from datetime import datetime+from typing import Any, Optional++from unstract.workflow_execution.enums import LogType++from unstract.core.constants import LogFieldName++logger = logging.getLogger(__name__)+++class LogDataDTO:+ """Log data DTO+ DTO for log data to store to QUEUE+ Attributes:+ execution_id: execution id+ organization_id: organization id+ timestamp: timestamp+ log_type: log type+ data: log data+ """++ def __init__(+ self,+ execution_id: str,+ organization_id: str,+ timestamp: int,+ log_type: str,+ data: dict[str, Any],+ ):+ self.execution_id: str = execution_id+ self.organization_id: str = organization_id+ self.timestamp: int = timestamp+ self.event_time: datetime = datetime.fromtimestamp(timestamp)+ self.log_type: LogType = log_type+ self.data: dict[str, Any] = data++ @classmethod+ def from_json(cls, json_data: str) -> Optional["LogDataDTO"]:+ try:+ json_data = json.loads(json_data)+ execution_id = json_data.get(LogFieldName.EXECUTION_ID)+ organization_id = json_data.get(LogFieldName.ORGANIZATION_ID)+ timestamp = json_data.get(LogFieldName.TIMESTAMP)+ log_type = json_data.get(LogFieldName.TYPE)+ data = json_data.get(LogFieldName.DATA)+ if all((execution_id, organization_id, timestamp, log_type, data)):+ return cls(execution_id, organization_id, timestamp, log_type, data)+ except (json.JSONDecodeError, AttributeError):+ logger.warning("Invalid log data: %s", json_data)+ return None++ def to_json(self) -> str:+ return json.dumps(+ {+ LogFieldName.EXECUTION_ID: self.execution_id,+ LogFieldName.ORGANIZATION_ID: self.organization_id,+ LogFieldName.TIMESTAMP: self.timestamp,+ LogFieldName.EVENT_TIME: self.event_time.isoformat(),+ LogFieldName.TYPE: self.log_type,+ LogFieldName.DATA: self.data,+ }+ )diff --git a/backend/utils/log_events.py b/backend/utils/log_events.pyindex d83cc03ee..c8ecfb989 100644--- a/backend/utils/log_events.py+++ b/backend/utils/log_events.py@@ -1,13 +1,19 @@+import jsonimport loggingimport osimport threadingimport time-from typing import Any+from typing import Any, Optionalimport redisimport socketiofrom django.conf import settingsfrom django.core.wsgi import WSGIHandler+from unstract.workflow_execution.enums import LogType+from utils.constants import ExecutionLogConstants+from utils.dto import LogDataDTO++from unstract.core.constants import LogFieldNamelogger = logging.getLogger(__name__)@@ -38,13 +44,105 @@ def disconnect(sid: str) -> None:logger.info(f"[{os.getpid()}] Client with SID:{sid} disconnected")-def _handle_pubsub_messages(message: Any) -> None:- channel = message["channel"].decode("utf-8")- data = message["data"]+def _get_validated_log_data(json_data: Any) -> Optional[LogDataDTO]:+ """Validate log data to persist history. This function takes log data in+ JSON format, validates it, and returns a `LogDataDTO` object if the data is+ valid. The validation process includes decoding bytes to string, parsing+ the string as JSON, and checking for required fields and log type.++ Args:+ json_data (Any): Log data in JSON format+ Returns:+ Optional[LogDataDTO]: Log data DTO object+ """+ if isinstance(json_data, bytes):+ json_data = json_data.decode("utf-8")++ if isinstance(json_data, str):+ try:+ # Parse the string as JSON+ json_data = json.loads(json_data)+ except json.JSONDecodeError:+ logger.error(f"Error decoding JSON data while validating {json_data}")+ return++ if not isinstance(json_data, dict):+ logger.warning(f"Getting invalid data type while validating {json_data}")+ return++ # Extract required fields from the JSON data+ execution_id = json_data.get(LogFieldName.EXECUTION_ID)+ organization_id = json_data.get(LogFieldName.ORGANIZATION_ID)+ timestamp = json_data.get(LogFieldName.TIMESTAMP)+ log_type = json_data.get(LogFieldName.TYPE)++ # Ensure the log type is LogType.LOG+ if log_type != LogType.LOG.value:+ return++ # Check if all required fields are present+ if not all((execution_id, organization_id, timestamp)):+ logger.warning(f"Missing required fields while validating {json_data}")+ return++ return LogDataDTO(+ execution_id=execution_id,+ organization_id=organization_id,+ timestamp=timestamp,+ log_type=log_type,+ data=json_data,+ )+++def _store_execution_log(data: bytes) -> None:+ """Store execution log in database+ Args:+ data (bytes): Execution log data in bytes format+ """+ if not ExecutionLogConstants.IS_ENABLED:+ return+ try:+ log_data = _get_validated_log_data(json_data=data)+ if log_data:+ redis_conn.rpush(ExecutionLogConstants.LOG_QUEUE_NAME, log_data.to_json())+ except Exception as e:+ logger.error(f"Error storing execution log: {e}")+++def _emit_websocket_event(channel: str, data: bytes) -> None:+ """Emit websocket event+ Args:+ channel (str): WebSocket channel+ data (bytes): Execution log data in bytes format+ """payload = {"data": data}+ try:+ logger.debug(f"[{os.getpid()}] Push websocket event: {channel}, {payload}")+ sio.emit(channel, payload)+ except Exception as e:+ logger.error(f"Error emitting WebSocket event: {e}")+++def _handle_pubsub_messages(message: dict[str, Any]) -> None:+ """Handle pubsub messages+ Args:+ message (dict[str, Any]): Pub sub message+ """+ channel = message.get("channel")+ data = message.get("data")++ if not channel or not data:+ logger.warning(f"Invalid message received: {message}")+ return++ try:+ channel_str = channel.decode("utf-8")+ except UnicodeDecodeError as e:+ logger.error(f"Error decoding channel: {e}")+ return- logger.debug(f"[{os.getpid()}] Push websocket event: {channel}, {payload}")- sio.emit(channel, {"data": data})+ _store_execution_log(data)+ _emit_websocket_event(channel_str, data)def _pubsub_listen_forever() -> None:diff --git a/backend/utils/pagination.py b/backend/utils/pagination.pynew file mode 100644index 000000000..c4a30b60e--- /dev/null+++ b/backend/utils/pagination.py@@ -0,0 +1,8 @@+from rest_framework.pagination import PageNumberPagination+from utils.constants import Pagination+++class CustomPagination(PageNumberPagination):+ page_size = Pagination.PAGE_SIZE+ page_size_query_param = Pagination.PAGE_SIZE_QUERY_PARAM+ max_page_size = Pagination.MAX_PAGE_SIZEdiff --git a/backend/utils/redis_cache.py b/backend/utils/redis_cache.pynew file mode 100644index 000000000..513d08edb--- /dev/null+++ b/backend/utils/redis_cache.py@@ -0,0 +1,9 @@+# KEY_FUNCTION for cache settings+def custom_key_function(key: str, key_prefix: str, version: int) -> str:+ version = int(version)+ if version > 1:+ return f"{key_prefix}:{version}:{key}"+ if key_prefix:+ return f"{key_prefix}:{key}"+ else:+ return keydiff --git a/backend/workflow_manager/workflow/apps.py b/backend/workflow_manager/workflow/apps.pyindex 71e894363..3c0d821d7 100644--- a/backend/workflow_manager/workflow/apps.py+++ b/backend/workflow_manager/workflow/apps.py@@ -4,3 +4,10 @@ from django.apps import AppConfigclass WorkflowConfig(AppConfig):default_auto_field = "django.db.models.BigAutoField"name = "workflow_manager.workflow"++ def ready(self):+ from workflow_manager.workflow.execution_log_utils import (+ create_log_consumer_scheduler_if_not_exists,+ )++ create_log_consumer_scheduler_if_not_exists()diff --git a/backend/workflow_manager/workflow/execution.py b/backend/workflow_manager/workflow/execution.pyindex acfff7fee..b332c9ecb 100644--- a/backend/workflow_manager/workflow/execution.py+++ b/backend/workflow_manager/workflow/execution.py@@ -145,7 +145,7 @@ class WorkflowExecutionServiceHelper(WorkflowExecutionService):workflow_execution.save()return workflow_execution- def __update_execution(+ def update_execution(self,status: Optional[ExecutionStatus] = None,execution_time: Optional[float] = None,@@ -171,11 +171,6 @@ class WorkflowExecutionServiceHelper(WorkflowExecutionService):detailed_message: str,execution_time: Optional[float] = None,) -> WorkflowExecutionError:- self.__update_execution(- status=ExecutionStatus.ERROR,- error=detailed_message,- execution_time=execution_time,- )return WorkflowExecutionError(detail=error_message)def has_successful_compilation(self) -> bool:@@ -190,12 +185,16 @@ class WorkflowExecutionServiceHelper(WorkflowExecutionService):def build(self) -> None:if self.compilation_result["success"] is True:self.build_workflow()- self.__update_execution(status=ExecutionStatus.READY)+ self.update_execution(status=ExecutionStatus.READY)else:logger.error("Errors while compiling workflow "f"{self.compilation_result['problems']}")+ self.update_execution(+ status=ExecutionStatus.ERROR,+ error=self.compilation_result["problems"][0],+ )raise self.__execution_error(error_message=self.compilation_result["problems"][0],detailed_message=self.compilation_result["problems"][0],@@ -207,29 +206,18 @@ class WorkflowExecutionServiceHelper(WorkflowExecutionService):execution_type = ExecutionType.STEPif self.compilation_result["success"] is True:- self.__update_execution(status=ExecutionStatus.READY)if (self.execution_mode == WorkflowExecution.Mode.INSTANTor self.execution_mode == WorkflowExecution.Mode.QUEUE):start_time = time.time()try:- self.__update_execution(increment_attempt=True)self.execute_workflow(execution_type=execution_type)end_time = time.time()execution_time = end_time - start_time- self.__update_execution(- status=ExecutionStatus.COMPLETED,- execution_time=execution_time,- )except StopExecution as exception:end_time = time.time()execution_time = end_time - start_time- self.__update_execution(- status=ExecutionStatus.STOPPED,- error=str(exception),- execution_time=execution_time,- )logger.info(f"Execution {self.execution_id} stopped")raise exceptionexcept Exception as exception:@@ -269,7 +257,9 @@ class WorkflowExecutionServiceHelper(WorkflowExecutionService):LogState.RUNNING, "Ready for execution", LogComponent.WORKFLOW)- def publish_final_workflow_logs(self) -> None:+ def publish_final_workflow_logs(+ self, total_files: int, processed_files: int+ ) -> None:"""Publishes the final logs for the workflow.Returns:@@ -279,6 +269,10 @@ class WorkflowExecutionServiceHelper(WorkflowExecutionService):self.publish_update_log(LogState.SUCCESS, "Executed successfully", LogComponent.WORKFLOW)+ self.publish_log(+ f"Execution completed successfully for the files {processed_files} "+ f"out of {total_files}"+ )def publish_initial_tool_execution_logs(self, current_step: int, total_step: int, file_name: str@@ -305,7 +299,7 @@ class WorkflowExecutionServiceHelper(WorkflowExecutionService):file_name: str,single_step: bool,file_history: Optional[FileHistory] = None,- ) -> None:+ ) -> bool:"""Executes the input file.Args:@@ -314,22 +308,24 @@ class WorkflowExecutionServiceHelper(WorkflowExecutionService):single step mode.file_history (Optional[FileHistory], optional):The file history object. Defaults to None.-Returns:- None+ bool: Flag indicating whether the file was executed."""execution_type = ExecutionType.COMPLETE+ is_executed = Falseif single_step:execution_type = ExecutionType.STEPif not (file_history and file_history.is_completed()):self.execute_uncached_input(file_name=file_name, single_step=single_step)+ self.publish_log(f"Tool executed successfully for {file_name}")+ is_executed = Trueelse:self.publish_log(f"Skipping file {file_name} as it is already processed.""Clear the cache to process it again")- self.publish_log(f"Tool executed successfully for {file_name}")self._handle_execution_type(execution_type)+ return is_executeddef execute_uncached_input(self, file_name: str, single_step: bool) -> None:"""Executes the uncached input file.diff --git a/backend/workflow_manager/workflow/execution_log_utils.py b/backend/workflow_manager/workflow/execution_log_utils.pynew file mode 100644index 000000000..6c2b7bb14--- /dev/null+++ b/backend/workflow_manager/workflow/execution_log_utils.py@@ -0,0 +1,106 @@+import logging+from collections import defaultdict++from account.models import Organization+from celery import shared_task+from django.db import IntegrityError+from django_celery_beat.models import IntervalSchedule, PeriodicTask+from django_tenants.utils import get_tenant_model, tenant_context+from utils.cache_service import CacheService+from utils.constants import ExecutionLogConstants+from utils.dto import LogDataDTO+from workflow_manager.workflow.models.execution_log import ExecutionLog++logger = logging.getLogger(__name__)+++@shared_task(bind=True)+def consume_log_history(self):+ organization_logs = defaultdict(list)+ logs_count = 0++ while logs_count < ExecutionLogConstants.LOGS_BATCH_LIMIT:+ log = CacheService.lpop(ExecutionLogConstants.LOG_QUEUE_NAME)+ if not log:+ break++ log_data = LogDataDTO.from_json(log)+ if not log_data:+ continue++ organization_id = log_data.organization_id+ organization_logs[organization_id].append(+ ExecutionLog(+ execution_id=log_data.execution_id,+ data=log_data.data,+ event_time=log_data.event_time,+ )+ )+ logs_count += 1+ logger.info(f"Logs count: {logs_count}")+ for organization_id, logs in organization_logs.items():+ store_to_db(organization_id, logs)+++def create_log_consumer_scheduler_if_not_exists():+ try:+ interval, _ = IntervalSchedule.objects.get_or_create(+ every=ExecutionLogConstants.CONSUMER_INTERVAL,+ period=IntervalSchedule.SECONDS,+ )+ except IntervalSchedule.MultipleObjectsReturned as error:+ logger.error(f"Error occurred while getting interval schedule: {error}")+ interval = IntervalSchedule.objects.filter(+ every=ExecutionLogConstants.CONSUMER_INTERVAL,+ period=IntervalSchedule.SECONDS,+ ).first()+ try:+ # Create the scheduler+ task, created = PeriodicTask.objects.get_or_create(+ name=ExecutionLogConstants.PERIODIC_TASK_NAME,+ task=ExecutionLogConstants.TASK,+ defaults={+ "interval": interval,+ "queue": ExecutionLogConstants.CELERY_QUEUE_NAME,+ "enabled": ExecutionLogConstants.IS_ENABLED,+ },+ )+ if not created:+ task.enabled = ExecutionLogConstants.IS_ENABLED+ task.interval = interval+ task.queue = ExecutionLogConstants.CELERY_QUEUE_NAME+ task.save()+ logger.info("Log consumer scheduler updated successfully.")+ else:+ logger.info("Log consumer scheduler created successfully.")+ except IntegrityError as error:+ logger.error(f"Error occurred while creating log consumer scheduler: {error}")+++def store_to_db(organization_id: str, execution_logs: list[ExecutionLog]) -> None:+ try:+ tenant: Organization = get_tenant_model().objects.get(+ schema_name=organization_id+ )+ except Organization.DoesNotExist:+ logger.error(f"Organization with ID {organization_id} does not exist.")+ return++ # Store the log data in the database within tenant context+ with tenant_context(tenant):+ ExecutionLog.objects.bulk_create(objs=execution_logs, ignore_conflicts=True)+++class ExecutionLogUtils:++ @staticmethod+ def get_execution_logs_by_execution_id(execution_id) -> list[ExecutionLog]:+ """Get all execution logs for a given execution ID.++ Args:+ execution_id (int): The ID of the execution.++ Returns:+ list[ExecutionLog]: A list of ExecutionLog objects.+ """+ return ExecutionLog.objects.filter(execution_id=execution_id)diff --git a/backend/workflow_manager/workflow/execution_log_view.py b/backend/workflow_manager/workflow/execution_log_view.pynew file mode 100644index 000000000..137349e72--- /dev/null+++ b/backend/workflow_manager/workflow/execution_log_view.py@@ -0,0 +1,27 @@+import logging++from permissions.permission import IsOwner+from rest_framework import viewsets+from rest_framework.versioning import URLPathVersioning+from utils.pagination import CustomPagination+from workflow_manager.workflow.models.execution_log import ExecutionLog+from workflow_manager.workflow.serializers import WorkflowExecutionLogSerializer++logger = logging.getLogger(__name__)+++class WorkflowExecutionLogViewSet(viewsets.ModelViewSet):+ versioning_class = URLPathVersioning+ permission_classes = [IsOwner]+ serializer_class = WorkflowExecutionLogSerializer+ pagination_class = CustomPagination++ EVENT_TIME_FELID_ASC = "event_time"++ def get_queryset(self):+ # Get the execution_id:pk from the URL path+ execution_id = self.kwargs.get("pk")+ queryset = ExecutionLog.objects.filter(execution_id=execution_id).order_by(+ self.EVENT_TIME_FELID_ASC+ )+ return querysetdiff --git a/backend/workflow_manager/workflow/execution_view.py b/backend/workflow_manager/workflow/execution_view.pynew file mode 100644index 000000000..7c5866d0c--- /dev/null+++ b/backend/workflow_manager/workflow/execution_view.py@@ -0,0 +1,25 @@+import logging++from permissions.permission import IsOwner+from rest_framework import viewsets+from rest_framework.versioning import URLPathVersioning+from workflow_manager.workflow.models.execution import WorkflowExecution+from workflow_manager.workflow.serializers import WorkflowExecutionSerializer++logger = logging.getLogger(__name__)+++class WorkflowExecutionViewSet(viewsets.ModelViewSet):+ versioning_class = URLPathVersioning+ permission_classes = [IsOwner]+ serializer_class = WorkflowExecutionSerializer++ CREATED_AT_FIELD_DESC = "-created_at"++ def get_queryset(self):+ # Get the uuid:pk from the URL path+ workflow_id = self.kwargs.get("pk")+ queryset = WorkflowExecution.objects.filter(workflow_id=workflow_id).order_by(+ self.CREATED_AT_FIELD_DESC+ )+ return querysetdiff --git a/backend/workflow_manager/workflow/migrations/0004_executionlog.py b/backend/workflow_manager/workflow/migrations/0004_executionlog.pynew file mode 100644index 000000000..63e25da3c--- /dev/null+++ b/backend/workflow_manager/workflow/migrations/0004_executionlog.py@@ -0,0 +1,40 @@+# Generated by Django 4.2.1 on 2024-04-22 06:10++import uuid++from django.db import migrations, models+++class Migration(migrations.Migration):++ dependencies = [+ ("workflow", "0003_workflowexecution_execution_log_id_and_more"),+ ]++ operations = [+ migrations.CreateModel(+ name="ExecutionLog",+ fields=[+ ("created_at", models.DateTimeField(auto_now_add=True)),+ ("modified_at", models.DateTimeField(auto_now=True)),+ (+ "id",+ models.UUIDField(+ default=uuid.uuid4,+ editable=False,+ primary_key=True,+ serialize=False,+ ),+ ),+ (+ "execution_id",+ models.UUIDField(db_comment="Execution ID", editable=False),+ ),+ ("data", models.JSONField()),+ ("event_time", models.DateTimeField()),+ ],+ options={+ "abstract": False,+ },+ ),+ ]diff --git a/backend/workflow_manager/workflow/models/__init__.py b/backend/workflow_manager/workflow/models/__init__.pyindex 57b3df1a1..938907762 100644--- a/backend/workflow_manager/workflow/models/__init__.py+++ b/backend/workflow_manager/workflow/models/__init__.py@@ -1,3 +1,4 @@from .execution import WorkflowExecution # noqa: F401+from .execution_log import ExecutionLog # noqa: F401from .file_history import FileHistory # noqa: F401from .workflow import Workflow # noqa: F401diff --git a/backend/workflow_manager/workflow/models/execution_log.py b/backend/workflow_manager/workflow/models/execution_log.pynew file mode 100644index 000000000..d3b2c5ae7--- /dev/null+++ b/backend/workflow_manager/workflow/models/execution_log.py@@ -0,0 +1,17 @@+import uuid++from django.db import models+from utils.models.base_model import BaseModel+++class ExecutionLog(BaseModel):+ id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)+ execution_id = models.UUIDField(+ editable=False,+ db_comment="Execution ID",+ )+ data = models.JSONField(db_comment="Execution log data")+ event_time = models.DateTimeField(db_comment="Execution log event time")++ def __str__(self):+ return f"Execution ID: {self.execution_id}, Message: {self.data}"diff --git a/backend/workflow_manager/workflow/serializers.py b/backend/workflow_manager/workflow/serializers.pyindex 180884887..04dfcdac4 100644--- a/backend/workflow_manager/workflow/serializers.py+++ b/backend/workflow_manager/workflow/serializers.py@@ -17,6 +17,8 @@ from workflow_manager.endpoint.models import WorkflowEndpointfrom workflow_manager.workflow.constants import WorkflowExecutionKey, WorkflowKeyfrom workflow_manager.workflow.exceptions import WorkflowGenerationErrorfrom workflow_manager.workflow.generator import WorkflowGenerator+from workflow_manager.workflow.models.execution import WorkflowExecution+from workflow_manager.workflow.models.execution_log import ExecutionLogfrom workflow_manager.workflow.models.workflow import Workflowfrom backend.constants import RequestKey@@ -149,3 +151,15 @@ class WorkflowEndpointSerializer(ModelSerializer):class Meta:model = WorkflowEndpointfields = "__all__"+++class WorkflowExecutionSerializer(ModelSerializer):+ class Meta:+ model = WorkflowExecution+ fields = "__all__"+++class WorkflowExecutionLogSerializer(ModelSerializer):+ class Meta:+ model = ExecutionLog+ fields = "__all__"diff --git a/backend/workflow_manager/workflow/urls.py b/backend/workflow_manager/workflow/urls.pyindex 144ad09b7..d75b5fc43 100644--- a/backend/workflow_manager/workflow/urls.py+++ b/backend/workflow_manager/workflow/urls.py@@ -1,5 +1,7 @@from django.urls import pathfrom rest_framework.urlpatterns import format_suffix_patterns+from workflow_manager.workflow.execution_log_view import WorkflowExecutionLogViewSet+from workflow_manager.workflow.execution_view import WorkflowExecutionViewSetfrom workflow_manager.workflow.views import WorkflowViewSetworkflow_list = WorkflowViewSet.as_view(@@ -19,7 +21,9 @@ workflow_detail = WorkflowViewSet.as_view(# fmt: on)workflow_execute = WorkflowViewSet.as_view({"post": "execute", "put": "activate"})-execution_entity = WorkflowViewSet.as_view({"get": "get_execution"})+execution_entity = WorkflowExecutionViewSet.as_view({"get": "retrieve"})+execution_list = WorkflowExecutionViewSet.as_view({"get": "list"})+execution_log_list = WorkflowExecutionLogViewSet.as_view({"get": "list"})workflow_clear_cache = WorkflowViewSet.as_view({"get": "clear_cache"})workflow_clear_file_marker = WorkflowViewSet.as_view({"get": "clear_file_marker"})workflow_schema = WorkflowViewSet.as_view({"get": "get_schema"})@@ -49,11 +53,21 @@ urlpatterns = format_suffix_patterns(workflow_execute,name="active-workflow",),+ path(+ "<uuid:pk>/execution/",+ execution_list,+ name="execution-list",+ ),path("execution/<uuid:pk>/",execution_entity,name="workflow-detail",),+ path(+ "execution/<uuid:pk>/logs/",+ execution_log_list,+ name="execution-log",+ ),path("schema/",workflow_schema,diff --git a/backend/workflow_manager/workflow/workflow_helper.py b/backend/workflow_manager/workflow/workflow_helper.pyindex db8a73d35..c74f2ccb2 100644--- a/backend/workflow_manager/workflow/workflow_helper.py+++ b/backend/workflow_manager/workflow/workflow_helper.py@@ -18,7 +18,7 @@ from rest_framework import serializersfrom tool_instance.constants import ToolInstanceKeyfrom tool_instance.models import ToolInstancefrom tool_instance.tool_instance_helper import ToolInstanceHelper-from unstract.workflow_execution.enums import LogComponent, LogState+from unstract.workflow_execution.enums import LogComponent, LogLevel, LogStatefrom unstract.workflow_execution.exceptions import StopExecutionfrom utils.cache_service import CacheServicefrom workflow_manager.endpoint.destination import DestinationConnector@@ -112,11 +112,16 @@ class WorkflowHelper:) -> WorkflowExecution:input_files = source.list_files_from_source()total_files = len(input_files)+ processed_files = 0+ error_raised = 0execution_service.publish_initial_workflow_logs(total_files)+ execution_service.update_execution(+ ExecutionStatus.EXECUTING, increment_attempt=True+ )for index, input_file in enumerate(input_files):file_number = index + 1try:- WorkflowHelper.process_file(+ is_executed, error = WorkflowHelper.process_file(current_step=file_number,total_step=total_files,input_file=input_file,@@ -127,9 +132,23 @@ class WorkflowHelper:single_step=single_step,hash_values_of_files=hash_values_of_files,)- except StopExecution:+ if is_executed:+ processed_files += 1+ if error:+ error_raised += 1+ except StopExecution as exception:+ execution_service.update_execution(+ ExecutionStatus.STOPPED, error=str(exception)+ )break- execution_service.publish_final_workflow_logs()+ if error_raised and error_raised == total_files:+ execution_service.update_execution(ExecutionStatus.ERROR)+ else:+ execution_service.update_execution(ExecutionStatus.COMPLETED)++ execution_service.publish_final_workflow_logs(+ total_files=total_files, processed_files=processed_files+ )return execution_service.get_execution_instance()@staticmethod@@ -143,9 +162,10 @@ class WorkflowHelper:execution_service: WorkflowExecutionServiceHelper,single_step: bool,hash_values_of_files: dict[str, str],- ) -> None:+ ) -> tuple[bool, Optional[str]]:file_history = Noneerror = None+ is_executed = Falsefile_name, file_hash = source.add_file_to_volume(input_file_path=input_file,hash_values_of_files=hash_values_of_files,@@ -157,7 +177,7 @@ class WorkflowHelper:file_history = FileHistoryHelper.get_file_history(workflow=workflow, cache_key=file_hash)- execution_service.execute_input_file(+ is_executed = execution_service.execute_input_file(file_name=file_name,single_step=single_step,file_history=file_history,@@ -166,7 +186,8 @@ class WorkflowHelper:raiseexcept Exception as e:execution_service.publish_log(- f"Error processing file {input_file}: {str(e)}"+ f"Error processing file {input_file}: {str(e)}",+ level=LogLevel.ERROR,)error = str(e)execution_service.publish_update_log(@@ -186,6 +207,7 @@ class WorkflowHelper:f"{file_name} is Processed successfully",LogComponent.DESTINATION,)+ return is_executed, error@staticmethoddef validate_tool_instances_meta(diff --git a/frontend/src/components/pipelines-or-deployments/log-modal/LogsModal.jsx b/frontend/src/components/pipelines-or-deployments/log-modal/LogsModal.jsxnew file mode 100644index 000000000..f8d520fbb--- /dev/null+++ b/frontend/src/components/pipelines-or-deployments/log-modal/LogsModal.jsx@@ -0,0 +1,175 @@+import { Table, Modal, Button } from "antd";+import PropTypes from "prop-types";+import { useState } from "react";++import { useSessionStore } from "../../../store/session-store.js";+import { useAxiosPrivate } from "../../../hooks/useAxiosPrivate.js";+import { useAlertStore } from "../../../store/alert-store.js";+import { useExceptionHandler } from "../../../hooks/useExceptionHandler.jsx";+import "./LogsModel.css";++const LogsModal = ({+ open,+ setOpen,+ logRecord,+ totalLogs,+ fetchExecutionLogs,+}) => {+ const [selectedLogId, setSelectedLogId] = useState(null);+ const [logDescModalOpen, setLogDescModalOpen] = useState(false);+ const [logDetails, setLogDetails] = useState([]);+ const { sessionDetails } = useSessionStore();+ const { setAlertDetails } = useAlertStore();+ const axiosPrivate = useAxiosPrivate();+ const handleException = useExceptionHandler();+ const [totalCount, setTotalCount] = useState(0);++ const fetchLogDetails = async (logId, page = 1, pageSize = 10) => {+ const requestOptions = {+ method: "GET",+ url: `/api/v1/unstract/${sessionDetails?.orgId}/workflow/execution/${logId}/logs/`,+ headers: {+ "X-CSRFToken": sessionDetails?.csrfToken,+ },+ params: {+ page: page,+ page_size: pageSize,+ },+ };+ axiosPrivate(requestOptions)+ .then((res) => {+ const logDetails = res.data.results.map((item) => ({+ id: item.id,+ log: item.data?.log,+ type: item.data?.type,+ stage: item.data?.stage,+ level: item.data?.level,+ event_time: item.event_time,+ }));+ setLogDetails(logDetails);+ setTotalCount(res.data.count);+ })+ .catch((err) => {+ setAlertDetails(handleException(err));+ });+ };++ const handleLogIdClick = (logId) => {+ setSelectedLogId(logId);+ fetchLogDetails(logId);+ setLogDescModalOpen(true);+ };++ const logColumns = [+ {+ title: "Executed At",+ dataIndex: "created_at",+ key: "created_at",+ },+ {+ title: "Execution ID",+ dataIndex: "execution_id",+ key: "execution_id",+ render: (text, record) => {+ return (+ <Button+ type="link"+ onClick={() => handleLogIdClick(record.execution_id)}+ >+ {text}+ </Button>+ );+ },+ },+ {+ title: "Status",+ dataIndex: "status",+ key: "status",+ render: (level) => <span className={level.toLowerCase()}>{level}</span>,+ },+ ];++ const logDetailsColumns = [+ {+ title: "Event Time",+ dataIndex: "event_time",+ key: "event_time",+ },+ {+ title: "Event Stage",+ dataIndex: "stage",+ key: "stage",+ },+ {+ title: "Log Level",+ dataIndex: "level",+ key: "level",+ render: (level) => <span className={level.toLowerCase()}>{level}</span>,+ },+ {+ title: "Log",+ dataIndex: "log",+ key: "log",+ },+ ];+ return (+ <>+ <Modal+ title="Execution Logs"+ centered+ open={open}+ onCancel={() => setOpen(false)}+ footer={null}+ width="80%"+ >+ <Table+ columns={logColumns}+ dataSource={logRecord}+ rowKey="id"+ align="left"+ pagination={{+ pageSize: 10,+ total: totalLogs,+ onChange: (page, pageSize) => {+ fetchExecutionLogs(page, pageSize);+ },+ }}+ />+ </Modal>++ <Modal+ title={`Execution Log Details - ${selectedLogId}`}+ centered+ open={logDescModalOpen}+ onCancel={() => setLogDescModalOpen(false)}+ footer={null}+ width="80%"+ wrapClassName="modal-body"+ >+ <Table+ columns={logDetailsColumns}+ dataSource={logDetails}+ rowKey="id"+ align="left"+ pagination={{+ pageSize: 10,+ total: totalCount,+ onChange: (page, pageSize) => {+ fetchLogDetails(selectedLogId, page, pageSize);+ },+ }}+ />+ </Modal>+ </>+ );+};++LogsModal.propTypes = {+ open: PropTypes.bool.isRequired,+ setOpen: PropTypes.func.isRequired,+ logRecord: PropTypes.array.isRequired,+ totalLogs: PropTypes.number.isRequired,+ fetchExecutionLogs: PropTypes.func.isRequired,+};++export { LogsModal };diff --git a/frontend/src/components/pipelines-or-deployments/log-modal/LogsModel.css b/frontend/src/components/pipelines-or-deployments/log-modal/LogsModel.cssnew file mode 100644index 000000000..704a775a2--- /dev/null+++ b/frontend/src/components/pipelines-or-deployments/log-modal/LogsModel.css@@ -0,0 +1,15 @@+.error {+ color: #FF0000;+}++.completed {+ color: #034e03;+}++.warning {+ color: #af6714;+}++table .ant-btn {+ padding: 0;+}diff --git a/frontend/src/components/pipelines-or-deployments/pipelines/Pipelines.jsx b/frontend/src/components/pipelines-or-deployments/pipelines/Pipelines.jsxindex 49fed9536..c9ff548f0 100644--- a/frontend/src/components/pipelines-or-deployments/pipelines/Pipelines.jsx+++ b/frontend/src/components/pipelines-or-deployments/pipelines/Pipelines.jsx@@ -4,6 +4,7 @@ import {EllipsisOutlined,SyncOutlined,HighlightOutlined,+ FileSearchOutlined,} from "@ant-design/icons";import { Dropdown, Image, Space, Switch, Typography } from "antd";import PropTypes from "prop-types";@@ -17,6 +18,7 @@ import { useSessionStore } from "../../../store/session-store.js";import { Layout } from "../../deployments/layout/Layout.jsx";import { SpinnerLoader } from "../../widgets/spinner-loader/SpinnerLoader.jsx";import { DeleteModal } from "../delete-modal/DeleteModal.jsx";+import { LogsModal } from "../log-modal/LogsModal.jsx";import { EtlTaskDeploy } from "../etl-task-deploy/EtlTaskDeploy.jsx";import "./Pipelines.css";import { useExceptionHandler } from "../../../hooks/useExceptionHandler.jsx";@@ -32,6 +34,9 @@ function Pipelines({ type }) {const axiosPrivate = useAxiosPrivate();const handleException = useExceptionHandler();const [isEdit, setIsEdit] = useState(false);+ const [openLogsModal, setOpenLogsModal] = useState(false);+ const [executionLogs, setExecutionLogs] = useState([]);+ const [executionLogsTotalCount, setExecutionLogsTotalCount] = useState(0);useEffect(() => {getPipelineList();@@ -175,6 +180,34 @@ function Pipelines({ type }) {});};+ const fetchExecutionLogs = (page = 1, pageSize = 10) => {+ const requestOptions = {+ method: "GET",+ url: `/api/v1/unstract/${sessionDetails?.orgId}/pipeline/${selectedPorD.id}/executions/`,+ headers: {+ "X-CSRFToken": sessionDetails?.csrfToken,+ },+ params: {+ page: page,+ page_size: pageSize,+ },+ };+ axiosPrivate(requestOptions)+ .then((res) => {+ const logs = res?.data?.results?.map((result) => ({+ created_at: result.created_at,+ execution_id: result.id,+ status: result.status,+ execution_time: result.execution_time,+ }));+ setExecutionLogs(logs);+ setExecutionLogsTotalCount(res.data.count);+ })+ .catch((err) => {+ setAlertDetails(handleException(err));+ });+ };+const clearCache = () => {const requestOptions = {method: "GET",@@ -237,6 +270,26 @@ function Pipelines({ type }) {},{key: "2",+ label: (+ <Space+ direction="horizontal"+ className="action-items"+ onClick={() => {+ setOpenLogsModal(true);+ fetchExecutionLogs();+ }}+ >+ <div>+ <FileSearchOutlined />+ </div>+ <div>+ <Typography.Text>View Logs</Typography.Text>+ </div>+ </Space>+ ),+ },+ {+ key: "3",label: (<Spacedirection="horizontal"@@ -253,7 +306,7 @@ function Pipelines({ type }) {),},{- key: "3",+ key: "4",label: (<Spacedirection="horizontal"@@ -270,7 +323,7 @@ function Pipelines({ type }) {),},{- key: "4",+ key: "5",label: (<Spacedirection="horizontal"@@ -431,6 +484,13 @@ function Pipelines({ type }) {setSelectedRow={setSelectedPorD}/>)}+ <LogsModal+ open={openLogsModal}+ setOpen={setOpenLogsModal}+ logRecord={executionLogs}+ totalLogs={executionLogsTotalCount}+ fetchExecutionLogs={fetchExecutionLogs}+ /><DeleteModalopen={openDeleteModal}setOpen={setOpenDeleteModal}diff --git a/unstract/core/src/unstract/core/constants.py b/unstract/core/src/unstract/core/constants.pynew file mode 100644index 000000000..5f397e7f6--- /dev/null+++ b/unstract/core/src/unstract/core/constants.py@@ -0,0 +1,7 @@+class LogFieldName:+ EXECUTION_ID = "execution_id"+ ORGANIZATION_ID = "organization_id"+ TIMESTAMP = "timestamp"+ TYPE = "type"+ DATA = "data"+ EVENT_TIME = "event_time"diff --git a/unstract/core/src/unstract/core/pubsub_helper.py b/unstract/core/src/unstract/core/pubsub_helper.pyindex 855a4d3f6..5658c0076 100644--- a/unstract/core/src/unstract/core/pubsub_helper.py+++ b/unstract/core/src/unstract/core/pubsub_helper.py@@ -43,6 +43,8 @@ class LogPublisher:step: Optional[int] = None,iteration: Optional[int] = None,iteration_total: Optional[int] = None,+ execution_id: Optional[str] = None,+ organization_id: Optional[str] = None,) -> dict[str, Any]:return {"timestamp": datetime.now(timezone.utc).timestamp(),@@ -56,6 +58,8 @@ class LogPublisher:"step": step,"iteration": iteration,"iteration_total": iteration_total,+ "execution_id": execution_id,+ "organization_id": organization_id,}@staticmethoddiff --git a/unstract/workflow-execution/src/unstract/workflow_execution/tools_utils.py b/unstract/workflow-execution/src/unstract/workflow_execution/tools_utils.pyindex b8a08d0a4..c915078ab 100644--- a/unstract/workflow-execution/src/unstract/workflow_execution/tools_utils.py+++ b/unstract/workflow-execution/src/unstract/workflow_execution/tools_utils.py@@ -106,6 +106,8 @@ class ToolsUtils:LogPublisher.log_workflow("BUILD",f"------ Building step {tool_instance.step}/{tool_uid}",+ execution_id=execution_id,+ organization_id=self.organization_id,),)@@ -117,7 +119,10 @@ class ToolsUtils:LogPublisher.publish(self.messaging_channel,LogPublisher.log_workflow(- "BUILD", f"Building the tool {tool_uid} now..."+ "BUILD",+ f"Building the tool {tool_uid} now...",+ execution_id=execution_id,+ organization_id=self.organization_id,),)tool_sandbox = ToolSandbox(diff --git a/unstract/workflow-execution/src/unstract/workflow_execution/workflow_execution.py b/unstract/workflow-execution/src/unstract/workflow_execution/workflow_execution.pyindex 2c3448dd9..7e3d73273 100644--- a/unstract/workflow-execution/src/unstract/workflow_execution/workflow_execution.py+++ b/unstract/workflow-execution/src/unstract/workflow_execution/workflow_execution.py@@ -375,6 +375,8 @@ class WorkflowExecutionService:step=step,iteration=iteration,iteration_total=iteration_total,+ execution_id=self.execution_id,+ organization_id=self.organization_id,)LogPublisher.publish(self.messaging_channel, log_details)diff --git a/worker/src/unstract/worker/worker.py b/worker/src/unstract/worker/worker.pyindex 70ef1c183..b79f95913 100644--- a/worker/src/unstract/worker/worker.py+++ b/worker/src/unstract/worker/worker.py@@ -2,6 +2,7 @@ import astimport jsonimport osimport uuid+from datetime import datetime, timezonefrom typing import Any, Optionalfrom docker.errors import APIError@@ -12,6 +13,7 @@ from unstract.worker.utils import Utilsimport dockerfrom docker import DockerClient # type: ignore[attr-defined]+from unstract.core.constants import LogFieldNamefrom unstract.core.pubsub_helper import LogPublisherload_dotenv()@@ -137,6 +139,8 @@ class UnstractWorker:self,container: Any,tool_instance_id: str,+ execution_id: str,+ organization_id: str,channel: Optional[str] = None,) -> None:for line in container.logs(stream=True, follow=True):@@ -146,6 +150,8 @@ class UnstractWorker:log_message=log_message,tool_instance_id=tool_instance_id,channel=channel,+ execution_id=execution_id,+ organization_id=organization_id,)def get_valid_log_message(self, log_message: str) -> Optional[dict[str, Any]]:@@ -168,6 +174,8 @@ class UnstractWorker:self,log_message: str,tool_instance_id: str,+ execution_id: str,+ organization_id: str,channel: Optional[str] = None,) -> Optional[dict[str, Any]]:log_dict = self.get_valid_log_message(log_message)@@ -176,7 +184,7 @@ class UnstractWorker:log_type = self.get_log_type(log_dict)if not self.is_valid_log_type(log_type):self.logger.warning(- f"Received invalid logType: {log_type} with log message: " f"{log_dict}"+ f"Received invalid logType: {log_type} with log message: {log_dict}")return Noneif log_type == LogType.RESULT:@@ -184,6 +192,10 @@ class UnstractWorker:if log_type == LogType.UPDATE:log_dict["component"] = tool_instance_idif channel:+ log_dict[LogFieldName.EXECUTION_ID] = execution_id+ log_dict[LogFieldName.ORGANIZATION_ID] = organization_id+ log_dict[LogFieldName.TIMESTAMP] = datetime.now(timezone.utc).timestamp()+# Publish to channel of socket ioLogPublisher.publish(channel, log_dict)return None@@ -366,6 +378,8 @@ class UnstractWorker:container=container,tool_instance_id=tool_instance_id,channel=self.messaging_channel,+ execution_id=execution_id,+ organization_id=organization_id,)except Exception as e:self.logger.error(f"Failed to run docker container: {e}")