LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
UN-3011 [FEAT] Limit maximum file execution count with file history viewer (#1676)
ali987271c35ec6
diff --git a/backend/backend/settings/base.py b/backend/backend/settings/base.pyindex b6cbb2599..94cb55b3a 100644--- a/backend/backend/settings/base.py+++ b/backend/backend/settings/base.py@@ -200,6 +200,8 @@ MAX_PARALLEL_FILE_BATCHES = int(os.environ.get("MAX_PARALLEL_FILE_BATCHES", 1))MAX_PARALLEL_FILE_BATCHES_MAX_VALUE = int(os.environ.get("MAX_PARALLEL_FILE_BATCHES_MAX_VALUE", 100))+# Maximum number of times a file can be executed in a workflow+MAX_FILE_EXECUTION_COUNT = int(os.environ.get("MAX_FILE_EXECUTION_COUNT", 3))CELERY_RESULT_CHORD_RETRY_INTERVAL = float(os.environ.get("CELERY_RESULT_CHORD_RETRY_INTERVAL", "3")diff --git a/backend/sample.env b/backend/sample.envindex 805d29ea4..1d361d0a8 100644--- a/backend/sample.env+++ b/backend/sample.env@@ -217,6 +217,10 @@ FILE_EXECUTION_TRACKER_TTL_IN_SECOND=18000# File execution tracker completed TTL in seconds (10 minutes)FILE_EXECUTION_TRACKER_COMPLETED_TTL_IN_SECOND=600+# Maximum number of times a file can be executed in ETL/TASK workflows+# Default: 3 (file is permanently skipped after 3 execution attempts)+MAX_FILE_EXECUTION_COUNT=3+# Runner polling timeout (3 hours)MAX_RUNNER_POLLING_WAIT_SECONDS=10800# Runner polling interval (2 seconds)diff --git a/backend/workflow_manager/internal_views.py b/backend/workflow_manager/internal_views.pyindex 411604488..94ecd7f76 100644--- a/backend/workflow_manager/internal_views.py+++ b/backend/workflow_manager/internal_views.py@@ -502,6 +502,7 @@ class WorkflowExecutionInternalViewSet(viewsets.ReadOnlyModelViewSet):from workflow_manager.workflow_v2.enums import ExecutionStatusstatus_enum = ExecutionStatus(validated_data["status"])+ logger.info(f"Updating status for execution {id} to {status_enum}")execution.update_execution(status=status_enum,error=error_message,@@ -2305,6 +2306,9 @@ class FileHistoryBatchCheckView(APIView):file_history_queryset, request, "workflow__organization")+ # Get max execution count for this workflow (for worker decision making)+ max_execution_count = workflow.get_max_execution_count()+# Get full file history details for cached resultsfile_histories = file_history_queryset.values("cache_key",@@ -2313,6 +2317,8 @@ class FileHistoryBatchCheckView(APIView):"error","file_path","provider_file_uuid",+ "execution_count",+ "status",)# Build response with both processed hashes (for compatibility) and full details@@ -2328,16 +2334,21 @@ class FileHistoryBatchCheckView(APIView):"error": fh["error"],"file_path": fh["file_path"],"provider_file_uuid": fh["provider_file_uuid"],+ "execution_count": fh["execution_count"],+ "status": fh["status"],+ "max_execution_count": max_execution_count, # Include for worker logic}logger.info(- f"File history batch check: {len(processed_file_hashes)}/{len(file_hashes)} files already processed"+ f"File history batch check: {len(processed_file_hashes)}/{len(file_hashes)} files already processed "+ f"(max_execution_count: {max_execution_count})")return Response({"processed_file_hashes": processed_file_hashes, # For backward compatibility"file_history_details": file_history_details, # Full details for cached results+ "max_execution_count": max_execution_count, # Global max for this workflow})@@ -2470,10 +2481,17 @@ class FileHistoryCreateView(APIView):)logger.info(- f"Created file history entry {file_history.id} for file {file_name}"+ f"Created/updated file history entry {file_history.id} for file {file_name} "+ f"(execution_count: {file_history.execution_count})")- return Response({"created": True, "file_history_id": str(file_history.id)})+ return Response(+ {+ "created": True,+ "file_history_id": str(file_history.id),+ "execution_count": file_history.execution_count,+ }+ )except Exception as e:logger.error(f"File history creation failed: {str(e)}")diff --git a/backend/workflow_manager/workflow_v2/file_history_helper.py b/backend/workflow_manager/workflow_v2/file_history_helper.pyindex 0929f1557..687f88dfe 100644--- a/backend/workflow_manager/workflow_v2/file_history_helper.py+++ b/backend/workflow_manager/workflow_v2/file_history_helper.py@@ -3,7 +3,7 @@ from datetime import timedeltafrom typing import Anyfrom django.conf import settings-from django.db.models import Q+from django.db.models import F, Qfrom django.db.utils import IntegrityErrorfrom django.utils import timezonefrom utils.cache_service import CacheService@@ -238,6 +238,61 @@ class FileHistoryHelper:workflow_log.log_error(logger=logger, message=error_msg)return None+ @staticmethod+ def _safe_str(value: Any) -> str:+ """Convert value to string, return empty string if None.++ Args:+ value: Value to convert++ Returns:+ str: String representation or empty string+ """+ return "" if value is None else str(value)++ @staticmethod+ def _truncate_hash(file_hash: str | None) -> str:+ """Truncate hash for logging purposes.++ Args:+ file_hash: Hash string to truncate++ Returns:+ str: Truncated hash (first 16 chars) or 'None' if missing+ """+ return file_hash[:16] if file_hash else "None"++ @staticmethod+ def _increment_file_history(+ file_history: FileHistory,+ status: ExecutionStatus,+ result: Any,+ metadata: str | None,+ error: str | None,+ ) -> FileHistory:+ """Update existing file history with incremented execution count.++ Args:+ file_history: FileHistory instance to update+ status: New execution status+ result: Execution result+ metadata: Execution metadata+ error: Error message if any++ Returns:+ FileHistory: Updated file history instance+ """+ FileHistory.objects.filter(id=file_history.id).update(+ execution_count=F("execution_count") + 1,+ status=status,+ result=str(result),+ metadata=FileHistoryHelper._safe_str(metadata),+ error=FileHistoryHelper._safe_str(error),+ )+ # Refresh from DB to get updated values+ file_history.refresh_from_db()+ return file_history+@staticmethoddef create_file_history(file_hash: FileHash,@@ -248,7 +303,11 @@ class FileHistoryHelper:error: str | None = None,is_api: bool = False,) -> FileHistory:- """Create a new file history record or return existing one.+ """Create a new file history record or increment existing one's execution count.++ This method implements execution count tracking:+ - If file history exists: increments execution_count atomically+ - If file history is new: creates with execution_count=1Args:file_hash (FileHash): The file hash for the file.@@ -260,44 +319,64 @@ class FileHistoryHelper:is_api (bool): Whether this is for API workflow (affects file_path handling).Returns:- FileHistory: Either newly created or existing file history record.+ FileHistory: Either newly created or updated file history record."""file_path = file_hash.file_path if not is_api else None- # Prepare data for creation+ # Check if file history already exists+ existing_history = FileHistoryHelper.get_file_history(+ workflow=workflow,+ cache_key=file_hash.file_hash,+ provider_file_uuid=file_hash.provider_file_uuid,+ file_path=file_path,+ )++ if existing_history:+ # File history exists - increment execution count atomically+ updated_history = FileHistoryHelper._increment_file_history(+ existing_history, status, result, metadata, error+ )+ logger.info(+ f"Updated FileHistory record (execution_count: {updated_history.execution_count}) - "+ f"file_name='{file_hash.file_name}', file_path='{file_hash.file_path}', "+ f"file_hash='{FileHistoryHelper._truncate_hash(file_hash.file_hash)}', "+ f"workflow={workflow}"+ )+ return updated_history++ # File history doesn't exist - create new record with execution_count=1create_data = {"workflow": workflow,"cache_key": file_hash.file_hash,"provider_file_uuid": file_hash.provider_file_uuid,"status": status,"result": str(result),- "metadata": str(metadata) if metadata else "",- "error": str(error) if error else "",+ "metadata": FileHistoryHelper._safe_str(metadata),+ "error": FileHistoryHelper._safe_str(error),"file_path": file_path,+ "execution_count": 1,}try:- # Try to create the file history recordfile_history = FileHistory.objects.create(**create_data)logger.info(- f"Created new FileHistory record - "+ f"Created new FileHistory record (execution_count: 1) - "f"file_name='{file_hash.file_name}', file_path='{file_hash.file_path}', "- f"file_hash='{file_hash.file_hash[:16] if file_hash.file_hash else 'None'}', "+ f"file_hash='{FileHistoryHelper._truncate_hash(file_hash.file_hash)}', "f"workflow={workflow}")return file_historyexcept IntegrityError as e:- # Race condition detected - another worker created the record- # Try to retrieve the existing record+ # Race condition: another worker created the record between our check and createlogger.info(- f"FileHistory constraint violation (expected in concurrent environment) - "+ f"FileHistory constraint violation (race condition) - "f"file_name='{file_hash.file_name}', file_path='{file_hash.file_path}', "- f"file_hash='{file_hash.file_hash[:16] if file_hash.file_hash else 'None'}', "- f"workflow={workflow}. Error: {str(e)}"+ f"file_hash='{FileHistoryHelper._truncate_hash(file_hash.file_hash)}', "+ f"workflow={workflow}. Error: {e!s}")- # Use the existing get_file_history method to retrieve the record+ # Retrieve the record created by another worker and increment itexisting_record = FileHistoryHelper.get_file_history(workflow=workflow,cache_key=file_hash.file_hash,@@ -306,18 +385,22 @@ class FileHistoryHelper:)if existing_record:- logger.info(- f"Retrieved existing FileHistory record after constraint violation - "- f"ID: {existing_record.id}, workflow={workflow}"+ # Increment the existing record+ updated_record = FileHistoryHelper._increment_file_history(+ existing_record, status, result, metadata, error)- return existing_record- else:- # This should rarely happen, but if we can't find the existing record,- # log the issue and re-raise the original exception- logger.error(- f"Failed to retrieve existing FileHistory record after constraint violation - "- f"file_name='{file_hash.file_name}', workflow={workflow}"+ logger.info(+ f"Retrieved and updated existing FileHistory record (execution_count: {updated_record.execution_count}) - "+ f"ID: {updated_record.id}, workflow={workflow}")+ return updated_record++ # This should rarely happen - existing record not found after IntegrityError+ logger.exception(+ f"Failed to retrieve existing FileHistory record after constraint violation - "+ f"file_name='{file_hash.file_name}', workflow={workflow}"+ )+ raise@staticmethoddef clear_history_for_workflow(diff --git a/backend/workflow_manager/workflow_v2/file_history_views.py b/backend/workflow_manager/workflow_v2/file_history_views.pynew file mode 100644index 000000000..316cec593--- /dev/null+++ b/backend/workflow_manager/workflow_v2/file_history_views.py@@ -0,0 +1,178 @@+import logging++from django.shortcuts import get_object_or_404+from rest_framework import status, viewsets+from rest_framework.decorators import action+from rest_framework.exceptions import ValidationError+from rest_framework.permissions import IsAuthenticated+from rest_framework.response import Response+from utils.pagination import CustomPagination++from workflow_manager.workflow_v2.models.file_history import FileHistory+from workflow_manager.workflow_v2.models.workflow import Workflow+from workflow_manager.workflow_v2.permissions import IsWorkflowOwnerOrShared+from workflow_manager.workflow_v2.serializers import FileHistorySerializer++logger = logging.getLogger(__name__)++MAX_BULK_DELETE_LIMIT = 100+++class FileHistoryViewSet(viewsets.ReadOnlyModelViewSet):+ """ViewSet for file history operations with filtering support."""++ serializer_class = FileHistorySerializer+ lookup_field = "id"+ permission_classes = [IsAuthenticated, IsWorkflowOwnerOrShared]+ pagination_class = CustomPagination++ def _validate_execution_count(self, value, param_name):+ """Validate execution count parameter is a non-negative integer.++ Args:+ value: The value to validate+ param_name: Name of the parameter for error messages++ Returns:+ int: The validated integer value++ Raises:+ ValidationError: If value is invalid+ """+ try:+ int_value = int(value)+ if int_value < 0:+ raise ValidationError({param_name: "Must be a non-negative integer"})+ return int_value+ except (ValueError, TypeError):+ raise ValidationError({param_name: "Must be a valid integer"})++ def get_queryset(self):+ """Get file histories for workflow with filters."""+ # Reuse cached workflow from permission check+ if hasattr(self.request, "_workflow_cache"):+ workflow = self.request._workflow_cache+ else:+ # Fallback if permission didn't run (shouldn't happen)+ workflow_id = self.kwargs.get("workflow_id")+ workflow = get_object_or_404(Workflow, id=workflow_id)++ queryset = FileHistory.objects.filter(workflow=workflow)++ # Apply simple filters from query parameters+ status_param = self.request.query_params.get("status")+ if status_param:+ status_list = [s.strip() for s in status_param.split(",")]+ queryset = queryset.filter(status__in=status_list)++ exec_min = self.request.query_params.get("execution_count_min")+ if exec_min:+ exec_min_val = self._validate_execution_count(exec_min, "execution_count_min")+ queryset = queryset.filter(execution_count__gte=exec_min_val)++ exec_max = self.request.query_params.get("execution_count_max")+ if exec_max:+ exec_max_val = self._validate_execution_count(exec_max, "execution_count_max")+ queryset = queryset.filter(execution_count__lte=exec_max_val)++ file_path_param = self.request.query_params.get("file_path")+ if file_path_param:+ # Support partial matching (case-insensitive)+ queryset = queryset.filter(file_path__icontains=file_path_param)++ return queryset.order_by("-created_at")++ def destroy(self, request, workflow_id=None, id=None):+ """Delete single file history by ID."""+ file_history = self.get_object()+ file_history_id = file_history.id++ file_history.delete()++ logger.info(f"Deleted file history {file_history_id} for workflow {workflow_id}")++ return Response(+ {"message": "File history deleted successfully"}, status=status.HTTP_200_OK+ )++ @action(detail=False, methods=["post"])+ def clear(self, request, workflow_id=None):+ """Clear file histories with filters or by specific IDs.++ Supports two modes:+ 1. ID-based deletion: Pass {"ids": [...]} to delete specific records (max 100)+ 2. Filter-based deletion: Pass filters like status, execution_count_min, etc.++ At least one filter or IDs must be provided to prevent accidental deletion+ of all records.+ """+ # Extract all filter parameters upfront+ ids = request.data.get("ids", [])+ status_list = request.data.get("status", [])+ exec_min = request.data.get("execution_count_min")+ exec_max = request.data.get("execution_count_max")+ file_path_param = request.data.get("file_path")++ # Safeguard: require at least one filter or IDs to prevent accidental mass deletion+ has_criteria = bool(+ ids+ or status_list+ or exec_min is not None+ or exec_max is not None+ or file_path_param+ )+ if not has_criteria:+ return Response(+ {+ "error": "At least one filter (ids, status, execution_count_min, "+ "execution_count_max, or file_path) must be provided"+ },+ status=status.HTTP_400_BAD_REQUEST,+ )++ workflow = get_object_or_404(Workflow, id=workflow_id)+ queryset = FileHistory.objects.filter(workflow=workflow)++ # Check for ID-based deletion+ if ids:+ if len(ids) > MAX_BULK_DELETE_LIMIT:+ return Response(+ {+ "error": f"Cannot delete more than {MAX_BULK_DELETE_LIMIT} "+ f"items at once. Received {len(ids)} IDs."+ },+ status=status.HTTP_400_BAD_REQUEST,+ )+ queryset = queryset.filter(id__in=ids)+ else:+ # Apply filters from request body (filter-based deletion)+ if status_list:+ queryset = queryset.filter(status__in=status_list)++ if exec_min is not None:+ exec_min_val = self._validate_execution_count(+ exec_min, "execution_count_min"+ )+ queryset = queryset.filter(execution_count__gte=exec_min_val)++ if exec_max is not None:+ exec_max_val = self._validate_execution_count(+ exec_max, "execution_count_max"+ )+ queryset = queryset.filter(execution_count__lte=exec_max_val)++ if file_path_param:+ queryset = queryset.filter(file_path__icontains=file_path_param)++ deleted_count, _ = queryset.delete()++ logger.info(+ f"Cleared {deleted_count} file history records for workflow {workflow_id}"+ )++ return Response(+ {+ "deleted_count": deleted_count,+ "message": f"{deleted_count} file history records deleted",+ }+ )diff --git a/backend/workflow_manager/workflow_v2/migrations/0018_filehistory_execution_count_and_more.py b/backend/workflow_manager/workflow_v2/migrations/0018_filehistory_execution_count_and_more.pynew file mode 100644index 000000000..1e02961d7--- /dev/null+++ b/backend/workflow_manager/workflow_v2/migrations/0018_filehistory_execution_count_and_more.py@@ -0,0 +1,59 @@+# Generated by Django 4.2.1 on 2025-11-24 12:41++import django.core.validators+from django.contrib.postgres.indexes import GinIndex+from django.contrib.postgres.operations import TrigramExtension+from django.db import migrations, models+++class Migration(migrations.Migration):+ dependencies = [+ ("workflow_v2", "0017_workflow_shared_to_org_workflow_shared_users"),+ ]++ operations = [+ # Enable PostgreSQL trigram extension for fast partial text matching+ TrigramExtension(),+ migrations.AddField(+ model_name="filehistory",+ name="execution_count",+ field=models.IntegerField(+ db_comment="Number of times this file has been processed", default=1+ ),+ ),+ migrations.AddField(+ model_name="workflow",+ name="max_file_execution_count",+ field=models.IntegerField(+ blank=True,+ db_comment="Maximum times a file can be executed. null=use org/global default. Only enforced for ETL/TASK workflows.",+ null=True,+ validators=[django.core.validators.MinValueValidator(1)],+ ),+ ),+ migrations.AddIndex(+ model_name="filehistory",+ index=models.Index(fields=["workflow", "status"], name="idx_fh_wf_status"),+ ),+ migrations.AddIndex(+ model_name="filehistory",+ index=models.Index(+ fields=["workflow", "execution_count"], name="idx_fh_wf_exec_cnt"+ ),+ ),+ migrations.AddIndex(+ model_name="filehistory",+ index=models.Index(+ fields=["workflow", "file_path"], name="idx_fh_workflow_filepath"+ ),+ ),+ # Add GIN trigram index for fast partial file_path search (10-250x faster)+ migrations.AddIndex(+ model_name="filehistory",+ index=GinIndex(+ fields=["file_path"],+ name="idx_fh_file_path_trgm",+ opclasses=["gin_trgm_ops"],+ ),+ ),+ ]diff --git a/backend/workflow_manager/workflow_v2/models/file_history.py b/backend/workflow_manager/workflow_v2/models/file_history.pyindex 1935b246c..ca0094e93 100644--- a/backend/workflow_manager/workflow_v2/models/file_history.py+++ b/backend/workflow_manager/workflow_v2/models/file_history.py@@ -1,5 +1,6 @@import uuid+from django.contrib.postgres.indexes import GinIndexfrom django.db import modelsfrom django.db.models import Qfrom utils.models.base_model import BaseModel@@ -20,6 +21,29 @@ class FileHistory(BaseModel):"""return self.status is not None and self.status == ExecutionStatus.COMPLETED.value+ def __str__(self) -> str:+ """String representation of FileHistory."""+ return f"FileHistory({self.id}, CacheKey: {self.cache_key}, Status: {self.status}, Count: {self.execution_count})"++ def has_exceeded_limit(self, workflow: "Workflow") -> bool:+ """Check if this file has exceeded its maximum execution count.++ For API workflows, this always returns False (no limit enforcement).+ For ETL/TASK workflows, checks against configured limit.++ Args:+ workflow: The workflow being executed.++ Returns:+ bool: True if file has exceeded limit and should be skipped.+ """+ # API workflows don't enforce execution limits (track count but don't skip)+ if workflow.deployment_type == Workflow.WorkflowType.API:+ return False++ max_count = workflow.get_max_execution_count()+ return self.execution_count >= max_count+id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)cache_key = models.CharField(max_length=HASH_LENGTH,@@ -39,6 +63,10 @@ class FileHistory(BaseModel):choices=ExecutionStatus.choices,db_comment="Latest status of execution",)+ execution_count = models.IntegerField(+ default=1,+ db_comment="Number of times this file has been processed",+ )error = models.TextField(blank=True,default="",@@ -59,6 +87,18 @@ class FileHistory(BaseModel):models.Index(fields=["workflow", "created_at"], name="idx_fh_workflow_created"),+ models.Index(fields=["workflow", "status"], name="idx_fh_wf_status"),+ models.Index(+ fields=["workflow", "execution_count"], name="idx_fh_wf_exec_cnt"+ ),+ models.Index(+ fields=["workflow", "file_path"], name="idx_fh_workflow_filepath"+ ),+ GinIndex(+ fields=["file_path"],+ name="idx_fh_file_path_trgm",+ opclasses=["gin_trgm_ops"],+ ),]constraints = [# Legacy behavior: file_path is not present or is nulldiff --git a/backend/workflow_manager/workflow_v2/models/workflow.py b/backend/workflow_manager/workflow_v2/models/workflow.pyindex 01f4e036a..a286662a1 100644--- a/backend/workflow_manager/workflow_v2/models/workflow.py+++ b/backend/workflow_manager/workflow_v2/models/workflow.py@@ -1,6 +1,8 @@import uuidfrom account_v2.models import User+from django.conf import settings+from django.core.validators import MinValueValidatorfrom django.db import modelsfrom utils.models.base_model import BaseModelfrom utils.models.organization_mixin import (@@ -67,6 +69,12 @@ class Workflow(DefaultOrganizationMixin, BaseModel):destination_settings = models.JSONField(null=True, db_comment="Settings for the Destination module")+ max_file_execution_count = models.IntegerField(+ null=True,+ blank=True,+ validators=[MinValueValidator(1)],+ db_comment="Maximum times a file can be executed. null=use org/global default. Only enforced for ETL/TASK workflows.",+ )created_by = models.ForeignKey(User,@@ -98,6 +106,21 @@ class Workflow(DefaultOrganizationMixin, BaseModel):def __str__(self) -> str:return f"{self.id}, name: {self.workflow_name}"+ def get_max_execution_count(self) -> int:+ """Get maximum execution count from configuration hierarchy.++ Priority: workflow setting > organization setting > global Django setting++ Returns:+ int: Maximum execution count limit.+ """+ # Check workflow-level setting first+ if self.max_file_execution_count is not None:+ return self.max_file_execution_count++ # Fall back to global Django setting (from backend.settings.execution_config)+ return settings.MAX_FILE_EXECUTION_COUNT+class Meta:verbose_name = "Workflow"verbose_name_plural = "Workflows"diff --git a/backend/workflow_manager/workflow_v2/permissions.py b/backend/workflow_manager/workflow_v2/permissions.pynew file mode 100644index 000000000..8d3843515--- /dev/null+++ b/backend/workflow_manager/workflow_v2/permissions.py@@ -0,0 +1,45 @@+from django.shortcuts import get_object_or_404+from rest_framework.permissions import BasePermission++from workflow_manager.workflow_v2.models.workflow import Workflow+++class IsWorkflowOwnerOrShared(BasePermission):+ """Permission class to check if user has access to a workflow.++ Checks:+ 1. User is the workflow owner (created_by)+ 2. User has shared access (in shared_users)+ 3. Workflow is shared to user's organization (shared_to_org)++ Caches the workflow on request object to avoid duplicate fetching.+ """++ message = "You do not have permission to access this workflow"++ def has_permission(self, request, view):+ workflow_id = view.kwargs.get("workflow_id")++ if not workflow_id:+ return False++ # Cache workflow on request to avoid re-fetching in get_queryset+ if not hasattr(request, "_workflow_cache"):+ request._workflow_cache = get_object_or_404(+ Workflow.objects.select_related(+ "created_by", "organization"+ ).prefetch_related("shared_users"),+ id=workflow_id,+ )++ workflow = request._workflow_cache+ user = request.user++ # Check access: owner OR shared user OR shared to organization+ has_access = (+ workflow.created_by == user+ or user in workflow.shared_users.all()+ or (workflow.shared_to_org and workflow.organization == user.organization)+ )++ return has_accessdiff --git a/backend/workflow_manager/workflow_v2/serializers.py b/backend/workflow_manager/workflow_v2/serializers.pyindex 26e377a6a..6442d1e1f 100644--- a/backend/workflow_manager/workflow_v2/serializers.py+++ b/backend/workflow_manager/workflow_v2/serializers.py@@ -131,10 +131,35 @@ class WorkflowExecutionLogSerializer(ModelSerializer):class FileHistorySerializer(ModelSerializer):+ max_execution_count = SerializerMethodField()+ has_exceeded_limit = SerializerMethodField()+class Meta:model = FileHistoryfields = "__all__"+ def get_max_execution_count(self, obj: FileHistory) -> int:+ """Get the maximum execution count from the associated workflow.++ Args:+ obj: FileHistory instance++ Returns:+ int: Maximum execution count from workflow configuration+ """+ return obj.workflow.get_max_execution_count()++ def get_has_exceeded_limit(self, obj: FileHistory) -> bool:+ """Check if this file has exceeded its execution limit.++ Args:+ obj: FileHistory instance++ Returns:+ bool: True if file has exceeded limit and should be skipped+ """+ return obj.has_exceeded_limit(obj.workflow)+class SharedUserListSerializer(ModelSerializer):"""Serializer for returning workflow with shared user details."""diff --git a/backend/workflow_manager/workflow_v2/urls/workflow.py b/backend/workflow_manager/workflow_v2/urls/workflow.pyindex 50f6de1f6..e9f404fed 100644--- a/backend/workflow_manager/workflow_v2/urls/workflow.py+++ b/backend/workflow_manager/workflow_v2/urls/workflow.py@@ -3,6 +3,7 @@ from rest_framework.urlpatterns import format_suffix_patternsfrom workflow_manager.workflow_v2.execution_log_view import WorkflowExecutionLogViewSetfrom workflow_manager.workflow_v2.execution_view import WorkflowExecutionViewSet+from workflow_manager.workflow_v2.file_history_views import FileHistoryViewSetfrom workflow_manager.workflow_v2.views import WorkflowViewSetworkflow_list = WorkflowViewSet.as_view(@@ -24,6 +25,12 @@ workflow_clear_file_marker = WorkflowViewSet.as_view({"get": "clear_file_marker"workflow_schema = WorkflowViewSet.as_view({"get": "get_schema"})can_update = WorkflowViewSet.as_view({"get": "can_update"})list_shared_users = WorkflowViewSet.as_view({"get": "list_of_shared_users"})++# File History views+file_history_list = FileHistoryViewSet.as_view({"get": "list"})+file_history_detail = FileHistoryViewSet.as_view({"get": "retrieve", "delete": "destroy"})+file_history_clear = FileHistoryViewSet.as_view({"post": "clear"})+urlpatterns = format_suffix_patterns([path("", workflow_list, name="workflow-list"),@@ -69,5 +76,21 @@ urlpatterns = format_suffix_patterns(workflow_schema,name="workflow-schema",),+ # File History nested routes+ path(+ "<uuid:workflow_id>/file-histories/",+ file_history_list,+ name="workflow-file-history-list",+ ),+ path(+ "<uuid:workflow_id>/file-histories/<uuid:id>/",+ file_history_detail,+ name="workflow-file-history-detail",+ ),+ path(+ "<uuid:workflow_id>/file-histories/clear/",+ file_history_clear,+ name="workflow-file-history-clear",+ ),])diff --git a/backend/workflow_manager/workflow_v2/views.py b/backend/workflow_manager/workflow_v2/views.pyindex e2d8e21ac..ec6fb46b7 100644--- a/backend/workflow_manager/workflow_v2/views.py+++ b/backend/workflow_manager/workflow_v2/views.py@@ -1067,25 +1067,31 @@ def file_history_batch_lookup_internal(request):f"cache_key={fh.cache_key}, provider_file_uuid={fh.provider_file_uuid}, file_path={fh.file_path}")+ # Serialize once for all matches (optimization)+ is_completed_result = fh.is_completed()+ serializer = FileHistorySerializer(fh)+ file_history_data = serializer.data+# Update response data for all matchesfor result_identifier in matched_identifiers:- is_completed_result = fh.is_completed()logger.info(f"DEBUG: FileHistoryBatch - Found record for UUID: {fh.provider_file_uuid}, "f"Path: {fh.file_path}, Status: {fh.status}, is_completed(): {is_completed_result}, "+ f"has_exceeded_limit: {file_history_data.get('has_exceeded_limit')}, "f"result_identifier: {result_identifier}")- serializer = FileHistorySerializer(fh)response_data[result_identifier] = {"found": True,"is_completed": is_completed_result,- "file_history": serializer.data,+ "file_history": file_history_data,}logger.info(f"DEBUG: FileHistoryBatch - Response updated for {result_identifier}: "- f"found=True, is_completed={is_completed_result}, status={fh.status}"+ f"found=True, is_completed={is_completed_result}, status={fh.status}, "+ f"max_execution_count={file_history_data.get('max_execution_count')}, "+ f"has_exceeded_limit={file_history_data.get('has_exceeded_limit')}")logger.info(diff --git a/frontend/src/components/agency/agency/Agency.jsx b/frontend/src/components/agency/agency/Agency.jsxindex 98040134f..8d4059f49 100644--- a/frontend/src/components/agency/agency/Agency.jsx+++ b/frontend/src/components/agency/agency/Agency.jsx@@ -42,6 +42,7 @@ import { pipelineService } from "../../pipelines-or-deployments/pipeline-serviceimport { InputOutput } from "../input-output/InputOutput";import { ToolSelectionSidebar } from "../tool-selection-sidebar/ToolSelectionSidebar.jsx";import { SpinnerLoader } from "../../widgets/spinner-loader/SpinnerLoader.jsx";+import FileHistoryModal from "../../pipelines-or-deployments/file-history-modal/FileHistoryModal.jsx";function Agency() {const [steps, setSteps] = useState([]);const [workflowProgress, setWorkflowProgress] = useState(0);@@ -96,6 +97,7 @@ function Agency() {const [openFileUploadModal, setOpenFileUploadModal] = useState(false);const [fileList, setFileList] = useState([]);const [wfExecutionParams, setWfExecutionParams] = useState([]);+ const [fileHistoryModalOpen, setFileHistoryModalOpen] = useState(false);const { setPostHogCustomEvent } = usePostHogEvents();@@ -1002,6 +1004,9 @@ function Agency() {case "run-workflow":handleRunWorkflow();break;+ case "view-file-history":+ setFileHistoryModalOpen(true);+ break;case "clear-history":handleClearFileMarker();break;@@ -1017,6 +1022,12 @@ function Agency() {icon: <PlayCircleOutlined />,disabled: isClearingFileHistory || loadingType === "EXECUTE",},+ {+ key: "view-file-history",+ label: "View File History",+ icon: <HistoryOutlined />,+ disabled: isClearingFileHistory || loadingType === "EXECUTE",+ },{key: "clear-history",label: "Clear Processed File History",@@ -1318,6 +1329,14 @@ function Agency() {onToolSelect={handleToolSelection}onSave={() => setShowToolSelectionSidebar(false)}/>++ {/* File History Modal */}+ <FileHistoryModal+ open={fileHistoryModalOpen}+ setOpen={setFileHistoryModalOpen}+ workflowId={details?.id}+ workflowName={projectName}+ /></div></IslandLayout>);diff --git a/frontend/src/components/pipelines-or-deployments/file-history-modal/FileHistoryModal.css b/frontend/src/components/pipelines-or-deployments/file-history-modal/FileHistoryModal.cssnew file mode 100644index 000000000..814c87901--- /dev/null+++ b/frontend/src/components/pipelines-or-deployments/file-history-modal/FileHistoryModal.css@@ -0,0 +1,122 @@+.file-history-modal .file-history-content {+ display: flex;+ flex-direction: column;+ gap: 16px;+}++.file-history-modal .filters-section {+ padding: 10px 12px;+ background-color: #fafafa;+ border: 1px solid #e8e8e8;+ border-radius: 6px;+ margin-bottom: 10px;+}++.file-history-modal .action-buttons {+ display: flex;+ justify-content: flex-start;+ padding: 8px 0;+}++.file-history-modal .ant-table-wrapper {+ margin-top: 8px;+}++/* Status tag colors */+.file-history-modal .ant-tag {+ font-weight: 500;+}++/* Mobile responsiveness - Small screens (< 576px) */+@media (max-width: 575px) {+ .file-history-modal .filters-section {+ padding: 8px 10px;+ }++ .file-history-modal .filter-buttons {+ margin-top: 4px !important;+ width: 100%;+ }++ .file-history-modal .filter-buttons .ant-btn {+ flex: 1;+ }++ .file-history-modal .ant-space-compact {+ display: flex;+ width: 100%;+ }++ .file-history-modal .action-buttons {+ flex-direction: column;+ gap: 8px;+ }++ .file-history-modal .action-buttons .ant-space {+ width: 100%;+ }++ .file-history-modal .action-buttons .ant-btn {+ width: 100%;+ justify-content: center;+ }++ /* Adjust modal padding on mobile */+ .file-history-modal .ant-modal-body {+ padding: 12px;+ }++ .file-history-modal .ant-modal-header {+ padding: 12px 16px;+ }+}++/* Tablet responsiveness - Medium screens (576px - 767px) */+@media (min-width: 576px) and (max-width: 767px) {+ .file-history-modal .filters-section {+ padding: 10px 12px;+ }++ .file-history-modal .filter-buttons {+ margin-top: 16px !important;+ }+}++/* Desktop - Large screens (>= 768px) */+@media (min-width: 768px) {+ .file-history-modal .filter-buttons {+ justify-content: flex-start;+ }+}++/* Utility classes for inline style replacement */+.file-history-modal .filter-label {+ font-size: 13px;+}++.file-history-modal .full-width {+ width: 100%;+}++.file-history-modal .flex-button {+ flex: 1;+}++/* Table cell content styles */+.file-history-modal .file-path-text {+ flex: 1;+ max-width: 400px;+}++.file-history-modal .error-text {+ flex: 1;+ max-width: 230px;+}++.file-history-modal .cell-content {+ width: 100%;+}++.warning-icon {+ color: #faad14;+}diff --git a/frontend/src/components/pipelines-or-deployments/file-history-modal/FileHistoryModal.jsx b/frontend/src/components/pipelines-or-deployments/file-history-modal/FileHistoryModal.jsxnew file mode 100644index 000000000..e8dc422ec--- /dev/null+++ b/frontend/src/components/pipelines-or-deployments/file-history-modal/FileHistoryModal.jsx@@ -0,0 +1,707 @@+import {+ Table,+ Modal,+ Button,+ Select,+ InputNumber,+ Input,+ Space,+ Typography,+ Popconfirm,+ Tag,+ Row,+ Col,+ message,+} from "antd";+import {+ DeleteOutlined,+ ReloadOutlined,+ FilterOutlined,+ ClearOutlined,+ CopyOutlined,+ ExclamationCircleFilled,+} from "@ant-design/icons";+import PropTypes from "prop-types";+import { useState, useEffect } from "react";++import { workflowService } from "../../workflows/workflow/workflow-service.js";+import { useAlertStore } from "../../../store/alert-store.js";+import { useExceptionHandler } from "../../../hooks/useExceptionHandler.jsx";+import { copyToClipboard } from "../../../helpers/GetStaticData";+import "./FileHistoryModal.css";++const { Text } = Typography;++const MAX_BULK_DELETE = 100;++const FileHistoryModal = ({ open, setOpen, workflowId, workflowName }) => {+ const [fileHistories, setFileHistories] = useState([]);+ const [loading, setLoading] = useState(false);+ const [selectedRowKeys, setSelectedRowKeys] = useState([]);+ const { setAlertDetails } = useAlertStore();+ const handleException = useExceptionHandler();+ const workflowApiService = workflowService();++ // Filter states+ const [statusFilter, setStatusFilter] = useState([]);+ const [executionCountMin, setExecutionCountMin] = useState(null);+ const [executionCountMax, setExecutionCountMax] = useState(null);+ const [filePathFilter, setFilePathFilter] = useState("");++ // Bulk delete confirmation states+ const [showBulkDeleteConfirm, setShowBulkDeleteConfirm] = useState(false);+ const [fetchingCount, setFetchingCount] = useState(false);+ const [bulkDeleteCount, setBulkDeleteCount] = useState(0);++ // Track applied filter values (used for data fetching, pagination, and bulk clear)+ // These are separate from input values to prevent fetching with unapplied filters+ const [appliedFilters, setAppliedFilters] = useState({+ status: [],+ executionCountMin: null,+ executionCountMax: null,+ filePath: "",+ });++ // Pagination states+ const [pagination, setPagination] = useState({+ current: 1,+ pageSize: 10,+ total: 0,+ });++ const statusOptions = [+ { label: "Error", value: "ERROR" },+ { label: "Completed", value: "COMPLETED" },+ ];++ // Check if filters have been applied (for "Clear with Filters" button)+ const hasAppliedFilters = Boolean(+ appliedFilters.status?.length > 0 ||+ appliedFilters.executionCountMin !== null ||+ appliedFilters.executionCountMax !== null ||+ appliedFilters.filePath+ );++ // Check if input values differ from applied values (for Apply button indicator)+ const hasUnappliedChanges = (() => {+ // Compare status arrays (order-independent)+ const statusEqual =+ statusFilter.length === appliedFilters.status.length &&+ statusFilter.every((s) => appliedFilters.status.includes(s));++ return (+ !statusEqual ||+ executionCountMin !== appliedFilters.executionCountMin ||+ executionCountMax !== appliedFilters.executionCountMax ||+ filePathFilter !== appliedFilters.filePath+ );+ })();++ // Copy to clipboard helper+ const handleCopy = async (text, label = "Text") => {+ const success = await copyToClipboard(text);+ if (success) {+ message.success(`${label} copied to clipboard!`);+ } else {+ message.error("Failed to copy to clipboard");+ }+ };++ // Fetch file histories with optional filters parameter+ // If filters are passed, use them; otherwise use appliedFilters from state+ const fetchFileHistoriesWithFilters = async (+ page = 1,+ pageSize = 10,+ filters = null+ ) => {+ if (!workflowId) {+ setAlertDetails({+ type: "error",+ content: "Workflow ID is missing. Unable to fetch file histories.",+ });+ return;+ }++ const filtersToUse = filters || appliedFilters;++ setLoading(true);+ try {+ const params = {+ page,+ page_size: pageSize,+ };++ // Use provided filters for data fetching+ if (filtersToUse.status?.length > 0) {+ params.status = filtersToUse.status.join(",");+ }+ if (+ filtersToUse.executionCountMin !== null &&+ filtersToUse.executionCountMin !== undefined+ ) {+ params.execution_count_min = filtersToUse.executionCountMin;+ }+ if (+ filtersToUse.executionCountMax !== null &&+ filtersToUse.executionCountMax !== undefined+ ) {+ params.execution_count_max = filtersToUse.executionCountMax;+ }+ if (filtersToUse.filePath) {+ params.file_path = filtersToUse.filePath;+ }++ const response = await workflowApiService.getFileHistories(+ workflowId,+ params+ );++ const data = response?.data?.results || [];+ setFileHistories(data);+ setPagination({+ current: page,+ pageSize: pageSize,+ total: response?.data?.count || 0,+ });+ } catch (err) {+ // Extract more detailed error message+ const errorMessage =+ err?.response?.data?.message ||+ err?.response?.data?.error ||+ err?.response?.data?.detail ||+ err?.message ||+ "Failed to fetch file histories";++ setAlertDetails({+ type: "error",+ content: errorMessage,+ });+ } finally {+ setLoading(false);+ }+ };++ // Alias for backward compatibility (pagination, refresh, etc.)+ const fetchFileHistories = (page, pageSize) =>+ fetchFileHistoriesWithFilters(page, pageSize);++ // Load data on open or filter change+ useEffect(() => {+ if (open && workflowId) {+ fetchFileHistories(1, pagination.pageSize);+ setSelectedRowKeys([]);+ }+ }, [open, workflowId]);++ // Handle filter apply+ const handleApplyFilters = () => {+ // Save current input values as applied filters+ const newFilters = {+ status: statusFilter,+ executionCountMin: executionCountMin,+ executionCountMax: executionCountMax,+ filePath: filePathFilter,+ };+ setAppliedFilters(newFilters);+ // Pass new filters directly (state update is async, so can't rely on appliedFilters yet)+ fetchFileHistoriesWithFilters(1, pagination.pageSize, newFilters);+ setSelectedRowKeys([]);+ };++ // Handle filter reset+ const handleResetFilters = () => {+ setStatusFilter([]);+ setExecutionCountMin(null);+ setExecutionCountMax(null);+ setFilePathFilter("");+ // Reset applied filters+ const emptyFilters = {+ status: [],+ executionCountMin: null,+ executionCountMax: null,+ filePath: "",+ };+ setAppliedFilters(emptyFilters);+ // Fetch with empty filters+ fetchFileHistoriesWithFilters(1, pagination.pageSize, emptyFilters);+ };++ // Handle pagination change+ const handleTableChange = (paginationConfig) => {+ fetchFileHistories(paginationConfig.current, paginationConfig.pageSize);+ };++ // Delete single file history+ const handleDeleteSingle = async (fileHistoryId) => {+ try {+ await workflowApiService.deleteFileHistory(workflowId, fileHistoryId);+ setAlertDetails({+ type: "success",+ message: "File history deleted successfully",+ });+ // Refresh data+ fetchFileHistories(pagination.current, pagination.pageSize);+ setSelectedRowKeys([]);+ } catch (err) {+ setAlertDetails(handleException(err));+ }+ };++ // Delete selected file histories (bulk delete by IDs)+ const handleDeleteSelected = async () => {+ if (selectedRowKeys.length === 0) return;++ if (selectedRowKeys.length > MAX_BULK_DELETE) {+ setAlertDetails({+ type: "error",+ content: `Cannot delete more than ${MAX_BULK_DELETE} items at once. Please select fewer items.`,+ });+ return;+ }++ setLoading(true);+ try {+ const response = await workflowApiService.bulkDeleteFileHistoriesByIds(+ workflowId,+ selectedRowKeys+ );++ setAlertDetails({+ type: "success",+ message:+ response?.data?.message ||+ `${selectedRowKeys.length} file histories deleted successfully`,+ });++ // Refresh data+ fetchFileHistories(pagination.current, pagination.pageSize);+ setSelectedRowKeys([]);+ } catch (err) {+ setAlertDetails(handleException(err));+ } finally {+ setLoading(false);+ }+ };++ // Prepare bulk clear - fetch count for confirmation using applied filters+ const handlePrepareBulkClear = async () => {+ setFetchingCount(true);+ try {+ const params = {+ page: 1,+ page_size: 1, // We only need count, not actual data+ };++ // Use applied filters (not input values)+ if (appliedFilters.status?.length > 0) {+ params.status = appliedFilters.status.join(",");+ }+ if (+ appliedFilters.executionCountMin !== null &&+ appliedFilters.executionCountMin !== undefined+ ) {+ params.execution_count_min = appliedFilters.executionCountMin;+ }+ if (+ appliedFilters.executionCountMax !== null &&+ appliedFilters.executionCountMax !== undefined+ ) {+ params.execution_count_max = appliedFilters.executionCountMax;+ }+ if (appliedFilters.filePath) {+ params.file_path = appliedFilters.filePath;+ }++ const response = await workflowApiService.getFileHistories(+ workflowId,+ params+ );++ const count = response?.data?.count || 0;+ if (count > 0) {+ setBulkDeleteCount(count);+ setShowBulkDeleteConfirm(true);+ }+ // Note: count === 0 case is handled by disabling the button when pagination.total === 0+ } catch (err) {+ setAlertDetails(handleException(err));+ } finally {+ setFetchingCount(false);+ }+ };++ // Perform bulk clear after user confirms using applied filters+ const performBulkClear = async () => {+ setShowBulkDeleteConfirm(false);+ setLoading(true);+ try {+ const filters = {};++ // Use applied filters (not input values)+ if (appliedFilters.status?.length > 0) {+ filters.status = appliedFilters.status;+ }+ if (+ appliedFilters.executionCountMin !== null &&+ appliedFilters.executionCountMin !== undefined+ ) {+ filters.execution_count_min = appliedFilters.executionCountMin;+ }+ if (+ appliedFilters.executionCountMax !== null &&+ appliedFilters.executionCountMax !== undefined+ ) {+ filters.execution_count_max = appliedFilters.executionCountMax;+ }+ if (appliedFilters.filePath) {+ filters.file_path = appliedFilters.filePath;+ }++ const response = await workflowApiService.bulkClearFileHistories(+ workflowId,+ filters+ );++ setAlertDetails({+ type: "success",+ message:+ response?.data?.message ||+ `${response?.data?.deleted_count} file histories cleared`,+ });++ // Refresh data+ fetchFileHistories(1, pagination.pageSize);+ setSelectedRowKeys([]);+ } catch (err) {+ setAlertDetails(handleException(err));+ } finally {+ setLoading(false);+ }+ };++ // Row selection config+ const rowSelection = {+ selectedRowKeys,+ onChange: (selectedKeys) => {+ setSelectedRowKeys(selectedKeys);+ },+ };++ // Table columns+ const columns = [+ {+ title: "File Path",+ dataIndex: "file_path",+ key: "file_path",+ width: "40%",+ ellipsis: true,+ render: (text) => (+ <Space size="small" className="cell-content">+ <Text ellipsis={{ tooltip: text }} className="file-path-text">+ {text || "N/A"}+ </Text>+ {text && (+ <Button+ type="text"+ size="small"+ icon={<CopyOutlined />}+ onClick={(e) => {+ e.stopPropagation();+ handleCopy(text, "File path");+ }}+ title="Copy file path"+ />+ )}+ </Space>+ ),+ },+ {+ title: "Status",+ dataIndex: "status",+ key: "status",+ width: "8%",+ render: (status) => {+ let color = "default";+ if (status === "SUCCESS" || status === "COMPLETED") {+ color = "success";+ } else if (status === "ERROR" || status === "FAILURE") {+ color = "error";+ } else if (status === "PROCESSING") {+ color = "processing";+ } else if (status === "PENDING") {+ color = "warning";+ }+ return <Tag color={color}>{status}</Tag>;+ },+ },+ {+ title: "Execution Count",+ dataIndex: "execution_count",+ key: "execution_count",+ width: "10%",+ render: (count, record) => {+ const max = record?.max_execution_count;+ const exceeded = record?.has_exceeded_limit;+ return (+ <Text type={exceeded ? "danger" : "secondary"}>+ {count} / {max || "∞"}+ </Text>+ );+ },+ },+ {+ title: "Last Run",+ dataIndex: "modified_at",+ key: "modified_at",+ width: "12%",+ responsive: ["md"],+ render: (date) => {+ if (!date) return "N/A";+ return new Date(date).toLocaleString();+ },+ },+ {+ title: "Error",+ dataIndex: "error",+ key: "error",+ width: "25%",+ ellipsis: true,+ responsive: ["md"],+ render: (error) =>+ error ? (+ <Space size="small" className="cell-content">+ <Text+ type="danger"+ ellipsis={{ tooltip: error }}+ className="error-text"+ >+ {error}+ </Text>+ <Button+ type="text"+ size="small"+ icon={<CopyOutlined />}+ onClick={(e) => {+ e.stopPropagation();+ handleCopy(error, "Error message");+ }}+ title="Copy error message"+ />+ </Space>+ ) : (+ "-"+ ),+ },+ {+ title: "Action",+ key: "action",+ width: "8%",+ render: (_, record) => (+ <Popconfirm+ title="Delete this file history?"+ description="This action cannot be undone."+ onConfirm={() => handleDeleteSingle(record.id)}+ okText="Yes"+ cancelText="No"+ >+ <Button type="link" danger icon={<DeleteOutlined />} size="small">+ Delete+ </Button>+ </Popconfirm>+ ),+ },+ ];++ return (+ <Modal+ title={`File History - ${workflowName || "Workflow"}`}+ centered+ open={open}+ onCancel={() => setOpen(false)}+ footer={null}+ width="90%"+ className="file-history-modal"+ >+ <div className="file-history-content">+ {/* Filters Section */}+ <div className="filters-section">+ <Row gutter={[12, 12]} align="bottom">+ <Col xs={24} sm={12} md={10}>+ <Space direction="vertical" size="small" className="full-width">+ <Text strong className="filter-label">+ File Path:+ </Text>+ <Input+ placeholder="Search by file path..."+ value={filePathFilter}+ onChange={(e) => setFilePathFilter(e.target.value)}+ allowClear+ onPressEnter={handleApplyFilters}+ />+ </Space>+ </Col>+ <Col xs={12} sm={8} md={4}>+ <Space direction="vertical" size="small" className="full-width">+ <Text strong className="filter-label">+ Status:+ </Text>+ <Select+ mode="multiple"+ placeholder="Select status"+ value={statusFilter}+ onChange={setStatusFilter}+ options={statusOptions}+ className="full-width"+ allowClear+ />+ </Space>+ </Col>+ <Col xs={6} sm={4} md={3}>+ <Space direction="vertical" size="small" className="full-width">+ <Text strong className="filter-label">+ Min Count:+ </Text>+ <InputNumber+ placeholder="0"+ value={executionCountMin}+ onChange={setExecutionCountMin}+ min={0}+ className="full-width"+ />+ </Space>+ </Col>+ <Col xs={6} sm={4} md={3}>+ <Space direction="vertical" size="small" className="full-width">+ <Text strong className="filter-label">+ Max Count:+ </Text>+ <InputNumber+ placeholder="∞"+ value={executionCountMax}+ onChange={setExecutionCountMax}+ min={executionCountMin || 0}+ className="full-width"+ />+ </Space>+ </Col>+ <Col xs={24} sm={12} md={4}>+ <Space size="small" className="full-width">+ <Button+ type="primary"+ icon={<FilterOutlined />}+ onClick={handleApplyFilters}+ className="flex-button"+ >+ {hasUnappliedChanges ? "Apply *" : "Apply"}+ </Button>+ <Button+ icon={<ReloadOutlined />}+ onClick={handleResetFilters}+ className="flex-button"+ >+ Reset+ </Button>+ </Space>+ </Col>+ </Row>+ </div>++ {/* Action Buttons */}+ <div className="action-buttons">+ <Space wrap>+ <Popconfirm+ title={`Delete ${selectedRowKeys.length} selected file histories?`}+ description="This action cannot be undone."+ onConfirm={handleDeleteSelected}+ okText="Yes"+ cancelText="No"+ disabled={selectedRowKeys.length === 0}+ >+ <Button+ danger+ icon={<DeleteOutlined />}+ disabled={selectedRowKeys.length === 0}+ >+ Delete Selected ({selectedRowKeys.length})+ </Button>+ </Popconfirm>++ <Button+ icon={<ClearOutlined />}+ onClick={handlePrepareBulkClear}+ loading={fetchingCount}+ disabled={!hasAppliedFilters || pagination.total === 0}+ >+ Clear with Filters+ </Button>++ {/* Clear with Filters confirmation modal */}+ <Modal+ title={+ <Space>+ <ExclamationCircleFilled className="warning-icon" />+ <span>Clear with filters</span>+ </Space>+ }+ open={showBulkDeleteConfirm}+ onCancel={() => setShowBulkDeleteConfirm(false)}+ footer={[+ <Button+ key="cancel"+ onClick={() => setShowBulkDeleteConfirm(false)}+ >+ Cancel+ </Button>,+ <Button key="clear" type="primary" onClick={performBulkClear}>+ Clear+ </Button>,+ ]}+ centered+ width={400}+ >+ <Typography.Text>+ This will delete {bulkDeleteCount} record+ {bulkDeleteCount !== 1 ? "s" : ""} matching the active filters.+ This action cannot be undone.+ </Typography.Text>+ </Modal>++ <Button+ icon={<ReloadOutlined />}+ onClick={() =>+ fetchFileHistories(pagination.current, pagination.pageSize)+ }+ >+ Refresh+ </Button>+ </Space>+ </div>++ {/* Table */}+ <Table+ columns={columns}+ dataSource={fileHistories}+ rowKey="id"+ loading={loading}+ rowSelection={rowSelection}+ pagination={{+ ...pagination,+ showSizeChanger: true,+ showTotal: (total) => `Total ${total} records`,+ }}+ onChange={handleTableChange}+ scroll={{ x: 1200, y: "calc(100vh - 450px)" }}+ />+ </div>+ </Modal>+ );+};++FileHistoryModal.propTypes = {+ open: PropTypes.bool.isRequired,+ setOpen: PropTypes.func.isRequired,+ workflowId: PropTypes.string,+ workflowName: PropTypes.string,+};++export default FileHistoryModal;diff --git a/frontend/src/components/pipelines-or-deployments/pipelines/Pipelines.jsx b/frontend/src/components/pipelines-or-deployments/pipelines/Pipelines.jsxindex 0a105862d..4cf1afc5c 100644--- a/frontend/src/components/pipelines-or-deployments/pipelines/Pipelines.jsx+++ b/frontend/src/components/pipelines-or-deployments/pipelines/Pipelines.jsx@@ -12,6 +12,7 @@ import {CopyOutlined,LoadingOutlined,ShareAltOutlined,+ HistoryOutlined,} from "@ant-design/icons";import {Button,@@ -39,6 +40,7 @@ import { SpinnerLoader } from "../../widgets/spinner-loader/SpinnerLoader.jsx";import { DeleteModal } from "../delete-modal/DeleteModal.jsx";import { LogsModal } from "../log-modal/LogsModal.jsx";import { EtlTaskDeploy } from "../etl-task-deploy/EtlTaskDeploy.jsx";+import FileHistoryModal from "../file-history-modal/FileHistoryModal.jsx";import "./Pipelines.css";import { useExceptionHandler } from "../../../hooks/useExceptionHandler.jsx";import { pipelineService } from "../pipeline-service.js";@@ -71,6 +73,7 @@ function Pipelines({ type }) {const [openLogsModal, setOpenLogsModal] = useState(false);const [executionLogs, setExecutionLogs] = useState([]);const [executionLogsTotalCount, setExecutionLogsTotalCount] = useState(0);+ const [openFileHistoryModal, setOpenFileHistoryModal] = useState(false);const { fetchExecutionLogs } = require("../log-modal/fetchExecutionLogs.js");const [openManageKeysModal, setOpenManageKeysModal] = useState(false);const [apiKeys, setApiKeys] = useState([]);@@ -468,6 +471,29 @@ function Pipelines({ type }) {</Space>),},+ {+ key: "view-file-history",+ label: (+ <Space+ direction="horizontal"+ className="action-items"+ onClick={() => {+ if (!selectedPorD?.workflow_id) {+ setAlertDetails({+ type: "error",+ content: "Cannot view file history: Workflow ID not found",+ });+ return;+ }++ setOpenFileHistoryModal(true);+ }}+ >+ <HistoryOutlined />+ <Typography.Text>View File History</Typography.Text>+ </Space>+ ),+ },{key: "divider-operation",type: "divider",@@ -750,6 +776,12 @@ function Pipelines({ type }) {type={deploymentApiTypes.pipeline}id={selectedPorD?.id}/>+ <FileHistoryModal+ open={openFileHistoryModal}+ setOpen={setOpenFileHistoryModal}+ workflowId={selectedPorD?.workflow_id}+ workflowName={selectedPorD?.pipeline_name}+ />{openShareModal && (<SharePermissionopen={openShareModal}diff --git a/frontend/src/components/workflows/workflow/workflow-service.js b/frontend/src/components/workflows/workflow/workflow-service.jsindex cf0beeaf1..5dfd0a76d 100644--- a/frontend/src/components/workflows/workflow/workflow-service.js+++ b/frontend/src/components/workflows/workflow/workflow-service.js@@ -68,6 +68,48 @@ function workflowService() {};return axiosPrivate(options);},+ getFileHistories: (workflowId, params = {}) => {+ options = {+ url: `${path}/workflow/${workflowId}/file-histories/`,+ method: "GET",+ params: params,+ };+ return axiosPrivate(options);+ },+ deleteFileHistory: (workflowId, fileHistoryId) => {+ options = {+ url: `${path}/workflow/${workflowId}/file-histories/${fileHistoryId}/`,+ method: "DELETE",+ headers: {+ "X-CSRFToken": csrfToken,+ },+ };+ return axiosPrivate(options);+ },+ bulkClearFileHistories: (workflowId, filters) => {+ options = {+ url: `${path}/workflow/${workflowId}/file-histories/clear/`,+ method: "POST",+ headers: {+ "X-CSRFToken": csrfToken,+ "Content-Type": "application/json",+ },+ data: filters,+ };+ return axiosPrivate(options);+ },+ bulkDeleteFileHistoriesByIds: (workflowId, ids) => {+ options = {+ url: `${path}/workflow/${workflowId}/file-histories/clear/`,+ method: "POST",+ headers: {+ "X-CSRFToken": csrfToken,+ "Content-Type": "application/json",+ },+ data: { ids },+ };+ return axiosPrivate(options);+ },canUpdate: (id) => {options = {url: `${path}/workflow/${id}/can-update`,diff --git a/frontend/src/helpers/GetStaticData.js b/frontend/src/helpers/GetStaticData.jsindex fc00b28ca..dcb620990 100644--- a/frontend/src/helpers/GetStaticData.js+++ b/frontend/src/helpers/GetStaticData.js@@ -664,6 +664,16 @@ const formatTimeDisplay = (seconds) => {return parts.join(" ");};+const copyToClipboard = async (text) => {+ try {+ await navigator.clipboard.writeText(text);+ return true;+ } catch {+ // Clipboard API may fail in insecure contexts or unsupported browsers+ return false;+ }+};+const UNSTRACT_ADMIN = "unstract_admin";const logsStaticContent = {@@ -744,4 +754,5 @@ export {formatTimeDisplay,formattedDateTimeWithSeconds,logsStaticContent,+ copyToClipboard,};diff --git a/unstract/core/src/unstract/core/data_models.py b/unstract/core/src/unstract/core/data_models.pyindex c8d23d375..7e8e984a0 100644--- a/unstract/core/src/unstract/core/data_models.py+++ b/unstract/core/src/unstract/core/data_models.py@@ -1181,6 +1181,7 @@ class FileHistoryData:cache_key: strprovider_file_uuid: str | None = Nonestatus: str = ExecutionStatus.PENDING.value+ execution_count: int = 1result: str | None = Nonemetadata: str | None = Noneerror: str | None = None@@ -1207,6 +1208,7 @@ class FileHistoryData:cache_key=data.get("cache_key"),provider_file_uuid=data.get("provider_file_uuid"),status=data.get("status", ExecutionStatus.PENDING.value),+ execution_count=data.get("execution_count", 1),result=data.get("result"),metadata=data.get("metadata"),error=data.get("error"),diff --git a/workers/api-deployment/tasks.py b/workers/api-deployment/tasks.pyindex 31367066c..46f8704ba 100644--- a/workers/api-deployment/tasks.py+++ b/workers/api-deployment/tasks.py@@ -25,6 +25,7 @@ from shared.workflow.execution.tool_validation import validate_workflow_tool_insfrom worker import appfrom unstract.core.data_models import ExecutionStatus, FileHashData, WorkerFileData+from unstract.core.worker_models import ApiDeploymentResultStatuslogger = WorkerLogger.get_logger(__name__)@@ -523,15 +524,27 @@ def _run_workflow_api()# Keep as string if all parsing fails+ # Map ExecutionStatus to ApiDeploymentResultStatus+ cached_status = cached_result.get("status")+ if cached_status == ExecutionStatus.COMPLETED.value:+ api_status = ApiDeploymentResultStatus.SUCCESS.value+ else:+ # Defensive: shouldn't happen (we only cache COMPLETED)+ api_status = ApiDeploymentResultStatus.FAILED.value+ logger.warning(+ f"Unexpected status {cached_status} in cached result for {hash_data.file_name}"+ )+api_result = {"file": hash_data.file_name,"file_execution_id": hash_data.provider_file_uuid or "",- "status": "Success", # Cached results are always successful+ "status": api_status,"result": cached_result_data,- "error": None,+ "error": cached_result.get("error"),"metadata": {"processing_time": 0.0, # Cached files take no time"source": "file_history_cache",+ "cached_status": cached_status,},}@@ -1152,19 +1165,34 @@ def _check_file_history_api(else:file_name = file_key- # Store cached result details+ # Check status before caching - only cache COMPLETED resultsif file_hash_str in file_history_details:- cached_results[file_hash_str] = {- "file_name": file_name,- "file_key": file_key,- "file_hash_data": file_hash_data,- **file_history_details[file_hash_str], # result, metadata, etc.- }-- skipped_files.append(file_name)- logger.info(- f"Using cached result for file: {file_name} (hash: {file_hash_str[:16]}...)"- )+ file_history = file_history_details[file_hash_str]+ status = file_history.get("status")++ # Only cache COMPLETED results, execute fresh for ERROR/STOPPED/etc.+ if status == ExecutionStatus.COMPLETED.value:+ cached_results[file_hash_str] = {+ "file_name": file_name,+ "file_key": file_key,+ "file_hash_data": file_hash_data,+ **file_history,+ }+ skipped_files.append(file_name)+ logger.info(+ f"Using cached COMPLETED result for file: {file_name} "+ f"(hash: {file_hash_str[:16]}...)"+ )+ else:+ # Non-COMPLETED status (ERROR, STOPPED, etc.) - execute fresh+ files_to_process[file_key] = file_hash_data+ logger.info(+ f"Skipping cached result with status={status} for file: {file_name}, "+ f"will execute fresh (hash: {file_hash_str[:16]}...)"+ )+ else:+ # Hash in processed_hashes but no file_history_details (shouldn't happen)+ files_to_process[file_key] = file_hash_dataelse:files_to_process[file_key] = file_hash_datadiff --git a/workers/shared/processing/files/processor.py b/workers/shared/processing/files/processor.pyindex 0a66a67dd..90009d3a5 100644--- a/workers/shared/processing/files/processor.py+++ b/workers/shared/processing/files/processor.py@@ -83,9 +83,6 @@ class CachedFileHandler:# Handle both legacy format (result field) and new format (file_history field)if history_result.get("found") and history_result.get("file_history"):- # Legacy format - direct result field- logger.info(f"✓ Retrieved cached result for {context.file_name}")-file_history_data = history_result.get("file_history")if not file_history_data:@@ -102,6 +99,20 @@ class CachedFileHandler:from_cache=True,)+ # Check status - only return COMPLETED results+ status = file_history_data.get("status")++ if status != ExecutionStatus.COMPLETED.value:+ logger.info(+ f"File {context.file_name} found in history with status={status}, "+ f"skipping cached result and executing fresh"+ )+ return None # Continue to fresh execution++ logger.info(+ f"✓ Retrieved cached COMPLETED result for {context.file_name}"+ )+# Parse cached JSON resulttry:cached_result = json.loads(file_history_data.get("result", "{}"))diff --git a/workers/shared/processing/filter_pipeline.py b/workers/shared/processing/filter_pipeline.pyindex 8287e4893..7a47bc7d4 100644--- a/workers/shared/processing/filter_pipeline.py+++ b/workers/shared/processing/filter_pipeline.py@@ -389,56 +389,66 @@ class FileHistoryFilter(FileFilter):if file_history:status = file_history.get("status", "UNKNOWN")history_path = file_history.get("file_path")+ execution_count = file_history.get("execution_count", 0)+ max_execution_count = file_history.get("max_execution_count", 3)+ has_exceeded_limit = file_history.get("has_exceeded_limit")logger.info(f"FileHistoryFilter - {file_hash.file_name}: History details: "- f"status={status}, path={history_path}, current_path={file_path}"+ f"status={status}, path={history_path}, current_path={file_path}, "+ f"execution_count={execution_count}/{max_execution_count}, "+ f"has_exceeded_limit={has_exceeded_limit}")++ # Check execution limit using backend's has_exceeded_limit flag+ # Backend determines if limit applies based on workflow type (e.g., API workflows are exempt)+ # Prefer backend's flag; only fall back to raw count if flag is not provided+ if has_exceeded_limit is True:+ logger.warning(+ f"FileHistoryFilter - {file_hash.file_name}: SKIP "+ f"(backend reports has_exceeded_limit=True)"+ )+ return True # Skip file - backend determined limit exceeded+ elif (+ has_exceeded_limit is None+ and max_execution_count is not None+ and execution_count >= max_execution_count+ ):+ # Backward compatibility: if backend doesn't provide has_exceeded_limit flag,+ # fall back to raw count comparison+ logger.warning(+ f"FileHistoryFilter - {file_hash.file_name}: SKIP "+ f"(max execution count reached: {execution_count}/{max_execution_count})"+ )+ return True # Skip file - max execution count exceededelse:logger.warning(f"FileHistoryFilter - {file_hash.file_name}: Found=True but no file_history data!")- # Check using proper status-based logic instead of just is_completed+ # Check using proper status-based logicstatus = (file_history.get("status", "UNKNOWN") if file_history else "NO_HISTORY_DATA")- # Import ExecutionStatus for proper status checking-- # Check if file should be skipped based on status- try:- if status in [- ExecutionStatus.EXECUTING.value,- ExecutionStatus.PENDING.value,- ExecutionStatus.COMPLETED.value,- ]:- logger.info(- f"FileHistoryFilter - {file_hash.file_name}: SKIP "- f"(status={status} is in skip-processing list)"- )- else:- logger.info(- f"FileHistoryFilter - {file_hash.file_name}: ACCEPT "- f"(status={status} allows reprocessing)"- )- return False- except Exception as e:- logger.warning(f"FileHistoryFilter - Error checking status {status}: {e}")- # Fallback to original is_completed logic if status checking fails- if not is_completed:- logger.info(- f"FileHistoryFilter - {file_hash.file_name}: ACCEPT "- f"(fallback: history exists but not completed, status={status})"- )- return False+ # If status allows reprocessing, accept immediately+ if status not in [+ ExecutionStatus.EXECUTING.value,+ ExecutionStatus.PENDING.value,+ ExecutionStatus.COMPLETED.value,+ ]:+ logger.info(+ f"FileHistoryFilter - {file_hash.file_name}: ACCEPT "+ f"(status={status} allows reprocessing)"+ )+ return False- # Now check path matching - only skip if paths match+ # Status requires skip check - only skip if path also matcheshistory_path = file_history.get("file_path") if file_history else Noneif history_path == file_path:logger.info(f"FileHistoryFilter - {file_hash.file_name}: SKIP "- f"(status={status} requires skip and same path: {file_path})"+ f"(status={status} and same path: {file_path})")return Trueelse:diff --git a/workers/shared/workflow/execution/service.py b/workers/shared/workflow/execution/service.pyindex f8146ec22..3b7e7a74b 100644--- a/workers/shared/workflow/execution/service.py+++ b/workers/shared/workflow/execution/service.py@@ -20,7 +20,7 @@ from shared.models.file_processing import FileProcessingContext# Import shared dataclasses for type safety and consistencyfrom unstract.core.data_models import (- # DestinationConfig, # remove once verified+ ExecutionStatus,FileHashData,FileOperationConstants,WorkflowDefinitionResponseData,@@ -1450,7 +1450,15 @@ class WorkerWorkflowExecutionService:logger.warning(f"Failed to serialize result: {e}")result_json = str(output_result)- # Create file history via API+ # Determine status and error based on processing outcome+ if processing_error or self._last_execution_error:+ file_status = ExecutionStatus.ERROR.value+ error_message = str(processing_error or self._last_execution_error)+ else:+ file_status = ExecutionStatus.COMPLETED.value+ error_message = ""++ # Create file history via API (for both success and error)file_history_response = self.api_client.create_file_history(file_path=file_hash.file_path if not destination.is_api else None,file_name=file_hash.file_name,@@ -1461,7 +1469,8 @@ class WorkerWorkflowExecutionService:mime_type=getattr(file_hash, "mime_type", ""),result=result_json,metadata=metadata,- status="COMPLETED",+ status=file_status,+ error=error_message,provider_file_uuid=getattr(file_hash, "provider_file_uuid", None),is_api=destination.is_api,)@@ -1504,26 +1513,25 @@ class WorkerWorkflowExecutionService:) -> bool:"""Determine if file history should be created.- File history creation rules:- - API workflows: Create WITH results only when use_file_history=True- - ETL/TASK/MANUAL_REVIEW workflows: Always create WITHOUT results (for tracking)+ File history creation rules (updated to track execution counts):+ - Always create file_history for execution count tracking (both success and error)+ - API workflows: Only create if use_file_history=True+ - API workflows with success: Only create if there's output result+ - API workflows with error: Always create for execution count tracking+ - ETL/TASK workflows: Always create (for both success and error)"""- # Don't create if there is a tool execution error- if self._last_execution_error:- return False-- # Don't create if there's a processing error- if processing_error:- return False-# For API workflows, only create if use_file_history is enabledif destination.is_api and not destination.use_file_history:return False- # For API workflows, only create if there's a valid output result- if destination.is_api and not output_result:- return False+ # For API workflows with success, only create if there's a valid output result+ # For API workflows with errors, always create for execution count tracking+ if destination.is_api:+ has_error = processing_error or self._last_execution_error+ if not has_error and not output_result:+ return False+ # Create file_history for both COMPLETED and ERROR statusesreturn Truedef _get_destination_config(