Skip to content

LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows

UN-2584 [FEAT] Configurable ETL schedule intervals with conflict prevention and 30-minute support (#1426)

alib169d0e9fec6
diff --git a/backend/backend/settings/base.py b/backend/backend/settings/base.py
index 8e4169412..871f1c3fc 100644
--- a/backend/backend/settings/base.py
+++ b/backend/backend/settings/base.py
@@ -155,6 +155,10 @@ INSTANT_WF_POLLING_TIMEOUT = int(
os.environ.get("INSTANT_WF_POLLING_TIMEOUT", "300")
) # 5 minutes
+# ETL Pipeline minimum schedule interval (in seconds)
+# Default: 1800 seconds (30 minutes)
+MIN_SCHEDULE_INTERVAL_SECONDS = int(os.environ.get("MIN_SCHEDULE_INTERVAL_SECONDS", 1800))
+
# File processing batches
MAX_PARALLEL_FILE_BATCHES = int(os.environ.get("MAX_PARALLEL_FILE_BATCHES", 1))
# Upper limit for batch validation
diff --git a/backend/pipeline_v2/constants.py b/backend/pipeline_v2/constants.py
index a234983b6..75a030eff 100644
--- a/backend/pipeline_v2/constants.py
+++ b/backend/pipeline_v2/constants.py
@@ -1,3 +1,6 @@
+from django.conf import settings
+
+
class PipelineConstants:
"""Constants for Pipelines."""
@@ -53,6 +56,20 @@ class PipelineErrors:
INVALID_WF = "The provided workflow does not exist"
+class PipelineScheduling:
+ """Constants for pipeline scheduling configuration."""
+
+ @classmethod
+ def get_min_interval_seconds(cls) -> int:
+ """Get minimum schedule interval in seconds from settings."""
+ return settings.MIN_SCHEDULE_INTERVAL_SECONDS
+
+ @classmethod
+ def get_min_interval_minutes(cls) -> int:
+ """Get minimum schedule interval in minutes from settings."""
+ return cls.get_min_interval_seconds() // 60
+
+
class PipelineURL:
"""Constants for URL names."""
diff --git a/backend/pipeline_v2/serializers/crud.py b/backend/pipeline_v2/serializers/crud.py
index a2cba45c3..d69f616fd 100644
--- a/backend/pipeline_v2/serializers/crud.py
+++ b/backend/pipeline_v2/serializers/crud.py
@@ -8,6 +8,7 @@ from croniter import croniter
from django.conf import settings
from pipeline_v2.constants import PipelineConstants as PC
from pipeline_v2.constants import PipelineKey as PK
+from pipeline_v2.constants import PipelineScheduling
from pipeline_v2.models import Pipeline
from rest_framework import serializers
from rest_framework.serializers import SerializerMethodField
@@ -43,6 +44,112 @@ class PipelineSerializer(IntegrityErrorMixin, AuditSerializer):
},
}
+ # Constants for validation
+ MINUTE_RANGE = (0, 59)
+ HOUR_MINUTES = 60
+
+ def _get_validation_examples(self, min_interval: int) -> str:
+ """Generate examples for error messages with exact min_interval gaps."""
+ first = 0
+ second = (first + min_interval) % self.HOUR_MINUTES
+ third = (second + min_interval) % self.HOUR_MINUTES
+ return f"'{first},{second}', '{second},{third}'"
+
+ def _get_cron_error_too_frequent(self) -> str:
+ min_interval = PipelineScheduling.get_min_interval_minutes()
+ return f"Configured cron schedule is too frequent. Ensure that consecutive runs are atleast {min_interval} minutes or longer."
+
+ def _get_cron_error_complex_pattern(self) -> str:
+ min_interval = PipelineScheduling.get_min_interval_minutes()
+ examples = self._get_validation_examples(min_interval)
+ return (
+ f"Intervals shorter than {min_interval} minutes detected. "
+ f"Ensure consecutive values are ≥{min_interval} minutes apart. Examples: {examples}"
+ )
+
+ def _get_cron_error_range_pattern(self) -> str:
+ min_interval = PipelineScheduling.get_min_interval_minutes()
+ examples = self._get_validation_examples(min_interval)
+ return f"Range patterns not supported. Use comma-separated values ≥{min_interval} minutes apart. Examples: {examples}"
+
+ def _validate_basic_cron_format(self, value: str | None) -> str | None:
+ """Validate basic cron format and handle None/empty cases."""
+ if value is None:
+ return None
+
+ cron_string = value.strip()
+ if not cron_string:
+ return None
+
+ try:
+ croniter(cron_string)
+ except Exception as error:
+ logger.error(f"Invalid cron string '{cron_string}': {error}")
+ raise serializers.ValidationError("Invalid cron string format.")
+
+ return cron_string
+
+ def _validate_step_pattern(self, minute_field: str) -> None:
+ """Validate step patterns like */15."""
+ parts = minute_field.split("/")
+ if len(parts) == 2 and parts[1].isdigit():
+ step = int(parts[1])
+ if step < PipelineScheduling.get_min_interval_minutes():
+ raise serializers.ValidationError(self._get_cron_error_too_frequent())
+
+ def _parse_and_validate_minutes(self, parts: list[str]) -> list[int]:
+ """Parse minute parts and validate range."""
+ if not all(
+ p.isdigit() and self.MINUTE_RANGE[0] <= int(p) <= self.MINUTE_RANGE[1]
+ for p in parts
+ ):
+ raise serializers.ValidationError(self._get_cron_error_complex_pattern())
+ return sorted([int(p) for p in parts])
+
+ def _check_consecutive_intervals(self, minutes: list[int], min_interval: int) -> None:
+ """Check intervals between consecutive minute values."""
+ for i in range(len(minutes) - 1):
+ if minutes[i + 1] - minutes[i] < min_interval:
+ raise serializers.ValidationError(self._get_cron_error_complex_pattern())
+
+ def _check_wraparound_interval(self, minutes: list[int], min_interval: int) -> None:
+ """Check wraparound interval from last to first minute of next hour."""
+ if len(minutes) > 1:
+ wraparound = (self.HOUR_MINUTES - minutes[-1]) + minutes[0]
+ if wraparound < min_interval:
+ raise serializers.ValidationError(self._get_cron_error_complex_pattern())
+
+ def _validate_comma_pattern(self, minute_field: str) -> None:
+ """Validate comma patterns like 0,30 or 0,10,20,30,40,50."""
+ parts = [p.strip() for p in minute_field.split(",")]
+ minutes = self._parse_and_validate_minutes(parts)
+
+ min_interval = PipelineScheduling.get_min_interval_minutes()
+ self._check_consecutive_intervals(minutes, min_interval)
+ self._check_wraparound_interval(minutes, min_interval)
+
+ def _validate_range_pattern(self, minute_field: str) -> None:
+ """Validate range patterns like 0-30.
+
+ Range patterns are not supported because they can create intervals
+ shorter than the minimum required interval. For example, with a
+ 30-minute minimum, "0-30" would create executions every minute
+ from 0 to 30, violating the minimum interval constraint.
+ """
+ raise serializers.ValidationError(self._get_cron_error_range_pattern())
+
+ def _validate_minute_field(self, minute_field: str) -> None:
+ """Validate the minute field for configurable minimum intervals."""
+ if minute_field == "*":
+ raise serializers.ValidationError(self._get_cron_error_too_frequent())
+
+ if "/" in minute_field:
+ self._validate_step_pattern(minute_field)
+ elif "," in minute_field:
+ self._validate_comma_pattern(minute_field)
+ elif "-" in minute_field:
+ self._validate_range_pattern(minute_field)
+
def validate_cron_string(self, value: str | None = None) -> str | None:
"""Validate the cron string provided in the serializer data.
@@ -64,28 +171,14 @@ class PipelineSerializer(IntegrityErrorMixin, AuditSerializer):
Optional[str]: The validated cron string if it is valid,
otherwise None.
"""
- if value is None:
+ cron_string = self._validate_basic_cron_format(value)
+ if cron_string is None:
return None
- cron_string = value.strip()
- # Check if the string is empty
- if not cron_string:
- return None
-
- # Validate the cron string
- try:
- croniter(cron_string)
- except Exception as error:
- logger.error(f"Invalid cron string '{cron_string}': {error}")
- raise serializers.ValidationError("Invalid cron string format.")
- # Check if the frequency is less than 1 hour
cron_parts = cron_string.split()
minute_field = cron_parts[0]
- if minute_field == "*" or any(char in minute_field for char in [",", "-", "/"]):
- raise serializers.ValidationError(
- "Cron schedule can not be more than once per hour. Please provide a "
- "cron schedule to run at an hourly or less frequent interval."
- )
+
+ self._validate_minute_field(minute_field)
return cron_string
diff --git a/backend/sample.env b/backend/sample.env
index 78007a9a7..9c55b2034 100644
--- a/backend/sample.env
+++ b/backend/sample.env
@@ -188,3 +188,8 @@ FILE_EXECUTION_TRACKER_COMPLETED_TTL_IN_SECOND=600 # 10 minutes
# Runner polling timeout
MAX_RUNNER_POLLING_WAIT_SECONDS=10800 # 3 hours
RUNNER_POLLING_INTERVAL_SECONDS=2 # 2 seconds
+
+# ETL Pipeline minimum schedule interval (in seconds)
+# Default: 1800 seconds (30 minutes)
+# Examples: 900 (15 min), 1800 (30 min), 3600 (60 min)
+MIN_SCHEDULE_INTERVAL_SECONDS=1800
diff --git a/backend/workflow_manager/endpoint_v2/source.py b/backend/workflow_manager/endpoint_v2/source.py
index 36542a79c..4098963da 100644
--- a/backend/workflow_manager/endpoint_v2/source.py
+++ b/backend/workflow_manager/endpoint_v2/source.py
@@ -37,7 +37,9 @@ from workflow_manager.endpoint_v2.exceptions import (
from workflow_manager.endpoint_v2.models import WorkflowEndpoint
from workflow_manager.file_execution.models import WorkflowFileExecution
from workflow_manager.utils.workflow_log import WorkflowLog
+from workflow_manager.workflow_v2.enums import ExecutionStatus
from workflow_manager.workflow_v2.file_history_helper import FileHistoryHelper
+from workflow_manager.workflow_v2.models.execution import WorkflowExecution
from workflow_manager.workflow_v2.models.file_history import FileHistory
from workflow_manager.workflow_v2.models.workflow import Workflow
@@ -372,6 +374,104 @@ class SourceConnector(BaseConnector):
count += 1
return count
+ def _is_file_being_processed(self, file_hash: FileHash) -> bool:
+ """Check if file is currently being processed or should be skipped
+ Uses direct database queries instead of FileExecutionStatusTracker
+
+ Args:
+ file_hash (FileHash): The file hash to check
+
+ Returns:
+ bool: True if file should be skipped (being processed or already completed), False otherwise
+ """
+ active_executions = self._get_active_workflow_executions()
+ logger.info(
+ f"Found {len(active_executions)} active executions for workflow {self.workflow.id}"
+ )
+
+ for execution in active_executions:
+ if self._has_blocking_file_execution(execution, file_hash):
+ return True
+
+ return False
+
+ def _get_active_workflow_executions(self):
+ """Get active executions for this workflow with organization filtering for security."""
+ organization = UserContext.get_organization()
+ return WorkflowExecution.objects.filter(
+ workflow=self.workflow,
+ workflow__organization_id=organization.id, # Security: Organization isolation
+ status__in=[ExecutionStatus.EXECUTING, ExecutionStatus.PENDING],
+ )
+
+ def _has_blocking_file_execution(self, execution, file_hash: FileHash) -> bool:
+ """Check if there's a blocking file execution that prevents processing."""
+ try:
+ # Try to find blocking file execution using file_hash
+ file_exec = self._find_blocking_file_execution_by_hash(execution, file_hash)
+ if file_exec:
+ self._log_duplicate_file_execution_skip(file_hash, file_exec, execution)
+ return True
+
+ # Try to find blocking file execution using provider_file_uuid
+ file_exec = self._find_blocking_file_execution_by_provider_uuid(
+ execution, file_hash
+ )
+ if file_exec:
+ self._log_duplicate_file_execution_skip(file_hash, file_exec, execution)
+ return True
+
+ except Exception as e:
+ logger.warning(
+ f"Error checking file execution status for {file_hash.file_name}: {e}\n"
+ "Allowing its execution to continue anyway"
+ )
+
+ return False
+
+ def _find_blocking_file_execution_by_hash(self, execution, file_hash: FileHash):
+ """Find blocking file execution by file hash if it exists."""
+ if not file_hash.file_hash:
+ return None
+
+ try:
+ return WorkflowFileExecution.objects.get(
+ workflow_execution=execution,
+ file_hash=file_hash.file_hash,
+ file_path=file_hash.file_path,
+ status__in=ExecutionStatus.get_skip_processing_statuses(),
+ )
+ except WorkflowFileExecution.DoesNotExist:
+ return None
+
+ def _find_blocking_file_execution_by_provider_uuid(
+ self, execution, file_hash: FileHash
+ ):
+ """Find blocking file execution by provider UUID if it exists."""
+ if not file_hash.provider_file_uuid:
+ return None
+
+ try:
+ return WorkflowFileExecution.objects.get(
+ workflow_execution=execution,
+ provider_file_uuid=file_hash.provider_file_uuid,
+ file_path=file_hash.file_path,
+ status__in=ExecutionStatus.get_skip_processing_statuses(),
+ )
+ except WorkflowFileExecution.DoesNotExist:
+ return None
+
+ def _log_duplicate_file_execution_skip(
+ self, file_hash: FileHash, file_exec, execution
+ ):
+ """Log message when skipping file due to duplicate/blocking execution."""
+ message = (
+ f"Skipping file '{file_hash.file_name}' - status: {file_exec.status} "
+ f"in execution {execution.id}, file execution id: {file_exec.id}"
+ )
+ self.workflow_log.publish_log(message)
+ logger.info(message)
+
def _should_skip_file(
self,
file_hash: FileHash,
@@ -387,12 +487,20 @@ class SourceConnector(BaseConnector):
bool: True if the file should be skipped, False otherwise.
"""
file_name = os.path.basename(file_hash.file_path)
- return not self._should_process_file(
- file_name, patterns
- ) or not self._is_new_file(
- file_hash=file_hash,
- workflow=self.endpoint.workflow,
- )
+
+ # Existing pattern check
+ if not self._should_process_file(file_name, patterns):
+ return True
+
+ # Existing file history check
+ if not self._is_new_file(file_hash=file_hash, workflow=self.endpoint.workflow):
+ return True
+
+ # NEW: Check if file is being processed
+ if self._is_file_being_processed(file_hash):
+ return True
+
+ return False
def _is_duplicate(self, file_hash: FileHash, unique_file_paths: set[str]) -> bool:
return (
diff --git a/backend/workflow_manager/file_execution/migrations/0006_workflowfileexecution_wf_file_hash_path_status_idx_and_more.py b/backend/workflow_manager/file_execution/migrations/0006_workflowfileexecution_wf_file_hash_path_status_idx_and_more.py
new file mode 100644
index 000000000..12ffd4f52
--- /dev/null
+++ b/backend/workflow_manager/file_execution/migrations/0006_workflowfileexecution_wf_file_hash_path_status_idx_and_more.py
@@ -0,0 +1,31 @@
+# Generated by Django 4.2.1 on 2025-07-14 14:58
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+ dependencies = [
+ ("file_execution", "0005_workflowfileexecution_fs_metadata_and_more"),
+ ]
+
+ operations = [
+ migrations.AddIndex(
+ model_name="workflowfileexecution",
+ index=models.Index(
+ fields=["workflow_execution", "file_hash", "file_path", "status"],
+ name="wf_file_hash_path_status_idx",
+ ),
+ ),
+ migrations.AddIndex(
+ model_name="workflowfileexecution",
+ index=models.Index(
+ fields=[
+ "workflow_execution",
+ "provider_file_uuid",
+ "file_path",
+ "status",
+ ],
+ name="wf_provider_uuid_path_stat_idx",
+ ),
+ ),
+ ]
diff --git a/backend/workflow_manager/file_execution/models.py b/backend/workflow_manager/file_execution/models.py
index ce8434b77..fd75b163f 100644
--- a/backend/workflow_manager/file_execution/models.py
+++ b/backend/workflow_manager/file_execution/models.py
@@ -187,6 +187,19 @@ class WorkflowFileExecution(BaseModel):
fields=["workflow_execution", "provider_file_uuid"],
name="workflow_exec_p_uuid_idx",
),
+ models.Index(
+ fields=["workflow_execution", "file_hash", "file_path", "status"],
+ name="wf_file_hash_path_status_idx",
+ ),
+ models.Index(
+ fields=[
+ "workflow_execution",
+ "provider_file_uuid",
+ "file_path",
+ "status",
+ ],
+ name="wf_provider_uuid_path_stat_idx",
+ ),
]
constraints = [
models.UniqueConstraint(
diff --git a/backend/workflow_manager/workflow_v2/enums.py b/backend/workflow_manager/workflow_v2/enums.py
index 7fb75af88..baebbf358 100644
--- a/backend/workflow_manager/workflow_v2/enums.py
+++ b/backend/workflow_manager/workflow_v2/enums.py
@@ -44,6 +44,70 @@ class ExecutionStatus(TextChoices):
)
return status_enum in [cls.COMPLETED, cls.STOPPED, cls.ERROR]
+ @classmethod
+ def is_active(cls, status: str | ExecutionStatus) -> bool:
+ """Check if the workflow execution status is active (in progress)."""
+ try:
+ status_enum = cls(status)
+ except ValueError:
+ raise ValueError(
+ f"Invalid status: {status}. Must be a valid ExecutionStatus."
+ )
+ return status_enum in [cls.PENDING, cls.EXECUTING]
+
+ @classmethod
+ def get_skip_processing_statuses(cls) -> list[ExecutionStatus]:
+ """Get list of statuses that should skip file processing.
+
+ Skip processing if:
+ - EXECUTING: File is currently being processed
+ - PENDING: File is queued to be processed
+ - COMPLETED: File has already been successfully processed
+
+ Returns:
+ list[ExecutionStatus]: List of statuses where file processing should be skipped
+ """
+ return [cls.EXECUTING, cls.PENDING, cls.COMPLETED]
+
+ @classmethod
+ def should_skip_file_processing(cls, status: str | ExecutionStatus) -> bool:
+ """Check if file processing should be skipped based on status.
+
+ Allow processing (retry) if:
+ - STOPPED: Processing was stopped, can retry
+ - ERROR: Processing failed, can retry
+ """
+ try:
+ status_enum = cls(status)
+ except ValueError:
+ raise ValueError(
+ f"Invalid status: {status}. Must be a valid ExecutionStatus."
+ )
+ return status_enum in cls.get_skip_processing_statuses()
+
+ @classmethod
+ def can_update_to_pending(cls, status: str | ExecutionStatus) -> bool:
+ """Check if a status can be updated to PENDING.
+
+ Allow updating to PENDING if:
+ - Status is STOPPED or ERROR (can retry)
+ - Status is None (new record)
+
+ Don't allow updating to PENDING if:
+ - Status is EXECUTING (currently processing)
+ - Status is COMPLETED (already done)
+ - Status is already PENDING (no change needed)
+ """
+ if status is None:
+ return True
+
+ try:
+ status_enum = cls(status)
+ except ValueError:
+ return True # Invalid status, allow update
+
+ return status_enum in [cls.STOPPED, cls.ERROR]
+
class SchemaType(Enum):
"""Possible types for workflow module's JSON schema.
diff --git a/backend/workflow_manager/workflow_v2/file_execution_tasks.py b/backend/workflow_manager/workflow_v2/file_execution_tasks.py
index 85616bc52..463212a7c 100644
--- a/backend/workflow_manager/workflow_v2/file_execution_tasks.py
+++ b/backend/workflow_manager/workflow_v2/file_execution_tasks.py
@@ -179,37 +179,33 @@ class FileExecutionTasks:
logger.info(f"Processing {total_files} files of execution {execution_id}")
- for file_number, (file_name, file_hash_dict) in enumerate(
- file_batch_data.files, 1
+ # Pre-create all WorkflowFileExecution records with PENDING status to prevent race conditions
+ pre_created_data = FileExecutionTasks._pre_create_file_executions(
+ file_batch_data.files, workflow, workflow_execution
+ )
+
+ for file_number, (file_name, (workflow_file_execution, file_hash)) in enumerate(
+ pre_created_data.items(), 1
):
logger.info(
f"[{celery_task_id}][{file_number}/{total_files}] Processing file '{file_name}'"
)
- file_hash = FileHash(
- file_path=file_hash_dict.get("file_path"),
- file_name=file_hash_dict.get("file_name"),
- source_connection_type=file_hash_dict.get("source_connection_type"),
- file_hash=file_hash_dict.get("file_hash"),
- file_size=file_hash_dict.get("file_size"),
- provider_file_uuid=file_hash_dict.get("provider_file_uuid"),
- mime_type=file_hash_dict.get("mime_type"),
- fs_metadata=file_hash_dict.get("fs_metadata"),
- file_destination=file_hash_dict.get("file_destination"),
- is_executed=file_hash_dict.get("is_executed"),
- file_number=file_hash_dict.get("file_number"),
- )
+
+ # Apply file destination processing
file_hash = WorkflowUtil.add_file_destination_filehash(
file_hash.file_number,
q_file_no_list,
file_hash,
)
logger.info(f"File hash for file {file_name}: {file_hash.to_json()}")
+
file_execution_result = FileExecutionTasks._process_file(
current_file_idx=file_number,
total_files=total_files,
file_data=file_data,
file_hash=file_hash,
workflow_execution=workflow_execution,
+ workflow_file_execution=workflow_file_execution,
)
if file_execution_result.error:
failed_files += 1
@@ -232,6 +228,65 @@ class FileExecutionTasks:
failed_files=failed_files,
).to_dict()
+ @classmethod
+ def _pre_create_file_executions(
+ cls,
+ files: list[tuple[str, dict]],
+ workflow: Workflow,
+ workflow_execution: WorkflowExecution,
+ ) -> dict[str, tuple[WorkflowFileExecution, FileHash]]:
+ """Pre-create WorkflowFileExecution records with PENDING status to prevent race conditions.
+
+ Args:
+ files: List of (file_name, file_hash_dict) tuples
+ workflow: Workflow instance
+ workflow_execution: WorkflowExecution instance
+
+ Returns:
+ dict: Mapping of file names to (WorkflowFileExecution, FileHash) tuples
+ """
+ pre_created_data = {}
+
+ # Get endpoint information
+ endpoint: WorkflowEndpoint = WorkflowEndpoint.objects.get(
+ workflow=workflow,
+ endpoint_type=WorkflowEndpoint.EndpointType.SOURCE,
+ )
+ is_api = endpoint.connection_type == WorkflowEndpoint.ConnectionType.API
+
+ for file_name, file_hash_dict in files:
+ file_hash = FileHash(
+ file_path=file_hash_dict.get("file_path"),
+ file_name=file_hash_dict.get("file_name"),
+ source_connection_type=file_hash_dict.get("source_connection_type"),
+ file_hash=file_hash_dict.get("file_hash"),
+ file_size=file_hash_dict.get("file_size"),
+ provider_file_uuid=file_hash_dict.get("provider_file_uuid"),
+ mime_type=file_hash_dict.get("mime_type"),
+ fs_metadata=file_hash_dict.get("fs_metadata"),
+ file_destination=file_hash_dict.get("file_destination"),
+ is_executed=file_hash_dict.get("is_executed"),
+ file_number=file_hash_dict.get("file_number"),
+ )
+
+ # Create WorkflowFileExecution record
+ workflow_file_execution = (
+ WorkflowFileExecution.objects.get_or_create_file_execution(
+ workflow_execution=workflow_execution,
+ file_hash=file_hash,
+ is_api=is_api,
+ )
+ )
+
+ # Update status to PENDING if allowed
+ if ExecutionStatus.can_update_to_pending(workflow_file_execution.status):
+ workflow_file_execution.update_status(ExecutionStatus.PENDING)
+
+ pre_created_data[file_name] = (workflow_file_execution, file_hash)
+
+ logger.info(f"Pre-created {len(pre_created_data)} WorkflowFileExecution records")
+ return pre_created_data
+
@file_processing_callback_app.task(
bind=True,
max_retries=0, # Maximum number of retries
@@ -337,6 +392,7 @@ class FileExecutionTasks:
file_data: FileData,
file_hash: FileHash,
workflow_execution: WorkflowExecution,
+ workflow_file_execution: WorkflowFileExecution | None = None,
) -> FileExecutionResult:
"""Process a single file in the workflow.
@@ -356,7 +412,7 @@ class FileExecutionTasks:
)
# Initialization Phase
execution_context = cls._initialize_execution_context(
- file_data, file_hash, workflow_execution
+ file_data, file_hash, workflow_execution, workflow_file_execution
)
workflow_log = execution_context.workflow_log
workflow_file_execution = execution_context.workflow_file_execution
@@ -614,6 +670,7 @@ class FileExecutionTasks:
file_data: FileData,
file_hash: FileHash,
workflow_execution: WorkflowExecution,
+ workflow_file_execution: WorkflowFileExecution | None = None,
) -> ExecutionContext:
"""Set up all required execution context objects and configurations."""
try:
@@ -625,14 +682,17 @@ class FileExecutionTasks:
is_api = endpoint.connection_type == WorkflowEndpoint.ConnectionType.API
- # Create execution record
- file_execution: WorkflowFileExecution = (
- WorkflowFileExecution.objects.get_or_create_file_execution(
- workflow_execution=workflow_execution,
- file_hash=file_hash,
- is_api=is_api,
+ # Use pre-created execution record or create new one if not provided
+ if workflow_file_execution is None:
+ file_execution: WorkflowFileExecution = (
+ WorkflowFileExecution.objects.get_or_create_file_execution(
+ workflow_execution=workflow_execution,
+ file_hash=file_hash,
+ is_api=is_api,
+ )
)
- )
+ else:
+ file_execution = workflow_file_execution
# Create configurations
source_config = SourceConfig.from_json(file_data.source_config)