LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
UN-2584 [FEAT] Configurable ETL schedule intervals with conflict prevention and 30-minute support (#1426)
alib169d0e9fec6
diff --git a/backend/backend/settings/base.py b/backend/backend/settings/base.pyindex 8e4169412..871f1c3fc 100644--- a/backend/backend/settings/base.py+++ b/backend/backend/settings/base.py@@ -155,6 +155,10 @@ INSTANT_WF_POLLING_TIMEOUT = int(os.environ.get("INSTANT_WF_POLLING_TIMEOUT", "300")) # 5 minutes+# ETL Pipeline minimum schedule interval (in seconds)+# Default: 1800 seconds (30 minutes)+MIN_SCHEDULE_INTERVAL_SECONDS = int(os.environ.get("MIN_SCHEDULE_INTERVAL_SECONDS", 1800))+# File processing batchesMAX_PARALLEL_FILE_BATCHES = int(os.environ.get("MAX_PARALLEL_FILE_BATCHES", 1))# Upper limit for batch validationdiff --git a/backend/pipeline_v2/constants.py b/backend/pipeline_v2/constants.pyindex a234983b6..75a030eff 100644--- a/backend/pipeline_v2/constants.py+++ b/backend/pipeline_v2/constants.py@@ -1,3 +1,6 @@+from django.conf import settings++class PipelineConstants:"""Constants for Pipelines."""@@ -53,6 +56,20 @@ class PipelineErrors:INVALID_WF = "The provided workflow does not exist"+class PipelineScheduling:+ """Constants for pipeline scheduling configuration."""++ @classmethod+ def get_min_interval_seconds(cls) -> int:+ """Get minimum schedule interval in seconds from settings."""+ return settings.MIN_SCHEDULE_INTERVAL_SECONDS++ @classmethod+ def get_min_interval_minutes(cls) -> int:+ """Get minimum schedule interval in minutes from settings."""+ return cls.get_min_interval_seconds() // 60++class PipelineURL:"""Constants for URL names."""diff --git a/backend/pipeline_v2/serializers/crud.py b/backend/pipeline_v2/serializers/crud.pyindex a2cba45c3..d69f616fd 100644--- a/backend/pipeline_v2/serializers/crud.py+++ b/backend/pipeline_v2/serializers/crud.py@@ -8,6 +8,7 @@ from croniter import croniterfrom django.conf import settingsfrom pipeline_v2.constants import PipelineConstants as PCfrom pipeline_v2.constants import PipelineKey as PK+from pipeline_v2.constants import PipelineSchedulingfrom pipeline_v2.models import Pipelinefrom rest_framework import serializersfrom rest_framework.serializers import SerializerMethodField@@ -43,6 +44,112 @@ class PipelineSerializer(IntegrityErrorMixin, AuditSerializer):},}+ # Constants for validation+ MINUTE_RANGE = (0, 59)+ HOUR_MINUTES = 60++ def _get_validation_examples(self, min_interval: int) -> str:+ """Generate examples for error messages with exact min_interval gaps."""+ first = 0+ second = (first + min_interval) % self.HOUR_MINUTES+ third = (second + min_interval) % self.HOUR_MINUTES+ return f"'{first},{second}', '{second},{third}'"++ def _get_cron_error_too_frequent(self) -> str:+ min_interval = PipelineScheduling.get_min_interval_minutes()+ return f"Configured cron schedule is too frequent. Ensure that consecutive runs are atleast {min_interval} minutes or longer."++ def _get_cron_error_complex_pattern(self) -> str:+ min_interval = PipelineScheduling.get_min_interval_minutes()+ examples = self._get_validation_examples(min_interval)+ return (+ f"Intervals shorter than {min_interval} minutes detected. "+ f"Ensure consecutive values are ≥{min_interval} minutes apart. Examples: {examples}"+ )++ def _get_cron_error_range_pattern(self) -> str:+ min_interval = PipelineScheduling.get_min_interval_minutes()+ examples = self._get_validation_examples(min_interval)+ return f"Range patterns not supported. Use comma-separated values ≥{min_interval} minutes apart. Examples: {examples}"++ def _validate_basic_cron_format(self, value: str | None) -> str | None:+ """Validate basic cron format and handle None/empty cases."""+ if value is None:+ return None++ cron_string = value.strip()+ if not cron_string:+ return None++ try:+ croniter(cron_string)+ except Exception as error:+ logger.error(f"Invalid cron string '{cron_string}': {error}")+ raise serializers.ValidationError("Invalid cron string format.")++ return cron_string++ def _validate_step_pattern(self, minute_field: str) -> None:+ """Validate step patterns like */15."""+ parts = minute_field.split("/")+ if len(parts) == 2 and parts[1].isdigit():+ step = int(parts[1])+ if step < PipelineScheduling.get_min_interval_minutes():+ raise serializers.ValidationError(self._get_cron_error_too_frequent())++ def _parse_and_validate_minutes(self, parts: list[str]) -> list[int]:+ """Parse minute parts and validate range."""+ if not all(+ p.isdigit() and self.MINUTE_RANGE[0] <= int(p) <= self.MINUTE_RANGE[1]+ for p in parts+ ):+ raise serializers.ValidationError(self._get_cron_error_complex_pattern())+ return sorted([int(p) for p in parts])++ def _check_consecutive_intervals(self, minutes: list[int], min_interval: int) -> None:+ """Check intervals between consecutive minute values."""+ for i in range(len(minutes) - 1):+ if minutes[i + 1] - minutes[i] < min_interval:+ raise serializers.ValidationError(self._get_cron_error_complex_pattern())++ def _check_wraparound_interval(self, minutes: list[int], min_interval: int) -> None:+ """Check wraparound interval from last to first minute of next hour."""+ if len(minutes) > 1:+ wraparound = (self.HOUR_MINUTES - minutes[-1]) + minutes[0]+ if wraparound < min_interval:+ raise serializers.ValidationError(self._get_cron_error_complex_pattern())++ def _validate_comma_pattern(self, minute_field: str) -> None:+ """Validate comma patterns like 0,30 or 0,10,20,30,40,50."""+ parts = [p.strip() for p in minute_field.split(",")]+ minutes = self._parse_and_validate_minutes(parts)++ min_interval = PipelineScheduling.get_min_interval_minutes()+ self._check_consecutive_intervals(minutes, min_interval)+ self._check_wraparound_interval(minutes, min_interval)++ def _validate_range_pattern(self, minute_field: str) -> None:+ """Validate range patterns like 0-30.++ Range patterns are not supported because they can create intervals+ shorter than the minimum required interval. For example, with a+ 30-minute minimum, "0-30" would create executions every minute+ from 0 to 30, violating the minimum interval constraint.+ """+ raise serializers.ValidationError(self._get_cron_error_range_pattern())++ def _validate_minute_field(self, minute_field: str) -> None:+ """Validate the minute field for configurable minimum intervals."""+ if minute_field == "*":+ raise serializers.ValidationError(self._get_cron_error_too_frequent())++ if "/" in minute_field:+ self._validate_step_pattern(minute_field)+ elif "," in minute_field:+ self._validate_comma_pattern(minute_field)+ elif "-" in minute_field:+ self._validate_range_pattern(minute_field)+def validate_cron_string(self, value: str | None = None) -> str | None:"""Validate the cron string provided in the serializer data.@@ -64,28 +171,14 @@ class PipelineSerializer(IntegrityErrorMixin, AuditSerializer):Optional[str]: The validated cron string if it is valid,otherwise None."""- if value is None:+ cron_string = self._validate_basic_cron_format(value)+ if cron_string is None:return None- cron_string = value.strip()- # Check if the string is empty- if not cron_string:- return None-- # Validate the cron string- try:- croniter(cron_string)- except Exception as error:- logger.error(f"Invalid cron string '{cron_string}': {error}")- raise serializers.ValidationError("Invalid cron string format.")- # Check if the frequency is less than 1 hourcron_parts = cron_string.split()minute_field = cron_parts[0]- if minute_field == "*" or any(char in minute_field for char in [",", "-", "/"]):- raise serializers.ValidationError(- "Cron schedule can not be more than once per hour. Please provide a "- "cron schedule to run at an hourly or less frequent interval."- )++ self._validate_minute_field(minute_field)return cron_stringdiff --git a/backend/sample.env b/backend/sample.envindex 78007a9a7..9c55b2034 100644--- a/backend/sample.env+++ b/backend/sample.env@@ -188,3 +188,8 @@ FILE_EXECUTION_TRACKER_COMPLETED_TTL_IN_SECOND=600 # 10 minutes# Runner polling timeoutMAX_RUNNER_POLLING_WAIT_SECONDS=10800 # 3 hoursRUNNER_POLLING_INTERVAL_SECONDS=2 # 2 seconds++# ETL Pipeline minimum schedule interval (in seconds)+# Default: 1800 seconds (30 minutes)+# Examples: 900 (15 min), 1800 (30 min), 3600 (60 min)+MIN_SCHEDULE_INTERVAL_SECONDS=1800diff --git a/backend/workflow_manager/endpoint_v2/source.py b/backend/workflow_manager/endpoint_v2/source.pyindex 36542a79c..4098963da 100644--- a/backend/workflow_manager/endpoint_v2/source.py+++ b/backend/workflow_manager/endpoint_v2/source.py@@ -37,7 +37,9 @@ from workflow_manager.endpoint_v2.exceptions import (from workflow_manager.endpoint_v2.models import WorkflowEndpointfrom workflow_manager.file_execution.models import WorkflowFileExecutionfrom workflow_manager.utils.workflow_log import WorkflowLog+from workflow_manager.workflow_v2.enums import ExecutionStatusfrom workflow_manager.workflow_v2.file_history_helper import FileHistoryHelper+from workflow_manager.workflow_v2.models.execution import WorkflowExecutionfrom workflow_manager.workflow_v2.models.file_history import FileHistoryfrom workflow_manager.workflow_v2.models.workflow import Workflow@@ -372,6 +374,104 @@ class SourceConnector(BaseConnector):count += 1return count+ def _is_file_being_processed(self, file_hash: FileHash) -> bool:+ """Check if file is currently being processed or should be skipped+ Uses direct database queries instead of FileExecutionStatusTracker++ Args:+ file_hash (FileHash): The file hash to check++ Returns:+ bool: True if file should be skipped (being processed or already completed), False otherwise+ """+ active_executions = self._get_active_workflow_executions()+ logger.info(+ f"Found {len(active_executions)} active executions for workflow {self.workflow.id}"+ )++ for execution in active_executions:+ if self._has_blocking_file_execution(execution, file_hash):+ return True++ return False++ def _get_active_workflow_executions(self):+ """Get active executions for this workflow with organization filtering for security."""+ organization = UserContext.get_organization()+ return WorkflowExecution.objects.filter(+ workflow=self.workflow,+ workflow__organization_id=organization.id, # Security: Organization isolation+ status__in=[ExecutionStatus.EXECUTING, ExecutionStatus.PENDING],+ )++ def _has_blocking_file_execution(self, execution, file_hash: FileHash) -> bool:+ """Check if there's a blocking file execution that prevents processing."""+ try:+ # Try to find blocking file execution using file_hash+ file_exec = self._find_blocking_file_execution_by_hash(execution, file_hash)+ if file_exec:+ self._log_duplicate_file_execution_skip(file_hash, file_exec, execution)+ return True++ # Try to find blocking file execution using provider_file_uuid+ file_exec = self._find_blocking_file_execution_by_provider_uuid(+ execution, file_hash+ )+ if file_exec:+ self._log_duplicate_file_execution_skip(file_hash, file_exec, execution)+ return True++ except Exception as e:+ logger.warning(+ f"Error checking file execution status for {file_hash.file_name}: {e}\n"+ "Allowing its execution to continue anyway"+ )++ return False++ def _find_blocking_file_execution_by_hash(self, execution, file_hash: FileHash):+ """Find blocking file execution by file hash if it exists."""+ if not file_hash.file_hash:+ return None++ try:+ return WorkflowFileExecution.objects.get(+ workflow_execution=execution,+ file_hash=file_hash.file_hash,+ file_path=file_hash.file_path,+ status__in=ExecutionStatus.get_skip_processing_statuses(),+ )+ except WorkflowFileExecution.DoesNotExist:+ return None++ def _find_blocking_file_execution_by_provider_uuid(+ self, execution, file_hash: FileHash+ ):+ """Find blocking file execution by provider UUID if it exists."""+ if not file_hash.provider_file_uuid:+ return None++ try:+ return WorkflowFileExecution.objects.get(+ workflow_execution=execution,+ provider_file_uuid=file_hash.provider_file_uuid,+ file_path=file_hash.file_path,+ status__in=ExecutionStatus.get_skip_processing_statuses(),+ )+ except WorkflowFileExecution.DoesNotExist:+ return None++ def _log_duplicate_file_execution_skip(+ self, file_hash: FileHash, file_exec, execution+ ):+ """Log message when skipping file due to duplicate/blocking execution."""+ message = (+ f"Skipping file '{file_hash.file_name}' - status: {file_exec.status} "+ f"in execution {execution.id}, file execution id: {file_exec.id}"+ )+ self.workflow_log.publish_log(message)+ logger.info(message)+def _should_skip_file(self,file_hash: FileHash,@@ -387,12 +487,20 @@ class SourceConnector(BaseConnector):bool: True if the file should be skipped, False otherwise."""file_name = os.path.basename(file_hash.file_path)- return not self._should_process_file(- file_name, patterns- ) or not self._is_new_file(- file_hash=file_hash,- workflow=self.endpoint.workflow,- )++ # Existing pattern check+ if not self._should_process_file(file_name, patterns):+ return True++ # Existing file history check+ if not self._is_new_file(file_hash=file_hash, workflow=self.endpoint.workflow):+ return True++ # NEW: Check if file is being processed+ if self._is_file_being_processed(file_hash):+ return True++ return Falsedef _is_duplicate(self, file_hash: FileHash, unique_file_paths: set[str]) -> bool:return (diff --git a/backend/workflow_manager/file_execution/migrations/0006_workflowfileexecution_wf_file_hash_path_status_idx_and_more.py b/backend/workflow_manager/file_execution/migrations/0006_workflowfileexecution_wf_file_hash_path_status_idx_and_more.pynew file mode 100644index 000000000..12ffd4f52--- /dev/null+++ b/backend/workflow_manager/file_execution/migrations/0006_workflowfileexecution_wf_file_hash_path_status_idx_and_more.py@@ -0,0 +1,31 @@+# Generated by Django 4.2.1 on 2025-07-14 14:58++from django.db import migrations, models+++class Migration(migrations.Migration):+ dependencies = [+ ("file_execution", "0005_workflowfileexecution_fs_metadata_and_more"),+ ]++ operations = [+ migrations.AddIndex(+ model_name="workflowfileexecution",+ index=models.Index(+ fields=["workflow_execution", "file_hash", "file_path", "status"],+ name="wf_file_hash_path_status_idx",+ ),+ ),+ migrations.AddIndex(+ model_name="workflowfileexecution",+ index=models.Index(+ fields=[+ "workflow_execution",+ "provider_file_uuid",+ "file_path",+ "status",+ ],+ name="wf_provider_uuid_path_stat_idx",+ ),+ ),+ ]diff --git a/backend/workflow_manager/file_execution/models.py b/backend/workflow_manager/file_execution/models.pyindex ce8434b77..fd75b163f 100644--- a/backend/workflow_manager/file_execution/models.py+++ b/backend/workflow_manager/file_execution/models.py@@ -187,6 +187,19 @@ class WorkflowFileExecution(BaseModel):fields=["workflow_execution", "provider_file_uuid"],name="workflow_exec_p_uuid_idx",),+ models.Index(+ fields=["workflow_execution", "file_hash", "file_path", "status"],+ name="wf_file_hash_path_status_idx",+ ),+ models.Index(+ fields=[+ "workflow_execution",+ "provider_file_uuid",+ "file_path",+ "status",+ ],+ name="wf_provider_uuid_path_stat_idx",+ ),]constraints = [models.UniqueConstraint(diff --git a/backend/workflow_manager/workflow_v2/enums.py b/backend/workflow_manager/workflow_v2/enums.pyindex 7fb75af88..baebbf358 100644--- a/backend/workflow_manager/workflow_v2/enums.py+++ b/backend/workflow_manager/workflow_v2/enums.py@@ -44,6 +44,70 @@ class ExecutionStatus(TextChoices):)return status_enum in [cls.COMPLETED, cls.STOPPED, cls.ERROR]+ @classmethod+ def is_active(cls, status: str | ExecutionStatus) -> bool:+ """Check if the workflow execution status is active (in progress)."""+ try:+ status_enum = cls(status)+ except ValueError:+ raise ValueError(+ f"Invalid status: {status}. Must be a valid ExecutionStatus."+ )+ return status_enum in [cls.PENDING, cls.EXECUTING]++ @classmethod+ def get_skip_processing_statuses(cls) -> list[ExecutionStatus]:+ """Get list of statuses that should skip file processing.++ Skip processing if:+ - EXECUTING: File is currently being processed+ - PENDING: File is queued to be processed+ - COMPLETED: File has already been successfully processed++ Returns:+ list[ExecutionStatus]: List of statuses where file processing should be skipped+ """+ return [cls.EXECUTING, cls.PENDING, cls.COMPLETED]++ @classmethod+ def should_skip_file_processing(cls, status: str | ExecutionStatus) -> bool:+ """Check if file processing should be skipped based on status.++ Allow processing (retry) if:+ - STOPPED: Processing was stopped, can retry+ - ERROR: Processing failed, can retry+ """+ try:+ status_enum = cls(status)+ except ValueError:+ raise ValueError(+ f"Invalid status: {status}. Must be a valid ExecutionStatus."+ )+ return status_enum in cls.get_skip_processing_statuses()++ @classmethod+ def can_update_to_pending(cls, status: str | ExecutionStatus) -> bool:+ """Check if a status can be updated to PENDING.++ Allow updating to PENDING if:+ - Status is STOPPED or ERROR (can retry)+ - Status is None (new record)++ Don't allow updating to PENDING if:+ - Status is EXECUTING (currently processing)+ - Status is COMPLETED (already done)+ - Status is already PENDING (no change needed)+ """+ if status is None:+ return True++ try:+ status_enum = cls(status)+ except ValueError:+ return True # Invalid status, allow update++ return status_enum in [cls.STOPPED, cls.ERROR]+class SchemaType(Enum):"""Possible types for workflow module's JSON schema.diff --git a/backend/workflow_manager/workflow_v2/file_execution_tasks.py b/backend/workflow_manager/workflow_v2/file_execution_tasks.pyindex 85616bc52..463212a7c 100644--- a/backend/workflow_manager/workflow_v2/file_execution_tasks.py+++ b/backend/workflow_manager/workflow_v2/file_execution_tasks.py@@ -179,37 +179,33 @@ class FileExecutionTasks:logger.info(f"Processing {total_files} files of execution {execution_id}")- for file_number, (file_name, file_hash_dict) in enumerate(- file_batch_data.files, 1+ # Pre-create all WorkflowFileExecution records with PENDING status to prevent race conditions+ pre_created_data = FileExecutionTasks._pre_create_file_executions(+ file_batch_data.files, workflow, workflow_execution+ )++ for file_number, (file_name, (workflow_file_execution, file_hash)) in enumerate(+ pre_created_data.items(), 1):logger.info(f"[{celery_task_id}][{file_number}/{total_files}] Processing file '{file_name}'")- file_hash = FileHash(- file_path=file_hash_dict.get("file_path"),- file_name=file_hash_dict.get("file_name"),- source_connection_type=file_hash_dict.get("source_connection_type"),- file_hash=file_hash_dict.get("file_hash"),- file_size=file_hash_dict.get("file_size"),- provider_file_uuid=file_hash_dict.get("provider_file_uuid"),- mime_type=file_hash_dict.get("mime_type"),- fs_metadata=file_hash_dict.get("fs_metadata"),- file_destination=file_hash_dict.get("file_destination"),- is_executed=file_hash_dict.get("is_executed"),- file_number=file_hash_dict.get("file_number"),- )++ # Apply file destination processingfile_hash = WorkflowUtil.add_file_destination_filehash(file_hash.file_number,q_file_no_list,file_hash,)logger.info(f"File hash for file {file_name}: {file_hash.to_json()}")+file_execution_result = FileExecutionTasks._process_file(current_file_idx=file_number,total_files=total_files,file_data=file_data,file_hash=file_hash,workflow_execution=workflow_execution,+ workflow_file_execution=workflow_file_execution,)if file_execution_result.error:failed_files += 1@@ -232,6 +228,65 @@ class FileExecutionTasks:failed_files=failed_files,).to_dict()+ @classmethod+ def _pre_create_file_executions(+ cls,+ files: list[tuple[str, dict]],+ workflow: Workflow,+ workflow_execution: WorkflowExecution,+ ) -> dict[str, tuple[WorkflowFileExecution, FileHash]]:+ """Pre-create WorkflowFileExecution records with PENDING status to prevent race conditions.++ Args:+ files: List of (file_name, file_hash_dict) tuples+ workflow: Workflow instance+ workflow_execution: WorkflowExecution instance++ Returns:+ dict: Mapping of file names to (WorkflowFileExecution, FileHash) tuples+ """+ pre_created_data = {}++ # Get endpoint information+ endpoint: WorkflowEndpoint = WorkflowEndpoint.objects.get(+ workflow=workflow,+ endpoint_type=WorkflowEndpoint.EndpointType.SOURCE,+ )+ is_api = endpoint.connection_type == WorkflowEndpoint.ConnectionType.API++ for file_name, file_hash_dict in files:+ file_hash = FileHash(+ file_path=file_hash_dict.get("file_path"),+ file_name=file_hash_dict.get("file_name"),+ source_connection_type=file_hash_dict.get("source_connection_type"),+ file_hash=file_hash_dict.get("file_hash"),+ file_size=file_hash_dict.get("file_size"),+ provider_file_uuid=file_hash_dict.get("provider_file_uuid"),+ mime_type=file_hash_dict.get("mime_type"),+ fs_metadata=file_hash_dict.get("fs_metadata"),+ file_destination=file_hash_dict.get("file_destination"),+ is_executed=file_hash_dict.get("is_executed"),+ file_number=file_hash_dict.get("file_number"),+ )++ # Create WorkflowFileExecution record+ workflow_file_execution = (+ WorkflowFileExecution.objects.get_or_create_file_execution(+ workflow_execution=workflow_execution,+ file_hash=file_hash,+ is_api=is_api,+ )+ )++ # Update status to PENDING if allowed+ if ExecutionStatus.can_update_to_pending(workflow_file_execution.status):+ workflow_file_execution.update_status(ExecutionStatus.PENDING)++ pre_created_data[file_name] = (workflow_file_execution, file_hash)++ logger.info(f"Pre-created {len(pre_created_data)} WorkflowFileExecution records")+ return pre_created_data+@file_processing_callback_app.task(bind=True,max_retries=0, # Maximum number of retries@@ -337,6 +392,7 @@ class FileExecutionTasks:file_data: FileData,file_hash: FileHash,workflow_execution: WorkflowExecution,+ workflow_file_execution: WorkflowFileExecution | None = None,) -> FileExecutionResult:"""Process a single file in the workflow.@@ -356,7 +412,7 @@ class FileExecutionTasks:)# Initialization Phaseexecution_context = cls._initialize_execution_context(- file_data, file_hash, workflow_execution+ file_data, file_hash, workflow_execution, workflow_file_execution)workflow_log = execution_context.workflow_logworkflow_file_execution = execution_context.workflow_file_execution@@ -614,6 +670,7 @@ class FileExecutionTasks:file_data: FileData,file_hash: FileHash,workflow_execution: WorkflowExecution,+ workflow_file_execution: WorkflowFileExecution | None = None,) -> ExecutionContext:"""Set up all required execution context objects and configurations."""try:@@ -625,14 +682,17 @@ class FileExecutionTasks:is_api = endpoint.connection_type == WorkflowEndpoint.ConnectionType.API- # Create execution record- file_execution: WorkflowFileExecution = (- WorkflowFileExecution.objects.get_or_create_file_execution(- workflow_execution=workflow_execution,- file_hash=file_hash,- is_api=is_api,+ # Use pre-created execution record or create new one if not provided+ if workflow_file_execution is None:+ file_execution: WorkflowFileExecution = (+ WorkflowFileExecution.objects.get_or_create_file_execution(+ workflow_execution=workflow_execution,+ file_hash=file_hash,+ is_api=is_api,+ ))- )+ else:+ file_execution = workflow_file_execution# Create configurationssource_config = SourceConfig.from_json(file_data.source_config)