LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
UN-2866 [MISC] Add database duplicate file check in file_processing worker with ERROR record creation for audit trail (#1575)
alif62b2d6d8636
diff --git a/workers/file_processing/tasks.py b/workers/file_processing/tasks.pyindex 7587c3873..d8cb88d7a 100644--- a/workers/file_processing/tasks.py+++ b/workers/file_processing/tasks.py@@ -485,24 +485,27 @@ def _refactored_pre_create_file_executions(# CRITICAL: Pre-create all WorkflowFileExecution records to prevent duplicates# This matches the backend's _pre_create_file_executions pattern for ALL workflow types- pre_created_file_executions: dict[str, PreCreatedFileData] = (- _pre_create_file_executions(- file_data=file_data,- files=files,- workflow_id=workflow_id,- execution_id=execution_id,- api_client=api_client,- workflow_type=workflow_type,- is_api=is_api_workflow,- use_file_history=context.get_setting("use_file_history", True),- )+ # Includes double safeguard: database check for active files+ workflow_logger = context.metadata.get("workflow_logger")+ pre_created_file_executions, skipped_files = _pre_create_file_executions(+ file_data=file_data,+ files=files,+ workflow_id=workflow_id,+ execution_id=execution_id,+ api_client=api_client,+ workflow_type=workflow_type,+ is_api=is_api_workflow,+ use_file_history=context.get_setting("use_file_history", True),+ workflow_logger=workflow_logger,)logger.info(- f"Pre-created {len(pre_created_file_executions)} WorkflowFileExecution records for {workflow_type} workflow"+ f"Pre-created {len(pre_created_file_executions)} WorkflowFileExecution records for {workflow_type} workflow "+ f"(skipped {len(skipped_files)} duplicates)")# Add to metadatacontext.pre_created_file_executions = pre_created_file_executions+ context.metadata["skipped_files"] = skipped_filesreturn context@@ -524,6 +527,7 @@ def _process_individual_files(context: WorkflowContextData) -> WorkflowContextDaapi_client = context.organization_context.api_clientworkflow_execution = context.metadata["workflow_execution"]pre_created_file_executions = context.pre_created_file_executions+ skipped_files = context.metadata.get("skipped_files", [])result = context.metadata["result"]successful_files_for_manual_review = context.metadata["successful_files_for_manual_review"@@ -558,8 +562,24 @@ def _process_individual_files(context: WorkflowContextData) -> WorkflowContextDapre_created_file_execution = pre_created_file_executions.get(file_name)if not pre_created_file_execution:+ # Check if this file was intentionally skipped as a duplicate+ # Construct identifier from file_hash_dict to match skipped_files format+ provider_uuid = file_hash_dict.get("provider_file_uuid")+ file_path = file_hash_dict.get("file_path")+ if provider_uuid and file_path:+ file_identifier = f"{provider_uuid}:{file_path}"+ if file_identifier in skipped_files:+ # File was already handled in pre-creation (marked as ERROR in DB, logged to UI)+ # Just count it as a failure and move on+ logger.debug(+ f"File '{file_name}' (identifier: {file_identifier}) was skipped as duplicate - already counted as ERROR"+ )+ result.increment_failure()+ continue++ # Truly missing - this is an error conditionlogger.error(- f"No pre-created WorkflowFileExecution found for file '{file_name}' - skipping"+ f"No pre-created WorkflowFileExecution found for file '{file_name}' - unexpected error")result.increment_failure()continue@@ -801,6 +821,7 @@ def _compile_batch_result(context: WorkflowContextData) -> dict[str, Any]:"""result = context.metadata["result"]workflow_logger = context.metadata.get("workflow_logger")+ skipped_files = context.metadata.get("skipped_files", [])# Send execution completion summary to UIif workflow_logger:@@ -810,11 +831,22 @@ def _compile_batch_result(context: WorkflowContextData) -> dict[str, Any]:total_time=result.execution_time,)- logger.info(- f"Function tasks.process_file_batch completed successfully. "- f"Batch execution time: {result.execution_time:.2f}s for "- f"{result.successful_files + result.failed_files} files"- )+ # Log batch completion summary+ # Note: Skipped duplicate files are marked as ERROR in database and counted in failed_files+ if skipped_files:+ logger.info(+ f"Function tasks.process_file_batch completed. "+ f"Successful: {result.successful_files} files, "+ f"Failed: {result.failed_files} files "+ f"(including {len(skipped_files)} duplicate(s) marked as ERROR). "+ f"Batch execution time: {result.execution_time:.2f}s"+ )+ else:+ logger.info(+ f"Function tasks.process_file_batch completed successfully. "+ f"Batch execution time: {result.execution_time:.2f}s for "+ f"{result.successful_files + result.failed_files} files"+ )# CRITICAL: Clean up StateStore to prevent data leaks between taskstry:@@ -824,10 +856,12 @@ def _compile_batch_result(context: WorkflowContextData) -> dict[str, Any]:logger.warning(f"Failed to cleanup StateStore context: {cleanup_error}")# Return the final result matching Django backend format+ # Note: Skipped duplicate files are in database as ERROR status, not counted separatelyreturn {"successful_files": result.successful_files,- "failed_files": result.failed_files,+ "failed_files": result.failed_files, # Includes duplicates marked as ERROR"total_files": result.successful_files + result.failed_files,+ "skipped_duplicates": len(skipped_files), # For informational purposes"execution_time": result.execution_time,"organization_id": context.organization_context.organization_id,}@@ -874,6 +908,112 @@ def _cleanup_file_cache_entry(# Don't raise - cache will expire anyway+def _check_file_already_active(+ file_hash: FileHashData,+ workflow_id: str,+ execution_id: str,+ api_client: InternalAPIClient,+ file_name: str,+) -> bool:+ """Check if file is already being processed (Redis-first with DB fallback).++ This provides a secondary check at the file_processing worker level to catch+ any files that slipped through the general worker's filter due to race conditions.++ OPTIMIZATION: Checks Redis cache first (fastest path, highest hit rate since general+ worker just added the file), then falls back to database if Redis check fails or+ returns no result.++ Args:+ file_hash: File hash data containing provider_file_uuid and file_path+ workflow_id: Workflow ID+ execution_id: Current execution ID to exclude+ api_client: Internal API client+ file_name: File name for logging++ Returns:+ True if file is already active in a different execution, False otherwise+ """+ # Only check files with provider_file_uuid (files from external sources)+ if not file_hash.provider_file_uuid:+ return False++ # STEP 1: Check Redis cache first (fastest path, most likely to find duplicates)+ try:+ from shared.cache.cache_backends import RedisCacheBackend+ from shared.workflow.execution.active_file_manager import ActiveFileManager++ cache = RedisCacheBackend()+ if cache.available:+ # Construct cache key using ActiveFileManager method for consistency+ cache_key = ActiveFileManager._create_cache_key(+ workflow_id,+ file_hash.provider_file_uuid,+ file_hash.file_path,+ )+ cached_data = cache.get(cache_key)++ if cached_data and isinstance(cached_data, dict):+ # Extract execution_id from cache wrapper structure+ cached_execution_id = cached_data.get("data", {}).get("execution_id")++ if cached_execution_id and cached_execution_id != execution_id:+ # Found in Redis - different execution is processing this file+ logger.warning(+ f"DUPLICATE DETECTED: File '{file_name}' (UUID: {file_hash.provider_file_uuid}) "+ f"is already being processed by execution {cached_execution_id} (Redis cache check)"+ )+ return True+ elif cached_execution_id == execution_id:+ # Same execution - not a duplicate+ logger.debug(+ f"File '{file_name}' found in Redis cache for same execution {execution_id}"+ )+ return False+ # If no execution_id or invalid data, fall through to DB check++ # Cache miss - fall through to DB check+ logger.info(+ f"File '{file_name}' not found in Redis cache, falling back to DB check for execution {execution_id}"+ )++ except Exception as redis_error:+ # Redis check failed, fall back to DB+ logger.warning(+ f"Redis check failed for '{file_name}': {redis_error}. Falling back to DB check"+ )++ # STEP 2: DB fallback (if Redis unavailable, cache miss, or invalid data)+ try:+ response = api_client.check_files_active_processing(+ workflow_id=workflow_id,+ files=[{"uuid": file_hash.provider_file_uuid, "path": file_hash.file_path}],+ current_execution_id=execution_id,+ )+ if response.success and response.data:+ # Check if this specific file is in active_identifiers+ active_identifiers = response.data.get("active_identifiers", [])++ file_identifier = f"{file_hash.provider_file_uuid}:{file_hash.file_path}"++ if file_identifier in active_identifiers:+ logger.warning(+ f"DUPLICATE DETECTED: File '{file_name}' (UUID: {file_hash.provider_file_uuid}) "+ f"is already being processed in another execution (DB check)"+ )+ return True++ return False++ except Exception as e:+ logger.warning(+ f"DB check failed for '{file_name}': {e}. "+ f"Proceeding with file processing (fail-safe)"+ )+ # Fail-safe: if both checks fail, allow processing+ return False++def _pre_create_file_executions(file_data: WorkerFileData,files: list[Any],@@ -883,24 +1023,33 @@ def _pre_create_file_executions(workflow_type: str,is_api: bool = False,use_file_history: bool = True,-) -> dict[str, Any]:+ workflow_logger: WorkerWorkflowLogger | None = None,+) -> tuple[dict[str, Any], list[str]]:"""Pre-create WorkflowFileExecution records with PENDING status to prevent race conditions.This matches the backend's _pre_create_file_executions pattern for ALL workflow typesand includes file history deduplication for ETL workflows.+ Includes double safeguard: checks database for duplicate active files before creating records.+Args:+ file_data: File data contextfiles: List of file items (can be tuples, lists, or dicts)workflow_id: Workflow IDexecution_id: Workflow execution IDapi_client: Internal API clientworkflow_type: Workflow type (API/ETL/TASK)is_api: Whether this is an API workflow+ use_file_history: Whether to use file history+ workflow_logger: Workflow logger for UI logging (optional)Returns:- Dict mapping file names to {'id': str, 'object': WorkflowFileExecutionData}+ Tuple of (pre_created_data dict, skipped_files list)+ - pre_created_data: Dict mapping file names to PreCreatedFileData+ - skipped_files: List of file identifiers (uuid:path) that were skipped (already active)"""pre_created_data: dict[str, PreCreatedFileData] = {}+ skipped_files: list[str] = [] # List of identifiers (uuid:path), not file names# Use the file history flag passed from execution parameterslogger.info(@@ -934,7 +1083,18 @@ def _pre_create_file_executions(# Convert to dict for APIfile_hash_dict_for_api = file_hash.to_dict()+ # DOUBLE SAFEGUARD: Check if file is already being processed (Redis-first with DB fallback)+ # This catches race conditions that slipped through general worker's filter+ is_duplicate = _check_file_already_active(+ file_hash=file_hash,+ workflow_id=workflow_id,+ execution_id=execution_id,+ api_client=api_client,+ file_name=file_name,+ )+try:+ # Create WorkflowFileExecution record for ALL files (including duplicates for audit trail)# Create WorkflowFileExecution record for ALL workflow types# CRITICAL FIX: For use_file_history=False, force create fresh records to prevent# reusing completed records from previous executions@@ -945,27 +1105,76 @@ def _pre_create_file_executions(force_create=not use_file_history, # Force create when file history is disabled)- # PROGRESSIVE STATUS UPDATE: Set initial status to PENDING (FIXED)- # This ensures file executions start with PENDING status before processing begins- try:- api_client.file_client.update_file_execution_status(- file_execution_id=workflow_file_execution.id,- status=ExecutionStatus.PENDING.value,+ # Handle duplicate files vs normal files differently+ if is_duplicate:+ # DUPLICATE DETECTED: Mark as ERROR with explanation for audit trail+ error_message = (+ "File skipped - already being processed in another execution "+ "(duplicate prevention safeguard)")- except Exception as status_error:- logger.warning(- f"Failed to set initial PENDING status for file {file_name}: {status_error}"++ try:+ api_client.file_client.update_file_execution_status(+ file_execution_id=workflow_file_execution.id,+ status=ExecutionStatus.ERROR.value,+ error_message=error_message,+ )+ logger.warning(+ f"DUPLICATE: File '{file_name}' marked as ERROR - {error_message}"+ )++ # Log to UI with error context+ if workflow_logger:+ from shared.infrastructure.logging.helpers import (+ log_file_processing_error,+ )++ log_file_processing_error(+ workflow_logger,+ str(workflow_file_execution.id),+ file_name,+ f"⏭️ {error_message}",+ )++ # Track as skipped using stable identifier (uuid:path), not just file_name+ # This prevents misclassification when different files share the same display name+ file_identifier = (+ f"{file_hash.provider_file_uuid}:{file_hash.file_path}"+ )+ skipped_files.append(file_identifier)++ except Exception as status_error:+ logger.error(+ f"Failed to mark duplicate file '{file_name}' as ERROR: {status_error}"+ )++ # Don't add to pre_created_data - file won't be processed+ logger.info(+ f"Created WorkflowFileExecution {workflow_file_execution.id} for duplicate file '{file_name}' (marked as ERROR)")- # Don't fail the entire creation if status update fails- pre_created_data[file_name] = PreCreatedFileData(- id=str(workflow_file_execution.id),- object=workflow_file_execution,- file_hash=file_hash,- )- logger.info(- f"Pre-created WorkflowFileExecution {workflow_file_execution.id} for {workflow_type} file '{file_name}'"- )+ else:+ # NORMAL FILE: Set initial status to PENDING for processing+ try:+ api_client.file_client.update_file_execution_status(+ file_execution_id=workflow_file_execution.id,+ status=ExecutionStatus.PENDING.value,+ )+ except Exception as status_error:+ logger.warning(+ f"Failed to set initial PENDING status for file {file_name}: {status_error}"+ )+ # Don't fail the entire creation if status update fails++ # Add to pre_created_data for processing+ pre_created_data[file_name] = PreCreatedFileData(+ id=str(workflow_file_execution.id),+ object=workflow_file_execution,+ file_hash=file_hash,+ )+ logger.info(+ f"Pre-created WorkflowFileExecution {workflow_file_execution.id} for {workflow_type} file '{file_name}'"+ )except Exception as e:logger.error(@@ -979,7 +1188,25 @@ def _pre_create_file_executions(# File history deduplication now handled during individual file processing- return pre_created_data+ # Log summary if files were skipped+ if skipped_files:+ logger.warning(+ f"Skipped {len(skipped_files)} duplicate file(s) already being processed: {skipped_files}"+ )+ if workflow_logger:+ from shared.infrastructure.logging.helpers import log_file_info++ log_file_info(+ workflow_logger,+ None,+ f"⏭️ Total {len(skipped_files)} duplicate file(s) skipped (already active in other executions)",+ )++ logger.info(+ f"Pre-created {len(pre_created_data)} file execution(s), skipped {len(skipped_files)} duplicate(s)"+ )++ return pre_created_data, skipped_filesdef _create_file_hash_from_dict(