LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
"""Active File Manager Utility
This module provides utilities for managing active file processing state and preventing
race conditions in concurrent workflow executions.
Key Features:
- Filter files that are already being processed by other executions
- Create cache entries to prevent race conditions
- Provide detailed statistics for monitoring and debugging
- Graceful error handling that never fails the entire execution
"""
import hashlib
import os
import time
from typing import Any, Protocol
from ...cache.cache_backends import RedisCacheBackend
from ...infrastructure.logging import WorkerLogger
# Constants for cache configuration
DEFAULT_ACTIVE_FILE_CACHE_TTL = 300 # 5 minutes
# Cap the active-file marker TTL at the PG stuck-batch reaper's DEFAULT timeout
# (WORKER_PG_BATCH_STUCK_TIMEOUT_SECONDS, 9000s) so a marker can be configured to
# outlive a stalled-but-not-yet-reaped batch — below it there is a window where
# the marker has expired but the reaper hasn't yet marked the execution terminal.
# NOTE: the reaper timeout is env-tunable; if it's raised above 9000, raise this
# too or that window reopens. Default marker TTL stays 300s; operators opt in to
# longer.
MAX_ACTIVE_FILE_CACHE_TTL = 9000 # 2.5 hours maximum
def get_active_file_cache_ttl() -> int:
"""Get the configurable TTL for active file cache entries.
Returns:
TTL in seconds, clamped to [60, MAX_ACTIVE_FILE_CACHE_TTL] (1 minute to
2.5 hours). A non-integer or out-of-range ``ACTIVE_FILE_CACHE_TTL`` is
surfaced as a warning rather than silently swallowed — a silently
shortened TTL re-opens the very window a raised value was meant to close.
"""
raw = os.environ.get("ACTIVE_FILE_CACHE_TTL")
if raw is None:
return DEFAULT_ACTIVE_FILE_CACHE_TTL
try:
ttl = int(raw)
except (ValueError, TypeError):
logger.warning(
f"ACTIVE_FILE_CACHE_TTL={raw!r} is not an integer; "
f"using default {DEFAULT_ACTIVE_FILE_CACHE_TTL}s."
)
return DEFAULT_ACTIVE_FILE_CACHE_TTL
clamped = min(max(ttl, 60), MAX_ACTIVE_FILE_CACHE_TTL)
if clamped != ttl:
logger.warning(
f"ACTIVE_FILE_CACHE_TTL={ttl}s out of range; clamped to {clamped}s "
f"(allowed 60..{MAX_ACTIVE_FILE_CACHE_TTL})."
)
return clamped
class LoggerProtocol(Protocol):
"""Protocol for logger objects to provide proper type hints."""
def debug(self, msg: str) -> None: ...
def info(self, msg: str) -> None: ...
def warning(self, msg: str) -> None: ...
def error(self, msg: str) -> None: ...
logger = WorkerLogger.get_logger(__name__)
class ActiveFileManager:
"""Utility class for managing active file processing state and race condition prevention.
**USAGE SCOPE**:
- ✅ USE: ETL/TASK workflows in @workers/general/
- ❌ DON'T USE: API deployments in @workers/api-deployment/
API deployments have different concurrency patterns and should not use the file_active
cache pattern. They handle duplicate processing through their own mechanisms.
"""
@staticmethod
def create_cache_entries(
source_files: dict[str, Any],
files_to_cache: dict[str, Any],
workflow_id: str,
execution_id: str,
logger_instance: LoggerProtocol | None = None,
) -> dict[str, Any]:
"""Create cache entries for files to prevent race conditions (cache-only, no filtering).
This method ONLY creates cache entries for race condition prevention. It does NOT
filter files or modify the source_files dictionary. Use this after FilterPipeline
has already applied all necessary filtering including ActiveFileFilter.
Args:
source_files: Dictionary of all source files (used for file_tracking_data)
files_to_cache: Dictionary of specific files to create cache entries for
workflow_id: Workflow identifier
execution_id: Current execution identifier
logger_instance: Optional logger override (uses module logger if None)
Returns:
Cache statistics dictionary with creation results
Example:
>>> # After FilterPipeline has filtered files
>>> cache_stats = ActiveFileManager.create_cache_entries(
... source_files=all_files, # Original files for tracking data
... files_to_cache=filtered_files, # Only cache these specific files
... workflow_id="workflow-123",
... execution_id="exec-456",
... )
>>> print(f"Created {cache_stats['cache_created']} cache entries")
"""
log = logger_instance or logger
if not files_to_cache:
return {
"cache_created": 0,
"cache_errors": 0,
"processing_files": [],
}
cache_stats = {
"cache_created": 0,
"cache_errors": 0,
"processing_files": [],
}
try:
# Extract provider_file_uuids and file paths from source files for tracking data
file_tracking_data = {} # file_key -> {provider_uuid, file_path, file_data} mapping
for file_key, file_data in source_files.items():
provider_uuid = ActiveFileManager._extract_provider_uuid(file_data)
file_path = ActiveFileManager._extract_file_path(file_data)
if provider_uuid:
file_tracking_data[file_key] = {
"provider_uuid": provider_uuid,
"file_path": file_path
or file_key, # fallback to file_key if no file_path
"file_data": file_data,
}
if not file_tracking_data:
log.warning(
"No provider_file_uuid found in source files, skipping cache creation"
)
return cache_stats
log.info(
f"🔒 Creating cache entries for {len(files_to_cache)} files to prevent race conditions"
)
# Create cache entries only for the specified files
ActiveFileManager._create_cache_entries_for_selected_files(
final_files_to_process=files_to_cache,
file_tracking_data=file_tracking_data,
workflow_id=workflow_id,
execution_id=execution_id,
log=log,
filtering_stats=cache_stats,
)
log.info(
f"✅ Cache creation complete: {cache_stats['cache_created']} entries created, "
f"{cache_stats['cache_errors']} errors"
)
except Exception as cache_error:
log.warning(f"Cache entry creation failed: {cache_error}")
cache_stats["error"] = str(cache_error)
return cache_stats
@staticmethod
def create_cache_entries_simple(
files_to_cache: dict[str, Any],
workflow_id: str,
execution_id: str,
logger_instance: LoggerProtocol | None = None,
) -> dict[str, Any]:
"""Create cache entries for files to prevent race conditions (simplified API).
This is a simplified version of create_cache_entries() for cases where you only
have the final filtered files to cache (e.g., in discovery methods after FilterPipeline).
Use this when source_files and files_to_cache would be the same dictionary.
Args:
files_to_cache: Dictionary of files to create cache entries for
workflow_id: Workflow identifier
execution_id: Current execution identifier
logger_instance: Optional logger override (uses module logger if None)
Returns:
Cache statistics dictionary with creation results
Example:
>>> # After FilterPipeline in discovery methods
>>> cache_stats = ActiveFileManager.create_cache_entries_simple(
... files_to_cache=final_filtered_files,
... workflow_id="workflow-123",
... execution_id="exec-456",
... )
>>> print(f"Created {cache_stats['cache_created']} cache entries")
"""
# Delegate to the full method with same dict for both parameters
return ActiveFileManager.create_cache_entries(
source_files=files_to_cache,
files_to_cache=files_to_cache,
workflow_id=workflow_id,
execution_id=execution_id,
logger_instance=logger_instance,
)
@staticmethod
def _extract_provider_uuid(file_data: Any) -> str | None:
"""Extract provider_file_uuid from file data, handling different formats."""
if hasattr(file_data, "provider_file_uuid") and file_data.provider_file_uuid:
return file_data.provider_file_uuid
elif isinstance(file_data, dict) and file_data.get("provider_file_uuid"):
return file_data["provider_file_uuid"]
return None
@staticmethod
def _extract_file_path(file_data: Any) -> str | None:
"""Extract file_path from file data, handling different formats."""
if hasattr(file_data, "file_path") and file_data.file_path:
return file_data.file_path
elif isinstance(file_data, dict) and file_data.get("file_path"):
return file_data["file_path"]
return None
@staticmethod
def _generate_file_path_hash(file_path: str) -> str:
"""Generate a short hash for file path to differentiate files with same provider_uuid.
Uses SHA256 with 12 characters for better collision resistance while keeping
cache keys reasonably short.
"""
return hashlib.sha256(file_path.encode("utf-8")).hexdigest()[:12]
@staticmethod
def _create_cache_key(workflow_id: str, provider_uuid: str, file_path: str) -> str:
"""Create cache key that uniquely identifies a file by provider_uuid AND file_path.
This prevents conflicts when multiple files with same content (same provider_uuid)
but different paths exist.
"""
file_path_hash = ActiveFileManager._generate_file_path_hash(file_path)
return f"file_active:{workflow_id}:{provider_uuid}:{file_path_hash}"
@staticmethod
def _handle_cache_check(
file_tracking_data: dict[str, dict],
workflow_id: str,
execution_id: str,
log: LoggerProtocol,
) -> dict[str, Any]:
"""Handle cache checking operations using batch Redis operations for performance."""
cache = RedisCacheBackend()
active_files_to_skip = set() # Set of file_keys to skip
stats = {
"cache_active": [],
"processing_files": [],
"cache_created": 0,
"cache_errors": 0,
}
if not file_tracking_data:
return {"active_files": active_files_to_skip, "stats": stats}
try:
# BATCH OPTIMIZATION: Pre-compute all cache keys and fetch in single operation
cache_key_to_file_key = {} # cache_key -> file_key mapping
file_key_to_tracking = {} # file_key -> tracking_info mapping
# Pre-compute all cache keys
for file_key, tracking_info in file_tracking_data.items():
provider_uuid = tracking_info["provider_uuid"]
file_path = tracking_info["file_path"]
cache_key = ActiveFileManager._create_cache_key(
workflow_id, provider_uuid, file_path
)
cache_key_to_file_key[cache_key] = file_key
file_key_to_tracking[file_key] = tracking_info
# Single batch Redis call instead of N individual calls
cache_keys = list(cache_key_to_file_key.keys())
log.debug(f"Batch checking {len(cache_keys)} cache keys for active files")
cached_results = cache.mget(cache_keys)
# Process batch results
for cache_key, cached_data in cached_results.items():
file_key = cache_key_to_file_key[cache_key]
tracking_info = file_key_to_tracking[file_key]
if cached_data and isinstance(cached_data, dict):
# Extract data from cache wrapper
cached_active = cached_data.get("data", {})
if isinstance(cached_active, dict):
cached_execution_id = cached_active.get("execution_id")
cached_file_path = cached_active.get("file_path", "unknown")
current_file_path = tracking_info["file_path"]
log.debug(
f"File {file_key}: cached_execution_id={cached_execution_id}, current_execution_id={execution_id}"
)
log.debug(
f" Cache path: {cached_file_path}, Current path: {current_file_path}"
)
if cached_execution_id != execution_id:
active_files_to_skip.add(file_key)
stats["cache_active"].append(file_key)
log.debug(
f"File {file_key} already active by execution {cached_execution_id}, will skip"
)
else:
log.debug(
f"File {file_key} cached by same execution {execution_id}, will process"
)
if active_files_to_skip:
log.info(
f"⚡ Found {len(active_files_to_skip)} files already active in cache (batch check)"
)
except Exception as batch_error:
log.warning(
f"Batch cache check failed, falling back to individual checks: {batch_error}"
)
# Fallback to individual checks if batch fails
return ActiveFileManager._handle_cache_check_fallback(
file_tracking_data, workflow_id, execution_id, log
)
return {"active_files": active_files_to_skip, "stats": stats}
@staticmethod
def _handle_cache_check_fallback(
file_tracking_data: dict[str, dict],
workflow_id: str,
execution_id: str,
log: LoggerProtocol,
) -> dict[str, Any]:
"""Fallback method using individual Redis operations if batch fails."""
cache = RedisCacheBackend()
active_files_to_skip = set()
stats = {
"cache_active": [],
"processing_files": [],
"cache_created": 0,
"cache_errors": 0,
}
# Fallback to individual cache checks
for file_key, tracking_info in file_tracking_data.items():
provider_uuid = tracking_info["provider_uuid"]
file_path = tracking_info["file_path"]
try:
cache_key = ActiveFileManager._create_cache_key(
workflow_id, provider_uuid, file_path
)
cached_active = cache.get(cache_key)
if cached_active and isinstance(cached_active, dict):
cached_execution_id = cached_active.get("execution_id")
if cached_execution_id != execution_id:
active_files_to_skip.add(file_key)
stats["cache_active"].append(file_key)
except Exception as key_error:
log.debug(f"Individual cache check failed for {file_key}: {key_error}")
return {"active_files": active_files_to_skip, "stats": stats}
@staticmethod
def _create_cache_entries_for_selected_files(
final_files_to_process: dict[str, Any],
file_tracking_data: dict[str, dict],
workflow_id: str,
execution_id: str,
log: LoggerProtocol,
filtering_stats: dict[str, Any],
) -> None:
"""Create cache entries only for files that will actually be processed.
OPTIMIZED: Uses batch Redis operations for better performance with large file sets.
Uses file-path-aware cache keys to differentiate files with the same content
but different paths.
"""
if not final_files_to_process:
return
cache = RedisCacheBackend()
ttl = get_active_file_cache_ttl() # Use configurable TTL
log.info(
f"Creating cache entries for {len(final_files_to_process)} final selected files (TTL: {ttl}s)"
)
try:
# BATCH OPTIMIZATION: Prepare all cache entries for batch creation
batch_cache_data = {} # cache_key -> (cache_data, ttl)
processing_files = []
cache_errors = 0
for file_key, file_data in final_files_to_process.items():
# Get tracking info for this file
tracking_info = file_tracking_data.get(file_key)
if not tracking_info:
log.warning(f"No tracking info found for file: {file_key}")
cache_errors += 1
continue
provider_uuid = tracking_info["provider_uuid"]
file_path = tracking_info["file_path"]
if not provider_uuid:
cache_errors += 1
continue
try:
# Prepare cache entry data
cache_key = ActiveFileManager._create_cache_key(
workflow_id, provider_uuid, file_path
)
cache_data = {
"execution_id": execution_id,
"workflow_id": workflow_id,
"provider_file_uuid": provider_uuid,
"file_path": file_path,
"status": "PENDING",
"created_at": time.time(),
}
batch_cache_data[cache_key] = (cache_data, ttl)
processing_files.append(file_key)
log.debug(f"Prepared cache entry for: {file_key} ({provider_uuid})")
except Exception as prep_error:
log.warning(
f"Failed to prepare cache entry for {file_key}: {prep_error}"
)
cache_errors += 1
# Single batch Redis operation instead of N individual operations
if batch_cache_data:
cache_created = cache.mset(batch_cache_data)
log.info(
f"🔒 Batch created {cache_created}/{len(batch_cache_data)} cache entries "
f"for race condition prevention"
)
else:
cache_created = 0
log.warning("No valid cache entries prepared for batch creation")
except Exception as batch_error:
log.warning(
f"Batch cache creation failed, falling back to individual operations: {batch_error}"
)
# Fallback to individual cache creation
cache_created, cache_errors, processing_files = (
ActiveFileManager._create_cache_entries_fallback(
final_files_to_process=final_files_to_process,
file_tracking_data=file_tracking_data,
workflow_id=workflow_id,
execution_id=execution_id,
log=log,
ttl=ttl,
)
)
# Update statistics with the actual cache creation results
filtering_stats["cache_created"] = cache_created
filtering_stats["cache_errors"] = cache_errors
filtering_stats["processing_files"] = processing_files
@staticmethod
def _create_cache_entries_fallback(
final_files_to_process: dict[str, Any],
file_tracking_data: dict[str, dict],
workflow_id: str,
execution_id: str,
log: LoggerProtocol,
ttl: int,
) -> tuple[int, int, list[str]]:
"""Fallback method for individual cache creation if batch fails."""
cache = RedisCacheBackend()
cache_created = 0
cache_errors = 0
processing_files = []
for file_key, file_data in final_files_to_process.items():
tracking_info = file_tracking_data.get(file_key)
if not tracking_info:
cache_errors += 1
continue
provider_uuid = tracking_info["provider_uuid"]
file_path = tracking_info["file_path"]
if not provider_uuid:
cache_errors += 1
continue
success = ActiveFileManager._create_cache_entry(
cache=cache,
workflow_id=workflow_id,
execution_id=execution_id,
provider_uuid=provider_uuid,
file_path=file_path,
log=log,
ttl=ttl,
)
if success:
cache_created += 1
processing_files.append(file_key)
else:
cache_errors += 1
return cache_created, cache_errors, processing_files
@staticmethod
def _create_cache_entry(
cache: RedisCacheBackend,
workflow_id: str,
execution_id: str,
provider_uuid: str,
file_path: str,
log: LoggerProtocol,
ttl: int | None = None,
) -> bool:
"""Create a cache entry for an active file using file-path-aware key."""
try:
# Use configurable TTL if not provided
if ttl is None:
ttl = get_active_file_cache_ttl()
cache_key = ActiveFileManager._create_cache_key(
workflow_id, provider_uuid, file_path
)
cache_data = {
"execution_id": execution_id,
"workflow_id": workflow_id,
"provider_file_uuid": provider_uuid,
"file_path": file_path, # Include file path in cache data
"status": "PENDING",
"created_at": time.time(),
}
cache.set(cache_key, cache_data, ttl=ttl)
log.debug(f"Created cache entry for {provider_uuid} at {file_path}")
return True
except Exception as cache_set_error:
log.warning(
f"Failed to create cache entry for {provider_uuid}: {cache_set_error}"
)
return False
@staticmethod
def _filter_source_files_by_keys(
source_files: dict[str, Any],
active_file_keys_to_skip: set[str],
log: LoggerProtocol,
) -> tuple[dict[str, Any], int]:
"""Filter source_files dict to remove active files by file keys."""
for file_key in active_file_keys_to_skip:
if file_key in source_files:
del source_files[file_key]
log.debug(f"Removed active file: {file_key}")
new_count = len(source_files)
return source_files, new_count
@staticmethod
def _filter_source_files(
source_files: dict[str, Any],
active_files_to_skip: set[str],
provider_file_map: dict[str, str],
log: LoggerProtocol,
) -> tuple[dict[str, Any], int]:
"""Filter source_files dict to remove active files (legacy method)."""
files_to_remove = [
provider_file_map[provider_uuid]
for provider_uuid in active_files_to_skip
if provider_uuid in provider_file_map
]
for file_key in files_to_remove:
del source_files[file_key]
new_count = len(source_files)
return source_files, new_count
@staticmethod
def cleanup_cache_entries(
provider_file_uuids: list[str],
workflow_id: str,
log: LoggerProtocol | None = None,
) -> int:
"""Clean up file-path-aware cache entries for completed file processing.
OPTIMIZED: Uses non-blocking SCAN instead of blocking KEYS for production safety.
Uses pattern matching to find all cache entries for the given provider UUIDs
since we don't have the file paths available during cleanup.
Args:
provider_file_uuids: List of provider file UUIDs to clean up
workflow_id: Workflow ID
log: Optional logger instance
Returns:
Number of cache entries cleaned up
"""
logger_instance = log or logger
if not provider_file_uuids:
return 0
try:
cache = RedisCacheBackend()
cleaned_count = 0
for provider_uuid in provider_file_uuids:
# Find all file-path-aware cache entries for this provider_uuid
pattern = f"file_active:{workflow_id}:{provider_uuid}:*"
try:
# Use non-blocking SCAN instead of blocking KEYS
matching_keys = cache.scan_keys(
pattern, count=50
) # Small batches for safety
if matching_keys:
# Delete in batches to avoid large delete operations
batch_size = 100
for i in range(0, len(matching_keys), batch_size):
batch_keys = matching_keys[i : i + batch_size]
try:
# Batch delete for efficiency
deleted_count = cache.redis_client.delete(*batch_keys)
cleaned_count += deleted_count
logger_instance.debug(
f"Cleaned up {deleted_count} cache entries for {provider_uuid} (batch {i//batch_size + 1})"
)
except Exception as batch_error:
logger_instance.warning(
f"Failed to delete batch for {provider_uuid}: {batch_error}"
)
# Fallback to individual deletion
for key in batch_keys:
try:
if cache.delete(key):
cleaned_count += 1
except Exception:
pass # Continue with other keys
except Exception as pattern_error:
logger_instance.warning(
f"Failed to cleanup entries for {provider_uuid}: {pattern_error}"
)
if cleaned_count > 0:
logger_instance.info(
f"🧹 Cleaned up {cleaned_count} active file cache entries (non-blocking SCAN)"
)
return cleaned_count
except Exception as cleanup_error:
logger_instance.warning(f"Failed to cleanup cache entries: {cleanup_error}")
return 0
def cleanup_active_file_cache(
provider_file_uuids: list[str],
workflow_id: str,
logger_instance: LoggerProtocol | None = None,
) -> int:
"""Convenience function that delegates to ActiveFileManager.cleanup_cache_entries()."""
return ActiveFileManager.cleanup_cache_entries(
provider_file_uuids=provider_file_uuids,
workflow_id=workflow_id,
log=logger_instance,
)