LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
UN-2801 [FIX] Fix file batch distribution algorithm to respect configured batch limits (#1530)
ali64fa6c7edaba
diff --git a/backend/workflow_manager/workflow_v2/workflow_helper.py b/backend/workflow_manager/workflow_v2/workflow_helper.pyindex 8969c5414..696a257e4 100644--- a/backend/workflow_manager/workflow_v2/workflow_helper.py+++ b/backend/workflow_manager/workflow_v2/workflow_helper.py@@ -1,6 +1,5 @@import jsonimport logging-import mathimport osimport timeimport traceback@@ -89,7 +88,8 @@ class WorkflowHelper:def get_file_batches(cls, input_files: dict[str, FileHash]) -> list[list[tuple[str, FileHash]]]:- """_summary_+ """Split input files into batches for parallel processing.+ Distributes files as evenly as possible across the target number of batches.Args:input_files (dict[str, FileHash]): input files@@ -106,20 +106,31 @@ class WorkflowHelper:BATCH_SIZE = Configuration.get_value_by_organization(config_key=ConfigKey.MAX_PARALLEL_FILE_BATCHES, organization=organization) # Max number of batches-file_items = list(json_serializable_files.items())- # Calculate how many items per batch+ # Calculate distributionnum_files = len(file_items)+ # Target number of batches (can't exceed number of files)num_batches = min(BATCH_SIZE, num_files)- items_per_batch = math.ceil(num_files / num_batches)- # Split into batches+ # Distribute files as evenly as possible+ base_items_per_batch = num_files // num_batches+ remainder = num_files % num_batches++ # Create batchesbatches = []- for start_index in range(0, len(file_items), items_per_batch):- end_index = start_index + items_per_batch- batch = file_items[start_index:end_index]- batches.append(batch)+ start_index = 0++ for i in range(num_batches):+ # First 'remainder' batches get one extra item+ batch_size = base_items_per_batch + (1 if i < remainder else 0)++ if start_index < len(file_items):+ end_index = min(start_index + batch_size, len(file_items))+ batch = file_items[start_index:end_index]+ if batch: # Only add non-empty batches+ batches.append(batch)+ start_index = end_indexreturn batches