LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
UN-2784 [FEAT] Implement FIFO/LIFO file processing order with round-robin batching (#1531)
Chandrasekharan M51bea4ecf7a8
diff --git a/backend/workflow_manager/endpoint_v2/constants.py b/backend/workflow_manager/endpoint_v2/constants.pyindex cedf19728..cb0deb8c6 100644--- a/backend/workflow_manager/endpoint_v2/constants.py+++ b/backend/workflow_manager/endpoint_v2/constants.py@@ -1,5 +1,10 @@+import loggingimport os+from enum import Enumfrom fnmatch import fnmatch+from typing import Any++logger = logging.getLogger(__name__)class TableColumns:@@ -63,6 +68,60 @@ class SourceKey:PROCESS_SUB_DIRECTORIES = "processSubDirectories"MAX_FILES = "maxFiles"FOLDERS = "folders"+ FILE_PROCESSING_ORDER = "fileProcessingOrder"+++class FileProcessingOrder(str, Enum):+ """File processing order for SourceKey.FILE_PROCESSING_ORDER.++ Semantics:+ - oldest_first: ascending last-modified time (mtime).+ - newest_first: descending mtime.+ - unordered: no explicit ordering (OS enumeration order; may be nondeterministic).+ """++ UNORDERED = "unordered"+ OLDEST_FIRST = "oldest_first" # FIFO+ NEWEST_FIRST = "newest_first" # LIFO++ @classmethod+ def values(cls) -> list[str]:+ return [v.value for v in cls]++ @classmethod+ def from_value(+ cls, value: Any, default: "FileProcessingOrder" = None+ ) -> "FileProcessingOrder":+ """Convert a value to FileProcessingOrder enum, with fallback to default.++ Args:+ value: The value to convert (can be string, enum, or None)+ default: Default value if conversion fails (defaults to UNORDERED)++ Returns:+ FileProcessingOrder enum value+ """+ if default is None:+ default = cls.UNORDERED++ if not value:+ return default++ # Already an enum instance+ if isinstance(value, cls):+ return value++ # Try to convert string to enum+ if isinstance(value, str):+ try:+ return cls(value)+ except ValueError:+ logger.warning(+ f"Invalid file processing order '{value}', using {default.value}"+ )+ return default++ return defaultclass DestinationKey:@@ -136,6 +195,7 @@ class FilePattern:class SourceConstant:MAX_RECURSIVE_DEPTH = 10+ MAX_FILES_FOR_SORTING = 40000class ApiDeploymentResultStatus:diff --git a/backend/workflow_manager/endpoint_v2/source.py b/backend/workflow_manager/endpoint_v2/source.pyindex a0e2b6365..c1e75ba16 100644--- a/backend/workflow_manager/endpoint_v2/source.py+++ b/backend/workflow_manager/endpoint_v2/source.py@@ -18,6 +18,7 @@ from utils.user_context import UserContextfrom workflow_manager.endpoint_v2.base_connector import BaseConnectorfrom workflow_manager.endpoint_v2.constants import (FilePattern,+ FileProcessingOrder,FileSystemConnector,FileType,SourceConstant,@@ -46,7 +47,7 @@ from workflow_manager.workflow_v2.models.workflow import Workflowfrom unstract.connectors.filesystems.unstract_file_system import UnstractFileSystemfrom unstract.filesystem import FileStorageType, FileSystemfrom unstract.sdk.file_storage import FileStorage-from unstract.workflow_execution.enums import LogStage, LogState+from unstract.workflow_execution.enums import LogLevel, LogStage, LogStatelogger = logging.getLogger(__name__)@@ -146,40 +147,46 @@ class SourceConnector(BaseConnector):def list_file_from_api_storage(self, file_hashes: dict[str, FileHash]- ) -> tuple[dict[str, FileHash], int]:+ ) -> dict[str, FileHash]:"""List all files from the api_storage_dir directory."""- return file_hashes, len(file_hashes)+ return file_hashes- def list_files_from_file_connector(self) -> tuple[dict[str, FileHash], int]:+ def list_files_from_file_connector(self) -> dict[str, FileHash]:"""_summary_Raises:InvalidDirectory: _description_Returns:- tuple[dict[str, FileHash], int]: A dictionary of matched file paths- and their corresponding FileHash objects, along with the total count- of matched files.+ dict[str, FileHash]: A dictionary of matched file paths+ and their corresponding FileHash objects."""connector: ConnectorInstance = self.endpoint.connector_instanceconnector_settings: dict[str, Any] = connector.connector_metadata- source_configurations: dict[str, Any] = self.endpoint.configuration- required_patterns = list(source_configurations.get(SourceKey.FILE_EXTENSIONS, []))+ source_configuration: dict[str, Any] = self.endpoint.configuration+ logger.info("Source configuration: %s", source_configuration)+ required_patterns = list(source_configuration.get(SourceKey.FILE_EXTENSIONS, []))recursive = bool(- source_configurations.get(SourceKey.PROCESS_SUB_DIRECTORIES, False)+ source_configuration.get(SourceKey.PROCESS_SUB_DIRECTORIES, False))limit = int(- source_configurations.get(SourceKey.MAX_FILES, FileSystemConnector.MAX_FILES)+ source_configuration.get(SourceKey.MAX_FILES, FileSystemConnector.MAX_FILES)+ )+ file_processing_order = FileProcessingOrder.from_value(+ source_configuration.get(SourceKey.FILE_PROCESSING_ORDER))root_dir_path = connector_settings.get(ConnectorKeys.PATH, "")- folders_to_process = list(source_configurations.get(SourceKey.FOLDERS, ["/"]))+ folders_to_process = list(source_configuration.get(SourceKey.FOLDERS, ["/"]))# Process from root in case its user provided list is emptyif not folders_to_process:folders_to_process = ["/"]patterns = self.valid_file_patterns(required_patterns=required_patterns)- self.publish_user_sys_log(+ self.workflow_log.publish_log(f"Matching for patterns '{', '.join(patterns)}' from "- f"'{', '.join(folders_to_process)}'"+ f"'{', '.join(folders_to_process)}' to process upto {limit} files "+ "recursively"+ if recursive+ else "")source_fs = self.get_fs_connector(@@ -205,27 +212,26 @@ class SourceConnector(BaseConnector):raiseraise InvalidInputDirectory(detail=msg)- total_files_to_process = 0- total_matched_files = {}- unique_file_paths: set[str] = set()-- for input_directory in valid_directories:- logger.debug(f"Listing files from: {input_directory}")- matched_files, count = self._get_matched_files(+ if file_processing_order == FileProcessingOrder.UNORDERED:+ total_matched_files = self._process_without_sorting(source_fs,- input_directory,+ valid_directories,patterns,recursive,limit,- unique_file_paths,)- self.publish_user_sys_log(f"Matched '{count}' files from '{input_directory}'")- total_matched_files.update(matched_files)- total_files_to_process += count- self.publish_input_output_list_file_logs(- folders_to_process, total_matched_files, total_files_to_process- )- return total_matched_files, total_files_to_process+ else:+ total_matched_files = self._process_with_sorting(+ source_fs,+ valid_directories,+ patterns,+ recursive,+ limit,+ file_processing_order,+ )++ self.publish_input_output_list_file_logs(folders_to_process, total_matched_files)+ return total_matched_filesdef publish_user_sys_log(self, msg: str) -> None:"""Publishes log to the user and system.@@ -240,14 +246,14 @@ class SourceConnector(BaseConnector):self.workflow_log.publish_log(message=msg)def publish_input_output_list_file_logs(- self, folders: list[str], matched_files: dict[str, FileHash], count: int+ self, folders: list[str], matched_files: dict[str, FileHash]) -> None:folders_list = "\n".join(f"- `{folder.strip()}`" for folder in folders)input_log = f"## Folders to process:\n\n{folders_list}\n\n"self.workflow_log.publish_update_log(state=LogState.INPUT_UPDATE, message=input_log)- output_log = self._matched_files_component_log(matched_files, count)+ output_log = self._matched_files_component_log(matched_files)self.workflow_log.publish_update_log(state=LogState.OUTPUT_UPDATE, message=output_log)@@ -264,17 +270,179 @@ class SourceConnector(BaseConnector):state=LogState.OUTPUT_UPDATE, message=output_log_message)- def _matched_files_component_log(- self, matched_files: dict[str, FileHash], count: int- ) -> str:+ def _matched_files_component_log(self, matched_files: dict[str, FileHash]) -> str:output_log = "### Matched files \n```text\n\n\n"for file_path in islice(matched_files.keys(), 20):output_log += f"- {file_path}\n"output_log += "```\n\n"- output_log += f"""Total matched files: {count}+ output_log += f"""Total matched files: {len(matched_files)}\n\nPlease note that only the first 20 files are shown.\n\n"""return output_log+ def _collect_and_sort_files(+ self,+ source_fs: UnstractFileSystem,+ valid_directories: list[str],+ recursive: bool,+ file_processing_order: str,+ ) -> list[dict[str, Any]]:+ """Collect files from all directories and apply global sorting.++ Used when file_processing_order is not UNORDERED.+ """+ all_files_metadata = []+ max_depth = int(SourceConstant.MAX_RECURSIVE_DEPTH) if recursive else 1+ max_files_for_sorting = SourceConstant.MAX_FILES_FOR_SORTING+ total_collected = 0++ # Collect files from all directories+ for input_directory in valid_directories:+ logger.debug(f"Collecting files from: {input_directory}")++ # Calculate remaining limit for this directory+ remaining_limit = max_files_for_sorting - total_collected+ if remaining_limit <= 0:+ break++ try:+ files_metadata = source_fs.list_files(+ directory=input_directory,+ max_depth=max_depth,+ include_dirs=False,+ limit=remaining_limit,+ )+ all_files_metadata.extend(files_metadata)+ total_collected += len(files_metadata)++ # Report errors+ user_errors = source_fs.report_errors_to_user()+ for error_msg in user_errors[:5]:+ self.workflow_log.publish_log(error_msg, level=LogLevel.ERROR)+ if len(user_errors) > 5:+ self.workflow_log.publish_log(+ f"... and {len(user_errors) - 5} more errors",+ level=LogLevel.ERROR,+ )++ # Check if we've hit the limit+ if total_collected >= max_files_for_sorting:+ break++ except Exception as e:+ error_msg = f"Failed to collect files from {input_directory}"+ logger.error(f"{error_msg}: {e}")+ self.workflow_log.publish_log(error_msg, level=LogLevel.ERROR)++ # Log warning if collection limit was reached+ if total_collected >= max_files_for_sorting:+ warning_msg = (+ f"File collection limit of '{max_files_for_sorting}' files reached. "+ "Ordering may not reflect all available files from source file system"+ )+ logger.warning(warning_msg)+ self.workflow_log.publish_log(warning_msg, level=LogLevel.WARN)++ # Apply sorting+ if file_processing_order == FileProcessingOrder.OLDEST_FIRST:+ sorted_files = source_fs.sort_files_by_modified_date(+ all_files_metadata, ascending=True+ )+ order_desc = "FIFO (oldest first)"+ else:+ sorted_files = source_fs.sort_files_by_modified_date(+ all_files_metadata, ascending=False+ )+ order_desc = "LIFO (newest first)"++ self.workflow_log.publish_log(+ f"Collected {len(all_files_metadata)} files, processing in {order_desc} order"+ )++ # Debug logging for first few files in sorted order+ if sorted_files and logger.isEnabledFor(logging.DEBUG):+ logger.debug("File processing order (first 10):")+ for i, file_meta in enumerate(sorted_files[:10]):+ file_path = file_meta.get("name", "unknown")+ modified_date = source_fs.extract_modified_date(file_meta)+ date_str = modified_date.isoformat() if modified_date else "unknown"+ logger.debug(f" {i + 1}. {file_path} (modified: {date_str})")++ return sorted_files++ def _process_with_sorting(+ self,+ source_fs: UnstractFileSystem,+ valid_directories: list[str],+ patterns: list[str],+ recursive: bool,+ limit: int,+ file_processing_order: str,+ ) -> dict[str, FileHash]:+ """In case ordering is expected, collect and sort all files."""+ matched_files = {}+ unique_file_paths: set[str] = set()++ # Collect and sort all files+ sorted_files = self._collect_and_sort_files(+ source_fs, valid_directories, recursive, file_processing_order+ )++ self._process_file_fs_directory(+ fs_metadata_list=sorted_files,+ count=0,+ limit=limit,+ unique_file_paths=unique_file_paths,+ matched_files=matched_files,+ patterns=patterns,+ source_fs=source_fs,+ dirs=[], # Not needed for pre-collected files+ )++ return matched_files++ def _process_without_sorting(+ self,+ source_fs: UnstractFileSystem,+ valid_directories: list[str],+ patterns: list[str],+ recursive: bool,+ limit: int,+ ) -> dict[str, FileHash]:+ """Process files directory by directory while checking for limit."""+ total_matched_files = {}+ total_files_to_process = 0+ unique_file_paths: set[str] = set()++ for input_directory in valid_directories:+ logger.debug(f"Listing files from: {input_directory}")++ remaining_limit = limit - total_files_to_process+ if remaining_limit <= 0:+ break++ matched_files = self._get_matched_files(+ source_fs,+ input_directory,+ patterns,+ recursive,+ remaining_limit,+ unique_file_paths,+ )+ matched_file_count = len(matched_files)+ self.publish_user_sys_log(+ f"Matched '{matched_file_count}' files from '{input_directory}'"+ )+ total_matched_files.update(matched_files)+ total_files_to_process += matched_file_count++ if total_files_to_process >= limit:+ msg = f"Maximum limit of '{limit}' files to process reached"+ self.workflow_log.publish_log(msg)+ logger.info(msg)+ break++ return total_matched_files+def _get_matched_files(self,source_fs: UnstractFileSystem,@@ -283,7 +451,7 @@ class SourceConnector(BaseConnector):recursive: bool,limit: int,unique_file_paths: set[str],- ) -> tuple[dict[str, FileHash], int]:+ ) -> dict[str, FileHash]:"""Get a dictionary of matched files based on patterns in a directory.This method searches for files in the specified `input_directory` that@@ -299,13 +467,15 @@ class SourceConnector(BaseConnector):limit (int): The maximum number of matched files to return.Returns:- tuple[dict[str, FileHash], int]: A dictionary of matched file paths- and their corresponding FileHash objects, along with the total count- of matched files.+ dict[str, FileHash]: A dictionary of matched file paths+ and their corresponding FileHash objects."""matched_files: dict[str, FileHash] = {}count = 0++ # Get max depth for recursive traversalmax_depth = int(SourceConstant.MAX_RECURSIVE_DEPTH) if recursive else 1+fs_fsspec = source_fs.get_fsspec_fs()for root, dirs, _ in fs_fsspec.walk(input_directory, maxdepth=max_depth):try:@@ -326,7 +496,8 @@ class SourceConnector(BaseConnector):source_fs=source_fs,dirs=dirs,)- return matched_files, count++ return matched_filesdef _process_file_fs_directory(self,@@ -737,16 +908,16 @@ class SourceConnector(BaseConnector):"""connection_type = self.endpoint.connection_typeif connection_type == WorkflowEndpoint.ConnectionType.FILESYSTEM:- files, count = self.list_files_from_file_connector()+ files = self.list_files_from_file_connector()elif connection_type == WorkflowEndpoint.ConnectionType.API:- files, count = self.list_file_from_api_storage(file_hashes)+ files = self.list_file_from_api_storage(file_hashes)else:raise InvalidSourceConnectionType()# TODO: move this to where file is listed at source.for index, file_hash in enumerate(files.values(), start=1):file_hash.file_number = index- return files, count+ return files, len(files)def get_file_content_hash(self, source_fs: UnstractFileSystem, file_path: str) -> str:"""Generate a hash value from the file content.diff --git a/backend/workflow_manager/endpoint_v2/static/src/file.json b/backend/workflow_manager/endpoint_v2/static/src/file.jsonindex dc8d6ef0a..1075d3ae8 100644--- a/backend/workflow_manager/endpoint_v2/static/src/file.json+++ b/backend/workflow_manager/endpoint_v2/static/src/file.json@@ -46,6 +46,14 @@"default": "skip_duplicates","enum": ["skip_duplicates", "reprocess_after_interval"],"enumNames": ["Never delete", "Delete after"]+ },+ "fileProcessingOrder": {+ "type": "string",+ "title": "File processing order",+ "description": "Order in which files will be processed. This could add an overhead based on the number of files in the source. Ordering may not work as expected if the source has more than 40,000 files to consider",+ "default": "unordered",+ "enum": ["unordered", "oldest_first", "newest_first"],+ "enumNames": ["No specific order", "Oldest first (FIFO)", "Newest first (LIFO)"]}},"dependencies": {diff --git a/backend/workflow_manager/workflow_v2/execution.py b/backend/workflow_manager/workflow_v2/execution.pyindex 9e180386d..68dfe78f5 100644--- a/backend/workflow_manager/workflow_v2/execution.py+++ b/backend/workflow_manager/workflow_v2/execution.py@@ -286,7 +286,7 @@ class WorkflowExecutionServiceHelper(WorkflowExecutionService):cost_dict = UsageHelper.get_aggregated_token_count(run_id=run_id)if not cost_dict:self.publish_log(- f"No cost data available for file '{file_name}'", level=LogLevel.WARNING+ f"No cost data available for file '{file_name}'", level=LogLevel.WARN)returncost = round(cost_dict.get("cost_in_dollars") or 0, 5)diff --git a/backend/workflow_manager/workflow_v2/workflow_helper.py b/backend/workflow_manager/workflow_v2/workflow_helper.pyindex f9433c639..c75d7718b 100644--- a/backend/workflow_manager/workflow_v2/workflow_helper.py+++ b/backend/workflow_manager/workflow_v2/workflow_helper.py@@ -107,31 +107,28 @@ class WorkflowHelper:BATCH_SIZE = Configuration.get_value_by_organization(config_key=ConfigKey.MAX_PARALLEL_FILE_BATCHES, organization=organization) # Max number of batches+ BATCH_SIZE = (+ int(BATCH_SIZE)+ if isinstance(BATCH_SIZE, int) or str(BATCH_SIZE).isdigit()+ else 1+ )file_items = list(json_serializable_files.items())# Calculate distributionnum_files = len(file_items)# Target number of batches (can't exceed number of files)num_batches = min(BATCH_SIZE, num_files)-- # Distribute files as evenly as possible- base_items_per_batch = num_files // num_batches- remainder = num_files % num_batches-- # Create batches- batches = []- start_index = 0-- for i in range(num_batches):- # First 'remainder' batches get one extra item- batch_size = base_items_per_batch + (1 if i < remainder else 0)-- if start_index < len(file_items):- end_index = min(start_index + batch_size, len(file_items))- batch = file_items[start_index:end_index]- if batch: # Only add non-empty batches- batches.append(batch)- start_index = end_index+ # Guard against invalid batch sizes+ if num_batches <= 0:+ num_batches = 1++ # Round-robin distribution for maintaining order in case its sorted+ batches = [[] for _ in range(num_batches)]+ for i, file_item in enumerate(file_items):+ batch_index = i % num_batches+ batches[batch_index].append(file_item)+ # Remove empties when num_files < num_batches+ batches = [b for b in batches if b]return batches@@ -185,6 +182,7 @@ class WorkflowHelper:else str(execution_mode))result = None+logger.info(f"Execution {workflow_execution.id} processing {total_files} files in {len(batches)} batches")diff --git a/frontend/src/layouts/rjsf-form-layout/CustomObjectFieldTemplate.jsx b/frontend/src/layouts/rjsf-form-layout/CustomObjectFieldTemplate.jsxindex 4c72ad2a2..73f860be9 100644--- a/frontend/src/layouts/rjsf-form-layout/CustomObjectFieldTemplate.jsx+++ b/frontend/src/layouts/rjsf-form-layout/CustomObjectFieldTemplate.jsx@@ -17,6 +17,7 @@ const CustomObjectFieldTemplate = (props) => {"processSubDirectories","fileExtensions","maxFiles",+ "fileProcessingOrder","fileReprocessingHandling",// Conditional fields at bottom:"reprocessInterval",diff --git a/prompt-service/src/unstract/prompt_service/services/answer_prompt.py b/prompt-service/src/unstract/prompt_service/services/answer_prompt.pyindex bf92262da..bf39b9d77 100644--- a/prompt-service/src/unstract/prompt_service/services/answer_prompt.py+++ b/prompt-service/src/unstract/prompt_service/services/answer_prompt.py@@ -317,7 +317,7 @@ class AnswerPromptService:"prompt_key": prompt_key,"doc_name": doc_name,},- LogLevel.WARNING,+ LogLevel.WARN,RunLevel.RUN,"Unable to parse JSON response from LLM, try using our"" cloud / enterprise feature 'record' or 'table' type",diff --git a/unstract/connectors/src/unstract/connectors/filesystems/azure_cloud_storage/azure_cloud_storage.py b/unstract/connectors/src/unstract/connectors/filesystems/azure_cloud_storage/azure_cloud_storage.pyindex 17013b5a9..2a8827313 100644--- a/unstract/connectors/src/unstract/connectors/filesystems/azure_cloud_storage/azure_cloud_storage.py+++ b/unstract/connectors/src/unstract/connectors/filesystems/azure_cloud_storage/azure_cloud_storage.py@@ -1,5 +1,7 @@import loggingimport os+from datetime import UTC, datetime+from email.utils import parsedate_to_datetimefrom typing import Anyimport azure.core.exceptions as AzureException@@ -107,6 +109,57 @@ class AzureCloudStorageFS(UnstractFileSystem):is_dir = metadata.get("type") == "directory"return is_dir+ def extract_modified_date(self, metadata: dict[str, Any]) -> datetime | None:+ """Extract the last modified date from Azure metadata.++ Accepts both ISO-8601 and RFC 1123 strings, normalizes all returned+ datetimes to timezone-aware UTC.++ Args:+ metadata: File metadata dictionary from fsspec++ Returns:+ timezone-aware UTC datetime object or None if not available+ """+ last_modified = metadata.get("last_modified")++ if isinstance(last_modified, datetime):+ # Ensure datetime has timezone info+ if last_modified.tzinfo is None:+ # Naive datetime - assume UTC+ return last_modified.replace(tzinfo=UTC)+ else:+ # Convert to UTC+ return last_modified.astimezone(UTC)++ elif isinstance(last_modified, str):+ # Try ISO-8601 format first+ try:+ dt = datetime.fromisoformat(last_modified.replace("Z", "+00:00"))+ # Ensure timezone awareness and convert to UTC+ if dt.tzinfo is None:+ return dt.replace(tzinfo=UTC)+ else:+ return dt.astimezone(UTC)+ except ValueError:+ pass++ # Fall back to RFC 1123 format+ try:+ dt = parsedate_to_datetime(last_modified)+ # parsedate_to_datetime returns timezone-aware datetime in UTC+ return dt.astimezone(UTC)+ except (ValueError, TypeError):+ logger.warning(+ f"[Azure] Failed to parse datetime '{last_modified}' from metadata keys: {list(metadata.keys())}"+ )+ return None++ logger.debug(+ f"[Azure] No modified date found in metadata keys: {list(metadata.keys())}"+ )+ return None+def test_credentials(self) -> bool:"""To test credentials for Azure Cloud Storage."""try:diff --git a/unstract/connectors/src/unstract/connectors/filesystems/box/box.py b/unstract/connectors/src/unstract/connectors/filesystems/box/box.pyindex 8a6c03347..570bdf18b 100644--- a/unstract/connectors/src/unstract/connectors/filesystems/box/box.py+++ b/unstract/connectors/src/unstract/connectors/filesystems/box/box.py@@ -1,6 +1,7 @@import jsonimport loggingimport os+from datetime import UTC, datetimefrom json import JSONDecodeErrorfrom typing import Any@@ -131,7 +132,51 @@ class BoxFS(UnstractFileSystem):Returns:bool: True if the path is a directory, False otherwise."""- raise NotImplementedError+ return metadata.get("type") == "directory"++ def extract_modified_date(self, metadata: dict[str, Any]) -> datetime | None:+ """Extract the last modified date from metadata.++ Args:+ metadata: File metadata dictionary from Box/fsspec.++ Returns:+ A timezone-aware datetime (UTC) or None if not available.+ """+ # Common Box/fsspec keys in order of specificity+ value = (+ metadata.get("modified_at")+ or metadata.get("content_modified_at")+ or metadata.get("modified")+ or metadata.get("last_modified")+ or metadata.get("updated_at")+ or metadata.get("updated")+ or metadata.get("client_modified")+ or metadata.get("server_modified")+ or metadata.get("mtime")+ )+ if value is None:+ return None+ try:+ if isinstance(value, datetime):+ dt = value if value.tzinfo else value.replace(tzinfo=UTC)+ return dt.astimezone(UTC)+ if isinstance(value, (int, float)):+ return datetime.fromtimestamp(value, tz=UTC)+ if isinstance(value, str):+ # Normalize 'Z' suffix to '+00:00' for fromisoformat compatibility+ v = value.strip().replace("Z", "+00:00")+ dt = datetime.fromisoformat(v)+ if dt.tzinfo is None:+ dt = dt.replace(tzinfo=UTC)+ return dt.astimezone(UTC)+ except (ValueError, TypeError) as e:+ logger.debug(+ "[Box] Failed to parse modified date (keys=%s): %s",+ list(metadata.keys()),+ e,+ )+ return Nonedef test_credentials(self) -> bool:"""To test credentials for the Box connector."""diff --git a/unstract/connectors/src/unstract/connectors/filesystems/google_cloud_storage/google_cloud_storage.py b/unstract/connectors/src/unstract/connectors/filesystems/google_cloud_storage/google_cloud_storage.pyindex ec05a35c5..e77864192 100644--- a/unstract/connectors/src/unstract/connectors/filesystems/google_cloud_storage/google_cloud_storage.py+++ b/unstract/connectors/src/unstract/connectors/filesystems/google_cloud_storage/google_cloud_storage.py@@ -2,6 +2,7 @@ import base64import jsonimport loggingimport os+from datetime import UTC, datetimefrom typing import Anyfrom gcsfs import GCSFileSystem@@ -111,6 +112,46 @@ class GoogleCloudStorageFS(UnstractFileSystem):# Note: Here Metadata type seems to be always "file" even for directoriesreturn metadata.get("type") == "directory"+ def extract_modified_date(self, metadata: dict[str, Any]) -> datetime | None:+ """Extract the last modified date from GCS metadata.++ Args:+ metadata: File metadata dictionary from fsspec++ Returns:+ datetime object or None if not available+ """+ # Prefer explicit presence check so falsy values like 0 are not skipped.+ updated = (+ metadata.get("updated") if "updated" in metadata else metadata.get("mtime")+ )+ if isinstance(updated, datetime):+ # Normalize to timezone-aware (UTC) to avoid naive/aware comparison errors.+ if updated.tzinfo is None:+ logger.debug("[GCS] Naive datetime encountered; assuming UTC.")+ return updated.replace(tzinfo=UTC)+ return updated.astimezone(UTC)+ elif isinstance(updated, (int, float)):+ # Epoch in seconds, milliseconds, or microseconds -> normalize to seconds.+ ts = float(updated)+ if ts > 1e14: # microseconds since epoch+ ts /= 1e6+ elif ts > 1e11: # milliseconds since epoch+ ts /= 1e3+ return datetime.fromtimestamp(ts, tz=UTC)+ elif isinstance(updated, str):+ # ISO8601 string (e.g., 2023-01-01T00:00:00Z)+ try:+ dt = datetime.fromisoformat(updated.strip().replace("Z", "+00:00"))+ if dt.tzinfo is None:+ return dt.replace(tzinfo=UTC)+ return dt.astimezone(UTC)+ except ValueError:+ logger.warning(f"[GCS] Invalid datetime format: {updated}")+ return None+ logger.debug(f"[GCS] No modified date found in metadata: {metadata}")+ return None+def test_credentials(self) -> bool:"""Test Google Cloud Storage credentials by accessing the root path info.diff --git a/unstract/connectors/src/unstract/connectors/filesystems/google_drive/google_drive.py b/unstract/connectors/src/unstract/connectors/filesystems/google_drive/google_drive.pyindex a8cb313bb..ab4707f87 100644--- a/unstract/connectors/src/unstract/connectors/filesystems/google_drive/google_drive.py+++ b/unstract/connectors/src/unstract/connectors/filesystems/google_drive/google_drive.py@@ -1,6 +1,7 @@import jsonimport loggingimport os+from datetime import datetimefrom pathlib import Pathfrom typing import Any@@ -125,6 +126,27 @@ class GoogleDriveFS(UnstractFileSystem):"""return metadata.get("type") == "directory"+ def extract_modified_date(self, metadata: dict[str, Any]) -> datetime | None:+ """Extract the last modified date from Google Drive metadata.++ Args:+ metadata: File metadata dictionary from fsspec++ Returns:+ datetime object or None if not available+ """+ modified_time = metadata.get("modifiedTime") or metadata.get("modified")+ if isinstance(modified_time, datetime):+ return modified_time+ elif isinstance(modified_time, str):+ try:+ return datetime.fromisoformat(modified_time.replace("Z", "+00:00"))+ except ValueError:+ logger.warning(f"[Google Drive] Invalid datetime format: {modified_time}")+ return None+ logger.debug(f"[Google Drive] No modified date found in metadata: {metadata}")+ return None+def test_credentials(self) -> bool:"""To test credentials for Google Drive."""is_dir = Falsediff --git a/unstract/connectors/src/unstract/connectors/filesystems/http/http.py b/unstract/connectors/src/unstract/connectors/filesystems/http/http.pyindex e19b24e54..87e786a7a 100644--- a/unstract/connectors/src/unstract/connectors/filesystems/http/http.py+++ b/unstract/connectors/src/unstract/connectors/filesystems/http/http.py@@ -1,5 +1,6 @@import loggingimport os+from datetime import datetimefrom typing import Anyimport aiohttp@@ -90,7 +91,22 @@ class HttpFS(UnstractFileSystem):Returns:bool: True if the path is a directory, False otherwise."""- raise NotImplementedError+ return False # HTTP(S) connector doesn't support directories++ def extract_modified_date(self, metadata: dict[str, Any]) -> datetime | None:+ """Extract the last modified date from HTTP(S) metadata.++ Args:+ metadata: File metadata dictionary from fsspec++ Returns:+ datetime object or None if not available+ """+ # HTTP(S) connector - return None as files don't have modification times+ logger.debug(+ f"[HTTP(S)] Modified date not applicable for HTTP resources: {metadata}"+ )+ return Nonedef test_credentials(self) -> bool:"""To test credentials for HTTP(S)."""diff --git a/unstract/connectors/src/unstract/connectors/filesystems/local_storage/local_storage.py b/unstract/connectors/src/unstract/connectors/filesystems/local_storage/local_storage.pyindex 8a701b25c..34cfa7965 100644--- a/unstract/connectors/src/unstract/connectors/filesystems/local_storage/local_storage.py+++ b/unstract/connectors/src/unstract/connectors/filesystems/local_storage/local_storage.py@@ -1,5 +1,6 @@import loggingimport os+from datetime import UTC, datetimefrom typing import Anyfrom fsspec.implementations.local import LocalFileSystem@@ -80,7 +81,50 @@ class LocalStorageFS(UnstractFileSystem):Returns:bool: True if the path is a directory, False otherwise."""- raise NotImplementedError+ return metadata.get("type") == "directory"++ def extract_modified_date(self, metadata: dict[str, Any]) -> datetime | None:+ """Extract the last modified date from local filesystem metadata.++ Args:+ metadata: File metadata dictionary from fsspec++ Returns:+ datetime object or None if not available+ """+ # Try common keys across providers+ mtime = None+ for key in ("mtime", "modified", "last_modified", "LastModified"):+ if key in metadata:+ mtime = metadata[key]+ break+ if isinstance(mtime, datetime):+ dt = mtime if mtime.tzinfo else mtime.replace(tzinfo=UTC)+ return dt.astimezone(UTC)+ if isinstance(mtime, (int, float)):+ # Unix timestamp (seconds)+ return datetime.fromtimestamp(mtime, tz=UTC)+ if isinstance(mtime, str):+ s = mtime.strip()+ # Handle trailing 'Z'+ if s.endswith("Z"):+ s = s[:-1] + "+00:00"+ try:+ dt = datetime.fromisoformat(s)+ except ValueError:+ # Fall back: numeric string+ try:+ return datetime.fromtimestamp(float(s), tz=UTC)+ except Exception:+ logger.debug(+ f"[LocalStorage] Unparseable modified date in metadata: {metadata}"+ )+ return None+ if dt.tzinfo is None:+ dt = dt.replace(tzinfo=UTC)+ return dt.astimezone(UTC)+ logger.debug(f"[LocalStorage] No modified date found in metadata: {metadata}")+ return Nonedef test_credentials(self, *args, **kwargs) -> bool: # type:ignore"""To test credentials for LocalStorage."""diff --git a/unstract/connectors/src/unstract/connectors/filesystems/minio/minio.py b/unstract/connectors/src/unstract/connectors/filesystems/minio/minio.pyindex 60a365d16..fb9b30f75 100644--- a/unstract/connectors/src/unstract/connectors/filesystems/minio/minio.py+++ b/unstract/connectors/src/unstract/connectors/filesystems/minio/minio.py@@ -1,5 +1,7 @@import loggingimport os+from datetime import UTC, datetime+from email.utils import parsedate_to_datetimefrom typing import Anyfrom s3fs.core import S3FileSystem@@ -104,6 +106,82 @@ class MinioFS(UnstractFileSystem):"""return metadata.get("type") == "directory"+ def _find_modified_date_value(self, metadata: dict[str, Any]) -> Any | None:+ """Find the modified date value from metadata using common keys."""+ for key in ["LastModified", "last_modified", "modified", "mtime"]:+ last_modified = metadata.get(key)+ if last_modified is not None:+ return last_modified++ logger.debug(+ f"[S3/MinIO] No modified date found in metadata keys: {list(metadata.keys())}"+ )+ return None++ def _normalize_datetime_to_utc(self, dt: datetime) -> datetime:+ """Normalize datetime object to timezone-aware UTC."""+ if dt.tzinfo is None:+ return dt.replace(tzinfo=UTC)+ return dt.astimezone(UTC)++ def _parse_string_datetime(+ self, date_str: str, metadata_keys: list[str]+ ) -> datetime | None:+ """Parse string datetime using multiple formats."""+ # Try ISO-8601 format first+ try:+ dt = datetime.fromisoformat(date_str.replace("Z", "+00:00"))+ return self._normalize_datetime_to_utc(dt)+ except ValueError:+ pass++ # Fall back to RFC 1123 format+ try:+ dt = parsedate_to_datetime(date_str)+ return dt.astimezone(UTC)+ except (ValueError, TypeError):+ logger.warning(+ f"[S3/MinIO] Failed to parse datetime '{date_str}' from metadata keys: {metadata_keys}"+ )+ return None++ def _parse_numeric_timestamp(self, timestamp: float) -> datetime | None:+ """Parse numeric epoch timestamp."""+ try:+ return datetime.fromtimestamp(timestamp, tz=UTC)+ except (ValueError, OSError):+ logger.warning(f"[S3/MinIO] Invalid epoch timestamp: {timestamp}")+ return None++ def extract_modified_date(self, metadata: dict[str, Any]) -> datetime | None:+ """Extract the last modified date from S3/MinIO metadata.++ Accepts multiple date formats and ensures timezone-aware UTC datetime.++ Args:+ metadata: File metadata dictionary from fsspec++ Returns:+ timezone-aware UTC datetime object or None if not available+ """+ last_modified = self._find_modified_date_value(metadata)+ if last_modified is None:+ return None++ if isinstance(last_modified, datetime):+ return self._normalize_datetime_to_utc(last_modified)++ if isinstance(last_modified, str):+ return self._parse_string_datetime(last_modified, list(metadata.keys()))++ if isinstance(last_modified, (int, float)):+ return self._parse_numeric_timestamp(last_modified)++ logger.debug(+ f"[S3/MinIO] Unsupported datetime type '{type(last_modified)}' in metadata keys: {list(metadata.keys())}"+ )+ return None+def get_fsspec_fs(self) -> S3FileSystem:return self.s3diff --git a/unstract/connectors/src/unstract/connectors/filesystems/sftp/sftp.py b/unstract/connectors/src/unstract/connectors/filesystems/sftp/sftp.pyindex e6ba61b82..3e3915deb 100644--- a/unstract/connectors/src/unstract/connectors/filesystems/sftp/sftp.py+++ b/unstract/connectors/src/unstract/connectors/filesystems/sftp/sftp.py@@ -1,5 +1,6 @@import loggingimport os+from datetime import UTC, datetimefrom typing import Anyfrom fsspec.implementations.sftp import SFTPFileSystem@@ -86,7 +87,25 @@ class SftpFS(UnstractFileSystem):Returns:bool: True if the path is a directory, False otherwise."""- raise NotImplementedError+ return metadata.get("type") == "directory"++ def extract_modified_date(self, metadata: dict[str, Any]) -> datetime | None:+ """Extract the last modified date from SFTP metadata.++ Args:+ metadata: File metadata dictionary from fsspec++ Returns:+ datetime object or None if not available+ """+ mtime = metadata.get("mtime")+ if isinstance(mtime, datetime):+ return mtime+ elif isinstance(mtime, (int, float)):+ # Unix timestamp+ return datetime.fromtimestamp(mtime, tz=UTC)+ logger.debug(f"[SFTP] No modified date found in metadata: {metadata}")+ return None# TODO: Check if this method can be removed, and use it from parent class# (class UnstractFileSystem)diff --git a/unstract/connectors/src/unstract/connectors/filesystems/unstract_file_system.py b/unstract/connectors/src/unstract/connectors/filesystems/unstract_file_system.pyindex d7debda77..c8b75b20e 100644--- a/unstract/connectors/src/unstract/connectors/filesystems/unstract_file_system.py+++ b/unstract/connectors/src/unstract/connectors/filesystems/unstract_file_system.py@@ -2,7 +2,7 @@ import base64import loggingimport osfrom abc import ABC, abstractmethod-from datetime import date, datetime+from datetime import UTC, date, datetimefrom typing import Anyfrom fsspec import AbstractFileSystem@@ -103,6 +103,124 @@ class UnstractFileSystem(UnstractConnector, ABC):"""pass+ @abstractmethod+ def extract_modified_date(self, metadata: dict[str, Any]) -> datetime | None:+ """Extract the last modified date from file metadata.++ Args:+ metadata: File metadata dictionary from fsspec++ Returns:+ datetime object or None if not available+ """+ pass++ def sort_files_by_modified_date(+ self, file_metadata_list: list[dict[str, Any]], ascending: bool = True+ ) -> list[dict[str, Any]]:+ """Sort files by their last modified date.++ Args:+ file_metadata_list: List of file metadata dictionaries+ ascending: If True, sort oldest first (FIFO); if False, newest first (LIFO)++ Returns:+ Sorted list of file metadata+ """++ def get_modified_date(metadata: dict[str, Any]) -> datetime:+ try:+ dt = self.extract_modified_date(metadata)+ if dt is None:+ # Fallback to epoch for files without timestamp+ logger.warning(+ f"No modified date found for file: {metadata['name']}, "+ "falling back to epoch for such files with no timestamp"+ )+ return datetime.fromtimestamp(0, tz=UTC)+ # Ensure the extracted date is normalized to UTC and timezone-aware+ if dt.tzinfo is None:+ # Naive datetime - assume UTC+ return dt.replace(tzinfo=UTC)+ else:+ # Convert to UTC+ return dt.astimezone(UTC)+ except Exception as e:+ # Log per-file warning and store error for this specific metadata entry+ file_name = metadata.get("name", "unknown file")+ msg = (+ f"Failed to extract modified date for file: {file_name}, "+ "falling back to epoch for such files and continuing execution"+ )+ logger.exception(f"{msg}: {e}")+ self._store_user_error(msg)+ # Return epoch as fallback so sorting never fails+ return datetime.fromtimestamp(0, tz=UTC)++ return sorted(file_metadata_list, key=get_modified_date, reverse=not ascending)++ def _store_user_error(self, error_msg: str) -> None:+ """Store user-friendly error message for later reporting.++ Args:+ error_msg: User-friendly error message without technical details+ """+ if not hasattr(self, "_user_errors"):+ self._user_errors = []+ self._user_errors.append(error_msg)++ def list_files(+ self,+ directory: str,+ max_depth: int = 1,+ include_dirs: bool = False,+ limit: int | None = None,+ ) -> list[dict[str, Any]]:+ """List files in a directory with optional sorting.++ Args:+ directory: Directory path to list+ max_depth: Maximum depth for recursive traversal+ include_dirs: Whether to include directories in results+ limit: Maximum number of files to collect (for performance)++ Returns:+ List of file metadata dictionaries+ """+ all_files = []+ fs_fsspec = self.get_fsspec_fs()++ for root, dirs, _ in fs_fsspec.walk(directory, maxdepth=max_depth):+ try:+ fs_metadata_list = fs_fsspec.listdir(root, detail=True)+ for metadata in fs_metadata_list:+ if not include_dirs and self.is_dir_by_metadata(metadata):+ continue+ all_files.append(metadata)++ if limit is not None and len(all_files) >= limit:+ return all_files+ except Exception as e:+ logger.warning(f"Failed to list directory {root}: {e}")+ self._store_user_error(f"Could not access directory: {root}")+ continue+ return all_files++ def report_errors_to_user(self) -> list[str]:+ """Get accumulated errors and clear the list.++ This method will be called from source.py after list_files() completes,+ to get any accumulated errors for reporting to the user.++ Returns:+ List of user-friendly error messages+ """+ if hasattr(self, "_user_errors") and self._user_errors:+ errors = self._user_errors.copy()+ self._user_errors.clear()+ return errors+ return []+@staticmethoddef get_connector_root_dir(input_dir: str, **kwargs: Any) -> str:"""Override to get root dir of a connector."""diff --git a/unstract/connectors/src/unstract/connectors/filesystems/zs_dropbox/zs_dropbox.py b/unstract/connectors/src/unstract/connectors/filesystems/zs_dropbox/zs_dropbox.pyindex 306357185..99151eaa3 100644--- a/unstract/connectors/src/unstract/connectors/filesystems/zs_dropbox/zs_dropbox.py+++ b/unstract/connectors/src/unstract/connectors/filesystems/zs_dropbox/zs_dropbox.py@@ -1,5 +1,6 @@import loggingimport os+from datetime import UTC, datetimefrom typing import Anyfrom dropbox.exceptions import ApiError as DropBoxApiError@@ -91,6 +92,44 @@ class DropboxFS(UnstractFileSystem):"""return metadata.get("type") == "directory"+ def extract_modified_date(self, metadata: dict[str, Any]) -> datetime | None:+ """Extract the last modified date from Dropbox metadata.++ Args:+ metadata: File metadata dictionary from fsspec++ Returns:+ timezone-aware UTC datetime object or None if not available+ """+ client_modified = metadata.get("client_modified") or metadata.get(+ "server_modified"+ )+ if isinstance(client_modified, datetime):+ # Ensure datetime has timezone info+ if client_modified.tzinfo is None:+ # Naive datetime - assume UTC+ return client_modified.replace(tzinfo=UTC)+ else:+ # Convert to UTC+ return client_modified.astimezone(UTC)+ elif isinstance(client_modified, str):+ try:+ # Only handle trailing 'Z' for UTC timezone+ if client_modified.endswith("Z"):+ client_modified = client_modified[:-1] + "+00:00"++ dt = datetime.fromisoformat(client_modified)+ # Ensure timezone awareness and convert to UTC+ if dt.tzinfo is None:+ return dt.replace(tzinfo=UTC)+ else:+ return dt.astimezone(UTC)+ except ValueError:+ logger.warning(f"[Dropbox] Invalid datetime format: {client_modified}")+ return None+ logger.debug(f"[Dropbox] No modified date found in metadata: {metadata}")+ return None+def test_credentials(self) -> bool:"""To test credentials for Dropbox."""try:diff --git a/unstract/workflow-execution/src/unstract/workflow_execution/enums.py b/unstract/workflow-execution/src/unstract/workflow_execution/enums.pyindex dae4550de..a87e57df4 100644--- a/unstract/workflow-execution/src/unstract/workflow_execution/enums.py+++ b/unstract/workflow-execution/src/unstract/workflow_execution/enums.py@@ -97,6 +97,8 @@ class LogType(Enum):UPDATE = "UPDATE"+# TODO: Use Enum from SDK insteadclass LogLevel(Enum):INFO = "INFO"ERROR = "ERROR"+ WARN = "WARN"