Skip to content

LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows

fix: Set chunk size for remote storage file transfer to 4MB (#1215)

Chandrasekharan M3e1659781fd9
diff --git a/backend/workflow_manager/endpoint_v2/source.py b/backend/workflow_manager/endpoint_v2/source.py
index fa8a384fc..619621b7f 100644
--- a/backend/workflow_manager/endpoint_v2/source.py
+++ b/backend/workflow_manager/endpoint_v2/source.py
@@ -574,7 +574,7 @@ class SourceConnector(BaseConnector):
destination_storage: Any,
source_path: str,
destination_paths: list[str],
- chunk_size: int = 4096,
+ chunk_size: int = 4194304,
) -> None:
"""
Copy a file from a source storage to one or more paths in a
@@ -593,7 +593,7 @@ class SourceConnector(BaseConnector):
destination_paths (list[str]): A list of paths where the file will be
copied in the destination storage.
chunk_size (int, optional): The number of bytes to read per chunk.
- Default is 4096 bytes.
+ (default is 4194304 bytes = 4 MB).
"""
seek_position = 0 # Start from the beginning
end_of_file = False