Skip to content

LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows

Update source.py by increasing file read chunk size (#1214)

aliafb4601cb378
diff --git a/backend/workflow_manager/endpoint_v2/source.py b/backend/workflow_manager/endpoint_v2/source.py
index 19358e86f..fa8a384fc 100644
--- a/backend/workflow_manager/endpoint_v2/source.py
+++ b/backend/workflow_manager/endpoint_v2/source.py
@@ -460,14 +460,14 @@ class SourceConnector(BaseConnector):
raise ValueError(f"Unsupported hash_method: {hash_method}")
def get_file_content(
- self, input_file_path: str, chunk_size: int = 8192
+ self, input_file_path: str, chunk_size: int = 4194304
) -> tuple[bytes, Optional[int]]:
"""Read the content of a file from a remote filesystem in chunks.
Args:
input_file_path (str): The path of the input file.
chunk_size (int): The size of the chunks to read at a time
- (default is 8192 bytes).
+ (default is 4194304 bytes = 4 MB).
Returns:
tuple[bytes, int]: