Skip to content

LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows

Added util for loading file based on the path (#451)

Rahul Johnyfb567217c0cf
diff --git a/backend/workflow_manager/endpoint/source.py b/backend/workflow_manager/endpoint/source.py
index e755a697c..4dd096044 100644
--- a/backend/workflow_manager/endpoint/source.py
+++ b/backend/workflow_manager/endpoint/source.py
@@ -3,6 +3,7 @@ import logging
import os
import shutil
from hashlib import md5, sha256
+from io import BytesIO
from pathlib import Path
from typing import Any, Optional
@@ -384,6 +385,18 @@ class SourceConnector(BaseConnector):
if connection_type == WorkflowEndpoint.ConnectionType.API:
results.append({"file": file_name, "result": result})
+ def load_file(self, input_file_path: str) -> tuple[str, BytesIO]:
+ connector: ConnectorInstance = self.endpoint.connector_instance
+ connector_settings: dict[str, Any] = connector.connector_metadata
+ source_fs: fsspec.AbstractFileSystem = self.get_fsspec(
+ settings=connector_settings, connector_id=connector.connector_id
+ )
+ with source_fs.open(input_file_path, "rb") as remote_file:
+ file_content = remote_file.read()
+ file_stream = BytesIO(file_content)
+
+ return remote_file.key, file_stream
+
@classmethod
def add_input_file_to_api_storage(
cls, workflow_id: str, execution_id: str, file_objs: list[UploadedFile]