LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
Added util for loading file based on the path (#451)
Rahul Johnyfb567217c0cf
diff --git a/backend/workflow_manager/endpoint/source.py b/backend/workflow_manager/endpoint/source.pyindex e755a697c..4dd096044 100644--- a/backend/workflow_manager/endpoint/source.py+++ b/backend/workflow_manager/endpoint/source.py@@ -3,6 +3,7 @@ import loggingimport osimport shutilfrom hashlib import md5, sha256+from io import BytesIOfrom pathlib import Pathfrom typing import Any, Optional@@ -384,6 +385,18 @@ class SourceConnector(BaseConnector):if connection_type == WorkflowEndpoint.ConnectionType.API:results.append({"file": file_name, "result": result})+ def load_file(self, input_file_path: str) -> tuple[str, BytesIO]:+ connector: ConnectorInstance = self.endpoint.connector_instance+ connector_settings: dict[str, Any] = connector.connector_metadata+ source_fs: fsspec.AbstractFileSystem = self.get_fsspec(+ settings=connector_settings, connector_id=connector.connector_id+ )+ with source_fs.open(input_file_path, "rb") as remote_file:+ file_content = remote_file.read()+ file_stream = BytesIO(file_content)++ return remote_file.key, file_stream+@classmethoddef add_input_file_to_api_storage(cls, workflow_id: str, execution_id: str, file_objs: list[UploadedFile]