Skip to content

LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows

UN-1648 [FIX] - Fixed issue with row insertion in destination DB for failure scenarios (#1533)

Praveen Kumarbb1aee0ef02e
diff --git a/backend/workflow_manager/endpoint_v2/destination.py b/backend/workflow_manager/endpoint_v2/destination.py
index e86374b21..8c1ef0322 100644
--- a/backend/workflow_manager/endpoint_v2/destination.py
+++ b/backend/workflow_manager/endpoint_v2/destination.py
@@ -321,7 +321,7 @@ class DestinationConnector(BaseConnector):
except ConnectorError as e:
raise UnstractFSException(core_err=e) from e
- def insert_into_db(self, input_file_path: str, error: str | None) -> None:
+ def insert_into_db(self, input_file_path: str, error: str | None = None) -> None:
"""Insert data into the database."""
connector_instance: ConnectorInstance = self.endpoint.connector_instance
connector_settings: dict[str, Any] = connector_instance.connector_metadata
@@ -345,17 +345,19 @@ class DestinationConnector(BaseConnector):
destination_configurations.get(DestinationKey.EXECUTION_ID, "execution_id")
)
- data = self.get_tool_execution_result()
+ data = self.get_tool_execution_result() if not error else None
metadata = self.get_combined_metadata()
- # If no data and no error, don't execute CREATE or INSERT query
if not data and not error:
logger.info("No data obtained from tool to insert into destination DB.")
return
- # Remove metadata from result
- # Tool text-extractor returns data in the form of string.
- # Don't pop out metadata in this case.
+ # Log when we're proceeding with error insertion
+ if error and not data:
+ logger.info(
+ f"Proceeding with error record insertion for {input_file_path}: {error}"
+ )
+
if isinstance(data, dict):
data.pop("metadata", None)