LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
UN-2938 [FIX] Ensure database data column is always created and handle NULL values for error records (#1624)
ali399a3a787b3b
diff --git a/workers/shared/infrastructure/database/utils.py b/workers/shared/infrastructure/database/utils.pyindex e3d78f93b..d79ad2b4a 100644--- a/workers/shared/infrastructure/database/utils.py+++ b/workers/shared/infrastructure/database/utils.py@@ -349,10 +349,14 @@ class WorkerDatabaseUtils:v2_col_name = f"{single_column_name}_v2"has_v2_col = WorkerDatabaseUtils._has_table_column(table_info, v2_col_name)- # Safety check: Handle None data (from failed tool executions)+ # Handle None data (from failed tool executions)+ # IMPORTANT: Always add column to values dict to ensure it exists in table schema+ # Without this, if first record is an error, table is created without data column+ # and subsequent successful inserts failif data is None:- # Don't add data columns - let database handle as NULL- # This prevents 'None' string from being passed to JSON columns+ values[single_column_name] = None+ if has_v2_col:+ values[v2_col_name] = Nonereturnif isinstance(data, str):diff --git a/workers/shared/workflow/destination_connector.py b/workers/shared/workflow/destination_connector.pyindex cc125a2d0..a6d0a7e36 100644--- a/workers/shared/workflow/destination_connector.py+++ b/workers/shared/workflow/destination_connector.py@@ -898,6 +898,11 @@ class WorkerDestinationConnector:database_entry=values,)+ # Remove None values from INSERT to let database handle as NULL+ # Table schema already created with all columns (including data column)+ # Removing None values prevents "invalid JSON" errors when inserting error records+ values = {k: v for k, v in values.items() if v is not None}+logger.info(f"Preparing SQL query data for table {table_name}")sql_columns_and_values = WorkerDatabaseUtils.get_sql_query_data(conn_cls=db_class,