Skip to content

LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows

UN-2938 [FIX] Ensure database data column is always created and handle NULL values for error records (#1624)

ali399a3a787b3b
diff --git a/workers/shared/infrastructure/database/utils.py b/workers/shared/infrastructure/database/utils.py
index e3d78f93b..d79ad2b4a 100644
--- a/workers/shared/infrastructure/database/utils.py
+++ b/workers/shared/infrastructure/database/utils.py
@@ -349,10 +349,14 @@ class WorkerDatabaseUtils:
v2_col_name = f"{single_column_name}_v2"
has_v2_col = WorkerDatabaseUtils._has_table_column(table_info, v2_col_name)
- # Safety check: Handle None data (from failed tool executions)
+ # Handle None data (from failed tool executions)
+ # IMPORTANT: Always add column to values dict to ensure it exists in table schema
+ # Without this, if first record is an error, table is created without data column
+ # and subsequent successful inserts fail
if data is None:
- # Don't add data columns - let database handle as NULL
- # This prevents 'None' string from being passed to JSON columns
+ values[single_column_name] = None
+ if has_v2_col:
+ values[v2_col_name] = None
return
if isinstance(data, str):
diff --git a/workers/shared/workflow/destination_connector.py b/workers/shared/workflow/destination_connector.py
index cc125a2d0..a6d0a7e36 100644
--- a/workers/shared/workflow/destination_connector.py
+++ b/workers/shared/workflow/destination_connector.py
@@ -898,6 +898,11 @@ class WorkerDestinationConnector:
database_entry=values,
)
+ # Remove None values from INSERT to let database handle as NULL
+ # Table schema already created with all columns (including data column)
+ # Removing None values prevents "invalid JSON" errors when inserting error records
+ values = {k: v for k, v in values.items() if v is not None}
+
logger.info(f"Preparing SQL query data for table {table_name}")
sql_columns_and_values = WorkerDatabaseUtils.get_sql_query_data(
conn_cls=db_class,