Skip to content

LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows

UN-2882 [FIX] Fix BigQuery float precision issue in metadata serialization (#1589)

ali16d045f02c34
diff --git a/workers/shared/infrastructure/database/utils.py b/workers/shared/infrastructure/database/utils.py
index 1a112be6b..df9a66bf5 100644
--- a/workers/shared/infrastructure/database/utils.py
+++ b/workers/shared/infrastructure/database/utils.py
@@ -6,6 +6,7 @@ of backend/workflow_manager/endpoint_v2/database_utils.py without Django depende
import datetime
import json
+import math
from typing import Any
from shared.enums.status_enums import FileProcessingStatus
@@ -77,6 +78,46 @@ class WorkerDBException(Exception):
class WorkerDatabaseUtils:
"""Worker-compatible database utilities following production patterns."""
+ @staticmethod
+ def _sanitize_floats_for_database(data: Any, precision: int = 6) -> Any:
+ """Recursively sanitize float values for database JSON compatibility.
+
+ BigQuery's PARSE_JSON() requires floats that can "round-trip" through
+ string representation. This function normalizes floats to ensure they
+ serialize cleanly for all database types (BigQuery, PostgreSQL, MySQL, etc.).
+
+ Args:
+ data: The data structure to sanitize (dict, list, or primitive)
+ precision: Number of decimal places to preserve (default: 6)
+
+ Returns:
+ Sanitized data with normalized float values
+
+ Example:
+ >>> _sanitize_floats_for_database({"time": 22.770092, "count": 5})
+ {'time': 22.770092, 'count': 5}
+ """
+ if isinstance(data, float):
+ # Handle special float values that databases don't support in JSON
+ if math.isnan(data) or math.isinf(data):
+ return None
+ # Normalize float representation using string formatting
+ # This ensures clean binary representation that BigQuery accepts
+ return float(f"{data:.{precision}f}")
+ elif isinstance(data, dict):
+ return {
+ k: WorkerDatabaseUtils._sanitize_floats_for_database(v, precision)
+ for k, v in data.items()
+ }
+ elif isinstance(data, list):
+ return [
+ WorkerDatabaseUtils._sanitize_floats_for_database(item, precision)
+ for item in data
+ ]
+ else:
+ # Return other types unchanged (int, str, bool, None, etc.)
+ return data
+
@staticmethod
def get_sql_values_for_query(
conn_cls: Any,
@@ -292,7 +333,11 @@ class WorkerDatabaseUtils:
# Add metadata with safe JSON serialization
if metadata and has_metadata_col:
try:
- values[TableColumns.METADATA] = json.dumps(metadata)
+ # Sanitize floats for database JSON compatibility (BigQuery, PostgreSQL, etc.)
+ sanitized_metadata = WorkerDatabaseUtils._sanitize_floats_for_database(
+ metadata
+ )
+ values[TableColumns.METADATA] = json.dumps(sanitized_metadata)
except (TypeError, ValueError) as e:
logger.error(f"Failed to serialize metadata to JSON: {e}")
# Create a safe fallback error object
@@ -358,9 +403,11 @@ class WorkerDatabaseUtils:
if has_v2_col:
values[v2_col_name] = wrapped_dict
else:
- values[single_column_name] = data
+ # Sanitize floats for database JSON compatibility
+ sanitized_data = WorkerDatabaseUtils._sanitize_floats_for_database(data)
+ values[single_column_name] = sanitized_data
if has_v2_col:
- values[v2_col_name] = data
+ values[v2_col_name] = sanitized_data
@staticmethod
def _process_split_column_mode(
@@ -368,12 +415,16 @@ class WorkerDatabaseUtils:
) -> None:
"""Process data for split column mode."""
if isinstance(data, dict):
- values.update(data)
+ # Sanitize floats for database JSON compatibility
+ sanitized_data = WorkerDatabaseUtils._sanitize_floats_for_database(data)
+ values.update(sanitized_data)
elif isinstance(data, str):
values[single_column_name] = data
else:
try:
- values[single_column_name] = json.dumps(data)
+ # Sanitize floats for database JSON compatibility before serialization
+ sanitized_data = WorkerDatabaseUtils._sanitize_floats_for_database(data)
+ values[single_column_name] = json.dumps(sanitized_data)
except (TypeError, ValueError) as e:
logger.error(
f"Failed to serialize data to JSON in split column mode: {e}"