LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
UN-2882 [FIX] Fix BigQuery float precision issue in metadata serialization (#1589)
ali16d045f02c34
diff --git a/workers/shared/infrastructure/database/utils.py b/workers/shared/infrastructure/database/utils.pyindex 1a112be6b..df9a66bf5 100644--- a/workers/shared/infrastructure/database/utils.py+++ b/workers/shared/infrastructure/database/utils.py@@ -6,6 +6,7 @@ of backend/workflow_manager/endpoint_v2/database_utils.py without Django dependeimport datetimeimport json+import mathfrom typing import Anyfrom shared.enums.status_enums import FileProcessingStatus@@ -77,6 +78,46 @@ class WorkerDBException(Exception):class WorkerDatabaseUtils:"""Worker-compatible database utilities following production patterns."""+ @staticmethod+ def _sanitize_floats_for_database(data: Any, precision: int = 6) -> Any:+ """Recursively sanitize float values for database JSON compatibility.++ BigQuery's PARSE_JSON() requires floats that can "round-trip" through+ string representation. This function normalizes floats to ensure they+ serialize cleanly for all database types (BigQuery, PostgreSQL, MySQL, etc.).++ Args:+ data: The data structure to sanitize (dict, list, or primitive)+ precision: Number of decimal places to preserve (default: 6)++ Returns:+ Sanitized data with normalized float values++ Example:+ >>> _sanitize_floats_for_database({"time": 22.770092, "count": 5})+ {'time': 22.770092, 'count': 5}+ """+ if isinstance(data, float):+ # Handle special float values that databases don't support in JSON+ if math.isnan(data) or math.isinf(data):+ return None+ # Normalize float representation using string formatting+ # This ensures clean binary representation that BigQuery accepts+ return float(f"{data:.{precision}f}")+ elif isinstance(data, dict):+ return {+ k: WorkerDatabaseUtils._sanitize_floats_for_database(v, precision)+ for k, v in data.items()+ }+ elif isinstance(data, list):+ return [+ WorkerDatabaseUtils._sanitize_floats_for_database(item, precision)+ for item in data+ ]+ else:+ # Return other types unchanged (int, str, bool, None, etc.)+ return data+@staticmethoddef get_sql_values_for_query(conn_cls: Any,@@ -292,7 +333,11 @@ class WorkerDatabaseUtils:# Add metadata with safe JSON serializationif metadata and has_metadata_col:try:- values[TableColumns.METADATA] = json.dumps(metadata)+ # Sanitize floats for database JSON compatibility (BigQuery, PostgreSQL, etc.)+ sanitized_metadata = WorkerDatabaseUtils._sanitize_floats_for_database(+ metadata+ )+ values[TableColumns.METADATA] = json.dumps(sanitized_metadata)except (TypeError, ValueError) as e:logger.error(f"Failed to serialize metadata to JSON: {e}")# Create a safe fallback error object@@ -358,9 +403,11 @@ class WorkerDatabaseUtils:if has_v2_col:values[v2_col_name] = wrapped_dictelse:- values[single_column_name] = data+ # Sanitize floats for database JSON compatibility+ sanitized_data = WorkerDatabaseUtils._sanitize_floats_for_database(data)+ values[single_column_name] = sanitized_dataif has_v2_col:- values[v2_col_name] = data+ values[v2_col_name] = sanitized_data@staticmethoddef _process_split_column_mode(@@ -368,12 +415,16 @@ class WorkerDatabaseUtils:) -> None:"""Process data for split column mode."""if isinstance(data, dict):- values.update(data)+ # Sanitize floats for database JSON compatibility+ sanitized_data = WorkerDatabaseUtils._sanitize_floats_for_database(data)+ values.update(sanitized_data)elif isinstance(data, str):values[single_column_name] = dataelse:try:- values[single_column_name] = json.dumps(data)+ # Sanitize floats for database JSON compatibility before serialization+ sanitized_data = WorkerDatabaseUtils._sanitize_floats_for_database(data)+ values[single_column_name] = json.dumps(sanitized_data)except (TypeError, ValueError) as e:logger.error(f"Failed to serialize data to JSON in split column mode: {e}"