LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
UN-1648 [FIX] Fixed issue with destination connectors using JSON columns (#1529)
Kirtiman Mishrae137320c0c16
diff --git a/backend/workflow_manager/endpoint_v2/constants.py b/backend/workflow_manager/endpoint_v2/constants.pyindex 8fd4d5226..cedf19728 100644--- a/backend/workflow_manager/endpoint_v2/constants.py+++ b/backend/workflow_manager/endpoint_v2/constants.py@@ -25,6 +25,7 @@ class TableColumns:class DBConnectionClass:SNOWFLAKE = "SnowflakeDB"+ BIGQUERY = "BigQuery"class Snowflake:diff --git a/backend/workflow_manager/endpoint_v2/database_utils.py b/backend/workflow_manager/endpoint_v2/database_utils.pyindex dd88f1065..c5011bfe5 100644--- a/backend/workflow_manager/endpoint_v2/database_utils.py+++ b/backend/workflow_manager/endpoint_v2/database_utils.py@@ -1,12 +1,10 @@import datetimeimport jsonimport logging-import uuid-from enum import Enumfrom typing import Anyfrom utils.constants import Common-from workflow_manager.endpoint_v2.constants import DBConnectionClass, TableColumns+from workflow_manager.endpoint_v2.constants import TableColumnsfrom workflow_manager.endpoint_v2.enums import FileProcessingStatusfrom workflow_manager.endpoint_v2.exceptions import UnstractDBExceptionfrom workflow_manager.workflow_v2.enums import AgentName, ColumnModes@@ -40,89 +38,6 @@ class DatabaseUtils:"timestamp": datetime.datetime.now().isoformat(),}- @staticmethod- def get_sql_values_for_query(- values: dict[str, Any], column_types: dict[str, str], cls_name: str- ) -> dict[str, str]:- """Making Sql Columns and Values for Query.-- Args:- values (dict[str, Any]): dictionary of columns and values- column_types (dict[str,str]): types of columns- cls (Any, optional): The database connection class (e.g.,- DBConnectionClass.SNOWFLAKE) for handling database-specific- queries.- Defaults to None.-- Returns:- list[str]: _description_-- Note:- - If `cls` is not provided or is None, the function assumes a- Default SQL database and makes values accordingly.- - If `cls` is provided and matches DBConnectionClass.SNOWFLAKE,- the function makes values using Snowflake-specific syntax.-- - Unstract creates id by default if table not exists.- If there is column 'id' in db table, it will insert- 'id' as uuid into the db table.- Else it will GET table details from INFORMATION SCHEMA and- insert into the table accordingly- """- sql_values: dict[str, Any] = {}- for column in values:- if cls_name == DBConnectionClass.SNOWFLAKE:- col = column.lower()- type_x = column_types.get(col, "")- if type_x == "VARIANT":- payload = values[column]- if isinstance(payload, Enum):- payload = payload.value- # Ensure valid JSON for VARIANT- try:- payload_json = json.dumps(payload, default=str)- sql_values[column] = f"parse_json($${payload_json}$$)"- except (TypeError, ValueError) as e:- logger.error(- f"Failed to serialize payload to JSON for column {column}: {e}"- )- # Create a safe fallback error object- fallback_payload = DatabaseUtils._create_safe_error_json(- f"column_{column}", e- )- payload_json = json.dumps(fallback_payload)- sql_values[column] = f"parse_json($${payload_json}$$)"- else:- # Non-VARIANT Snowflake types remain unchanged- sql_values[column] = f"{values[column]}"- else:- # Handle JSON and Enum types for each database- value = values[column]- if isinstance(value, (dict, list)):- try:- sql_values[column] = json.dumps(value)- except (TypeError, ValueError) as e:- logger.error(- f"Failed to serialize value to JSON for column {column}: {e}"- )- # Create a safe fallback error object - consistent with Snowflake approach- fallback_value = DatabaseUtils._create_safe_error_json(- f"column_{column}", e- )- sql_values[column] = json.dumps(fallback_value)- elif isinstance(value, Enum):- sql_values[column] = value.value- else:- sql_values[column] = (- f"{value}" # Non-JSON/Enum types handled as before- )- # If table has a column 'id', unstract inserts a unique value to it- # Oracle db has column 'ID' instead of 'id'- if any(key in column_types for key in ["id", "ID"]):- uuid_id = str(uuid.uuid4())- sql_values["id"] = f"{uuid_id}"- return sql_values-@staticmethoddef get_column_types(conn_cls: Any,@@ -145,12 +60,24 @@ class DatabaseUtils:return conn_cls.get_information_schema(table_name=table_name)except ConnectorError as e:raise UnstractDBException(detail=e.message) from e- except Exception as e:- logger.error(- f"Error getting db-column-name and db-column-type "- f"for {table_name}: {str(e)}"- )- raise++ @staticmethod+ def get_sql_values_for_query(+ conn_cls: Any,+ values: dict[str, Any],+ column_types: dict[str, str],+ ) -> dict[str, Any]:+ """Function to prepare SQL values by calling connector method.++ Args:+ conn_cls (Any): DB Connection class+ values (dict[str, Any]): dictionary of columns and values+ column_types (dict[str, str]): types of columns from database schema++ Returns:+ dict[str, Any]: Dictionary of column names to SQL values+ """+ return conn_cls.get_sql_values_for_query(values=values, column_types=column_types)@staticmethoddef get_columns_and_values(@@ -236,7 +163,6 @@ class DatabaseUtils:values[TableColumns.STATUS] = (FileProcessingStatus.ERROR if error else FileProcessingStatus.SUCCESS)-if column_mode == ColumnModes.WRITE_JSON_TO_A_SINGLE_COLUMN:if isinstance(data, str):wrapped_dict = {"result": data}@@ -251,52 +177,9 @@ class DatabaseUtils:k.lower() == f"{single_column_name}_v2".lower() for k in table_info):values[f"{single_column_name}_v2"] = data- if column_mode == ColumnModes.SPLIT_JSON_INTO_COLUMNS:- if isinstance(data, dict):- values[single_column_name] = data- elif isinstance(data, str):- values[single_column_name] = data- # Only write to v2 if it exists- if table_info and any(- k.lower() == f"{single_column_name}_v2".lower() for k in table_info- ):- values[f"{single_column_name}_v2"] = data- else:- try:- values[single_column_name] = json.dumps(- data- ) # Legacy column gets JSON string- except (TypeError, ValueError) as e:- logger.error(- f"Failed to serialize data to JSON for column {single_column_name}: {e}"- )- # Create a safe fallback error object- fallback_data = DatabaseUtils._create_safe_error_json(- single_column_name, e- )- values[single_column_name] = json.dumps(fallback_data)-- # Only write to v2 if it exists- if table_info and any(- k.lower() == f"{single_column_name}_v2".lower() for k in table_info- ):- # Make sure v2 gets a serializable value too if needed- if isinstance(data, (dict, list)):- values[f"{single_column_name}_v2"] = data- else:- try:- # Test if it's JSON serializable before assigning- json.dumps(data)- values[f"{single_column_name}_v2"] = data- except (TypeError, ValueError) as e:- # Create a safe fallback error object- fallback_data = DatabaseUtils._create_safe_error_json(- f"{single_column_name}_v2", e- )- values[f"{single_column_name}_v2"] = fallback_data-values[file_path_name] = file_pathvalues[execution_id_name] = execution_id+ logger.debug(f"database_utils.py get_columns_and_values values: {values}")return values@staticmethod@@ -325,15 +208,20 @@ class DatabaseUtils:- For other SQL databases, it uses default SQL generationbased on column types."""- cls_name = conn_cls.__class__.__name__column_types: dict[str, str] = DatabaseUtils.get_column_types(conn_cls=conn_cls, table_name=table_name)+ logger.debug(f"database_utils.py get_sql_query_data column_types: {column_types}")+sql_columns_and_values = DatabaseUtils.get_sql_values_for_query(+ conn_cls=conn_cls,values=values,column_types=column_types,- cls_name=cls_name,)+ logger.debug(+ f"database_utils.py get_sql_query_data sql_columns_and_values: {sql_columns_and_values}"+ )+return sql_columns_and_values@staticmethod@@ -357,7 +245,9 @@ class DatabaseUtils:So we need to use INSERT INTO ... SELECT ... syntax- sql values can contain data with single quote. It needs to"""- sql = db_class.get_sql_insert_query(table_name=table_name, sql_keys=sql_keys)+ sql = db_class.get_sql_insert_query(+ table_name=table_name, sql_keys=sql_keys, sql_values=sql_values+ )logger.debug(f"inserting into table {table_name} with: {sql} query")logger.debug(f"sql_values: {sql_values}")@@ -399,6 +289,7 @@ class DatabaseUtils:"""sql = db_class.create_table_query(table=table_name, database_entry=database_entry)logger.debug(f"creating table {table_name} with: {sql} query")+try:db_class.execute_query(engine=engine, sql_query=sql, sql_values=None, table_name=table_namediff --git a/backend/workflow_manager/endpoint_v2/destination.py b/backend/workflow_manager/endpoint_v2/destination.pyindex dfff91ce2..e86374b21 100644--- a/backend/workflow_manager/endpoint_v2/destination.py+++ b/backend/workflow_manager/endpoint_v2/destination.py@@ -368,12 +368,13 @@ class DestinationConnector(BaseConnector):table_info = db_class.get_information_schema(table_name=table_name)- # Check whether to migrate table to include new columns+ logger.info(+ f"destination connector engine: {engine} with table info: {table_info}"+ )+if table_info:- if db_class.is_string_column(- table_info=table_info, column_name=single_column_name- ):- db_class.migrate_table_to_v2(+ if db_class.has_no_metadata(table_info=table_info):+ table_info = db_class.migrate_table_to_v2(table_name=table_name,column_name=single_column_name,engine=engine,@@ -395,6 +396,8 @@ class DestinationConnector(BaseConnector):error=error,)+ logger.debug(f"destination.py values: {values}")+try:# Reuse the same db_class and engine created earlier@@ -410,7 +413,9 @@ class DestinationConnector(BaseConnector):table_name=table_name,values=values,)-+ logger.info(+ f"destination.py sql_columns_and_values: {sql_columns_and_values}"+ )DatabaseUtils.execute_write_query(db_class=db_class,engine=engine,diff --git a/backend/workflow_manager/endpoint_v2/tests/__init__.py b/backend/workflow_manager/endpoint_v2/tests/__init__.pyindex fca2b2401..c7f1fbe4c 100644--- a/backend/workflow_manager/endpoint_v2/tests/__init__.py+++ b/backend/workflow_manager/endpoint_v2/tests/__init__.py@@ -1,3 +1,3 @@-from backend.celery import app as celery_app+from backend.celery_service import app as celery_app__all__ = ["celery_app"]diff --git a/backend/workflow_manager/endpoint_v2/tests/destination-connectors/__init__.py b/backend/workflow_manager/endpoint_v2/tests/destination-connectors/__init__.pynew file mode 100644index 000000000..0205067fe--- /dev/null+++ b/backend/workflow_manager/endpoint_v2/tests/destination-connectors/__init__.py@@ -0,0 +1 @@+# Tests for destination connectorsdiff --git a/backend/workflow_manager/endpoint_v2/tests/destination-connectors/test_destination_connector_bigquery.py b/backend/workflow_manager/endpoint_v2/tests/destination-connectors/test_destination_connector_bigquery.pynew file mode 100644index 000000000..dd28e4095--- /dev/null+++ b/backend/workflow_manager/endpoint_v2/tests/destination-connectors/test_destination_connector_bigquery.py@@ -0,0 +1,254 @@+import json+import os+from unittest.mock import Mock, patch++from django.test import TestCase+from workflow_manager.endpoint_v2.constants import DestinationKey+from workflow_manager.endpoint_v2.destination import DestinationConnector++from unstract.connectors.databases.bigquery import BigQuery+++class TestDestinationConnectorBigQuery(TestCase):+ """Integration test for insert_into_db method with real BigQuery connector."""++ def setUp(self) -> None:+ """Set up test data and real BigQuery configuration."""++ # BigQuery connection settings for testing+ service_account_json = os.getenv("SERVICE_ACCOUNT_JSON")+ if not service_account_json:+ self.skipTest("SERVICE_ACCOUNT_JSON environment variable not set")++ # Parse the service account JSON+ try:+ service_account_dict = json.loads(service_account_json)+ project_id = service_account_dict.get("project_id")+ if not project_id:+ self.skipTest("project_id not found in service account JSON")+ except json.JSONDecodeError:+ self.skipTest("Invalid JSON in SERVICE_ACCOUNT_JSON environment variable")++ self.bigquery_config = {"json_credentials": service_account_json}++ # Test data that will be inserted into the database+ self.test_data = {"key": "value", "result": "test_result", "status": "success"}+ self.test_metadata = {+ "file_execution_id": "test-file-exec-id",+ "extracted_text": "This is sample extracted text",+ "processing_time": 1.5,+ }+ self.input_file_path = "/path/to/test/file.pdf"++ # BigQuery table format: project.dataset.table+ dataset_name = "temp_dataset" # Use your dataset name+ self.test_table_name = f"{project_id}.{dataset_name}.output_1"++ # Create real BigQuery connector instance+ self.bigquery_connector = BigQuery(settings=self.bigquery_config)++ # Expected columns based on configuration (BigQuery data types)+ self.expected_columns = {+ "id": "STRING",+ "file_path": "STRING",+ "execution_id": "STRING",+ "data": "JSON",+ "created_at": "TIMESTAMP",+ "created_by": "STRING",+ "metadata": "JSON",+ "error_message": "STRING",+ "status": "STRING",+ "user_field_1": "BOOL",+ "user_field_2": "INT64",+ "user_field_3": "STRING",+ }++ def verify_table_columns(self, table_name: str) -> None:+ """Verify that all expected columns exist in the table with correct types."""+ table_info = self.bigquery_connector.get_information_schema(+ table_name=table_name+ )++ print(f"🔍 Actual table structure for '{table_name}': {table_info}")++ # Check that key columns exist (some columns may be created on-demand)+ required_columns = {+ "id": "STRING",+ "file_path": "STRING",+ "execution_id": "STRING",+ # "data": "STRING", # Legacy column remains STRING after migration+ # "data_v2": "JSON", # New JSON column added during migration+ "metadata": "JSON", # New JSON column added during migration+ "created_at": "TIMESTAMP",+ "created_by": "STRING",+ }++ for expected_column, expected_type in required_columns.items():+ self.assertIn(+ expected_column,+ table_info,+ f"Expected column '{expected_column}' not found in table '{table_name}'",+ )++ actual_type = table_info[expected_column]+ # BigQuery may return lowercase types, so normalize for comparison+ self.assertEqual(+ actual_type.upper(),+ expected_type.upper(),+ f"Column '{expected_column}' has type '{actual_type}', expected '{expected_type}'",+ )++ print(f"✅ All required columns verified in table '{table_name}'")++ def create_mock_workflow(self) -> Mock:+ """Create a mock workflow object."""+ workflow = Mock()+ workflow.id = "test-workflow-id"+ return workflow++ def create_mock_workflow_log(self) -> Mock:+ """Create a mock workflow log object."""+ return Mock()++ def create_real_connector_instance(self) -> Mock:+ """Create connector instance using real BigQuery connector."""+ connector = Mock()+ connector.connector_id = BigQuery.get_id()+ connector.connector_metadata = self.bigquery_config+ # Store reference to real connector for potential future use+ connector._real_bigquery_connector = self.bigquery_connector+ return connector++ def create_mock_endpoint(self, mock_connector_instance: Mock) -> Mock:+ """Create a mock endpoint with real database configuration."""+ endpoint = Mock()+ endpoint.connector_instance = mock_connector_instance+ endpoint.configuration = {+ DestinationKey.TABLE: self.test_table_name,+ DestinationKey.INCLUDE_AGENT: True,+ DestinationKey.INCLUDE_TIMESTAMP: True,+ DestinationKey.AGENT_NAME: "Unstract/DBWriter",+ DestinationKey.COLUMN_MODE: "Write JSON to a single column",+ DestinationKey.SINGLE_COLUMN_NAME: "data",+ DestinationKey.FILE_PATH: "file_path",+ DestinationKey.EXECUTION_ID: "execution_id",+ }+ return endpoint++ def create_destination_connector(+ self, mock_workflow: Mock, mock_workflow_log: Mock, mock_endpoint: Mock+ ) -> DestinationConnector:+ """Create a destination connector with mocked Django dependencies but real DB."""+ with patch(+ "workflow_manager.endpoint_v2.destination.UserContext"+ ) as mock_context:+ mock_context.get_organization_identifier.return_value = "test-org"++ with patch.object(+ DestinationConnector,+ "_get_endpoint_for_workflow",+ return_value=mock_endpoint,+ ):+ with patch.object(+ DestinationConnector,+ "_get_source_endpoint_for_workflow",+ return_value=mock_endpoint,+ ):+ connector = DestinationConnector(+ workflow=mock_workflow,+ execution_id="test-execution-id",+ workflow_log=mock_workflow_log,+ use_file_history=False,+ )+ return connector++ def test_insert_into_db_happy_path_bigquery(self) -> None:+ """Test successful insertion into real BigQuery database."""+ # Create mock objects for Django models+ mock_workflow = self.create_mock_workflow()+ mock_workflow_log = self.create_mock_workflow_log()+ mock_connector_instance = self.create_real_connector_instance()+ mock_endpoint = self.create_mock_endpoint(mock_connector_instance)++ # Create destination connector with real database connection+ destination_connector = self.create_destination_connector(+ mock_workflow, mock_workflow_log, mock_endpoint+ )++ # Mock only the methods that get data, let database operations be real+ with patch.object(+ destination_connector,+ "get_tool_execution_result",+ return_value=self.test_data,+ ):+ with patch.object(+ destination_connector,+ "get_combined_metadata",+ return_value=self.test_metadata,+ ):+ # This will execute real database operations using BigQuery connector+ destination_connector.insert_into_db(+ input_file_path=self.input_file_path, error=None+ )++ # Verify that all expected columns were created+ self.verify_table_columns(self.test_table_name)++ print(+ f"✅ Successfully inserted test data into BigQuery table: {self.test_table_name}"+ )++ def test_insert_into_db_with_error_bigquery(self) -> None:+ """Test insertion with error parameter into real BigQuery database."""+ # Create mock objects+ mock_workflow = self.create_mock_workflow()+ mock_workflow_log = self.create_mock_workflow_log()+ mock_connector_instance = self.create_real_connector_instance()+ mock_endpoint = self.create_mock_endpoint(mock_connector_instance)++ # Create destination connector+ destination_connector = self.create_destination_connector(+ mock_workflow, mock_workflow_log, mock_endpoint+ )++ error_message = "Test processing error occurred"++ # Mock the methods that get data+ with patch.object(+ destination_connector,+ "get_tool_execution_result",+ return_value=self.test_data,+ ):+ with patch.object(+ destination_connector,+ "get_combined_metadata",+ return_value=self.test_metadata,+ ):+ # Execute with error parameter+ destination_connector.insert_into_db(+ input_file_path=self.input_file_path, error=error_message+ )++ # Verify that all expected columns were created+ self.verify_table_columns(self.test_table_name)++ print(+ f"✅ Successfully inserted error data into BigQuery table: {self.test_table_name}"+ )++ def test_bigquery_connector_connection(self) -> None:+ """Test that the BigQuery connector can establish a connection."""+ # Test the real BigQuery connector directly+ try:+ engine = self.bigquery_connector.get_engine()+ self.assertIsNotNone(engine)+ print("✅ BigQuery connector successfully established connection")++ # Test a simple query to verify connection works+ test_query = "SELECT 1 as test_column"+ result = self.bigquery_connector.execute(test_query)+ self.assertIsNotNone(result)+ print("✅ BigQuery connector successfully executed test query")++ except Exception as e:+ self.fail(f"BigQuery connector failed to connect: {str(e)}")diff --git a/backend/workflow_manager/endpoint_v2/tests/destination-connectors/test_destination_connector_mariadb.py b/backend/workflow_manager/endpoint_v2/tests/destination-connectors/test_destination_connector_mariadb.pynew file mode 100644index 000000000..0f2bd39ab--- /dev/null+++ b/backend/workflow_manager/endpoint_v2/tests/destination-connectors/test_destination_connector_mariadb.py@@ -0,0 +1,675 @@+import json+import os+from unittest.mock import Mock, patch++from django.test import TestCase+from workflow_manager.endpoint_v2.constants import DestinationKey+from workflow_manager.endpoint_v2.destination import DestinationConnector++from unstract.connectors.databases.mariadb import MariaDB+++class TestDestinationConnectorMariaDB(TestCase):+ """Integration test for insert_into_db method with real MariaDB connector."""++ def setUp(self) -> None:+ """Set up test data and real MariaDB configuration."""++ # MariaDB connection settings for testing+ required_env_vars = [+ "MARIADB_HOST",+ "MARIADB_PORT",+ "MARIADB_DATABASE",+ "MARIADB_USER",+ "MARIADB_PASSWORD",+ ]++ # Check if all required environment variables are set+ missing_vars = [var for var in required_env_vars if not os.getenv(var)]+ if missing_vars:+ self.skipTest(+ f"Required MariaDB environment variables not set: {', '.join(missing_vars)}"+ )++ self.mariadb_config = {+ "host": os.getenv("MARIADB_HOST"),+ "port": os.getenv("MARIADB_PORT"),+ "database": os.getenv("MARIADB_DATABASE"),+ "user": os.getenv("MARIADB_USER"),+ "password": os.getenv("MARIADB_PASSWORD"),+ }++ # Test data that will be inserted into the database+ self.test_data = {"key": "value", "result": "test_result", "status": "success"}+ self.test_metadata = {+ "file_execution_id": "test-file-exec-id",+ "extracted_text": "This is sample extracted text",+ "processing_time": 1.5,+ }+ self.input_file_path = "/path/to/test/file.pdf"+ self.test_table_name = "MIGRATION"++ # Create real MariaDB connector instance+ self.mariadb_connector = MariaDB(settings=self.mariadb_config)++ # Expected columns based on configuration (MariaDB/MySQL data types)+ self.expected_columns = {+ "id": "longtext",+ "file_path": "longtext",+ "execution_id": "longtext",+ "data": "longtext",+ "created_at": "timestamp",+ "created_by": "longtext",+ "metadata": "json",+ "error_message": "longtext",+ "status": "enum",+ "user_field_1": "tinyint", # BOOLEAN maps to TINYINT in MariaDB+ "user_field_2": "bigint",+ "user_field_3": "longtext",+ }++ def verify_table_columns(self, table_name: str) -> None:+ """Verify that all expected columns exist in the table with correct types."""+ table_info = self.mariadb_connector.get_information_schema(+ table_name=table_name+ )++ print(f"🔍 Actual table structure for '{table_name}': {table_info}")++ # Check that key columns exist (some columns may be created on-demand)+ required_columns = {+ "id": "longtext",+ "file_path": "longtext",+ "execution_id": "longtext",+ "data": "longtext",+ "created_at": "timestamp",+ "created_by": "longtext",+ }++ for expected_column, expected_type in required_columns.items():+ self.assertIn(+ expected_column,+ table_info,+ f"Expected column '{expected_column}' not found in table '{table_name}'",+ )++ actual_type = table_info[expected_column].lower()+ expected_type_lower = expected_type.lower()++ # MariaDB may return slightly different type names, so we do flexible matching+ if expected_type_lower == "timestamp":+ self.assertIn(+ actual_type,+ ["timestamp", "datetime", "timestamp(0)"],+ f"Column '{expected_column}' has type '{actual_type}', expected timestamp-like type",+ )+ elif expected_type_lower == "tinyint":+ self.assertIn(+ actual_type,+ ["tinyint", "tinyint(1)", "boolean", "bool"],+ f"Column '{expected_column}' has type '{actual_type}', expected boolean/tinyint type",+ )+ else:+ self.assertEqual(+ actual_type,+ expected_type_lower,+ f"Column '{expected_column}' has type '{actual_type}', expected '{expected_type_lower}'",+ )++ print(f"✅ All required columns verified in table '{table_name}'")++ def create_mock_workflow(self) -> Mock:+ """Create a mock workflow object."""+ workflow = Mock()+ workflow.id = "test-workflow-id"+ return workflow++ def create_mock_workflow_log(self) -> Mock:+ """Create a mock workflow log object."""+ return Mock()++ def create_real_connector_instance(self) -> Mock:+ """Create connector instance using real MariaDB connector."""+ connector = Mock()+ connector.connector_id = MariaDB.get_id()+ connector.connector_metadata = self.mariadb_config+ # Store reference to real connector for potential future use+ connector._real_mariadb_connector = self.mariadb_connector+ return connector++ def create_mock_endpoint(self, mock_connector_instance: Mock) -> Mock:+ """Create a mock endpoint with real database configuration."""+ endpoint = Mock()+ endpoint.connector_instance = mock_connector_instance+ endpoint.configuration = {+ DestinationKey.TABLE: self.test_table_name,+ DestinationKey.INCLUDE_AGENT: True,+ DestinationKey.INCLUDE_TIMESTAMP: True,+ DestinationKey.AGENT_NAME: "Unstract/DBWriter",+ DestinationKey.COLUMN_MODE: "Write JSON to a single column",+ DestinationKey.SINGLE_COLUMN_NAME: "data",+ DestinationKey.FILE_PATH: "file_path",+ DestinationKey.EXECUTION_ID: "execution_id",+ }+ return endpoint++ def create_destination_connector(+ self, mock_workflow: Mock, mock_workflow_log: Mock, mock_endpoint: Mock+ ) -> DestinationConnector:+ """Create a destination connector with mocked Django dependencies but real DB."""+ with patch(+ "workflow_manager.endpoint_v2.destination.UserContext"+ ) as mock_context:+ mock_context.get_organization_identifier.return_value = "test-org"++ with patch.object(+ DestinationConnector,+ "_get_endpoint_for_workflow",+ return_value=mock_endpoint,+ ):+ with patch.object(+ DestinationConnector,+ "_get_source_endpoint_for_workflow",+ return_value=mock_endpoint,+ ):+ connector = DestinationConnector(+ workflow=mock_workflow,+ execution_id="test-execution-id",+ workflow_log=mock_workflow_log,+ use_file_history=False,+ )+ return connector++ def test_insert_into_db_happy_path_mariadb(self) -> None:+ """Test successful insertion into real MariaDB database."""+ # Create mock objects for Django models+ mock_workflow = self.create_mock_workflow()+ mock_workflow_log = self.create_mock_workflow_log()+ mock_connector_instance = self.create_real_connector_instance()+ mock_endpoint = self.create_mock_endpoint(mock_connector_instance)++ # Create destination connector with real database connection+ destination_connector = self.create_destination_connector(+ mock_workflow, mock_workflow_log, mock_endpoint+ )++ # Mock only the methods that get data, let database operations be real+ with patch.object(+ destination_connector,+ "get_tool_execution_result",+ return_value=self.test_data,+ ):+ with patch.object(+ destination_connector,+ "get_combined_metadata",+ return_value=self.test_metadata,+ ):+ # This will execute real database operations using MariaDB connector+ destination_connector.insert_into_db(+ input_file_path=self.input_file_path, error=None+ )++ # Verify that all expected columns were created+ self.verify_table_columns(self.test_table_name)++ print(+ f"✅ Successfully inserted test data into MariaDB table: {self.test_table_name}"+ )++ def test_insert_into_db_with_error_mariadb(self) -> None:+ """Test insertion with error parameter into real MariaDB database."""+ # Create mock objects+ mock_workflow = self.create_mock_workflow()+ mock_workflow_log = self.create_mock_workflow_log()+ mock_connector_instance = self.create_real_connector_instance()+ mock_endpoint = self.create_mock_endpoint(mock_connector_instance)++ # Create destination connector+ destination_connector = self.create_destination_connector(+ mock_workflow, mock_workflow_log, mock_endpoint+ )++ error_message = "Test processing error occurred"++ # Mock the methods that get data+ with patch.object(+ destination_connector,+ "get_tool_execution_result",+ return_value=self.test_data,+ ):+ with patch.object(+ destination_connector,+ "get_combined_metadata",+ return_value=self.test_metadata,+ ):+ # Execute with error parameter+ destination_connector.insert_into_db(+ input_file_path=self.input_file_path, error=error_message+ )++ # Verify that all expected columns were created+ self.verify_table_columns(self.test_table_name)++ print(+ f"✅ Successfully inserted error data into MariaDB table: {self.test_table_name}"+ )++ def test_mariadb_connector_connection(self) -> None:+ """Test that the MariaDB connector can establish a connection."""+ # Test the real MariaDB connector directly+ try:+ engine = self.mariadb_connector.get_engine()+ self.assertIsNotNone(engine)+ print("✅ MariaDB connector successfully established connection")++ # Test a simple query to verify connection works+ cursor = engine.cursor()+ cursor.execute("SELECT 1 as test_column")+ result = cursor.fetchall()+ self.assertIsNotNone(result)+ self.assertEqual(result[0][0], 1)+ cursor.close()+ print("✅ MariaDB connector successfully executed test query")++ # Clean up+ if hasattr(engine, "close"):+ engine.close()+ except Exception as e:+ self.fail(f"MariaDB connector failed to connect: {str(e)}")++ def test_comprehensive_column_data_verification(self) -> None:+ """Test that data is properly inserted into all columns with correct values."""+ # Create mock objects+ mock_workflow = self.create_mock_workflow()+ mock_workflow_log = self.create_mock_workflow_log()+ mock_connector_instance = self.create_real_connector_instance()+ mock_endpoint = self.create_mock_endpoint(mock_connector_instance)++ # Create destination connector+ destination_connector = self.create_destination_connector(+ mock_workflow, mock_workflow_log, mock_endpoint+ )++ # Mock the methods that get data+ with patch.object(+ destination_connector,+ "get_tool_execution_result",+ return_value=self.test_data,+ ):+ with patch.object(+ destination_connector,+ "get_combined_metadata",+ return_value=self.test_metadata,+ ):+ # Execute database insertion+ destination_connector.insert_into_db(+ input_file_path=self.input_file_path, error=None+ )++ # Verify table structure+ self.verify_table_columns(self.test_table_name)++ # Verify data was inserted correctly by querying the table+ engine = self.mariadb_connector.get_engine()+ try:+ cursor = engine.cursor()+ cursor.execute(+ f"SELECT created_by, file_path, execution_id, created_at, data, metadata FROM {self.test_table_name} ORDER BY created_at DESC LIMIT 1"+ )+ row = cursor.fetchone()++ self.assertIsNotNone(row, "No data found in table after insertion")++ # Map results to column names+ created_by, file_path, execution_id, created_at, data, metadata = row++ # Verify specific column values+ self.assertEqual(+ created_by,+ "Unstract/DBWriter",+ f"Expected created_by to be 'Unstract/DBWriter', got '{created_by}'",+ )++ self.assertEqual(+ file_path,+ self.input_file_path,+ f"Expected file_path to be '{self.input_file_path}', got '{file_path}'",+ )++ self.assertEqual(+ execution_id,+ "test-execution-id",+ f"Expected execution_id to be 'test-execution-id', got '{execution_id}'",+ )++ self.assertIsNotNone(created_at, "Expected created_at timestamp to be set")++ self.assertIsNotNone(data, "Expected data column to contain inserted data")++ self.assertIsNotNone(+ metadata, "Expected metadata column to contain metadata"+ )++ cursor.close()+ print("✅ All column data values verified successfully")++ finally:+ if hasattr(engine, "close"):+ engine.close()++ def test_mariadb_specific_data_types(self) -> None:+ """Test MariaDB-specific data types and constraints."""+ # Test that status column accepts ENUM values correctly+ engine = self.mariadb_connector.get_engine()+ try:+ cursor = engine.cursor()++ # Create the table first if it doesn't exist+ create_table_query = self.mariadb_connector.get_create_table_base_query(+ self.test_table_name+ )+ # Add the required columns for the test+ create_table_query += (+ "file_path LONGTEXT, " "execution_id LONGTEXT, " "data LONGTEXT)"+ )++ cursor.execute(create_table_query)++ # Test inserting valid ENUM values+ cursor.execute(+ f"INSERT INTO {self.test_table_name} (id, status, created_at) VALUES (%s, %s, NOW())",+ ("test-enum-1", "SUCCESS"),+ )++ cursor.execute(+ f"INSERT INTO {self.test_table_name} (id, status, created_at) VALUES (%s, %s, NOW())",+ ("test-enum-2", "ERROR"),+ )++ # Verify the data was inserted+ cursor.execute(+ f"SELECT id, status FROM {self.test_table_name} WHERE id IN (%s, %s)",+ ("test-enum-1", "test-enum-2"),+ )+ results = cursor.fetchall()++ self.assertEqual(len(results), 2, "Expected 2 rows to be inserted")++ # Verify ENUM values+ statuses = [row[1] for row in results]+ self.assertIn("SUCCESS", statuses)+ self.assertIn("ERROR", statuses)++ cursor.close()+ print("✅ MariaDB ENUM data type verification successful")++ finally:+ if hasattr(engine, "close"):+ engine.close()++ def test_mariadb_legacy_to_v2_migration(self) -> None:+ """Test complete legacy table to v2 migration workflow."""+ legacy_table_name = "output_1"++ # # Step 1: Create a legacy table manually (simulating old table structure)+ # print("🏗️ Step 1: Creating legacy table...")+ # self._create_legacy_table(legacy_table_name)++ # Step 2: Verify table is detected as legacy+ print("🔍 Step 2: Verifying table is detected as legacy...")+ is_legacy = self._verify_table_is_legacy(legacy_table_name)+ self.assertTrue(is_legacy, "Table should be detected as legacy")++ # Step 3: Trigger migration using the real migration flow+ print("⚡ Step 3: Initiating migration...")+ self._initiate_migration_via_workflow(legacy_table_name)++ # Step 4: Verify migration was successful+ print("✅ Step 4: Verifying migration success...")+ self._verify_migration_success(legacy_table_name)++ # Step 5: Test dual column writing (legacy + v2)+ print("📝 Step 5: Testing dual column writing...")+ self._test_dual_column_writing(legacy_table_name)++ print("🎉 Migration test completed successfully!")++ def _create_legacy_table(self, table_name: str) -> None:+ """Create a legacy table structure (data column as LONGTEXT, not JSON)."""+ engine = self.mariadb_connector.get_engine()+ try:+ cursor = engine.cursor()++ # FORCE DROP AND RECREATE to ensure clean legacy table+ cursor.execute(f"DROP TABLE IF EXISTS {table_name}")+ print(f"🗑️ Dropped existing table '{table_name}' if it existed")++ # Create fresh legacy table with LONGTEXT data column (not JSON)+ legacy_create_query = f"""+ CREATE TABLE {table_name} (+ id LONGTEXT,+ created_by LONGTEXT,+ created_at TIMESTAMP,+ data LONGTEXT,+ file_path LONGTEXT,+ execution_id LONGTEXT+ )+ """++ cursor.execute(legacy_create_query)+ print(+ f"📋 Created fresh legacy table '{table_name}' with LONGTEXT data column"+ )++ # Insert some legacy data to make it realistic+ cursor.execute(+ f"INSERT INTO {table_name} (id, created_by, created_at, data, file_path, execution_id) VALUES (%s, %s, NOW(), %s, %s, %s)",+ (+ "legacy-1",+ "Legacy/DBWriter",+ '{"old": "legacy_data"}',+ "/legacy/file.pdf",+ "legacy-exec-1",+ ),+ )+ print(f"📝 Inserted legacy test data into '{table_name}'")++ cursor.close()+ print(f"✅ Legacy table '{table_name}' setup completed")++ finally:+ if hasattr(engine, "close"):+ engine.close()++ def _verify_table_is_legacy(self, table_name: str) -> bool:+ """Verify that table is detected as legacy using the real detection logic."""+ try:+ # Use the actual detection logic from the codebase+ is_string_column = self.mariadb_connector.is_string_column(+ table_info=self.mariadb_connector.get_information_schema(table_name),+ column_name="data",+ )++ if is_string_column:+ print(+ f"✅ Table '{table_name}' correctly detected as legacy (data column is string type)"+ )+ return True+ else:+ print(f"❌ Table '{table_name}' not detected as legacy")+ return False++ except Exception as e:+ print(f"❌ Error checking legacy status: {e}")+ return False++ def _initiate_migration_via_workflow(self, table_name: str) -> None:+ """Initiate migration using the real workflow destination connector logic."""+ # Create mock objects for the workflow system+ mock_workflow = self.create_mock_workflow()+ mock_workflow_log = self.create_mock_workflow_log()+ mock_connector_instance = self.create_real_connector_instance()++ # Create endpoint with the legacy table name+ mock_endpoint = Mock()+ mock_endpoint.connector_instance = mock_connector_instance+ mock_endpoint.configuration = {+ DestinationKey.TABLE: table_name, # Use legacy table+ DestinationKey.INCLUDE_AGENT: True,+ DestinationKey.INCLUDE_TIMESTAMP: True,+ DestinationKey.AGENT_NAME: "Unstract/DBWriter",+ DestinationKey.COLUMN_MODE: "Write JSON to a single column",+ DestinationKey.SINGLE_COLUMN_NAME: "data",+ DestinationKey.FILE_PATH: "file_path",+ DestinationKey.EXECUTION_ID: "execution_id",+ }++ # Create destination connector+ destination_connector = self.create_destination_connector(+ mock_workflow, mock_workflow_log, mock_endpoint+ )++ # Mock the data methods to provide test data+ test_migration_data = {"migration_test": "v2_data", "status": "migrated"}+ test_migration_metadata = {+ "file_execution_id": "migration-test-exec-id",+ "migration_test": True,+ "extracted_text": "Migration test extracted text",+ }++ with patch.object(+ destination_connector,+ "get_tool_execution_result",+ return_value=test_migration_data,+ ):+ with patch.object(+ destination_connector,+ "get_combined_metadata",+ return_value=test_migration_metadata,+ ):+ # This should trigger the migration logic automatically+ destination_connector.insert_into_db(+ input_file_path="/migration/test/file.pdf", error=None+ )++ print(f"✅ Migration initiated for table '{table_name}'")++ def _verify_migration_success(self, table_name: str) -> None:+ """Verify that migration was successful by checking for v2 columns."""+ table_info = self.mariadb_connector.get_information_schema(table_name)++ # Check for v2 columns that should have been added during migration+ expected_v2_columns = {+ "data_v2": "json",+ "metadata": "json",+ "user_field_1": "tinyint",+ "user_field_2": "bigint",+ "user_field_3": "longtext",+ "status": "enum",+ "error_message": "longtext",+ }++ for column_name, expected_type in expected_v2_columns.items():+ self.assertIn(+ column_name,+ table_info,+ f"Migration failed: Column '{column_name}' not found after migration",+ )++ actual_type = table_info[column_name].lower()+ if expected_type == "json":+ # MariaDB might return different representations for JSON+ self.assertIn(+ actual_type,+ ["json", "longtext"], # MariaDB sometimes shows JSON as longtext+ f"Column '{column_name}' has type '{actual_type}', expected JSON-compatible type",+ )+ elif expected_type == "enum":+ self.assertIn(+ actual_type,+ ["enum"],+ f"Column '{column_name}' has type '{actual_type}', expected enum type",+ )+ else:+ self.assertEqual(+ actual_type,+ expected_type,+ f"Column '{column_name}' has type '{actual_type}', expected '{expected_type}'",+ )++ print(+ f"✅ Migration verification successful - all v2 columns present in '{table_name}'"+ )++ def _test_dual_column_writing(self, table_name: str) -> None:+ """Test that data is written to both legacy and v2 columns after migration."""+ engine = self.mariadb_connector.get_engine()+ try:+ cursor = engine.cursor()++ # Query the latest inserted row to verify dual writing+ cursor.execute(+ f"""+ SELECT data, data_v2, metadata, status, error_message+ FROM {table_name}+ WHERE file_path = '/migration/test/file.pdf'+ ORDER BY created_at DESC+ LIMIT 1+ """+ )+ row = cursor.fetchone()++ self.assertIsNotNone(row, "No migrated data found in table")++ data_legacy, data_v2, metadata, status, error_message = row++ # Verify legacy column has data+ self.assertIsNotNone(data_legacy, "Legacy data column should not be empty")++ # Verify v2 column has data+ self.assertIsNotNone(data_v2, "V2 data column should not be empty")++ # Verify metadata column has data+ self.assertIsNotNone(metadata, "Metadata column should not be empty")++ # Verify status is set correctly+ self.assertEqual(+ status, "SUCCESS", "Status should be SUCCESS for successful migration"+ )++ # Parse and verify JSON data (if stored as JSON string in legacy)+ if isinstance(data_legacy, str):+ try:+ parsed_legacy = json.loads(data_legacy)+ self.assertIn(+ "migration_test",+ parsed_legacy,+ "Legacy data should contain migration test data",+ )+ except json.JSONDecodeError:+ # If not JSON, just verify it contains our test data somehow+ self.assertIn("migration_test", str(data_legacy))++ cursor.close()+ print(f"✅ Dual column writing verification successful for '{table_name}'")++ finally:+ if hasattr(engine, "close"):+ engine.close()++ def tearDown(self) -> None:+ """Clean up test resources."""+ # Optionally clean up test tables+ # Note: Be careful with cleanup in production environments+ # TEMPORARILY DISABLED FOR DEBUGGING - Uncomment to clean up tables+ # try:+ # engine = self.mariadb_connector.get_engine()+ # cursor = engine.cursor()+ # # Clean up both regular test table and migration test table+ # cursor.execute(f"DROP TABLE IF EXISTS {self.test_table_name}")+ # cursor.execute(f"DROP TABLE IF EXISTS LEGACY_OUTPUT_TEST")+ # cursor.close()+ # engine.close()+ # except Exception:+ # # Ignore cleanup errors+ passdiff --git a/backend/workflow_manager/endpoint_v2/tests/destination-connectors/test_destination_connector_mssql.py b/backend/workflow_manager/endpoint_v2/tests/destination-connectors/test_destination_connector_mssql.pynew file mode 100644index 000000000..e9bafbc39--- /dev/null+++ b/backend/workflow_manager/endpoint_v2/tests/destination-connectors/test_destination_connector_mssql.py@@ -0,0 +1,302 @@+import os+from unittest.mock import Mock, patch++from django.test import TestCase+from workflow_manager.endpoint_v2.constants import DestinationKey+from workflow_manager.endpoint_v2.destination import DestinationConnector++from unstract.connectors.databases.mssql import MSSQL+++class TestDestinationConnectorMSSQL(TestCase):+ """Integration test for insert_into_db method with real MSSQL connector."""++ def setUp(self) -> None:+ """Set up test data and real MSSQL configuration."""++ # MSSQL connection settings for testing+ required_env_vars = [+ "MSSQL_SERVER",+ "MSSQL_USER",+ "MSSQL_PASSWORD",+ "MSSQL_DATABASE",+ ]++ # Check if all required environment variables are set+ missing_vars = [var for var in required_env_vars if not os.getenv(var)]+ if missing_vars:+ self.skipTest(+ f"Required MSSQL environment variables not set: {', '.join(missing_vars)}"+ )++ self.mssql_config = {+ "server": os.getenv("MSSQL_SERVER"),+ "port": os.getenv("MSSQL_PORT", "1433"),+ "user": os.getenv("MSSQL_USER"),+ "password": os.getenv("MSSQL_PASSWORD"),+ "database": os.getenv("MSSQL_DATABASE"),+ }++ # Test data that will be inserted into the database+ self.test_data = {"key": "value", "result": "test_result", "status": "success"}+ self.test_metadata = {+ "file_execution_id": "test-file-exec-id",+ "extracted_text": "This is sample extracted text",+ "processing_time": 1.5,+ }+ self.input_file_path = "/path/to/test/file.pdf"++ # MSSQL table naming (use schema.table format)+ self.test_table_name = "test_schema_1.MIGRATION"++ # Create real MSSQL connector instance+ self.mssql_connector = MSSQL(settings=self.mssql_config)++ # Expected columns based on configuration (MSSQL data types)+ self.expected_columns = {+ "id": "NVARCHAR",+ "file_path": "NVARCHAR",+ "execution_id": "NVARCHAR",+ "data": "NVARCHAR", # Legacy column remains NVARCHAR after migration+ "data_v2": "NVARCHAR", # New NVARCHAR column added during migration+ "metadata": "NVARCHAR", # New NVARCHAR column added during migration+ "created_at": "DATETIMEOFFSET",+ "created_by": "NVARCHAR",+ "status": "NVARCHAR",+ "error_message": "NVARCHAR",+ "user_field_1": "BIT",+ "user_field_2": "INT",+ "user_field_3": "NVARCHAR",+ }++ def verify_table_columns(self, table_name: str) -> None:+ """Verify that the table has expected columns with correct data types."""+ table_info = self.mssql_connector.get_information_schema(table_name=table_name)++ print(f"🔍 Actual table structure for '{table_name}': {table_info}")++ # Check that key columns exist (some columns may be created on-demand)+ required_columns = {+ "id": ["NVARCHAR", "TEXT"], # MSSQL uses NVARCHAR+ "file_path": ["NVARCHAR", "TEXT"], # MSSQL uses NVARCHAR+ "execution_id": ["NVARCHAR", "TEXT"], # MSSQL uses NVARCHAR+ "data": ["NVARCHAR", "TEXT"], # Can be NVARCHAR, migration might convert+ "metadata": [+ "NVARCHAR",+ "TEXT",+ ], # New NVARCHAR column added during migration+ "created_at": "DATETIMEOFFSET",+ "created_by": ["NVARCHAR", "TEXT"], # MSSQL uses NVARCHAR+ }++ # Optional columns that may exist in migration scenarios+ optional_columns = {+ "data_v2": ["NVARCHAR"], # Only exists in migration scenarios+ }++ for expected_column, expected_type in required_columns.items():+ self.assertIn(+ expected_column,+ table_info,+ f"Expected column '{expected_column}' not found in table '{table_name}'",+ )++ actual_type = table_info[expected_column]+ # Handle both single expected types and lists of acceptable types+ if isinstance(expected_type, list):+ self.assertIn(+ actual_type.upper(),+ [t.upper() for t in expected_type],+ f"Column '{expected_column}' has type '{actual_type}', expected one of {expected_type}",+ )+ else:+ # MSSQL may return different type names, normalize for comparison+ self.assertEqual(+ actual_type.upper(),+ expected_type.upper(),+ f"Column '{expected_column}' has type '{actual_type}', expected '{expected_type}'",+ )++ # Check optional columns if they exist+ for optional_column, expected_type in optional_columns.items():+ if optional_column in table_info:+ actual_type = table_info[optional_column]+ if isinstance(expected_type, list):+ self.assertIn(+ actual_type.upper(),+ [t.upper() for t in expected_type],+ f"Optional column '{optional_column}' has type '{actual_type}', expected one of {expected_type}",+ )+ else:+ self.assertEqual(+ actual_type.upper(),+ expected_type.upper(),+ f"Optional column '{optional_column}' has type '{actual_type}', expected '{expected_type}'",+ )++ def create_mock_workflow(self) -> Mock:+ """Create a mock workflow object."""+ workflow = Mock()+ workflow.id = "test-workflow-id"+ return workflow++ def create_mock_workflow_log(self) -> Mock:+ """Create a mock workflow log object."""+ return Mock()++ def create_real_connector_instance(self) -> Mock:+ """Create connector instance using real MSSQL connector."""+ connector = Mock()+ connector.connector_id = MSSQL.get_id()+ connector.connector_metadata = self.mssql_config+ # Store reference to real connector for potential future use+ connector._real_mssql_connector = self.mssql_connector+ return connector++ def create_mock_endpoint(self, mock_connector_instance: Mock) -> Mock:+ """Create a mock endpoint with real database configuration."""+ endpoint = Mock()+ endpoint.connector_instance = mock_connector_instance+ endpoint.configuration = {+ DestinationKey.TABLE: self.test_table_name,+ DestinationKey.INCLUDE_AGENT: True,+ DestinationKey.INCLUDE_TIMESTAMP: True,+ DestinationKey.AGENT_NAME: "Unstract/DBWriter",+ DestinationKey.COLUMN_MODE: "Write JSON to a single column",+ DestinationKey.SINGLE_COLUMN_NAME: "data",+ DestinationKey.FILE_PATH: "file_path",+ DestinationKey.EXECUTION_ID: "execution_id",+ }+ return endpoint++ def create_destination_connector(+ self, mock_workflow: Mock, mock_workflow_log: Mock, mock_endpoint: Mock+ ) -> DestinationConnector:+ """Create a destination connector with mocked Django dependencies but real DB."""+ with patch(+ "workflow_manager.endpoint_v2.destination.UserContext"+ ) as mock_context:+ mock_context.get_organization_identifier.return_value = "test-org"++ with patch.object(+ DestinationConnector,+ "_get_endpoint_for_workflow",+ return_value=mock_endpoint,+ ):+ with patch.object(+ DestinationConnector,+ "_get_source_endpoint_for_workflow",+ return_value=mock_endpoint,+ ):+ connector = DestinationConnector(+ workflow=mock_workflow,+ execution_id="test-execution-id",+ workflow_log=mock_workflow_log,+ use_file_history=False,+ )+ return connector++ def test_insert_into_db_happy_path_mssql(self) -> None:+ """Test successful insertion into real MSSQL database."""+ # Create mock objects for Django models+ mock_workflow = self.create_mock_workflow()+ mock_workflow_log = self.create_mock_workflow_log()+ mock_connector_instance = self.create_real_connector_instance()+ mock_endpoint = self.create_mock_endpoint(mock_connector_instance)++ # Create destination connector with real database connection+ destination_connector = self.create_destination_connector(+ mock_workflow, mock_workflow_log, mock_endpoint+ )++ # Test the insert_into_db method with real MSSQL+ try:+ # Mock only the methods that get data, let database operations be real+ with patch.object(+ destination_connector,+ "get_tool_execution_result",+ return_value=self.test_data,+ ):+ with patch.object(+ destination_connector,+ "get_combined_metadata",+ return_value=self.test_metadata,+ ):+ # This will execute real database operations using MSSQL connector+ destination_connector.insert_into_db(+ input_file_path=self.input_file_path, error=None+ )++ # Verify the table structure+ self.verify_table_columns(self.test_table_name)++ print("✅ MSSQL integration test completed successfully!")++ except Exception as e:+ print(f"❌ MSSQL integration test failed: {str(e)}")+ raise++ def test_insert_into_db_with_error_mssql(self) -> None:+ """Test insertion with error parameter into real MSSQL database."""+ # Create mock objects+ mock_workflow = self.create_mock_workflow()+ mock_workflow_log = self.create_mock_workflow_log()+ mock_connector_instance = self.create_real_connector_instance()+ mock_endpoint = self.create_mock_endpoint(mock_connector_instance)++ # Create destination connector+ destination_connector = self.create_destination_connector(+ mock_workflow, mock_workflow_log, mock_endpoint+ )++ error_message = "Test processing error occurred"++ # Mock the methods that get data+ with patch.object(+ destination_connector,+ "get_tool_execution_result",+ return_value=self.test_data,+ ):+ with patch.object(+ destination_connector,+ "get_combined_metadata",+ return_value=self.test_metadata,+ ):+ # Execute with error parameter+ destination_connector.insert_into_db(+ input_file_path=self.input_file_path, error=error_message+ )++ # Verify the table structure+ self.verify_table_columns(self.test_table_name)++ print(+ f"✅ Successfully inserted error data into MSSQL table: {self.test_table_name}"+ )++ def test_mssql_connector_connection(self) -> None:+ """Test that the MSSQL connector can establish a connection."""+ # Test the real MSSQL connector directly+ try:+ engine = self.mssql_connector.get_engine()+ self.assertIsNotNone(engine)+ print("✅ MSSQL connector successfully established connection")++ # Test a simple query to verify connection works+ with engine.cursor() as cursor:+ cursor.execute("SELECT 1 as test_column")+ result = cursor.fetchall()+ self.assertIsNotNone(result)+ print("✅ MSSQL connector successfully executed test query")++ # Clean up+ if hasattr(engine, "close"):+ engine.close()+ except Exception as e:+ self.fail(f"MSSQL connector failed to connect: {str(e)}")++ def tearDown(self) -> None:+ """Clean up test resources."""+ # Optionally clean up test table+ # Note: Be careful with cleanup in production environments+ passdiff --git a/backend/workflow_manager/endpoint_v2/tests/destination-connectors/test_destination_connector_mysql.py b/backend/workflow_manager/endpoint_v2/tests/destination-connectors/test_destination_connector_mysql.pynew file mode 100644index 000000000..2e418647d--- /dev/null+++ b/backend/workflow_manager/endpoint_v2/tests/destination-connectors/test_destination_connector_mysql.py@@ -0,0 +1,644 @@+import json+import os+from unittest.mock import Mock, patch++from django.test import TestCase+from workflow_manager.endpoint_v2.constants import DestinationKey+from workflow_manager.endpoint_v2.destination import DestinationConnector++from unstract.connectors.databases.mysql import MySQL+++class TestDestinationConnectorMySQL(TestCase):+ """Integration test for insert_into_db method with real MySQL connector."""++ def setUp(self) -> None:+ """Set up test data and real MySQL configuration."""++ # MySQL connection settings for testing+ required_env_vars = [+ "MYSQL_HOST",+ "MYSQL_PORT",+ "MYSQL_DATABASE",+ "MYSQL_USER",+ "MYSQL_PASSWORD",+ ]++ # Check if all required environment variables are set+ missing_vars = [var for var in required_env_vars if not os.getenv(var)]+ if missing_vars:+ self.skipTest(+ f"Required MySQL environment variables not set: {', '.join(missing_vars)}"+ )++ self.mysql_config = {+ "host": os.getenv("MYSQL_HOST"),+ "port": os.getenv("MYSQL_PORT", 3306),+ "database": os.getenv("MYSQL_DATABASE"),+ "user": os.getenv("MYSQL_USER"),+ "password": os.getenv("MYSQL_PASSWORD"),+ }++ # Test data that will be inserted into the database+ self.test_data = {"key": "value", "result": "test_result", "status": "success"}+ self.test_metadata = {+ "file_execution_id": "test-file-exec-id",+ "extracted_text": "This is sample extracted text",+ "processing_time": 1.5,+ }+ self.input_file_path = "/path/to/test/file.pdf"+ self.test_table_name = "MIGRATION"++ # Create real MySQL connector instance+ self.mysql_connector = MySQL(settings=self.mysql_config)++ # Expected columns based on configuration (MySQL data types)+ self.expected_columns = {+ "id": "longtext",+ "file_path": "longtext",+ "execution_id": "longtext",+ "data": "json",+ "created_at": "timestamp",+ "created_by": "longtext",+ "metadata": "json",+ "error_message": "longtext",+ "status": "enum",+ "user_field_1": "boolean", # BOOLEAN maps to TINYINT in MySQL+ "user_field_2": "bigint",+ "user_field_3": "longtext",+ }++ def verify_table_columns(self, table_name: str) -> None:+ """Verify that all expected columns exist in the table with correct types."""+ table_info = self.mysql_connector.get_information_schema(table_name=table_name)++ print(f"🔍 Actual table structure for '{table_name}': {table_info}")++ # Check that key columns exist (some columns may be created on-demand)+ required_columns = {+ "id": "longtext",+ "file_path": "longtext",+ "execution_id": "longtext",+ "data": "json",+ "created_at": "timestamp",+ "created_by": "longtext",+ }++ for expected_column, expected_type in required_columns.items():+ self.assertIn(+ expected_column,+ table_info,+ f"Expected column '{expected_column}' not found in table '{table_name}'",+ )++ actual_type = table_info[expected_column].lower()+ expected_type_lower = expected_type.lower()++ # MySQL may return slightly different type names, so we do flexible matching+ if expected_type_lower == "timestamp":+ self.assertIn(+ actual_type,+ ["timestamp", "datetime", "timestamp(0)"],+ f"Column '{expected_column}' has type '{actual_type}', expected timestamp-like type",+ )+ elif expected_type_lower == "tinyint":+ self.assertIn(+ actual_type,+ ["tinyint", "tinyint(1)", "boolean", "bool"],+ f"Column '{expected_column}' has type '{actual_type}', expected boolean/tinyint type",+ )+ else:+ self.assertEqual(+ actual_type,+ expected_type_lower,+ f"Column '{expected_column}' has type '{actual_type}', expected '{expected_type_lower}'",+ )++ print(f"✅ All required columns verified in table '{table_name}'")++ def create_mock_workflow(self) -> Mock:+ """Create a mock workflow object."""+ workflow = Mock()+ workflow.id = "test-workflow-id"+ return workflow++ def create_mock_workflow_log(self) -> Mock:+ """Create a mock workflow log object."""+ return Mock()++ def create_real_connector_instance(self) -> Mock:+ """Create connector instance using real MySQL connector."""+ connector = Mock()+ connector.connector_id = MySQL.get_id()+ connector.connector_metadata = self.mysql_config+ # Store reference to real connector for potential future use+ connector._real_mysql_connector = self.mysql_connector+ return connector++ def create_mock_endpoint(self, mock_connector_instance: Mock) -> Mock:+ """Create a mock endpoint with real database configuration."""+ endpoint = Mock()+ endpoint.connector_instance = mock_connector_instance+ endpoint.configuration = {+ DestinationKey.TABLE: self.test_table_name,+ DestinationKey.INCLUDE_AGENT: True,+ DestinationKey.INCLUDE_TIMESTAMP: True,+ DestinationKey.AGENT_NAME: "Unstract/DBWriter",+ DestinationKey.COLUMN_MODE: "Write JSON to a single column",+ DestinationKey.SINGLE_COLUMN_NAME: "data",+ DestinationKey.FILE_PATH: "file_path",+ DestinationKey.EXECUTION_ID: "execution_id",+ }+ return endpoint++ def create_destination_connector(+ self, mock_workflow: Mock, mock_workflow_log: Mock, mock_endpoint: Mock+ ) -> DestinationConnector:+ """Create a destination connector with mocked Django dependencies but real DB."""+ with patch(+ "workflow_manager.endpoint_v2.destination.UserContext"+ ) as mock_context:+ mock_context.get_organization_identifier.return_value = "test-org"++ with patch.object(+ DestinationConnector,+ "_get_endpoint_for_workflow",+ return_value=mock_endpoint,+ ):+ with patch.object(+ DestinationConnector,+ "_get_source_endpoint_for_workflow",+ return_value=mock_endpoint,+ ):+ connector = DestinationConnector(+ workflow=mock_workflow,+ execution_id="test-execution-id",+ workflow_log=mock_workflow_log,+ use_file_history=False,+ )+ return connector++ def test_insert_into_db_happy_path_mysql(self) -> None:+ """Test successful insertion into real MySQL database."""+ # Create mock objects for Django models+ mock_workflow = self.create_mock_workflow()+ mock_workflow_log = self.create_mock_workflow_log()+ mock_connector_instance = self.create_real_connector_instance()+ mock_endpoint = self.create_mock_endpoint(mock_connector_instance)++ # Create destination connector with real database connection+ destination_connector = self.create_destination_connector(+ mock_workflow, mock_workflow_log, mock_endpoint+ )++ # Mock only the methods that get data, let database operations be real+ with patch.object(+ destination_connector,+ "get_tool_execution_result",+ return_value=self.test_data,+ ):+ with patch.object(+ destination_connector,+ "get_combined_metadata",+ return_value=self.test_metadata,+ ):+ # This will execute real database operations using MySQL connector+ destination_connector.insert_into_db(+ input_file_path=self.input_file_path, error=None+ )++ # Verify that all expected columns were created+ self.verify_table_columns(self.test_table_name)++ print(+ f"✅ Successfully inserted test data into MySQL table: {self.test_table_name}"+ )++ def test_insert_into_db_with_error_mysql(self) -> None:+ """Test insertion with error parameter into real MySQL database."""+ # Create mock objects+ mock_workflow = self.create_mock_workflow()+ mock_workflow_log = self.create_mock_workflow_log()+ mock_connector_instance = self.create_real_connector_instance()+ mock_endpoint = self.create_mock_endpoint(mock_connector_instance)++ # Create destination connector+ destination_connector = self.create_destination_connector(+ mock_workflow, mock_workflow_log, mock_endpoint+ )++ error_message = "Test processing error occurred"++ # Mock the methods that get data+ with patch.object(+ destination_connector,+ "get_tool_execution_result",+ return_value=self.test_data,+ ):+ with patch.object(+ destination_connector,+ "get_combined_metadata",+ return_value=self.test_metadata,+ ):+ # Execute with error parameter+ destination_connector.insert_into_db(+ input_file_path=self.input_file_path, error=error_message+ )++ # Verify that all expected columns were created+ self.verify_table_columns(self.test_table_name)++ print(+ f"✅ Successfully inserted error data into MySQL table: {self.test_table_name}"+ )++ def test_mysql_connector_connection(self) -> None:+ """Test that the MySQL connector can establish a connection."""+ # Test the real MySQL connector directly+ try:+ engine = self.mysql_connector.get_engine()+ self.assertIsNotNone(engine)+ print("✅ MySQL connector successfully established connection")++ # Test a simple query to verify connection works+ cursor = engine.cursor()+ cursor.execute("SELECT 1 as test_column")+ result = cursor.fetchall()+ self.assertIsNotNone(result)+ self.assertEqual(result[0][0], 1)+ cursor.close()+ print("✅ MySQL connector successfully executed test query")++ # Clean up+ if hasattr(engine, "close"):+ engine.close()+ except Exception as e:+ self.fail(f"MySQL connector failed to connect: {str(e)}")++ def test_comprehensive_column_data_verification(self) -> None:+ """Test that data is properly inserted into all columns with correct values."""+ # Create mock objects+ mock_workflow = self.create_mock_workflow()+ mock_workflow_log = self.create_mock_workflow_log()+ mock_connector_instance = self.create_real_connector_instance()+ mock_endpoint = self.create_mock_endpoint(mock_connector_instance)++ # Create destination connector+ destination_connector = self.create_destination_connector(+ mock_workflow, mock_workflow_log, mock_endpoint+ )++ # Mock the methods that get data+ with patch.object(+ destination_connector,+ "get_tool_execution_result",+ return_value=self.test_data,+ ):+ with patch.object(+ destination_connector,+ "get_combined_metadata",+ return_value=self.test_metadata,+ ):+ # Execute database insertion+ destination_connector.insert_into_db(+ input_file_path=self.input_file_path, error=None+ )++ # Verify table structure+ self.verify_table_columns(self.test_table_name)++ # Verify data was inserted correctly by querying the table+ engine = self.mysql_connector.get_engine()+ try:+ cursor = engine.cursor()+ cursor.execute(+ f"SELECT created_by, file_path, execution_id, created_at, data, metadata FROM {self.test_table_name} ORDER BY created_at DESC LIMIT 1"+ )+ row = cursor.fetchone()++ self.assertIsNotNone(row, "No data found in table after insertion")++ # Map results to column names+ created_by, file_path, execution_id, created_at, data, metadata = row++ # Verify specific column values+ self.assertEqual(+ created_by,+ "Unstract/DBWriter",+ f"Expected created_by to be 'Unstract/DBWriter', got '{created_by}'",+ )++ self.assertEqual(+ file_path,+ self.input_file_path,+ f"Expected file_path to be '{self.input_file_path}', got '{file_path}'",+ )++ self.assertEqual(+ execution_id,+ "test-execution-id",+ f"Expected execution_id to be 'test-execution-id', got '{execution_id}'",+ )++ self.assertIsNotNone(created_at, "Expected created_at timestamp to be set")++ self.assertIsNotNone(data, "Expected data column to contain inserted data")++ self.assertIsNotNone(+ metadata, "Expected metadata column to contain metadata"+ )++ cursor.close()+ print("✅ All column data values verified successfully")++ finally:+ if hasattr(engine, "close"):+ engine.close()++ def test_mysql_specific_data_types(self) -> None:+ """Test MySQL-specific data types and constraints."""+ # Test that status column accepts ENUM values correctly+ engine = self.mysql_connector.get_engine()+ try:+ cursor = engine.cursor()++ # Create the table first if it doesn't exist+ create_table_query = self.mysql_connector.get_create_table_base_query(+ self.test_table_name+ )+ # Add the required columns for the test+ create_table_query += (+ "file_path LONGTEXT, " "execution_id LONGTEXT, " "data LONGTEXT)"+ )++ cursor.execute(create_table_query)++ # Test inserting valid ENUM values+ cursor.execute(+ f"INSERT INTO {self.test_table_name} (id, status, created_at) VALUES (%s, %s, NOW())",+ ("test-enum-1", "SUCCESS"),+ )++ cursor.execute(+ f"INSERT INTO {self.test_table_name} (id, status, created_at) VALUES (%s, %s, NOW())",+ ("test-enum-2", "ERROR"),+ )++ # Verify the data was inserted+ cursor.execute(+ f"SELECT id, status FROM {self.test_table_name} WHERE id IN (%s, %s)",+ ("test-enum-1", "test-enum-2"),+ )+ results = cursor.fetchall()++ self.assertEqual(len(results), 2, "Expected 2 rows to be inserted")++ # Verify ENUM values+ statuses = [row[1] for row in results]+ self.assertIn("SUCCESS", statuses)+ self.assertIn("ERROR", statuses)++ cursor.close()+ print("✅ MySQL ENUM data type verification successful")++ finally:+ if hasattr(engine, "close"):+ engine.close()++ def _create_legacy_table(self, table_name: str) -> None:+ """Create a legacy table structure (data column as LONGTEXT, not JSON)."""+ engine = self.mysql_connector.get_engine()+ try:+ cursor = engine.cursor()++ # FORCE DROP AND RECREATE to ensure clean legacy table+ cursor.execute(f"DROP TABLE IF EXISTS {table_name}")+ print(f"🗑️ Dropped existing table '{table_name}' if it existed")++ # Create fresh legacy table with LONGTEXT data column (not JSON)+ legacy_create_query = f"""+ CREATE TABLE {table_name} (+ id LONGTEXT,+ created_by LONGTEXT,+ created_at TIMESTAMP,+ data LONGTEXT,+ file_path LONGTEXT,+ execution_id LONGTEXT+ )+ """++ cursor.execute(legacy_create_query)+ print(+ f"📋 Created fresh legacy table '{table_name}' with LONGTEXT data column"+ )++ # Insert some legacy data to make it realistic+ cursor.execute(+ f"INSERT INTO {table_name} (id, created_by, created_at, data, file_path, execution_id) VALUES (%s, %s, NOW(), %s, %s, %s)",+ (+ "legacy-1",+ "Legacy/DBWriter",+ '{"old": "legacy_data"}',+ "/legacy/file.pdf",+ "legacy-exec-1",+ ),+ )+ print(f"📝 Inserted legacy test data into '{table_name}'")++ cursor.close()+ print(f"✅ Legacy table '{table_name}' setup completed")++ finally:+ if hasattr(engine, "close"):+ engine.close()++ def _verify_table_is_legacy(self, table_name: str) -> bool:+ """Verify that table is detected as legacy using the real detection logic."""+ try:+ # Use the actual detection logic from the codebase+ is_string_column = self.mysql_connector.is_string_column(+ table_info=self.mysql_connector.get_information_schema(table_name),+ column_name="data",+ )++ if is_string_column:+ print(+ f"✅ Table '{table_name}' correctly detected as legacy (data column is string type)"+ )+ return True+ else:+ print(f"❌ Table '{table_name}' not detected as legacy")+ return False++ except Exception as e:+ print(f"❌ Error checking legacy status: {e}")+ return False++ def _initiate_migration_via_workflow(self, table_name: str) -> None:+ """Initiate migration using the real workflow destination connector logic."""+ # Create mock objects for the workflow system+ mock_workflow = self.create_mock_workflow()+ mock_workflow_log = self.create_mock_workflow_log()+ mock_connector_instance = self.create_real_connector_instance()++ # Create endpoint with the legacy table name+ mock_endpoint = Mock()+ mock_endpoint.connector_instance = mock_connector_instance+ mock_endpoint.configuration = {+ DestinationKey.TABLE: table_name, # Use legacy table+ DestinationKey.INCLUDE_AGENT: True,+ DestinationKey.INCLUDE_TIMESTAMP: True,+ DestinationKey.AGENT_NAME: "Unstract/DBWriter",+ DestinationKey.COLUMN_MODE: "Write JSON to a single column",+ DestinationKey.SINGLE_COLUMN_NAME: "data",+ DestinationKey.FILE_PATH: "file_path",+ DestinationKey.EXECUTION_ID: "execution_id",+ }++ # Create destination connector+ destination_connector = self.create_destination_connector(+ mock_workflow, mock_workflow_log, mock_endpoint+ )++ # Mock the data methods to provide test data+ test_migration_data = {"migration_test": "v2_data", "status": "migrated"}+ test_migration_metadata = {+ "file_execution_id": "migration-test-exec-id",+ "migration_test": True,+ "extracted_text": "Migration test extracted text",+ }++ with patch.object(+ destination_connector,+ "get_tool_execution_result",+ return_value=test_migration_data,+ ):+ with patch.object(+ destination_connector,+ "get_combined_metadata",+ return_value=test_migration_metadata,+ ):+ # This should trigger the migration logic automatically+ destination_connector.insert_into_db(+ input_file_path="/migration/test/file.pdf", error=None+ )++ print(f"✅ Migration initiated for table '{table_name}'")++ def _verify_migration_success(self, table_name: str) -> None:+ """Verify that migration was successful by checking for v2 columns."""+ table_info = self.mysql_connector.get_information_schema(table_name)++ # Check for v2 columns that should have been added during migration+ expected_v2_columns = {+ "data_v2": "json",+ "metadata": "json",+ "user_field_1": "boolean",+ "user_field_2": "bigint",+ "user_field_3": "longtext",+ "status": "enum",+ "error_message": "longtext",+ }++ for column_name, expected_type in expected_v2_columns.items():+ self.assertIn(+ column_name,+ table_info,+ f"Migration failed: Column '{column_name}' not found after migration",+ )++ actual_type = table_info[column_name].lower()+ if expected_type == "json":+ # MySQL might return different representations for JSON+ self.assertIn(+ actual_type,+ ["json", "longtext"], # MySQL sometimes shows JSON as longtext+ f"Column '{column_name}' has type '{actual_type}', expected JSON-compatible type",+ )+ elif expected_type == "enum":+ self.assertIn(+ actual_type,+ ["enum"],+ f"Column '{column_name}' has type '{actual_type}', expected enum type",+ )+ else:+ self.assertEqual(+ actual_type,+ expected_type,+ f"Column '{column_name}' has type '{actual_type}', expected '{expected_type}'",+ )++ print(+ f"✅ Migration verification successful - all v2 columns present in '{table_name}'"+ )++ def _test_dual_column_writing(self, table_name: str) -> None:+ """Test that data is written to both legacy and v2 columns after migration."""+ engine = self.mysql_connector.get_engine()+ try:+ cursor = engine.cursor()++ # Query the latest inserted row to verify dual writing+ cursor.execute(+ f"""+ SELECT data, data_v2, metadata, status, error_message+ FROM {table_name}+ WHERE file_path = '/migration/test/file.pdf'+ ORDER BY created_at DESC+ LIMIT 1+ """+ )+ row = cursor.fetchone()++ self.assertIsNotNone(row, "No migrated data found in table")++ data_legacy, data_v2, metadata, status, error_message = row++ # Verify legacy column has data+ self.assertIsNotNone(data_legacy, "Legacy data column should not be empty")++ # Verify v2 column has data+ self.assertIsNotNone(data_v2, "V2 data column should not be empty")++ # Verify metadata column has data+ self.assertIsNotNone(metadata, "Metadata column should not be empty")++ # Verify status is set correctly+ self.assertEqual(+ status, "SUCCESS", "Status should be SUCCESS for successful migration"+ )++ # Parse and verify JSON data (if stored as JSON string in legacy)+ if isinstance(data_legacy, str):+ try:+ parsed_legacy = json.loads(data_legacy)+ self.assertIn(+ "migration_test",+ parsed_legacy,+ "Legacy data should contain migration test data",+ )+ except json.JSONDecodeError:+ # If not JSON, just verify it contains our test data somehow+ self.assertIn("migration_test", str(data_legacy))++ cursor.close()+ print(f"✅ Dual column writing verification successful for '{table_name}'")++ finally:+ if hasattr(engine, "close"):+ engine.close()++ def tearDown(self) -> None:+ """Clean up test resources."""+ # # Clean up test tables+ # try:+ # engine = self.mysql_connector.get_engine()+ # cursor = engine.cursor()+ # # Clean up both regular test table and migration test table+ # cursor.execute(f"DROP TABLE IF EXISTS {self.test_table_name}")+ # cursor.execute(f"DROP TABLE IF EXISTS LEGACY_OUTPUT_MYSQL")+ # cursor.close()+ # engine.close()+ # except Exception:+ # Ignore cleanup errors+ passdiff --git a/backend/workflow_manager/endpoint_v2/tests/destination-connectors/test_destination_connector_oracle.py b/backend/workflow_manager/endpoint_v2/tests/destination-connectors/test_destination_connector_oracle.pynew file mode 100644index 000000000..5698e722a--- /dev/null+++ b/backend/workflow_manager/endpoint_v2/tests/destination-connectors/test_destination_connector_oracle.py@@ -0,0 +1,313 @@+import os+from unittest.mock import Mock, patch++from django.test import TestCase+from workflow_manager.endpoint_v2.constants import DestinationKey+from workflow_manager.endpoint_v2.destination import DestinationConnector++from unstract.connectors.databases.oracle_db import OracleDB+++class TestDestinationConnectorOracle(TestCase):+ """Integration test for insert_into_db method with real Oracle DB connector."""++ def setUp(self) -> None:+ """Set up test data and real Oracle DB configuration."""++ # Oracle DB connection settings for testing+ required_env_vars = [+ "ORACLE_CONFIG_DIR",+ "ORACLE_USER",+ "ORACLE_PASSWORD",+ "ORACLE_DSN",+ "ORACLE_WALLET_LOCATION",+ "ORACLE_WALLET_PASSWORD",+ ]++ # Check if all required environment variables are set+ missing_vars = [var for var in required_env_vars if not os.getenv(var)]+ if missing_vars:+ self.skipTest(+ f"Required Oracle DB environment variables not set: {', '.join(missing_vars)}"+ )++ self.oracle_config = {+ "config_dir": os.getenv("ORACLE_CONFIG_DIR"),+ "user": os.getenv("ORACLE_USER"),+ "password": os.getenv("ORACLE_PASSWORD"),+ "dsn": os.getenv("ORACLE_DSN"),+ "wallet_location": os.getenv("ORACLE_WALLET_LOCATION"),+ "wallet_password": os.getenv("ORACLE_WALLET_PASSWORD"),+ }++ # Test data that will be inserted into the database+ self.test_data = {"key": "value", "result": "test_result", "status": "success"}+ self.test_metadata = {+ "file_execution_id": "test-file-exec-id",+ "extracted_text": "This is sample extracted text",+ "processing_time": 1.5,+ }+ self.input_file_path = "/path/to/test/file.pdf"++ # Oracle table naming (typically uppercase)+ self.test_table_name = "MIGRATION"++ # Create real Oracle DB connector instance+ self.oracle_connector = OracleDB(settings=self.oracle_config)++ # Expected columns based on configuration (Oracle data types)+ self.expected_columns = {+ "ID": "VARCHAR2",+ "FILE_PATH": "VARCHAR2",+ "EXECUTION_ID": "VARCHAR2",+ "DATA": "CLOB", # Legacy column remains CLOB after migration+ "DATA_V2": "CLOB", # New CLOB column added during migration+ "METADATA": "CLOB", # New CLOB column added during migration+ "CREATED_AT": "TIMESTAMP",+ "CREATED_BY": "VARCHAR2",+ "STATUS": "VARCHAR2",+ "ERROR_MESSAGE": "VARCHAR2",+ "USER_FIELD_1": "NUMBER",+ "USER_FIELD_2": "NUMBER",+ "USER_FIELD_3": "VARCHAR2",+ }++ def verify_table_columns(self, table_name: str) -> None:+ """Verify that the table has expected columns with correct data types."""+ table_info = self.oracle_connector.get_information_schema(table_name=table_name)++ print(f"🔍 Actual table structure for '{table_name}': {table_info}")++ # Check that key columns exist (some columns may be created on-demand)+ required_columns = {+ "ID": ["VARCHAR2"], # Oracle uses VARCHAR2+ "FILE_PATH": ["CLOB", "VARCHAR2"], # Oracle can use either CLOB or VARCHAR2+ "EXECUTION_ID": [+ "CLOB",+ "VARCHAR2",+ ], # Oracle can use either CLOB or VARCHAR2+ "DATA": [+ "CLOB",+ "VARCHAR2",+ ], # Can be CLOB or VARCHAR2, depends on migration scenario+ "METADATA": [+ "CLOB",+ ], # New CLOB column added during migration for JSON data+ "CREATED_AT": ["TIMESTAMP", "TIMESTAMP(6)"], # Oracle timestamp types+ "CREATED_BY": ["VARCHAR2"], # Oracle uses VARCHAR2+ }++ # Optional columns that may exist in migration scenarios+ optional_columns = {+ "DATA_V2": [+ "CLOB",+ "VARCHAR2",+ ], # Can be CLOB or VARCHAR2 in migration scenarios+ }++ for expected_column, expected_type in required_columns.items():+ self.assertIn(+ expected_column,+ table_info,+ f"Expected column '{expected_column}' not found in table '{table_name}'",+ )++ actual_type = table_info[expected_column]+ # Handle both single expected types and lists of acceptable types+ if isinstance(expected_type, list):+ self.assertIn(+ actual_type.upper(),+ [t.upper() for t in expected_type],+ f"Column '{expected_column}' has type '{actual_type}', expected one of {expected_type}",+ )+ else:+ # Oracle may return different type names, normalize for comparison+ self.assertEqual(+ actual_type.upper(),+ expected_type.upper(),+ f"Column '{expected_column}' has type '{actual_type}', expected '{expected_type}'",+ )++ # Check optional columns if they exist+ for optional_column, expected_type in optional_columns.items():+ if optional_column in table_info:+ actual_type = table_info[optional_column]+ if isinstance(expected_type, list):+ self.assertIn(+ actual_type.upper(),+ [t.upper() for t in expected_type],+ f"Optional column '{optional_column}' has type '{actual_type}', expected one of {expected_type}",+ )+ else:+ self.assertEqual(+ actual_type.upper(),+ expected_type.upper(),+ f"Optional column '{optional_column}' has type '{actual_type}', expected '{expected_type}'",+ )++ def create_mock_workflow(self) -> Mock:+ """Create a mock workflow object."""+ workflow = Mock()+ workflow.id = "test-workflow-id"+ return workflow++ def create_mock_workflow_log(self) -> Mock:+ """Create a mock workflow log object."""+ return Mock()++ def create_real_connector_instance(self) -> Mock:+ """Create connector instance using real Oracle DB connector."""+ connector = Mock()+ connector.connector_id = OracleDB.get_id()+ connector.connector_metadata = self.oracle_config+ # Store reference to real connector for potential future use+ connector._real_oracle_connector = self.oracle_connector+ return connector++ def create_mock_endpoint(self, mock_connector_instance: Mock) -> Mock:+ """Create a mock endpoint with real database configuration."""+ endpoint = Mock()+ endpoint.connector_instance = mock_connector_instance+ endpoint.configuration = {+ DestinationKey.TABLE: self.test_table_name,+ DestinationKey.INCLUDE_AGENT: True,+ DestinationKey.INCLUDE_TIMESTAMP: True,+ DestinationKey.AGENT_NAME: "Unstract/DBWriter",+ DestinationKey.COLUMN_MODE: "Write JSON to a single column",+ DestinationKey.SINGLE_COLUMN_NAME: "data",+ DestinationKey.FILE_PATH: "file_path",+ DestinationKey.EXECUTION_ID: "execution_id",+ }+ return endpoint++ def create_destination_connector(+ self, mock_workflow: Mock, mock_workflow_log: Mock, mock_endpoint: Mock+ ) -> DestinationConnector:+ """Create a destination connector with mocked Django dependencies but real DB."""+ with patch(+ "workflow_manager.endpoint_v2.destination.UserContext"+ ) as mock_context:+ mock_context.get_organization_identifier.return_value = "test-org"++ with patch.object(+ DestinationConnector,+ "_get_endpoint_for_workflow",+ return_value=mock_endpoint,+ ):+ with patch.object(+ DestinationConnector,+ "_get_source_endpoint_for_workflow",+ return_value=mock_endpoint,+ ):+ connector = DestinationConnector(+ workflow=mock_workflow,+ execution_id="test-execution-id",+ workflow_log=mock_workflow_log,+ use_file_history=False,+ )+ return connector++ def test_insert_into_db_happy_path_oracle(self) -> None:+ """Test successful insertion into real Oracle DB database."""+ # Create mock objects for Django models+ mock_workflow = self.create_mock_workflow()+ mock_workflow_log = self.create_mock_workflow_log()+ mock_connector_instance = self.create_real_connector_instance()+ mock_endpoint = self.create_mock_endpoint(mock_connector_instance)++ # Create destination connector with real database connection+ destination_connector = self.create_destination_connector(+ mock_workflow, mock_workflow_log, mock_endpoint+ )++ # Test the insert_into_db method with real Oracle DB+ try:+ # Mock only the methods that get data, let database operations be real+ with patch.object(+ destination_connector,+ "get_tool_execution_result",+ return_value=self.test_data,+ ):+ with patch.object(+ destination_connector,+ "get_combined_metadata",+ return_value=self.test_metadata,+ ):+ # This will execute real database operations using Oracle DB connector+ destination_connector.insert_into_db(+ input_file_path=self.input_file_path, error=None+ )++ # Verify the table structure+ self.verify_table_columns(self.test_table_name)++ print("✅ Oracle DB integration test completed successfully!")++ except Exception as e:+ print(f"❌ Oracle DB integration test failed: {str(e)}")+ raise++ def test_insert_into_db_with_error_oracle(self) -> None:+ """Test insertion with error parameter into real Oracle DB database."""+ # Create mock objects+ mock_workflow = self.create_mock_workflow()+ mock_workflow_log = self.create_mock_workflow_log()+ mock_connector_instance = self.create_real_connector_instance()+ mock_endpoint = self.create_mock_endpoint(mock_connector_instance)++ # Create destination connector+ destination_connector = self.create_destination_connector(+ mock_workflow, mock_workflow_log, mock_endpoint+ )++ error_message = "Test processing error occurred"++ # Mock the methods that get data+ with patch.object(+ destination_connector,+ "get_tool_execution_result",+ return_value=self.test_data,+ ):+ with patch.object(+ destination_connector,+ "get_combined_metadata",+ return_value=self.test_metadata,+ ):+ # Execute with error parameter+ destination_connector.insert_into_db(+ input_file_path=self.input_file_path, error=error_message+ )++ # Verify the table structure+ self.verify_table_columns(self.test_table_name)++ print(+ f"✅ Successfully inserted error data into Oracle DB table: {self.test_table_name}"+ )++ def test_oracle_connector_connection(self) -> None:+ """Test that the Oracle DB connector can establish a connection."""+ # Test the real Oracle DB connector directly+ try:+ engine = self.oracle_connector.get_engine()+ self.assertIsNotNone(engine)+ print("✅ Oracle DB connector successfully established connection")++ # Test a simple query to verify connection works+ with engine.cursor() as cursor:+ cursor.execute("SELECT 1 as test_column FROM DUAL")+ result = cursor.fetchall()+ self.assertIsNotNone(result)+ print("✅ Oracle DB connector successfully executed test query")++ # Clean up+ if hasattr(engine, "close"):+ engine.close()+ except Exception as e:+ self.fail(f"Oracle DB connector failed to connect: {str(e)}")++ def tearDown(self) -> None:+ """Clean up test resources."""+ # Optionally clean up test table+ # Note: Be careful with cleanup in production environments+ passdiff --git a/backend/workflow_manager/endpoint_v2/tests/destination-connectors/test_destination_connector_postgres.py b/backend/workflow_manager/endpoint_v2/tests/destination-connectors/test_destination_connector_postgres.pynew file mode 100644index 000000000..33f31f055--- /dev/null+++ b/backend/workflow_manager/endpoint_v2/tests/destination-connectors/test_destination_connector_postgres.py@@ -0,0 +1,326 @@+import os+from unittest.mock import Mock, patch++from django.test import TestCase+from workflow_manager.endpoint_v2.constants import DestinationKey+from workflow_manager.endpoint_v2.destination import DestinationConnector++from unstract.connectors.databases.postgresql import PostgreSQL+++class TestDestinationConnectorPostgreSQL(TestCase):+ """Integration test for insert_into_db method with real PostgreSQL connector."""++ def setUp(self) -> None:+ """Set up test data and real PostgreSQL configuration."""++ # Real PostgreSQL connection settings for testing+ self.postgres_config = {+ "host": os.getenv("DB_HOST", "localhost"),+ "port": os.getenv("DB_PORT", "5432"),+ "database": os.getenv("DB_NAME", "test_unstract"),+ "user": os.getenv("DB_USER", "postgres"),+ "password": os.getenv("DB_PASSWORD", "password"),+ "schema": "test", # Add schema to fix PostgreSQL issue+ }++ # Test data that will be inserted into the database+ self.test_data = {"key": "value", "result": "test_result", "status": "success"}+ self.test_metadata = {+ "file_execution_id": "test-file-exec-id",+ "extracted_text": "This is sample extracted text",+ "processing_time": 1.5,+ }+ self.input_file_path = "/path/to/test/file.pdf"+ self.test_table_name = "OUTPUT_3"++ # Create real PostgreSQL connector instance+ self.postgres_connector = PostgreSQL(settings=self.postgres_config)++ # Expected columns based on configuration+ self.expected_columns = {+ "id": "text",+ "file_path": "text",+ "execution_id": "text",+ "data": "jsonb",+ "created_at": "timestamp without time zone",+ "created_by": "text",+ "metadata": "jsonb",+ "error_message": "text",+ "status": "text",+ "user_field_1": "boolean",+ "user_field_2": "integer",+ "user_field_3": "text",+ }++ def verify_table_columns(self, table_name: str) -> None:+ """Verify that all expected columns exist in the table with correct types."""+ table_info = self.postgres_connector.get_information_schema(+ table_name=table_name+ )++ print(f"🔍 Actual table structure for '{table_name}': {table_info}")++ # Check that key columns exist (some columns may be created on-demand)+ required_columns = {+ "id": "text",+ "file_path": "text",+ "execution_id": "text",+ "data": ["text", "jsonb"], # Can be either, migration might convert+ "created_at": "timestamp without time zone",+ "created_by": "text",+ }++ for expected_column, expected_types in required_columns.items():+ self.assertIn(+ expected_column,+ table_info,+ f"Expected column '{expected_column}' not found in table '{table_name}'",+ )++ actual_type = table_info[expected_column]+ if isinstance(expected_types, list):+ self.assertIn(+ actual_type,+ expected_types,+ f"Column '{expected_column}' has type '{actual_type}', expected one of {expected_types}",+ )+ else:+ self.assertEqual(+ actual_type,+ expected_types,+ f"Column '{expected_column}' has type '{actual_type}', expected '{expected_types}'",+ )++ print(f"✅ All required columns verified in table '{table_name}'")++ def create_mock_workflow(self) -> Mock:+ """Create a mock workflow object."""+ workflow = Mock()+ workflow.id = "test-workflow-id"+ return workflow++ def create_mock_workflow_log(self) -> Mock:+ """Create a mock workflow log object."""+ return Mock()++ def create_real_connector_instance(self) -> Mock:+ """Create connector instance using real PostgreSQL connector."""+ connector = Mock()+ connector.connector_id = "postgresql|6db35f45-be11-4fd5-80c5-85c48183afbb"+ connector.connector_metadata = self.postgres_config+ # Store reference to real connector for potential future use+ connector._real_postgresql_connector = self.postgres_connector+ return connector++ def create_mock_endpoint(self, mock_connector_instance: Mock) -> Mock:+ """Create a mock endpoint with real database configuration."""+ endpoint = Mock()+ endpoint.connector_instance = mock_connector_instance+ endpoint.configuration = {+ DestinationKey.TABLE: self.test_table_name,+ DestinationKey.INCLUDE_AGENT: True,+ DestinationKey.INCLUDE_TIMESTAMP: True,+ DestinationKey.AGENT_NAME: "Unstract/DBWriter",+ DestinationKey.COLUMN_MODE: "Write JSON to a single column",+ DestinationKey.SINGLE_COLUMN_NAME: "data",+ DestinationKey.FILE_PATH: "file_path",+ DestinationKey.EXECUTION_ID: "execution_id",+ }+ return endpoint++ def create_destination_connector(+ self, mock_workflow: Mock, mock_workflow_log: Mock, mock_endpoint: Mock+ ) -> DestinationConnector:+ """Create a destination connector with mocked Django dependencies but real DB."""+ with patch(+ "workflow_manager.endpoint_v2.destination.UserContext"+ ) as mock_context:+ mock_context.get_organization_identifier.return_value = "test-org"++ with patch.object(+ DestinationConnector,+ "_get_endpoint_for_workflow",+ return_value=mock_endpoint,+ ):+ with patch.object(+ DestinationConnector,+ "_get_source_endpoint_for_workflow",+ return_value=mock_endpoint,+ ):+ connector = DestinationConnector(+ workflow=mock_workflow,+ execution_id="test-execution-id",+ workflow_log=mock_workflow_log,+ use_file_history=False,+ )+ return connector++ def test_insert_into_db_happy_path_postgresql(self) -> None:+ """Test successful insertion into real PostgreSQL database."""+ # Create mock objects for Django models+ mock_workflow = self.create_mock_workflow()+ mock_workflow_log = self.create_mock_workflow_log()+ mock_connector_instance = self.create_real_connector_instance()+ mock_endpoint = self.create_mock_endpoint(mock_connector_instance)++ # Create destination connector with real database connection+ destination_connector = self.create_destination_connector(+ mock_workflow, mock_workflow_log, mock_endpoint+ )++ # Mock only the methods that get data, let database operations be real+ with patch.object(+ destination_connector,+ "get_tool_execution_result",+ return_value=self.test_data,+ ):+ with patch.object(+ destination_connector,+ "get_combined_metadata",+ return_value=self.test_metadata,+ ):+ # This will execute real database operations using PostgreSQL connector+ destination_connector.insert_into_db(+ input_file_path=self.input_file_path, error=None+ )++ # Verify that all expected columns were created+ self.verify_table_columns(self.test_table_name)++ print(+ f"✅ Successfully inserted test data into PostgreSQL table: {self.test_table_name}"+ )++ def test_insert_into_db_with_error_postgresql(self) -> None:+ """Test insertion with error parameter into real PostgreSQL database."""+ # Create mock objects+ mock_workflow = self.create_mock_workflow()+ mock_workflow_log = self.create_mock_workflow_log()+ mock_connector_instance = self.create_real_connector_instance()+ mock_endpoint = self.create_mock_endpoint(mock_connector_instance)++ # Create destination connector+ destination_connector = self.create_destination_connector(+ mock_workflow, mock_workflow_log, mock_endpoint+ )++ error_message = "Test processing error occurred"++ # Mock the methods that get data+ with patch.object(+ destination_connector,+ "get_tool_execution_result",+ return_value=self.test_data,+ ):+ with patch.object(+ destination_connector,+ "get_combined_metadata",+ return_value=self.test_metadata,+ ):+ # Execute with error parameter+ destination_connector.insert_into_db(+ input_file_path=self.input_file_path, error=error_message+ )++ # Verify that all expected columns were created+ self.verify_table_columns(self.test_table_name)++ print(+ f"✅ Successfully inserted error data into PostgreSQL table: {self.test_table_name}"+ )++ def test_postgresql_connector_connection(self) -> None:+ """Test that the PostgreSQL connector can establish a connection."""+ # Test the real PostgreSQL connector directly+ try:+ engine = self.postgres_connector.get_engine()+ self.assertIsNotNone(engine)+ print("✅ PostgreSQL connector successfully established connection")++ # Clean up+ if hasattr(engine, "close"):+ engine.close()+ except Exception as e:+ self.fail(f"PostgreSQL connector failed to connect: {str(e)}")++ def test_comprehensive_column_data_verification(self) -> None:+ """Test that data is properly inserted into all columns with correct values."""+ # Create mock objects+ mock_workflow = self.create_mock_workflow()+ mock_workflow_log = self.create_mock_workflow_log()+ mock_connector_instance = self.create_real_connector_instance()+ mock_endpoint = self.create_mock_endpoint(mock_connector_instance)++ # Create destination connector+ destination_connector = self.create_destination_connector(+ mock_workflow, mock_workflow_log, mock_endpoint+ )++ # Mock the methods that get data+ with patch.object(+ destination_connector,+ "get_tool_execution_result",+ return_value=self.test_data,+ ):+ with patch.object(+ destination_connector,+ "get_combined_metadata",+ return_value=self.test_metadata,+ ):+ # Execute database insertion+ destination_connector.insert_into_db(+ input_file_path=self.input_file_path, error=None+ )++ # Verify table structure+ self.verify_table_columns(self.test_table_name)++ # Verify data was inserted correctly by querying the table+ engine = self.postgres_connector.get_engine()+ try:+ # Use cursor for raw psycopg2 connection+ cursor = engine.cursor()+ cursor.execute(+ f"SELECT created_by, file_path, execution_id, created_at, data, metadata FROM {self.test_table_name} ORDER BY created_at DESC LIMIT 1"+ )+ row = cursor.fetchone()++ self.assertIsNotNone(row, "No data found in table after insertion")++ # Map results to column names+ created_by, file_path, execution_id, created_at, data, metadata = row++ # Verify specific column values+ self.assertEqual(+ created_by,+ "Unstract/DBWriter",+ f"Expected created_by to be 'Unstract/DBWriter', got '{created_by}'",+ )++ self.assertEqual(+ file_path,+ self.input_file_path,+ f"Expected file_path to be '{self.input_file_path}', got '{file_path}'",+ )++ self.assertEqual(+ execution_id,+ "test-execution-id",+ f"Expected execution_id to be 'test-execution-id', got '{execution_id}'",+ )++ self.assertIsNotNone(created_at, "Expected created_at timestamp to be set")++ self.assertIsNotNone(data, "Expected data column to contain inserted data")++ self.assertIsNotNone(+ metadata, "Expected metadata column to contain metadata"+ )++ cursor.close()+ print("✅ All column data values verified successfully")++ finally:+ if hasattr(engine, "close"):+ engine.close()diff --git a/backend/workflow_manager/endpoint_v2/tests/destination-connectors/test_destination_connector_redshift.py b/backend/workflow_manager/endpoint_v2/tests/destination-connectors/test_destination_connector_redshift.pynew file mode 100644index 000000000..0dee96694--- /dev/null+++ b/backend/workflow_manager/endpoint_v2/tests/destination-connectors/test_destination_connector_redshift.py@@ -0,0 +1,326 @@+import os+from unittest.mock import Mock, patch++from django.test import TestCase+from workflow_manager.endpoint_v2.constants import DestinationKey+from workflow_manager.endpoint_v2.destination import DestinationConnector++from unstract.connectors.databases.redshift import Redshift+++class TestDestinationConnectorRedshift(TestCase):+ """Integration test for insert_into_db method with real Redshift connector."""++ def setUp(self) -> None:+ """Set up test data and real Redshift configuration."""++ # Redshift connection settings for testing+ required_env_vars = [+ "REDSHIFT_HOST",+ "REDSHIFT_USER",+ "REDSHIFT_PASSWORD",+ "REDSHIFT_DATABASE",+ ]++ # Check if all required environment variables are set+ missing_vars = [var for var in required_env_vars if not os.getenv(var)]+ if missing_vars:+ self.skipTest(+ f"Required Redshift environment variables not set: {', '.join(missing_vars)}"+ )++ self.redshift_config = {+ "host": os.getenv("REDSHIFT_HOST"),+ "port": os.getenv("REDSHIFT_PORT", "5439"),+ "user": os.getenv("REDSHIFT_USER"),+ "password": os.getenv("REDSHIFT_PASSWORD"),+ "database": os.getenv("REDSHIFT_DATABASE"),+ "schema": os.getenv("REDSHIFT_SCHEMA", "public"),+ }++ # Test data that will be inserted into the database+ self.test_data = {"key": "value", "result": "test_result", "status": "success"}+ self.test_metadata = {+ "file_execution_id": "test-file-exec-id",+ "extracted_text": "This is sample extracted text",+ "processing_time": 1.5,+ }+ self.input_file_path = "/path/to/test/file.pdf"++ # Redshift table naming (use schema.table format)+ self.test_table_name = "MIGRATION"++ # Create real Redshift connector instance+ self.redshift_connector = Redshift(settings=self.redshift_config)++ # Expected columns based on configuration (Redshift data types)+ self.expected_columns = {+ "id": "CHARACTER VARYING",+ "file_path": "CHARACTER VARYING",+ "execution_id": "CHARACTER VARYING",+ "data": "CHARACTER VARYING", # Legacy column remains VARCHAR after migration+ "data_v2": "CHARACTER VARYING", # New VARCHAR column added during migration+ "metadata": "CHARACTER VARYING", # New VARCHAR column added during migration+ "created_at": "TIMESTAMP WITHOUT TIME ZONE",+ "created_by": "CHARACTER VARYING",+ "status": "CHARACTER VARYING",+ "error_message": "CHARACTER VARYING",+ "user_field_1": "BOOLEAN",+ "user_field_2": "INTEGER",+ "user_field_3": "CHARACTER VARYING",+ }++ def verify_table_columns(self, table_name: str) -> None:+ """Verify that the table has expected columns with correct data types."""+ table_info = self.redshift_connector.get_information_schema(+ table_name=table_name+ )++ print(f"🔍 Actual table structure for '{table_name}': {table_info}")++ # Check that key columns exist (some columns may be created on-demand)+ required_columns = {+ "id": [+ "CHARACTER VARYING",+ "VARCHAR",+ ], # Redshift uses CHARACTER VARYING or VARCHAR+ "file_path": [+ "CHARACTER VARYING",+ "VARCHAR",+ ], # Redshift uses CHARACTER VARYING or VARCHAR+ "execution_id": [+ "CHARACTER VARYING",+ "VARCHAR",+ ], # Redshift uses CHARACTER VARYING or VARCHAR+ "data": [+ "CHARACTER VARYING",+ "VARCHAR",+ "SUPER",+ ], # Can be VARCHAR, migration might convert+ "metadata": [+ "CHARACTER VARYING",+ "VARCHAR",+ "SUPER",+ ], # New VARCHAR column added during migration, or SUPER for JSON+ "created_at": ["TIMESTAMP WITHOUT TIME ZONE", "TIMESTAMP"],+ "created_by": [+ "CHARACTER VARYING",+ "VARCHAR",+ ], # Redshift uses CHARACTER VARYING or VARCHAR+ }++ # Optional columns that may exist in migration scenarios+ optional_columns = {+ "data_v2": [+ "CHARACTER VARYING",+ "VARCHAR",+ "SUPER",+ ], # Only exists in migration scenarios+ }++ for expected_column, expected_type in required_columns.items():+ self.assertIn(+ expected_column,+ table_info,+ f"Expected column '{expected_column}' not found in table '{table_name}'",+ )++ actual_type = table_info[expected_column]+ # Handle both single expected types and lists of acceptable types+ if isinstance(expected_type, list):+ self.assertIn(+ actual_type.upper(),+ [t.upper() for t in expected_type],+ f"Column '{expected_column}' has type '{actual_type}', expected one of {expected_type}",+ )+ else:+ # Redshift may return different type names, normalize for comparison+ self.assertEqual(+ actual_type.upper(),+ expected_type.upper(),+ f"Column '{expected_column}' has type '{actual_type}', expected '{expected_type}'",+ )++ # Check optional columns if they exist+ for optional_column, expected_type in optional_columns.items():+ if optional_column in table_info:+ actual_type = table_info[optional_column]+ if isinstance(expected_type, list):+ self.assertIn(+ actual_type.upper(),+ [t.upper() for t in expected_type],+ f"Optional column '{optional_column}' has type '{actual_type}', expected one of {expected_type}",+ )+ else:+ self.assertEqual(+ actual_type.upper(),+ expected_type.upper(),+ f"Optional column '{optional_column}' has type '{actual_type}', expected '{expected_type}'",+ )++ def create_mock_workflow(self) -> Mock:+ """Create a mock workflow object."""+ workflow = Mock()+ workflow.id = "test-workflow-id"+ return workflow++ def create_mock_workflow_log(self) -> Mock:+ """Create a mock workflow log object."""+ return Mock()++ def create_real_connector_instance(self) -> Mock:+ """Create connector instance using real Redshift connector."""+ connector = Mock()+ connector.connector_id = Redshift.get_id()+ connector.connector_metadata = self.redshift_config+ # Store reference to real connector for potential future use+ connector._real_redshift_connector = self.redshift_connector+ return connector++ def create_mock_endpoint(self, mock_connector_instance: Mock) -> Mock:+ """Create a mock endpoint with real database configuration."""+ endpoint = Mock()+ endpoint.connector_instance = mock_connector_instance+ endpoint.configuration = {+ DestinationKey.TABLE: self.test_table_name,+ DestinationKey.INCLUDE_AGENT: True,+ DestinationKey.INCLUDE_TIMESTAMP: True,+ DestinationKey.AGENT_NAME: "Unstract/DBWriter",+ DestinationKey.COLUMN_MODE: "Write JSON to a single column",+ DestinationKey.SINGLE_COLUMN_NAME: "data",+ DestinationKey.FILE_PATH: "file_path",+ DestinationKey.EXECUTION_ID: "execution_id",+ }+ return endpoint++ def create_destination_connector(+ self, mock_workflow: Mock, mock_workflow_log: Mock, mock_endpoint: Mock+ ) -> DestinationConnector:+ """Create a destination connector with mocked Django dependencies but real DB."""+ with patch(+ "workflow_manager.endpoint_v2.destination.UserContext"+ ) as mock_context:+ mock_context.get_organization_identifier.return_value = "test-org"++ with patch.object(+ DestinationConnector,+ "_get_endpoint_for_workflow",+ return_value=mock_endpoint,+ ):+ with patch.object(+ DestinationConnector,+ "_get_source_endpoint_for_workflow",+ return_value=mock_endpoint,+ ):+ connector = DestinationConnector(+ workflow=mock_workflow,+ execution_id="test-execution-id",+ workflow_log=mock_workflow_log,+ use_file_history=False,+ )+ return connector++ def test_insert_into_db_happy_path_redshift(self) -> None:+ """Test successful insertion into real Redshift database."""+ # Create mock objects for Django models+ mock_workflow = self.create_mock_workflow()+ mock_workflow_log = self.create_mock_workflow_log()+ mock_connector_instance = self.create_real_connector_instance()+ mock_endpoint = self.create_mock_endpoint(mock_connector_instance)++ # Create destination connector with real database connection+ destination_connector = self.create_destination_connector(+ mock_workflow, mock_workflow_log, mock_endpoint+ )++ # Test the insert_into_db method with real Redshift+ try:+ # Mock only the methods that get data, let database operations be real+ with patch.object(+ destination_connector,+ "get_tool_execution_result",+ return_value=self.test_data,+ ):+ with patch.object(+ destination_connector,+ "get_combined_metadata",+ return_value=self.test_metadata,+ ):+ # This will execute real database operations using Redshift connector+ destination_connector.insert_into_db(+ input_file_path=self.input_file_path, error=None+ )++ # Verify the table structure+ self.verify_table_columns(self.test_table_name)++ print("✅ Redshift integration test completed successfully!")++ except Exception as e:+ print(f"❌ Redshift integration test failed: {str(e)}")+ raise++ def test_insert_into_db_with_error_redshift(self) -> None:+ """Test insertion with error parameter into real Redshift database."""+ # Create mock objects+ mock_workflow = self.create_mock_workflow()+ mock_workflow_log = self.create_mock_workflow_log()+ mock_connector_instance = self.create_real_connector_instance()+ mock_endpoint = self.create_mock_endpoint(mock_connector_instance)++ # Create destination connector+ destination_connector = self.create_destination_connector(+ mock_workflow, mock_workflow_log, mock_endpoint+ )++ error_message = "Test processing error occurred"++ # Mock the methods that get data+ with patch.object(+ destination_connector,+ "get_tool_execution_result",+ return_value=self.test_data,+ ):+ with patch.object(+ destination_connector,+ "get_combined_metadata",+ return_value=self.test_metadata,+ ):+ # Execute with error parameter+ destination_connector.insert_into_db(+ input_file_path=self.input_file_path, error=error_message+ )++ # Verify the table structure+ self.verify_table_columns(self.test_table_name)++ print(+ f"✅ Successfully inserted error data into Redshift table: {self.test_table_name}"+ )++ def test_redshift_connector_connection(self) -> None:+ """Test that the Redshift connector can establish a connection."""+ # Test the real Redshift connector directly+ try:+ engine = self.redshift_connector.get_engine()+ self.assertIsNotNone(engine)+ print("✅ Redshift connector successfully established connection")++ # Test a simple query to verify connection works+ with engine.cursor() as cursor:+ cursor.execute("SELECT 1 as test_column")+ result = cursor.fetchall()+ self.assertIsNotNone(result)+ print("✅ Redshift connector successfully executed test query")++ # Clean up+ if hasattr(engine, "close"):+ engine.close()+ except Exception as e:+ self.fail(f"Redshift connector failed to connect: {str(e)}")++ def tearDown(self) -> None:+ """Clean up test resources."""+ # Optionally clean up test table+ # Note: Be careful with cleanup in production environments+ passdiff --git a/backend/workflow_manager/endpoint_v2/tests/destination-connectors/test_destination_connector_snowflake.py b/backend/workflow_manager/endpoint_v2/tests/destination-connectors/test_destination_connector_snowflake.pynew file mode 100644index 000000000..d54ef8763--- /dev/null+++ b/backend/workflow_manager/endpoint_v2/tests/destination-connectors/test_destination_connector_snowflake.py@@ -0,0 +1,308 @@+import os+from unittest.mock import Mock, patch++from django.test import TestCase+from workflow_manager.endpoint_v2.constants import DestinationKey+from workflow_manager.endpoint_v2.destination import DestinationConnector++from unstract.connectors.databases.snowflake import SnowflakeDB+++class TestDestinationConnectorSnowflake(TestCase):+ """Integration test for insert_into_db method with real Snowflake connector."""++ def setUp(self) -> None:+ """Set up test data and real Snowflake configuration."""++ # Snowflake connection settings for testing+ required_env_vars = [+ "SNOWFLAKE_ACCOUNT",+ "SNOWFLAKE_USER",+ "SNOWFLAKE_PASSWORD",+ "SNOWFLAKE_DATABASE",+ "SNOWFLAKE_SCHEMA",+ "SNOWFLAKE_WAREHOUSE",+ "SNOWFLAKE_ROLE",+ ]++ # Check if all required environment variables are set+ missing_vars = [var for var in required_env_vars if not os.getenv(var)]+ if missing_vars:+ self.skipTest(+ f"Required Snowflake environment variables not set: {', '.join(missing_vars)}"+ )++ self.snowflake_config = {+ "account": os.getenv("SNOWFLAKE_ACCOUNT"),+ "user": os.getenv("SNOWFLAKE_USER"),+ "password": os.getenv("SNOWFLAKE_PASSWORD"),+ "database": os.getenv("SNOWFLAKE_DATABASE"),+ "schema": os.getenv("SNOWFLAKE_SCHEMA"),+ "warehouse": os.getenv("SNOWFLAKE_WAREHOUSE"),+ "role": os.getenv("SNOWFLAKE_ROLE"),+ }++ # Test data that will be inserted into the database+ self.test_data = {"key": "value", "result": "test_result", "status": "success"}+ self.test_metadata = {+ "file_execution_id": "test-file-exec-id",+ "extracted_text": "This is sample extracted text",+ "processing_time": 1.5,+ }+ self.input_file_path = "/path/to/test/file.pdf"++ # Snowflake table naming (use schema.table format)+ self.test_table_name = f"{self.snowflake_config['schema']}.migration"++ # Create real Snowflake connector instance+ self.snowflake_connector = SnowflakeDB(settings=self.snowflake_config)++ # Expected columns based on configuration (Snowflake data types)+ self.expected_columns = {+ "id": "TEXT",+ "file_path": "TEXT",+ "execution_id": "TEXT",+ "data": "TEXT", # Legacy column remains TEXT after migration+ "data_v2": "VARIANT", # New VARIANT column added during migration+ "metadata": "VARIANT", # New VARIANT column added during migration+ "created_at": "TIMESTAMP_NTZ",+ "created_by": "TEXT",+ "status": "TEXT",+ "error_message": "TEXT",+ "user_field_1": "BOOLEAN",+ "user_field_2": "NUMBER",+ "user_field_3": "TEXT",+ }++ def verify_table_columns(self, table_name: str) -> None:+ """Verify that the table has expected columns with correct data types."""+ table_info = self.snowflake_connector.get_information_schema(+ table_name=table_name+ )++ print(f"🔍 Actual table structure for '{table_name}': {table_info}")++ # Check that key columns exist (some columns may be created on-demand)+ required_columns = {+ "id": "VARCHAR", # Snowflake uses VARCHAR instead of TEXT+ "file_path": "VARCHAR", # Snowflake uses VARCHAR instead of TEXT+ "execution_id": "VARCHAR", # Snowflake uses VARCHAR instead of TEXT+ "data": ["VARCHAR", "VARIANT"], # Can be either, migration might convert+ "metadata": "VARIANT", # New VARIANT column added during migration+ "created_at": "TIMESTAMP_NTZ",+ "created_by": "VARCHAR", # Snowflake uses VARCHAR instead of TEXT+ }++ # Optional columns that may exist in migration scenarios+ optional_columns = {+ "data_v2": ["VARCHAR", "VARIANT"], # Only exists in migration scenarios+ }++ for expected_column, expected_type in required_columns.items():+ self.assertIn(+ expected_column,+ table_info,+ f"Expected column '{expected_column}' not found in table '{table_name}'",+ )++ actual_type = table_info[expected_column]+ # Handle both single expected types and lists of acceptable types+ if isinstance(expected_type, list):+ self.assertIn(+ actual_type.upper(),+ [t.upper() for t in expected_type],+ f"Column '{expected_column}' has type '{actual_type}', expected one of {expected_type}",+ )+ else:+ # Snowflake may return different type names, normalize for comparison+ self.assertEqual(+ actual_type.upper(),+ expected_type.upper(),+ f"Column '{expected_column}' has type '{actual_type}', expected '{expected_type}'",+ )++ # Check optional columns if they exist+ for optional_column, expected_type in optional_columns.items():+ if optional_column in table_info:+ actual_type = table_info[optional_column]+ if isinstance(expected_type, list):+ self.assertIn(+ actual_type.upper(),+ [t.upper() for t in expected_type],+ f"Optional column '{optional_column}' has type '{actual_type}', expected one of {expected_type}",+ )+ else:+ self.assertEqual(+ actual_type.upper(),+ expected_type.upper(),+ f"Optional column '{optional_column}' has type '{actual_type}', expected '{expected_type}'",+ )++ def create_mock_workflow(self) -> Mock:+ """Create a mock workflow object."""+ workflow = Mock()+ workflow.id = "test-workflow-id"+ return workflow++ def create_mock_workflow_log(self) -> Mock:+ """Create a mock workflow log object."""+ return Mock()++ def create_real_connector_instance(self) -> Mock:+ """Create connector instance using real Snowflake connector."""+ connector = Mock()+ connector.connector_id = SnowflakeDB.get_id()+ connector.connector_metadata = self.snowflake_config+ # Store reference to real connector for potential future use+ connector._real_snowflake_connector = self.snowflake_connector+ return connector++ def create_mock_endpoint(self, mock_connector_instance: Mock) -> Mock:+ """Create a mock endpoint with real database configuration."""+ endpoint = Mock()+ endpoint.connector_instance = mock_connector_instance+ endpoint.configuration = {+ DestinationKey.TABLE: self.test_table_name.split(".")[+ -1+ ], # Just table name+ DestinationKey.INCLUDE_AGENT: True,+ DestinationKey.INCLUDE_TIMESTAMP: True,+ DestinationKey.AGENT_NAME: "Unstract/DBWriter",+ DestinationKey.COLUMN_MODE: "Write JSON to a single column",+ DestinationKey.SINGLE_COLUMN_NAME: "data",+ DestinationKey.FILE_PATH: "file_path",+ DestinationKey.EXECUTION_ID: "execution_id",+ }+ return endpoint++ def create_destination_connector(+ self, mock_workflow: Mock, mock_workflow_log: Mock, mock_endpoint: Mock+ ) -> DestinationConnector:+ """Create a destination connector with mocked Django dependencies but real DB."""+ with patch(+ "workflow_manager.endpoint_v2.destination.UserContext"+ ) as mock_context:+ mock_context.get_organization_identifier.return_value = "test-org"++ with patch.object(+ DestinationConnector,+ "_get_endpoint_for_workflow",+ return_value=mock_endpoint,+ ):+ with patch.object(+ DestinationConnector,+ "_get_source_endpoint_for_workflow",+ return_value=mock_endpoint,+ ):+ connector = DestinationConnector(+ workflow=mock_workflow,+ execution_id="test-execution-id",+ workflow_log=mock_workflow_log,+ use_file_history=False,+ )+ return connector++ def test_insert_into_db_happy_path_snowflake(self) -> None:+ """Test successful insertion into real Snowflake database."""+ # Create mock objects for Django models+ mock_workflow = self.create_mock_workflow()+ mock_workflow_log = self.create_mock_workflow_log()+ mock_connector_instance = self.create_real_connector_instance()+ mock_endpoint = self.create_mock_endpoint(mock_connector_instance)++ # Create destination connector with real database connection+ destination_connector = self.create_destination_connector(+ mock_workflow, mock_workflow_log, mock_endpoint+ )++ # Test the insert_into_db method with real Snowflake+ try:+ # Mock only the methods that get data, let database operations be real+ with patch.object(+ destination_connector,+ "get_tool_execution_result",+ return_value=self.test_data,+ ):+ with patch.object(+ destination_connector,+ "get_combined_metadata",+ return_value=self.test_metadata,+ ):+ # This will execute real database operations using Snowflake connector+ destination_connector.insert_into_db(+ input_file_path=self.input_file_path, error=None+ )++ # Verify the table structure+ self.verify_table_columns(self.test_table_name)++ print("✅ Snowflake integration test completed successfully!")++ except Exception as e:+ print(f"❌ Snowflake integration test failed: {str(e)}")+ raise++ def test_insert_into_db_with_error_snowflake(self) -> None:+ """Test insertion with error parameter into real Snowflake database."""+ # Create mock objects+ mock_workflow = self.create_mock_workflow()+ mock_workflow_log = self.create_mock_workflow_log()+ mock_connector_instance = self.create_real_connector_instance()+ mock_endpoint = self.create_mock_endpoint(mock_connector_instance)++ # Create destination connector+ destination_connector = self.create_destination_connector(+ mock_workflow, mock_workflow_log, mock_endpoint+ )++ error_message = "Test processing error occurred"++ # Mock the methods that get data+ with patch.object(+ destination_connector,+ "get_tool_execution_result",+ return_value=self.test_data,+ ):+ with patch.object(+ destination_connector,+ "get_combined_metadata",+ return_value=self.test_metadata,+ ):+ # Execute with error parameter+ destination_connector.insert_into_db(+ input_file_path=self.input_file_path, error=error_message+ )++ # Verify the table structure+ self.verify_table_columns(self.test_table_name)++ print(+ f"✅ Successfully inserted error data into Snowflake table: {self.test_table_name}"+ )++ def test_snowflake_connector_connection(self) -> None:+ """Test that the Snowflake connector can establish a connection."""+ # Test the real Snowflake connector directly+ try:+ engine = self.snowflake_connector.get_engine()+ self.assertIsNotNone(engine)+ print("✅ Snowflake connector successfully established connection")++ # Test a simple query to verify connection works+ with engine.cursor() as cursor:+ cursor.execute("SELECT 1 as test_column")+ result = cursor.fetchall()+ self.assertIsNotNone(result)+ print("✅ Snowflake connector successfully executed test query")++ # Clean up+ if hasattr(engine, "close"):+ engine.close()+ except Exception as e:+ self.fail(f"Snowflake connector failed to connect: {str(e)}")++ def tearDown(self) -> None:+ """Clean up test resources."""+ # Optionally clean up test table+ # Note: Be careful with cleanup in production environments+ passdiff --git a/backend/workflow_manager/endpoint_v2/tests/test_database_utils/__init__.py b/backend/workflow_manager/endpoint_v2/tests/test_database_utils/__init__.pyindex e69de29bb..fca2b2401 100644--- a/backend/workflow_manager/endpoint_v2/tests/test_database_utils/__init__.py+++ b/backend/workflow_manager/endpoint_v2/tests/test_database_utils/__init__.py@@ -0,0 +1,3 @@+from backend.celery import app as celery_app++__all__ = ["celery_app"]diff --git a/backend/workflow_manager/endpoint_v2/tests/test_database_utils/base_test_db.py b/backend/workflow_manager/endpoint_v2/tests/test_database_utils/base_test_db.pydeleted file mode 100644index d83fc3a59..000000000--- a/backend/workflow_manager/endpoint_v2/tests/test_database_utils/base_test_db.py+++ /dev/null@@ -1,160 +0,0 @@-import datetime-import json-import os-from typing import Any--import pytest # type: ignore-from dotenv import load_dotenv--from unstract.connectors.databases.bigquery import BigQuery-from unstract.connectors.databases.mariadb import MariaDB-from unstract.connectors.databases.mssql import MSSQL-from unstract.connectors.databases.mysql import MySQL-from unstract.connectors.databases.postgresql import PostgreSQL-from unstract.connectors.databases.redshift import Redshift-from unstract.connectors.databases.snowflake import SnowflakeDB-from unstract.connectors.databases.unstract_db import UnstractDB--load_dotenv("test.env")---class BaseTestDB:- @pytest.fixture(autouse=True)- def base_setup(self) -> None:- self.postgres_creds = {- "user": os.getenv("DB_USER"),- "password": os.getenv("DB_PASSWORD"),- "host": os.getenv("DB_HOST"),- "port": os.getenv("DB_PORT"),- "database": os.getenv("DB_NAME"),- }- self.redshift_creds = {- "user": os.getenv("REDSHIFT_USER"),- "password": os.getenv("REDSHIFT_PASSWORD"),- "host": os.getenv("REDSHIFT_HOST"),- "port": os.getenv("REDSHIFT_PORT"),- "database": os.getenv("REDSHIFT_DB"),- }- self.snowflake_creds = {- "user": os.getenv("SNOWFLAKE_USER"),- "password": os.getenv("SNOWFLAKE_PASSWORD"),- "account": os.getenv("SNOWFLAKE_ACCOUNT"),- "role": os.getenv("SNOWFLAKE_ROLE"),- "database": os.getenv("SNOWFLAKE_DB"),- "schema": os.getenv("SNOWFLAKE_SCHEMA"),- "warehouse": os.getenv("SNOWFLAKE_WAREHOUSE"),- }- self.mssql_creds = {- "user": os.getenv("MSSQL_USER"),- "password": os.getenv("MSSQL_PASSWORD"),- "server": os.getenv("MSSQL_SERVER"),- "port": os.getenv("MSSQL_PORT"),- "database": os.getenv("MSSQL_DB"),- }- self.mysql_creds = {- "user": os.getenv("MYSQL_USER"),- "password": os.getenv("MYSQL_PASSWORD"),- "host": os.getenv("MYSQL_SERVER"),- "port": os.getenv("MYSQL_PORT"),- "database": os.getenv("MYSQL_DB"),- }- self.mariadb_creds = {- "user": os.getenv("MARIADB_USER"),- "password": os.getenv("MARIADB_PASSWORD"),- "host": os.getenv("MARIADB_SERVER"),- "port": os.getenv("MARIADB_PORT"),- "database": os.getenv("MARIADB_DB"),- }- self.database_entry = {- "created_by": "Unstract/DBWriter",- "created_at": datetime.datetime(2024, 5, 20, 7, 46, 57, 307998),- "data": '{"input_file": "simple.pdf", "result": "report"}',- }- valid_schema_name = "public"- invalid_schema_name = "public_1"- self.valid_postgres_creds = {**self.postgres_creds, "schema": valid_schema_name}- self.invalid_postgres_creds = {- **self.postgres_creds,- "schema": invalid_schema_name,- }- self.valid_redshift_creds = {**self.redshift_creds, "schema": valid_schema_name}- self.invalid_redshift_creds = {- **self.redshift_creds,- "schema": invalid_schema_name,- }- self.invalid_syntax_table_name = "invalid-syntax.name.test_output"- self.invalid_wrong_table_name = "database.schema.test_output"- self.valid_table_name = "test_output"- bigquery_json_str = os.getenv("BIGQUERY_CREDS", "{}")- self.bigquery_settings = json.loads(bigquery_json_str)- self.bigquery_settings["json_credentials"] = bigquery_json_str- self.valid_bigquery_table_name = "unstract.bigquery_test.bigquery_output"- self.invalid_snowflake_db = {**self.snowflake_creds, "database": "invalid"}- self.invalid_snowflake_schema = {**self.snowflake_creds, "schema": "invalid"}- self.invalid_snowflake_warehouse = {- **self.snowflake_creds,- "warehouse": "invalid",- }-- # Gets all valid db instances except- # Bigquery (table name needs to be writted separately for bigquery)- @pytest.fixture(- params=[- ("valid_postgres_creds", PostgreSQL),- ("snowflake_creds", SnowflakeDB),- ("mssql_creds", MSSQL),- ("mysql_creds", MySQL),- ("mariadb_creds", MariaDB),- ("valid_redshift_creds", Redshift),- ]- )- def valid_dbs_instance(self, request: Any) -> Any:- return self.get_db_instance(request=request)-- # Gets all valid db instances except:- # Bigquery (table name needs to be writted separately for bigquery)- # Redshift (can't process more than 64KB character type)- @pytest.fixture(- params=[- ("valid_postgres_creds", PostgreSQL),- ("snowflake_creds", SnowflakeDB),- ("mssql_creds", MSSQL),- ("mysql_creds", MySQL),- ("mariadb_creds", MariaDB),- ]- )- def valid_dbs_instance_to_handle_large_doc(self, request: Any) -> Any:- return self.get_db_instance(request=request)-- def get_db_instance(self, request: Any) -> UnstractDB:- creds_name, db_class = request.param- creds = getattr(self, creds_name)- if not creds:- pytest.fail(f"Unknown credentials: {creds_name}")- db_instance = db_class(settings=creds)- return db_instance-- # Gets all invalid-db instances for postgres, redshift:- @pytest.fixture(- params=[- ("invalid_postgres_creds", PostgreSQL),- ("invalid_redshift_creds", Redshift),- ]- )- def invalid_dbs_instance(self, request: Any) -> Any:- return self.get_db_instance(request=request)-- @pytest.fixture- def valid_bigquery_db_instance(self) -> Any:- return BigQuery(settings=self.bigquery_settings)-- # Gets all invalid-db instances for snowflake:- @pytest.fixture(- params=[- ("invalid_snowflake_db", SnowflakeDB),- ("invalid_snowflake_schema", SnowflakeDB),- ("invalid_snowflake_warehouse", SnowflakeDB),- ]- )- def invalid_snowflake_db_instance(self, request: Any) -> Any:- return self.get_db_instance(request=request)diff --git a/backend/workflow_manager/endpoint_v2/tests/test_database_utils/test_create_table_if_not_exists.py b/backend/workflow_manager/endpoint_v2/tests/test_database_utils/test_create_table_if_not_exists.pydeleted file mode 100644index cbf4f1619..000000000--- a/backend/workflow_manager/endpoint_v2/tests/test_database_utils/test_create_table_if_not_exists.py+++ /dev/null@@ -1,97 +0,0 @@-import pytest # type: ignore-from workflow_manager.endpoint_v2.database_utils import DatabaseUtils-from workflow_manager.endpoint_v2.exceptions import UnstractDBException--from unstract.connectors.databases.unstract_db import UnstractDB--from .base_test_db import BaseTestDB---class TestCreateTableIfNotExists(BaseTestDB):- def test_create_table_if_not_exists_valid(- self, valid_dbs_instance: UnstractDB- ) -> None:- engine = valid_dbs_instance.get_engine()- result = DatabaseUtils.create_table_if_not_exists(- db_class=valid_dbs_instance,- engine=engine,- table_name=self.valid_table_name,- database_entry=self.database_entry,- )- assert result is None-- def test_create_table_if_not_exists_bigquery_valid(- self, valid_bigquery_db_instance: UnstractDB- ) -> None:- engine = valid_bigquery_db_instance.get_engine()- result = DatabaseUtils.create_table_if_not_exists(- db_class=valid_bigquery_db_instance,- engine=engine,- table_name=self.valid_bigquery_table_name,- database_entry=self.database_entry,- )- assert result is None-- def test_create_table_if_not_exists_invalid_schema(- self, invalid_dbs_instance: UnstractDB- ) -> None:- engine = invalid_dbs_instance.get_engine()- with pytest.raises(UnstractDBException):- DatabaseUtils.create_table_if_not_exists(- db_class=invalid_dbs_instance,- engine=engine,- table_name=self.valid_table_name,- database_entry=self.database_entry,- )-- def test_create_table_if_not_exists_invalid_syntax(- self, valid_dbs_instance: UnstractDB- ) -> None:- engine = valid_dbs_instance.get_engine()- with pytest.raises(UnstractDBException):- DatabaseUtils.create_table_if_not_exists(- db_class=valid_dbs_instance,- engine=engine,- table_name=self.invalid_syntax_table_name,- database_entry=self.database_entry,- )-- def test_create_table_if_not_exists_wrong_table_name(- self, valid_dbs_instance: UnstractDB- ) -> None:- engine = valid_dbs_instance.get_engine()- with pytest.raises(UnstractDBException):- DatabaseUtils.create_table_if_not_exists(- db_class=valid_dbs_instance,- engine=engine,- table_name=self.invalid_wrong_table_name,- database_entry=self.database_entry,- )-- def test_create_table_if_not_exists_feature_not_supported(- self, invalid_dbs_instance: UnstractDB- ) -> None:- engine = invalid_dbs_instance.get_engine()- with pytest.raises(UnstractDBException):- DatabaseUtils.create_table_if_not_exists(- db_class=invalid_dbs_instance,- engine=engine,- table_name=self.invalid_wrong_table_name,- database_entry=self.database_entry,- )-- def test_create_table_if_not_exists_invalid_snowflake_db(- self, invalid_snowflake_db_instance: UnstractDB- ) -> None:- engine = invalid_snowflake_db_instance.get_engine()- with pytest.raises(UnstractDBException):- DatabaseUtils.create_table_if_not_exists(- db_class=invalid_snowflake_db_instance,- engine=engine,- table_name=self.invalid_wrong_table_name,- database_entry=self.database_entry,- )---if __name__ == "__main__":- pytest.main()diff --git a/backend/workflow_manager/endpoint_v2/tests/test_database_utils/test_execute_write_query.py b/backend/workflow_manager/endpoint_v2/tests/test_database_utils/test_execute_write_query.pydeleted file mode 100644index cb3686e4d..000000000--- a/backend/workflow_manager/endpoint_v2/tests/test_database_utils/test_execute_write_query.py+++ /dev/null@@ -1,167 +0,0 @@-import os-import uuid-from typing import Any--import pytest # type: ignore-from workflow_manager.endpoint_v2.database_utils import DatabaseUtils-from workflow_manager.endpoint_v2.exceptions import UnstractDBException--from unstract.connectors.databases.redshift import Redshift-from unstract.connectors.databases.unstract_db import UnstractDB--from .base_test_db import BaseTestDB---class TestExecuteWriteQuery(BaseTestDB):- @pytest.fixture(autouse=True)- def setup(self, base_setup: Any) -> None:- self.sql_columns_and_values = {- "created_by": "Unstract/DBWriter",- "created_at": "2024-05-20 10:36:25.362609",- "data": '{"input_file": "simple.pdf", "result": "report"}',- "id": str(uuid.uuid4()),- }-- def test_execute_write_query_valid(self, valid_dbs_instance: Any) -> None:- engine = valid_dbs_instance.get_engine()- result = DatabaseUtils.execute_write_query(- db_class=valid_dbs_instance,- engine=engine,- table_name=self.valid_table_name,- sql_keys=list(self.sql_columns_and_values.keys()),- sql_values=list(self.sql_columns_and_values.values()),- )- assert result is None-- def test_execute_write_query_invalid_schema(self, invalid_dbs_instance: Any) -> None:- engine = invalid_dbs_instance.get_engine()- with pytest.raises(UnstractDBException):- DatabaseUtils.execute_write_query(- db_class=invalid_dbs_instance,- engine=engine,- table_name=self.valid_table_name,- sql_keys=list(self.sql_columns_and_values.keys()),- sql_values=list(self.sql_columns_and_values.values()),- )-- def test_execute_write_query_invalid_syntax(self, valid_dbs_instance: Any) -> None:- engine = valid_dbs_instance.get_engine()- with pytest.raises(UnstractDBException):- DatabaseUtils.execute_write_query(- db_class=valid_dbs_instance,- engine=engine,- table_name=self.invalid_syntax_table_name,- sql_keys=list(self.sql_columns_and_values.keys()),- sql_values=list(self.sql_columns_and_values.values()),- )-- def test_execute_write_query_feature_not_supported(- self, invalid_dbs_instance: Any- ) -> None:- engine = invalid_dbs_instance.get_engine()- with pytest.raises(UnstractDBException):- DatabaseUtils.execute_write_query(- db_class=invalid_dbs_instance,- engine=engine,- table_name=self.invalid_wrong_table_name,- sql_keys=list(self.sql_columns_and_values.keys()),- sql_values=list(self.sql_columns_and_values.values()),- )-- def load_text_to_sql_values(self) -> dict[str, Any]:- file_path = os.path.join(os.path.dirname(__file__), "static", "large_doc.txt")- with open(file_path, encoding="utf-8") as file:- content = file.read()- sql_columns_and_values = self.sql_columns_and_values.copy()- sql_columns_and_values["data"] = content- return sql_columns_and_values-- @pytest.fixture- def valid_redshift_db_instance(self) -> Any:- return Redshift(self.valid_redshift_creds)-- def test_execute_write_query_datatype_too_large_redshift(- self, valid_redshift_db_instance: Any- ) -> None:- engine = valid_redshift_db_instance.get_engine()- sql_columns_and_values = self.load_text_to_sql_values()- with pytest.raises(UnstractDBException):- DatabaseUtils.execute_write_query(- db_class=valid_redshift_db_instance,- engine=engine,- table_name=self.valid_table_name,- sql_keys=list(sql_columns_and_values.keys()),- sql_values=list(sql_columns_and_values.values()),- )-- def test_execute_write_query_bigquery_valid(- self, valid_bigquery_db_instance: Any- ) -> None:- engine = valid_bigquery_db_instance.get_engine()- result = DatabaseUtils.execute_write_query(- db_class=valid_bigquery_db_instance,- engine=engine,- table_name=self.valid_bigquery_table_name,- sql_keys=list(self.sql_columns_and_values.keys()),- sql_values=list(self.sql_columns_and_values.values()),- )- assert result is None-- def test_execute_write_query_wrong_table_name(- self, valid_dbs_instance: UnstractDB- ) -> None:- engine = valid_dbs_instance.get_engine()- with pytest.raises(UnstractDBException):- DatabaseUtils.execute_write_query(- db_class=valid_dbs_instance,- engine=engine,- table_name=self.invalid_wrong_table_name,- sql_keys=list(self.sql_columns_and_values.keys()),- sql_values=list(self.sql_columns_and_values.values()),- )-- def test_execute_write_query_bigquery_large_doc(- self, valid_bigquery_db_instance: Any- ) -> None:- engine = valid_bigquery_db_instance.get_engine()- sql_columns_and_values = self.load_text_to_sql_values()- result = DatabaseUtils.execute_write_query(- db_class=valid_bigquery_db_instance,- engine=engine,- table_name=self.valid_bigquery_table_name,- sql_keys=list(sql_columns_and_values.keys()),- sql_values=list(sql_columns_and_values.values()),- )- assert result is None-- def test_execute_write_query_invalid_snowflake_db(- self, invalid_snowflake_db_instance: UnstractDB- ) -> None:- engine = invalid_snowflake_db_instance.get_engine()- with pytest.raises(UnstractDBException):- DatabaseUtils.execute_write_query(- db_class=invalid_snowflake_db_instance,- engine=engine,- table_name=self.invalid_wrong_table_name,- sql_keys=list(self.sql_columns_and_values.keys()),- sql_values=list(self.sql_columns_and_values.values()),- )-- # Make this function at last to cover all large doc- def test_execute_write_query_large_doc(- self, valid_dbs_instance_to_handle_large_doc: Any- ) -> None:- engine = valid_dbs_instance_to_handle_large_doc.get_engine()- sql_columns_and_values = self.load_text_to_sql_values()- result = DatabaseUtils.execute_write_query(- db_class=valid_dbs_instance_to_handle_large_doc,- engine=engine,- table_name=self.valid_table_name,- sql_keys=list(sql_columns_and_values.keys()),- sql_values=list(sql_columns_and_values.values()),- )- assert result is None---if __name__ == "__main__":- pytest.main()diff --git a/unstract/connectors/src/unstract/connectors/databases/bigquery/bigquery.py b/unstract/connectors/src/unstract/connectors/databases/bigquery/bigquery.pyindex 49d185cf7..2f42cd333 100644--- a/unstract/connectors/src/unstract/connectors/databases/bigquery/bigquery.py+++ b/unstract/connectors/src/unstract/connectors/databases/bigquery/bigquery.py@@ -2,6 +2,8 @@ import datetimeimport jsonimport loggingimport os+import uuid+from enum import Enumfrom typing import Anyimport google.api_core.exceptions@@ -59,9 +61,6 @@ class BigQuery(UnstractDB):def can_read() -> bool:return True- def get_string_type(self) -> str:- return "string"-def get_engine(self) -> Client:return bigquery.Client.from_service_account_info( # type: ignoreinfo=self.json_credentials@@ -121,8 +120,8 @@ class BigQuery(UnstractDB):f"user_field_1 BOOL DEFAULT FALSE, "f"user_field_2 INT64 DEFAULT 0, "f"user_field_3 STRING DEFAULT NULL, "- f"status STRING CHECK (status IN ('ERROR', 'SUCCESS')), "- f"error_message STRING"+ f"status STRING, "+ f"error_message STRING, ")return sql_query@@ -131,25 +130,29 @@ class BigQuery(UnstractDB):f"ALTER TABLE {table_name} "f"ADD COLUMN {column_name}_v2 JSON, "f"ADD COLUMN metadata JSON, "- f"ADD COLUMN user_field_1 BOOL DEFAULT FALSE, "- f"ADD COLUMN user_field_2 INT64 DEFAULT 0, "- f"ADD COLUMN user_field_3 STRING DEFAULT NULL, "- f"ADD COLUMN status STRING CHECK (status IN ('ERROR', 'SUCCESS')), "+ f"ADD COLUMN user_field_1 BOOL, "+ f"ADD COLUMN user_field_2 INT64, "+ f"ADD COLUMN user_field_3 STRING, "+ f"ADD COLUMN status STRING, "f"ADD COLUMN error_message STRING")return sql_query@staticmethod- def get_sql_insert_query(table_name: str, sql_keys: list[str]) -> str:+ def get_sql_insert_query(+ table_name: str, sql_keys: list[str], sql_values: list[str] = None+ ) -> str:"""Function to generate parameterised insert sql query.Args:table_name (str): db-connector table namesql_keys (list[str]): column names+ sql_values (list[str], optional): SQL values for database-specific handlingReturns:str: returns a string with parameterised insert sql query"""+ # BigQuery uses @ parameterization, ignore sql_values for nowkeys_str = ",".join(sql_keys)values_placeholder = ",".join(["@" + key for key in sql_keys])return f"INSERT INTO {table_name} ({keys_str}) VALUES ({values_placeholder})"@@ -172,15 +175,44 @@ class BigQuery(UnstractDB):table_name = kwargs.get("table_name", None)if table_name is None:raise ValueError("Please enter a valid table_name to to create/insert table")+sql_keys = list(kwargs.get("sql_keys", []))+ column_types = self.get_information_schema(table_name=table_name)+try:if sql_values:- query_parameters = [- bigquery.ScalarQueryParameter(key, "STRING", value)- for key, value in zip(sql_keys, sql_values, strict=False)- ]+ query_parameters = []+ # Modify SQL query to use PARSE_JSON for JSON columns+ modified_sql = sql_query++ for key, value in zip(sql_keys, sql_values, strict=False):+ column_type = column_types.get(key.lower(), "").upper()++ if isinstance(value, (dict, list)) and column_type == "JSON":+ # For JSON objects in JSON columns, convert to string and use PARSE_JSON+ json_str = json.dumps(value) if value else None+ if json_str:+ # Replace @key with PARSE_JSON(@key) in the SQL query+ modified_sql = modified_sql.replace(+ f"@{key}", f"PARSE_JSON(@{key})"+ )+ query_parameters.append(+ bigquery.ScalarQueryParameter(key, "STRING", json_str)+ )+ elif isinstance(value, (dict, list)):+ # For dict/list values in STRING columns, serialize to JSON string+ json_str = json.dumps(value) if value else None+ query_parameters.append(+ bigquery.ScalarQueryParameter(key, "STRING", json_str)+ )+ else:+ # For other values, use STRING as before+ query_parameters.append(+ bigquery.ScalarQueryParameter(key, "STRING", value)+ )+query_params = bigquery.QueryJobConfig(query_parameters=query_parameters)- query_job = engine.query(sql_query, job_config=query_params)+ query_job = engine.query(modified_sql, job_config=query_params)else:query_job = engine.query(sql_query)query_job.result()@@ -232,7 +264,52 @@ class BigQuery(UnstractDB):f"table_name = '{table}'")results = self.execute(query=query)++ # If table doesn't exist, execute returns None+ if results is None:+ logger.info(f"Table {table_name} does not exist, returning empty schema")+ return {}++ # Process the schema resultscolumn_types: dict[str, str] = self.get_db_column_types(columns_with_types=results)return column_types++ def get_sql_values_for_query(+ self, values: dict[str, Any], column_types: dict[str, str]+ ) -> dict[str, str]:+ """Prepare SQL values for BigQuery queries with JSON column support.++ Args:+ values (dict[str, Any]): dictionary of columns and values+ column_types (dict[str, str]): types of columns from database schema++ Returns:+ dict[str, str]: Dictionary of column names to SQL values for parameterized queries+ """+ sql_values: dict[str, Any] = {}+ for column in values:+ value = values[column]+ if isinstance(value, (dict, list)):+ # For BigQuery, keep dict/list objects as-is for JSON columns+ sql_values[column] = value+ elif isinstance(value, str):+ # Try to parse JSON strings back to objects for BigQuery+ try:+ parsed_value = json.loads(value)+ sql_values[column] = parsed_value+ except (TypeError, ValueError, json.JSONDecodeError):+ # Not a JSON string, keep as string+ sql_values[column] = f"{value}"+ elif isinstance(value, Enum):+ sql_values[column] = value.value+ else:+ sql_values[column] = f"{value}"++ # If table has a column 'id', unstract inserts a unique value to it+ if any(key in column_types for key in ["id", "ID"]):+ uuid_id = str(uuid.uuid4())+ sql_values["id"] = f"{uuid_id}"++ return sql_valuesdiff --git a/unstract/connectors/src/unstract/connectors/databases/mariadb/mariadb.py b/unstract/connectors/src/unstract/connectors/databases/mariadb/mariadb.pyindex 753336d54..06e2421fa 100644--- a/unstract/connectors/src/unstract/connectors/databases/mariadb/mariadb.py+++ b/unstract/connectors/src/unstract/connectors/databases/mariadb/mariadb.py@@ -59,7 +59,7 @@ class MariaDB(UnstractDB, MysqlHandler):)return con- def sql_to_db_mapping(self, value: str) -> str:+ def sql_to_db_mapping(self, value: Any) -> str:return str(MysqlHandler.sql_to_db_mapping(value=value))def execute_query(@@ -74,3 +74,51 @@ class MariaDB(UnstractDB, MysqlHandler):host=self.host,table_name=table_name,)++ def get_create_table_base_query(self, table: str) -> str:+ """Function to create a base create table sql query with MySQL specific types.++ Args:+ table (str): db-connector table name++ Returns:+ str: generates a create sql base query with the constant columns+ """+ sql_query = (+ f"CREATE TABLE IF NOT EXISTS {table} "+ f"(id LONGTEXT, "+ f"created_by LONGTEXT, created_at TIMESTAMP, "+ f"metadata JSON, "+ f"user_field_1 BOOLEAN DEFAULT FALSE, "+ f"user_field_2 BIGINT DEFAULT 0, "+ f"user_field_3 LONGTEXT DEFAULT NULL, "+ f"status ENUM('ERROR', 'SUCCESS'), "+ f"error_message LONGTEXT, "+ )+ return sql_query++ def get_information_schema(self, table_name: str) -> dict[str, str]:+ """Get information schema for MySQL database."""+ query = (+ "SELECT column_name, data_type FROM "+ "information_schema.columns WHERE "+ f"UPPER(table_name) = UPPER('{table_name}') AND table_schema = '{self.database}'"+ )+ results = self.execute(query=query)+ column_types: dict[str, str] = self.get_db_column_types(+ columns_with_types=results+ )+ return column_types++ def prepare_multi_column_migration(self, table_name: str, column_name: str) -> str:+ sql_query = (+ f"ALTER TABLE {table_name} "+ f"ADD COLUMN {column_name}_v2 JSON, "+ f"ADD COLUMN metadata JSON, "+ f"ADD COLUMN user_field_1 BOOLEAN DEFAULT FALSE, "+ f"ADD COLUMN user_field_2 BIGINT DEFAULT 0, "+ f"ADD COLUMN user_field_3 LONGTEXT DEFAULT NULL, "+ f"ADD COLUMN status ENUM('ERROR', 'SUCCESS'), "+ f"ADD COLUMN error_message LONGTEXT"+ )+ return sql_querydiff --git a/unstract/connectors/src/unstract/connectors/databases/mssql/mssql.py b/unstract/connectors/src/unstract/connectors/databases/mssql/mssql.pyindex 9dc426d62..c4ec0b1ab 100644--- a/unstract/connectors/src/unstract/connectors/databases/mssql/mssql.py+++ b/unstract/connectors/src/unstract/connectors/databases/mssql/mssql.py@@ -58,9 +58,6 @@ class MSSQL(UnstractDB):def can_read() -> bool:return True- def get_string_type(self) -> str:- return "text"-def sql_to_db_mapping(self, value: str) -> str:"""Gets the python datatype of value and converts python datatype tocorresponding DB datatype.@@ -131,22 +128,28 @@ class MSSQL(UnstractDB):f"user_field_2 INT DEFAULT 0, "f"user_field_3 NVARCHAR(MAX) DEFAULT NULL, "f"status NVARCHAR(10) CHECK (status IN ('ERROR', 'SUCCESS')), "- f"error_message NVARCHAR(MAX))"+ f"error_message NVARCHAR(MAX), ")return sql_query- def prepare_multi_column_migration(self, table_name: str, column_name: str) -> str:- sql_query = (- f"ALTER TABLE {table_name} "- f"ADD {column_name}_v2 NVARCHAR(MAX), "- + "metadata NVARCHAR(MAX), "- + "user_field_1 BIT DEFAULT 0, "- + "user_field_2 INT DEFAULT 0, "- + "user_field_3 NVARCHAR(MAX) DEFAULT NULL, "- + "status NVARCHAR(10) CHECK (status IN ('ERROR', 'SUCCESS')), "- + "error_message NVARCHAR(MAX)"- )- return sql_query+ def prepare_multi_column_migration(+ self, table_name: str, column_name: str+ ) -> list[str]:+ """Returns a list of ALTER TABLE statements for MSSQL column migration.++ MSSQL doesn't support adding multiple columns in a single ALTER TABLE statement,+ so we return a list of individual statements like Snowflake.+ """+ sql_statements = [+ f"ALTER TABLE {table_name} ADD {column_name}_v2 NVARCHAR(MAX)",+ f"ALTER TABLE {table_name} ADD metadata NVARCHAR(MAX)",+ f"ALTER TABLE {table_name} ADD user_field_1 BIT DEFAULT 0",+ f"ALTER TABLE {table_name} ADD user_field_2 INT DEFAULT 0",+ f"ALTER TABLE {table_name} ADD user_field_3 NVARCHAR(MAX) DEFAULT NULL",+ f"ALTER TABLE {table_name} ADD status NVARCHAR(10)",+ f"ALTER TABLE {table_name} ADD error_message NVARCHAR(MAX)",+ ]+ return sql_statementsdef execute_query(self, engine: Any, sql_query: str, sql_values: Any, **kwargs: Anydiff --git a/unstract/connectors/src/unstract/connectors/databases/mysql/mysql.py b/unstract/connectors/src/unstract/connectors/databases/mysql/mysql.pyindex deb666c11..4fcd2e8f9 100644--- a/unstract/connectors/src/unstract/connectors/databases/mysql/mysql.py+++ b/unstract/connectors/src/unstract/connectors/databases/mysql/mysql.py@@ -52,6 +52,20 @@ class MySQL(UnstractDB, MysqlHandler):def get_string_type(self) -> str:return "longtext"+ def get_information_schema(self, table_name: str) -> dict[str, str]:+ """Get information schema for MySQL database."""+ # Try case-insensitive search since MySQL table names can be case-sensitive+ query = (+ "SELECT column_name, data_type FROM "+ "information_schema.columns WHERE "+ f"UPPER(table_name) = UPPER('{table_name}') AND table_schema = '{self.database}'"+ )+ results = self.execute(query=query)+ column_types: dict[str, str] = self.get_db_column_types(+ columns_with_types=results+ )+ return column_types+def get_engine(self) -> Connection: # type: ignore[type-arg]con = pymysql.connect(host=self.host,@@ -62,7 +76,8 @@ class MySQL(UnstractDB, MysqlHandler):)return con- def sql_to_db_mapping(self, value: str) -> str:+ def sql_to_db_mapping(self, value: Any) -> str:+ """Override to handle JSON columns correctly for MySQL."""return str(MysqlHandler.sql_to_db_mapping(value=value))def get_create_table_base_query(self, table: str) -> str:diff --git a/unstract/connectors/src/unstract/connectors/databases/oracle_db/oracle_db.py b/unstract/connectors/src/unstract/connectors/databases/oracle_db/oracle_db.pyindex 6c1592995..f20dc2fa7 100644--- a/unstract/connectors/src/unstract/connectors/databases/oracle_db/oracle_db.py+++ b/unstract/connectors/src/unstract/connectors/databases/oracle_db/oracle_db.py@@ -1,4 +1,5 @@import datetime+import loggingimport osfrom typing import Any@@ -7,6 +8,8 @@ from oracledb.connection import Connectionfrom unstract.connectors.databases.unstract_db import UnstractDB+logger = logging.getLogger(__name__)+class OracleDB(UnstractDB):def __init__(self, settings: dict[str, Any]):@@ -59,9 +62,6 @@ class OracleDB(UnstractDB):def can_read() -> bool:return True- def get_string_type(self) -> str:- return "clob"-def get_engine(self) -> Connection:con = oracledb.connect(config_dir=self.config_dir,@@ -85,14 +85,14 @@ class OracleDB(UnstractDB):"""python_type = type(value)mapping = {- str: "CLOB",+ str: "VARCHAR2(32767)",int: "NUMBER",float: "LONG",datetime.datetime: "TIMESTAMP",- dict: "CLOB CHECK (IS JSON)",- list: "CLOB CHECK (IS JSON)",+ dict: "CLOB",+ list: "CLOB",}- return mapping.get(python_type, "CLOB")+ return mapping.get(python_type, "VARCHAR2(32767)")def get_create_table_base_query(self, table: str) -> str:"""Function to create a base create table sql query.@@ -104,38 +104,53 @@ class OracleDB(UnstractDB):str: generates a create sql base query with the constant columns"""sql_query = (- f"CREATE TABLE IF NOT EXISTS {table} "+ f"CREATE TABLE {table} "f"(id VARCHAR2(32767) , "f"created_by VARCHAR2(32767), created_at TIMESTAMP, "- f"metadata CLOB CHECK (metadata IS JSON), "+ f"metadata CLOB, "f"user_field_1 NUMBER(1) DEFAULT 0, "f"user_field_2 NUMBER DEFAULT 0, "f"user_field_3 VARCHAR2(32767) DEFAULT NULL, "- f"status VARCHAR2(10) CHECK (status IN ('ERROR', 'SUCCESS')), "+ f"status VARCHAR2(10), "f"error_message VARCHAR2(32767), ")return sql_query- def prepare_multi_column_migration(self, table_name: str, column_name: str) -> str:- sql_query = (- f"ALTER TABLE {table_name} "- f"ADD {column_name}_v2 CLOB CHECK ({column_name}_v2 IS JSON), "- f"ADD metadata CLOB CHECK (metadata IS JSON), "- f"ADD user_field_1 NUMBER(1) DEFAULT 0, "- f"ADD user_field_2 NUMBER DEFAULT 0, "- f"ADD user_field_3 VARCHAR2(32767) DEFAULT NULL, "- f"ADD status VARCHAR2(10) CHECK (status IN ('ERROR', 'SUCCESS')), "- f"ADD error_message VARCHAR2(32767)"- )- return sql_query+ def prepare_multi_column_migration(self, table_name: str, column_name: str) -> list:+ """Prepare ALTER TABLE statements for adding new columns to an existing table.++ Args:+ table_name (str): The name of the table to alter+ column_name (str): The base name of the column to add a _v2 version for++ Returns:+ list: List of ALTER TABLE statements, one per column addition++ Note:+ Oracle does not support multiple ADD clauses in a single ALTER TABLE statement.+ Each column addition requires a separate ALTER TABLE statement.+ """+ # Return one ALTER statement per column for Oracle compatibility+ return [+ f"ALTER TABLE {table_name} ADD {column_name}_v2 CLOB",+ f"ALTER TABLE {table_name} ADD metadata CLOB",+ f"ALTER TABLE {table_name} ADD user_field_1 NUMBER(1) DEFAULT 0",+ f"ALTER TABLE {table_name} ADD user_field_2 NUMBER DEFAULT 0",+ f"ALTER TABLE {table_name} ADD user_field_3 VARCHAR2(32767) DEFAULT NULL",+ f"ALTER TABLE {table_name} ADD status VARCHAR2(10)",+ f"ALTER TABLE {table_name} ADD error_message VARCHAR2(32767)",+ ]@staticmethod- def get_sql_insert_query(table_name: str, sql_keys: list[str]) -> str:+ def get_sql_insert_query(+ table_name: str, sql_keys: list[str], sql_values: list[str] | None = None+ ) -> str:"""Function to generate parameterised insert sql query.Args:table_name (str): db-connector table namesql_keys (list[str]): column names+ sql_values (list[str], optional): SQL values for database-specific handling (ignored for Oracle)Returns:str: returns a string with parameterised insert sql query@@ -152,7 +167,7 @@ class OracleDB(UnstractDB):def execute_query(self, engine: Any, sql_query: str, sql_values: Any, **kwargs: Any) -> None:- """Executes create/insert query.+ """Executes create/insert query with Oracle-specific error handling.Args:engine (Any): oracle db client engine@@ -161,12 +176,19 @@ class OracleDB(UnstractDB):"""sql_keys = list(kwargs.get("sql_keys", []))with engine.cursor() as cursor:- if sql_values:- params = dict(zip(sql_keys, sql_values, strict=False))- cursor.execute(sql_query, params)- else:- cursor.execute(sql_query)- engine.commit()+ try:+ if sql_values:+ params = dict(zip(sql_keys, sql_values, strict=False))+ cursor.execute(sql_query, params)+ else:+ cursor.execute(sql_query)+ engine.commit()+ except oracledb.DatabaseError as e:+ logger.debug(f"Oracle database error occurred: {str(e)}")+ if "ORA-00955" in str(e):+ logger.info("Table already exists (ORA-00955) for oracle-db")+ else:+ raisedef get_information_schema(self, table_name: str) -> dict[str, str]:"""Function to generate information schema of the big query table.diff --git a/unstract/connectors/src/unstract/connectors/databases/postgresql/postgresql.py b/unstract/connectors/src/unstract/connectors/databases/postgresql/postgresql.pyindex aba40f70e..21bee6a20 100644--- a/unstract/connectors/src/unstract/connectors/databases/postgresql/postgresql.py+++ b/unstract/connectors/src/unstract/connectors/databases/postgresql/postgresql.py@@ -67,9 +67,6 @@ class PostgreSQL(UnstractDB, PsycoPgHandler):def can_read() -> bool:return True- def get_string_type(self) -> str:- return "text"-def sql_to_db_mapping(self, value: str) -> str:"""Gets the python datatype of value and converts python datatype tocorresponding DB datatype.diff --git a/unstract/connectors/src/unstract/connectors/databases/redshift/redshift.py b/unstract/connectors/src/unstract/connectors/databases/redshift/redshift.pyindex 13dc31af7..d50764d3f 100644--- a/unstract/connectors/src/unstract/connectors/databases/redshift/redshift.py+++ b/unstract/connectors/src/unstract/connectors/databases/redshift/redshift.py@@ -53,9 +53,6 @@ class Redshift(UnstractDB, PsycoPgHandler):def can_read() -> bool:return True- def get_string_type(self) -> str:- return "character varying"-def get_engine(self) -> connection:return psycopg2.connect(host=self.host,@@ -69,14 +66,14 @@ class Redshift(UnstractDB, PsycoPgHandler):def sql_to_db_mapping(self, value: str) -> str:python_type = type(value)mapping = {- str: "VARCHAR(MAX)",+ str: "VARCHAR(65535)",int: "BIGINT",float: "DOUBLE PRECISION",datetime.datetime: "TIMESTAMP",- dict: "VARCHAR(MAX)",- list: "VARCHAR(MAX)",+ dict: "SUPER",+ list: "SUPER",}- return mapping.get(python_type, "VARCHAR(MAX)")+ return mapping.get(python_type, "VARCHAR(65535)")def get_create_table_base_query(self, table: str) -> str:sql_query = (@@ -87,7 +84,7 @@ class Redshift(UnstractDB, PsycoPgHandler):f"user_field_1 BOOLEAN DEFAULT FALSE, "f"user_field_2 INTEGER DEFAULT 0, "f"user_field_3 VARCHAR(65535) DEFAULT NULL, "- f"status VARCHAR(10) CHECK (status IN ('ERROR', 'SUCCESS')), "+ f"status VARCHAR(256), "f"error_message VARCHAR(65535), ")return sql_query@@ -109,12 +106,12 @@ class Redshift(UnstractDB, PsycoPgHandler):"""# Return one ALTER statement per column for Redshift compatibilityreturn [- f"ALTER TABLE {table_name} ADD COLUMN {column_name}_v2 VARCHAR(65535);",- f"ALTER TABLE {table_name} ADD COLUMN metadata VARCHAR(65535);",+ f"ALTER TABLE {table_name} ADD COLUMN {column_name}_v2 SUPER;",+ f"ALTER TABLE {table_name} ADD COLUMN metadata SUPER;",f"ALTER TABLE {table_name} ADD COLUMN user_field_1 BOOLEAN DEFAULT FALSE;",f"ALTER TABLE {table_name} ADD COLUMN user_field_2 INTEGER DEFAULT 0;",f"ALTER TABLE {table_name} ADD COLUMN user_field_3 VARCHAR(65535) DEFAULT NULL;",- f"ALTER TABLE {table_name} ADD COLUMN status VARCHAR(10) CHECK (status IN ('ERROR', 'SUCCESS'));",+ f"ALTER TABLE {table_name} ADD COLUMN status VARCHAR(256);",f"ALTER TABLE {table_name} ADD COLUMN error_message VARCHAR(65535);",]diff --git a/unstract/connectors/src/unstract/connectors/databases/snowflake/snowflake.py b/unstract/connectors/src/unstract/connectors/databases/snowflake/snowflake.pyindex 49dc42932..62167bdc7 100644--- a/unstract/connectors/src/unstract/connectors/databases/snowflake/snowflake.py+++ b/unstract/connectors/src/unstract/connectors/databases/snowflake/snowflake.py@@ -1,6 +1,9 @@import datetime+import jsonimport loggingimport os+import uuid+from enum import Enumfrom typing import Anyimport snowflake.connector@@ -9,6 +12,7 @@ from snowflake.connector.connection import SnowflakeConnectionfrom unstract.connectors.databases.exceptions import SnowflakeProgrammingExceptionfrom unstract.connectors.databases.unstract_db import UnstractDB+from unstract.connectors.exceptions import ConnectorErrorlogger = logging.getLogger(__name__)@@ -56,9 +60,6 @@ class SnowflakeDB(UnstractDB):def can_read() -> bool:return True- def get_string_type(self) -> str:- return "text"-def sql_to_db_mapping(self, value: str) -> str:"""Gets the python datatype of value and converts python datatype tocorresponding DB datatype.@@ -95,49 +96,67 @@ class SnowflakeDB(UnstractDB):def get_create_table_base_query(self, table: str) -> str:sql_query = (- f"CREATE TABLE {table} IF NOT EXISTS "+ f"CREATE TABLE IF NOT EXISTS {table} "f"(id TEXT ,"f"created_by TEXT, created_at TIMESTAMP, "f"metadata VARIANT, "f"user_field_1 BOOLEAN DEFAULT FALSE, "f"user_field_2 INT DEFAULT 0, "f"user_field_3 TEXT DEFAULT NULL, "- f"status VARCHAR CHECK (status IN ('ERROR', 'SUCCESS')), "+ f"status VARCHAR, "f"error_message VARCHAR, ")return sql_query- def prepare_multi_column_migration(self, table_name: str, column_name: str) -> str:- sql_query = (- f"ALTER TABLE {table_name} "- f"ADD COLUMN {column_name}_v2 VARIANT, "- f"ADD COLUMN metadata VARIANT, "- f"ADD COLUMN user_field_1 BOOLEAN DEFAULT FALSE, "- f"ADD COLUMN user_field_2 INT DEFAULT 0, "- f"ADD COLUMN user_field_3 TEXT DEFAULT NULL, "- f"ADD COLUMN status VARCHAR CHECK (status IN ('ERROR', 'SUCCESS')), "- f"ADD COLUMN error_message VARCHAR"- )- return sql_query+ def prepare_multi_column_migration(self, table_name: str, column_name: str) -> list:+ """Returns a list of separate ALTER TABLE statements for Snowflake.++ Snowflake doesn't support multiple ADD COLUMN clauses in a single statement,+ so we return a list of individual ALTER TABLE statements.+ """+ sql_statements = [+ f"ALTER TABLE {table_name} ADD COLUMN {column_name}_v2 VARIANT",+ f"ALTER TABLE {table_name} ADD COLUMN metadata VARIANT",+ f"ALTER TABLE {table_name} ADD COLUMN user_field_1 BOOLEAN DEFAULT FALSE",+ f"ALTER TABLE {table_name} ADD COLUMN user_field_2 INT DEFAULT 0",+ f"ALTER TABLE {table_name} ADD COLUMN user_field_3 TEXT DEFAULT NULL",+ f"ALTER TABLE {table_name} ADD COLUMN status VARCHAR",+ f"ALTER TABLE {table_name} ADD COLUMN error_message VARCHAR",+ ]+ return sql_statementsdef execute_query(self, engine: Any, sql_query: str, sql_values: Any, **kwargs: Any) -> None:table_name = kwargs.get("table_name", None)+ logger.debug(f"Snowflake execute_query called with sql_query: {sql_query}")+ logger.debug(f"sql_values: {sql_values}")+try:with engine.cursor() as cursor:- if sql_values:+ # Check if the SQL query is already complete (contains SELECT)+ if "SELECT" in sql_query.upper():+ # Complete SQL query - execute directly+ logger.debug("Executing complete SQL query")+ cursor.execute(sql_query)+ elif sql_values:+ # Parameterized query - execute with values+ logger.debug("Executing parameterized query")cursor.execute(sql_query, sql_values)else:+ # Direct SQL execution (used for DDL statements)+ logger.debug("Executing SQL directly (no parameters)")cursor.execute(sql_query)engine.commit()except SnowflakeError.ProgrammingError as e:logger.error(- f"snowflake programming error in crearing/inserting table: "+ f"snowflake programming error in creating/inserting table: "f"{e.msg} {e.errno}")+ logger.error(f"SQL Query: {sql_query}")+ logger.error(f"SQL Values: {sql_values}")raise SnowflakeProgrammingException(- detail=e.msg,+ detail=f"{e.msg} | SQL: {sql_query} | Values: {sql_values}",database=self.database,schema=self.schema,table_name=table_name,@@ -146,7 +165,176 @@ class SnowflakeDB(UnstractDB):def get_information_schema(self, table_name: str) -> dict[str, str]:query = f"describe table {table_name}"column_types: dict[str, str] = {}- results = self.execute(query=query)- for column in results:- column_types[column[0].lower()] = column[1].split("(")[0]+ try:+ results = self.execute(query=query)+ if results:+ for column in results:+ column_types[column[0].lower()] = column[1].split("(")[0]+ except (SnowflakeError.ProgrammingError, ConnectorError) as e:+ # Handle table not found errors gracefully+ error_str = str(e).lower()+ if (+ "002003" in error_str+ or "does not exist" in error_str+ or "not authorized" in error_str+ ):+ logger.info(f"Table {table_name} does not exist or not authorized: {e}")+ return {}+ else:+ # Re-raise other programming/connector errors+ raisereturn column_types++ def get_sql_values_for_query(+ self, values: dict[str, Any], column_types: dict[str, str]+ ) -> dict[str, str]:+ """Prepare SQL values for Snowflake queries with VARIANT column support.++ Args:+ values (dict[str, Any]): dictionary of columns and values+ column_types (dict[str, str]): types of columns from database schema++ Returns:+ dict[str, str]: Dictionary of column names to SQL values or SQL fragments++ Note:+ For VARIANT columns, this returns SQL fragments like PARSE_JSON('...')+ instead of parameterized values, since Snowflake needs special handling+ for JSON data in VARIANT columns.+ """+ sql_values: dict[str, Any] = {}+ has_variant_columns = any(+ column_types.get(col.lower(), "").upper() == "VARIANT"+ for col in values.keys()+ )++ for column in values:+ value = values[column]+ col = column.lower()+ type_x = column_types.get(col, "")++ if isinstance(value, Enum):+ if has_variant_columns and type_x.upper() == "VARIANT":+ # For VARIANT Enum values, create SQL fragment+ escaped_value = str(value.value).replace("'", "''")+ sql_values[column] = f"'{escaped_value}'"+ else:+ # For non-VARIANT Enum values, create SQL fragment when we have VARIANT columns+ if has_variant_columns:+ escaped_value = str(value.value).replace("'", "''")+ sql_values[column] = f"'{escaped_value}'"+ else:+ sql_values[column] = value.value+ elif isinstance(value, (dict, list)):+ # For dict/list values, check if this is a VARIANT column+ try:+ json_str = json.dumps(value)+ if has_variant_columns and type_x.upper() == "VARIANT":+ # For VARIANT columns, return SQL fragment with PARSE_JSON+ escaped_value = json_str.replace("'", "''")+ sql_values[column] = f"PARSE_JSON('{escaped_value}')"+ else:+ # For non-VARIANT columns with VARIANT columns present, create quoted SQL fragment+ if has_variant_columns:+ escaped_value = json_str.replace("'", "''")+ sql_values[column] = f"'{escaped_value}'"+ else:+ # No VARIANT columns, use regular parameterization+ sql_values[column] = json_str+ except (TypeError, ValueError) as e:+ logger.error(+ f"Failed to serialize value to JSON for column {column}: {e}"+ )+ fallback_value = {+ "error": "JSON serialization failed",+ "error_type": e.__class__.__name__,+ "error_message": str(e),+ "data_type": str(type(value)),+ "data_description": f"column_{column}",+ "timestamp": datetime.datetime.now().isoformat(),+ }+ fallback_json = json.dumps(fallback_value)+ if has_variant_columns and type_x.upper() == "VARIANT":+ escaped_value = fallback_json.replace("'", "''")+ sql_values[column] = f"PARSE_JSON('{escaped_value}')"+ else:+ # For non-VARIANT columns with VARIANT columns present, create quoted SQL fragment+ if has_variant_columns:+ escaped_value = fallback_json.replace("'", "''")+ sql_values[column] = f"'{escaped_value}'"+ else:+ # No VARIANT columns, use regular parameterization+ sql_values[column] = fallback_json+ elif type_x.upper() == "VARIANT" and isinstance(value, str):+ if has_variant_columns:+ try:+ # For VARIANT columns with string values, validate if it's already valid JSON+ json.loads(value)+ escaped_value = value.replace("'", "''")+ sql_values[column] = f"PARSE_JSON('{escaped_value}')"+ except (json.JSONDecodeError, TypeError):+ # Not JSON, convert to JSON string and create SQL fragment+ json_str = json.dumps(value)+ escaped_value = json_str.replace("'", "''")+ sql_values[column] = f"PARSE_JSON('{escaped_value}')"+ else:+ # No VARIANT columns, use regular parameterization+ try:+ json.loads(value)+ sql_values[column] = value+ except (json.JSONDecodeError, TypeError):+ sql_values[column] = json.dumps(value)+ else:+ # All other values+ if has_variant_columns:+ # When we have VARIANT columns, create SQL fragments for consistency+ if value is None:+ sql_values[column] = "NULL"+ else:+ escaped_value = str(value).replace("'", "''")+ sql_values[column] = f"'{escaped_value}'"+ else:+ # No VARIANT columns, use regular parameterization+ sql_values[column] = f"{value}"++ # If table has a column 'id', unstract inserts a unique value to it+ if any(key in column_types for key in ["id", "ID"]):+ uuid_id = str(uuid.uuid4())+ if has_variant_columns:+ sql_values["id"] = f"'{uuid_id}'"+ else:+ sql_values["id"] = uuid_id++ return sql_values++ def get_sql_insert_query(+ self, table_name: str, sql_keys: list[str], sql_values: list[str] = None+ ) -> str:+ """Generate SQL insert query for Snowflake with special handling for VARIANT columns.++ Args:+ table_name (str): db-connector table name+ sql_keys (list[str]): column names+ sql_values (list[str], optional): SQL values, may contain fragments like PARSE_JSON(...)++ Returns:+ str: Complete SQL insert query with VARIANT columns handled appropriately+ """+ keys_str = ",".join(sql_keys)++ if sql_values:+ # Check if we have SQL fragments that need SELECT format+ has_sql_fragments = any(+ isinstance(v, str)+ and ("PARSE_JSON(" in v or "NULL" == v or v.startswith("'"))+ for v in sql_values+ )++ if has_sql_fragments:+ # Build complete SQL with SELECT format for VARIANT columns+ values_str = ",".join(str(v) for v in sql_values)+ return f"INSERT INTO {table_name} ({keys_str}) SELECT {values_str}"++ # Fall back to parameterized format for standard queries+ values_placeholder = ",".join(["%s" for _ in sql_keys])+ return f"INSERT INTO {table_name} ({keys_str}) VALUES ({values_placeholder})"diff --git a/unstract/connectors/src/unstract/connectors/databases/unstract_db.py b/unstract/connectors/src/unstract/connectors/databases/unstract_db.pyindex 74a5ad6e6..95298d1f8 100644--- a/unstract/connectors/src/unstract/connectors/databases/unstract_db.py+++ b/unstract/connectors/src/unstract/connectors/databases/unstract_db.py@@ -1,6 +1,9 @@import datetime+import jsonimport logging+import uuidfrom abc import ABC, abstractmethod+from enum import Enumfrom typing import Anyfrom workflow_manager.endpoint_v2.constants import TableColumns@@ -65,28 +68,6 @@ class UnstractDB(UnstractConnector, ABC):def python_social_auth_backend() -> str:return ""- @abstractmethod- def get_string_type(self) -> str:- """Child classes implement this to return the string type name for their DB."""- pass-- @abstractmethod- def prepare_multi_column_migration(- self, table_name: str, column_name: str- ) -> str | list:- """Returns the ALTER TABLE query specific to the database.-- Args:- table_name (str): The name of the table to alter- column_name (str): The base name of the column to add a _v2 version for-- Returns:- str | list: Either a single SQL ALTER TABLE statement (str) or- a list of separate ALTER TABLE statements for databases- that don't support multiple column additions in one statement- """- pass-@abstractmethoddef get_engine(self) -> Any:pass@@ -107,6 +88,7 @@ class UnstractDB(UnstractConnector, ABC):except Exception as e:raise ConnectorError(str(e)) from e+ # TO DO: Remove if neededdef sql_to_db_mapping(self, value: str) -> str:"""Gets the python datatype of value and converts python datatypeto corresponding DB datatype@@ -125,6 +107,24 @@ class UnstractDB(UnstractConnector, ABC):}return mapping.get(python_type, "TEXT")+ @abstractmethod+ def prepare_multi_column_migration(+ self, table_name: str, column_name: str+ ) -> str | list:+ """Returns the ALTER TABLE query specific to the database.++ Args:+ table_name (str): The name of the table to alter+ column_name (str): The base name of the column to add a _v2 version for++ Returns:+ str | list: Either a single SQL ALTER TABLE statement (str) or+ a list of separate ALTER TABLE statements for databases+ that don't support multiple column additions in one statement+ """+ pass++ @abstractmethoddef get_create_table_base_query(self, table: str) -> str:"""Function to create a base create table sql query.@@ -134,13 +134,6 @@ class UnstractDB(UnstractConnector, ABC):Returns:str: generates a create sql base query with the constant columns"""- sql_query = (- f"CREATE TABLE IF NOT EXISTS {table} "- f"(id TEXT , "- f"created_by TEXT, created_at TIMESTAMP, "- f"metadata TEXT, "- )- return sql_querydef create_table_query(self, table: str, database_entry: dict[str, Any]) -> Any:"""Function to create a create table sql query.@@ -156,6 +149,8 @@ class UnstractDB(UnstractConnector, ABC):sql_query = ""create_table_query = self.get_create_table_base_query(table=table)+ logger.info(f"Create table base query {create_table_query}")+sql_query += create_table_queryfor key, val in database_entry.items():@@ -166,16 +161,20 @@ class UnstractDB(UnstractConnector, ABC):return sql_query.rstrip(", ") + ")"@staticmethod- def get_sql_insert_query(table_name: str, sql_keys: list[str]) -> str:+ def get_sql_insert_query(+ table_name: str, sql_keys: list[str], sql_values: list[str] = None+ ) -> str:"""Function to generate parameterised insert sql query.Args:table_name (str): db-connector table namesql_keys (list[str]): column names+ sql_values (list[str], optional): SQL values for database-specific handlingReturns:str: returns a string with parameterised insert sql query"""+ # Base implementation ignores sql_values and returns parameterized querykeys_str = ",".join(sql_keys)values_placeholder = ",".join(["%s" for _ in sql_keys])return f"INSERT INTO {table_name} ({keys_str}) VALUES ({values_placeholder})"@@ -229,39 +228,120 @@ class UnstractDB(UnstractConnector, ABC):column_types[column_name] = data_typereturn column_types- def is_string_column(self, table_info: dict[str, str], column_name: str) -> bool:- """Check if the column is a string type specific to the DB connector.+ def has_no_metadata(self, table_info: dict[str, str]) -> bool:+ """Check if metadata field exists in table_info (case-insensitive).Args:- table_info (dict): column_name -> column_type- column_name (str): name of column to check+ table_info: Dictionary containing table field names and their typesReturns:- bool: True if column is a string type+ bool: False if metadata exists, True if metadata does not exist"""- column_type = table_info.get(column_name)- if column_type is None:- return False+ logger.info("Checking if column 'metadata' exists in table_info")+ metadata_exists = any(key.lower() == "metadata" for key in table_info.keys())+ if metadata_exists:+ logger.info(+ "column 'metadata' exists for corresponding table. No migration needed"+ )+ return not metadata_exists++ def migrate_table_to_v2(+ self, table_name: str, column_name: str, engine: Any+ ) -> dict[str, str]:+ """Retruns the information schema of table after alteraring table+ This will add column _v2 to the table and return the information schema- # Skip migration if *_v2 column already exists- if f"{column_name}_v2" in table_info:- return False+ Args:+ table_name (str): _description_+ column_name (str): _description_+ engine (Any): _description_- return column_type.lower() == self.get_string_type().lower()+ Raises:+ UnstractDBException: _description_- def migrate_table_to_v2(self, table_name: str, column_name: str, engine: Any) -> None:- sql_query = self.prepare_multi_column_migration(+ Returns:+ dict[str, str]: _description_+ """+ sql_query_or_list = self.prepare_multi_column_migration(table_name=table_name, column_name=column_name)+ logger.info(+ "Running table migration for table %s by adding following columns %s",+ table_name,+ sql_query_or_list,+ )try:- self.execute_query(- engine=engine,- sql_query=sql_query,- sql_values=None,- table_name=table_name,- )+ if isinstance(sql_query_or_list, list):+ for sql_query in sql_query_or_list:+ self.execute_query(+ engine=engine,+ sql_query=sql_query,+ sql_values=None,+ table_name=table_name,+ )+ else:+ self.execute_query(+ engine=engine,+ sql_query=sql_query_or_list,+ sql_values=None,+ table_name=table_name,+ )+ logger.info(+ "successfully migrated table %s with: %s query",+ table_name,+ sql_query_or_list,+ )+ return self.get_information_schema(table_name=table_name)except UnstractDBConnectorException as e:raise UnstractDBException(detail=e.detail) from e- logger.debug(f"successfully migrated table {table_name} with: {sql_query} query")+ def get_sql_values_for_query(+ self, values: dict[str, Any], column_types: dict[str, str]+ ) -> dict[str, str]:+ """Prepare SQL values for query execution.++ Args:+ values (dict[str, Any]): dictionary of columns and values+ column_types (dict[str, str]): types of columns from database schema++ Returns:+ dict[str, str]: Dictionary of column names to SQL values for parameterized queries++ Note:+ This is the base implementation for standard databases.+ Database-specific connectors can override this method for custom logic.+ """+ sql_values: dict[str, Any] = {}+ for column in values:+ value = values[column]+ # Handle JSON and Enum types for standard databases+ if isinstance(value, (dict, list)):+ try:+ sql_values[column] = json.dumps(value)+ except (TypeError, ValueError) as e:+ logger.error(+ f"Failed to serialize value to JSON for column {column}: {e}"+ )+ # Create a safe fallback error object+ fallback_value = {+ "error": "JSON serialization failed",+ "error_type": e.__class__.__name__,+ "error_message": str(e),+ "data_type": str(type(value)),+ "data_description": f"column_{column}",+ "timestamp": datetime.datetime.now().isoformat(),+ }+ sql_values[column] = json.dumps(fallback_value)+ elif isinstance(value, Enum):+ sql_values[column] = value.value+ else:+ sql_values[column] = f"{value}"++ # If table has a column 'id', unstract inserts a unique value to it+ # Oracle db has column 'ID' instead of 'id'+ if any(key in column_types for key in ["id", "ID"]):+ uuid_id = str(uuid.uuid4())+ sql_values["id"] = f"{uuid_id}"++ return sql_values