Skip to content

LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows

UN-1648 [FIX] Fixed issue with destination connectors using JSON columns (#1529)

Kirtiman Mishrae137320c0c16
diff --git a/backend/workflow_manager/endpoint_v2/constants.py b/backend/workflow_manager/endpoint_v2/constants.py
index 8fd4d5226..cedf19728 100644
--- a/backend/workflow_manager/endpoint_v2/constants.py
+++ b/backend/workflow_manager/endpoint_v2/constants.py
@@ -25,6 +25,7 @@ class TableColumns:
class DBConnectionClass:
SNOWFLAKE = "SnowflakeDB"
+ BIGQUERY = "BigQuery"
class Snowflake:
diff --git a/backend/workflow_manager/endpoint_v2/database_utils.py b/backend/workflow_manager/endpoint_v2/database_utils.py
index dd88f1065..c5011bfe5 100644
--- a/backend/workflow_manager/endpoint_v2/database_utils.py
+++ b/backend/workflow_manager/endpoint_v2/database_utils.py
@@ -1,12 +1,10 @@
import datetime
import json
import logging
-import uuid
-from enum import Enum
from typing import Any
from utils.constants import Common
-from workflow_manager.endpoint_v2.constants import DBConnectionClass, TableColumns
+from workflow_manager.endpoint_v2.constants import TableColumns
from workflow_manager.endpoint_v2.enums import FileProcessingStatus
from workflow_manager.endpoint_v2.exceptions import UnstractDBException
from workflow_manager.workflow_v2.enums import AgentName, ColumnModes
@@ -40,89 +38,6 @@ class DatabaseUtils:
"timestamp": datetime.datetime.now().isoformat(),
}
- @staticmethod
- def get_sql_values_for_query(
- values: dict[str, Any], column_types: dict[str, str], cls_name: str
- ) -> dict[str, str]:
- """Making Sql Columns and Values for Query.
-
- Args:
- values (dict[str, Any]): dictionary of columns and values
- column_types (dict[str,str]): types of columns
- cls (Any, optional): The database connection class (e.g.,
- DBConnectionClass.SNOWFLAKE) for handling database-specific
- queries.
- Defaults to None.
-
- Returns:
- list[str]: _description_
-
- Note:
- - If `cls` is not provided or is None, the function assumes a
- Default SQL database and makes values accordingly.
- - If `cls` is provided and matches DBConnectionClass.SNOWFLAKE,
- the function makes values using Snowflake-specific syntax.
-
- - Unstract creates id by default if table not exists.
- If there is column 'id' in db table, it will insert
- 'id' as uuid into the db table.
- Else it will GET table details from INFORMATION SCHEMA and
- insert into the table accordingly
- """
- sql_values: dict[str, Any] = {}
- for column in values:
- if cls_name == DBConnectionClass.SNOWFLAKE:
- col = column.lower()
- type_x = column_types.get(col, "")
- if type_x == "VARIANT":
- payload = values[column]
- if isinstance(payload, Enum):
- payload = payload.value
- # Ensure valid JSON for VARIANT
- try:
- payload_json = json.dumps(payload, default=str)
- sql_values[column] = f"parse_json($${payload_json}$$)"
- except (TypeError, ValueError) as e:
- logger.error(
- f"Failed to serialize payload to JSON for column {column}: {e}"
- )
- # Create a safe fallback error object
- fallback_payload = DatabaseUtils._create_safe_error_json(
- f"column_{column}", e
- )
- payload_json = json.dumps(fallback_payload)
- sql_values[column] = f"parse_json($${payload_json}$$)"
- else:
- # Non-VARIANT Snowflake types remain unchanged
- sql_values[column] = f"{values[column]}"
- else:
- # Handle JSON and Enum types for each database
- value = values[column]
- if isinstance(value, (dict, list)):
- try:
- sql_values[column] = json.dumps(value)
- except (TypeError, ValueError) as e:
- logger.error(
- f"Failed to serialize value to JSON for column {column}: {e}"
- )
- # Create a safe fallback error object - consistent with Snowflake approach
- fallback_value = DatabaseUtils._create_safe_error_json(
- f"column_{column}", e
- )
- sql_values[column] = json.dumps(fallback_value)
- elif isinstance(value, Enum):
- sql_values[column] = value.value
- else:
- sql_values[column] = (
- f"{value}" # Non-JSON/Enum types handled as before
- )
- # If table has a column 'id', unstract inserts a unique value to it
- # Oracle db has column 'ID' instead of 'id'
- if any(key in column_types for key in ["id", "ID"]):
- uuid_id = str(uuid.uuid4())
- sql_values["id"] = f"{uuid_id}"
- return sql_values
-
@staticmethod
def get_column_types(
conn_cls: Any,
@@ -145,12 +60,24 @@ class DatabaseUtils:
return conn_cls.get_information_schema(table_name=table_name)
except ConnectorError as e:
raise UnstractDBException(detail=e.message) from e
- except Exception as e:
- logger.error(
- f"Error getting db-column-name and db-column-type "
- f"for {table_name}: {str(e)}"
- )
- raise
+
+ @staticmethod
+ def get_sql_values_for_query(
+ conn_cls: Any,
+ values: dict[str, Any],
+ column_types: dict[str, str],
+ ) -> dict[str, Any]:
+ """Function to prepare SQL values by calling connector method.
+
+ Args:
+ conn_cls (Any): DB Connection class
+ values (dict[str, Any]): dictionary of columns and values
+ column_types (dict[str, str]): types of columns from database schema
+
+ Returns:
+ dict[str, Any]: Dictionary of column names to SQL values
+ """
+ return conn_cls.get_sql_values_for_query(values=values, column_types=column_types)
@staticmethod
def get_columns_and_values(
@@ -236,7 +163,6 @@ class DatabaseUtils:
values[TableColumns.STATUS] = (
FileProcessingStatus.ERROR if error else FileProcessingStatus.SUCCESS
)
-
if column_mode == ColumnModes.WRITE_JSON_TO_A_SINGLE_COLUMN:
if isinstance(data, str):
wrapped_dict = {"result": data}
@@ -251,52 +177,9 @@ class DatabaseUtils:
k.lower() == f"{single_column_name}_v2".lower() for k in table_info
):
values[f"{single_column_name}_v2"] = data
- if column_mode == ColumnModes.SPLIT_JSON_INTO_COLUMNS:
- if isinstance(data, dict):
- values[single_column_name] = data
- elif isinstance(data, str):
- values[single_column_name] = data
- # Only write to v2 if it exists
- if table_info and any(
- k.lower() == f"{single_column_name}_v2".lower() for k in table_info
- ):
- values[f"{single_column_name}_v2"] = data
- else:
- try:
- values[single_column_name] = json.dumps(
- data
- ) # Legacy column gets JSON string
- except (TypeError, ValueError) as e:
- logger.error(
- f"Failed to serialize data to JSON for column {single_column_name}: {e}"
- )
- # Create a safe fallback error object
- fallback_data = DatabaseUtils._create_safe_error_json(
- single_column_name, e
- )
- values[single_column_name] = json.dumps(fallback_data)
-
- # Only write to v2 if it exists
- if table_info and any(
- k.lower() == f"{single_column_name}_v2".lower() for k in table_info
- ):
- # Make sure v2 gets a serializable value too if needed
- if isinstance(data, (dict, list)):
- values[f"{single_column_name}_v2"] = data
- else:
- try:
- # Test if it's JSON serializable before assigning
- json.dumps(data)
- values[f"{single_column_name}_v2"] = data
- except (TypeError, ValueError) as e:
- # Create a safe fallback error object
- fallback_data = DatabaseUtils._create_safe_error_json(
- f"{single_column_name}_v2", e
- )
- values[f"{single_column_name}_v2"] = fallback_data
-
values[file_path_name] = file_path
values[execution_id_name] = execution_id
+ logger.debug(f"database_utils.py get_columns_and_values values: {values}")
return values
@staticmethod
@@ -325,15 +208,20 @@ class DatabaseUtils:
- For other SQL databases, it uses default SQL generation
based on column types.
"""
- cls_name = conn_cls.__class__.__name__
column_types: dict[str, str] = DatabaseUtils.get_column_types(
conn_cls=conn_cls, table_name=table_name
)
+ logger.debug(f"database_utils.py get_sql_query_data column_types: {column_types}")
+
sql_columns_and_values = DatabaseUtils.get_sql_values_for_query(
+ conn_cls=conn_cls,
values=values,
column_types=column_types,
- cls_name=cls_name,
)
+ logger.debug(
+ f"database_utils.py get_sql_query_data sql_columns_and_values: {sql_columns_and_values}"
+ )
+
return sql_columns_and_values
@staticmethod
@@ -357,7 +245,9 @@ class DatabaseUtils:
So we need to use INSERT INTO ... SELECT ... syntax
- sql values can contain data with single quote. It needs to
"""
- sql = db_class.get_sql_insert_query(table_name=table_name, sql_keys=sql_keys)
+ sql = db_class.get_sql_insert_query(
+ table_name=table_name, sql_keys=sql_keys, sql_values=sql_values
+ )
logger.debug(f"inserting into table {table_name} with: {sql} query")
logger.debug(f"sql_values: {sql_values}")
@@ -399,6 +289,7 @@ class DatabaseUtils:
"""
sql = db_class.create_table_query(table=table_name, database_entry=database_entry)
logger.debug(f"creating table {table_name} with: {sql} query")
+
try:
db_class.execute_query(
engine=engine, sql_query=sql, sql_values=None, table_name=table_name
diff --git a/backend/workflow_manager/endpoint_v2/destination.py b/backend/workflow_manager/endpoint_v2/destination.py
index dfff91ce2..e86374b21 100644
--- a/backend/workflow_manager/endpoint_v2/destination.py
+++ b/backend/workflow_manager/endpoint_v2/destination.py
@@ -368,12 +368,13 @@ class DestinationConnector(BaseConnector):
table_info = db_class.get_information_schema(table_name=table_name)
- # Check whether to migrate table to include new columns
+ logger.info(
+ f"destination connector engine: {engine} with table info: {table_info}"
+ )
+
if table_info:
- if db_class.is_string_column(
- table_info=table_info, column_name=single_column_name
- ):
- db_class.migrate_table_to_v2(
+ if db_class.has_no_metadata(table_info=table_info):
+ table_info = db_class.migrate_table_to_v2(
table_name=table_name,
column_name=single_column_name,
engine=engine,
@@ -395,6 +396,8 @@ class DestinationConnector(BaseConnector):
error=error,
)
+ logger.debug(f"destination.py values: {values}")
+
try:
# Reuse the same db_class and engine created earlier
@@ -410,7 +413,9 @@ class DestinationConnector(BaseConnector):
table_name=table_name,
values=values,
)
-
+ logger.info(
+ f"destination.py sql_columns_and_values: {sql_columns_and_values}"
+ )
DatabaseUtils.execute_write_query(
db_class=db_class,
engine=engine,
diff --git a/backend/workflow_manager/endpoint_v2/tests/__init__.py b/backend/workflow_manager/endpoint_v2/tests/__init__.py
index fca2b2401..c7f1fbe4c 100644
--- a/backend/workflow_manager/endpoint_v2/tests/__init__.py
+++ b/backend/workflow_manager/endpoint_v2/tests/__init__.py
@@ -1,3 +1,3 @@
-from backend.celery import app as celery_app
+from backend.celery_service import app as celery_app
__all__ = ["celery_app"]
diff --git a/backend/workflow_manager/endpoint_v2/tests/destination-connectors/__init__.py b/backend/workflow_manager/endpoint_v2/tests/destination-connectors/__init__.py
new file mode 100644
index 000000000..0205067fe
--- /dev/null
+++ b/backend/workflow_manager/endpoint_v2/tests/destination-connectors/__init__.py
@@ -0,0 +1 @@
+# Tests for destination connectors
diff --git a/backend/workflow_manager/endpoint_v2/tests/destination-connectors/test_destination_connector_bigquery.py b/backend/workflow_manager/endpoint_v2/tests/destination-connectors/test_destination_connector_bigquery.py
new file mode 100644
index 000000000..dd28e4095
--- /dev/null
+++ b/backend/workflow_manager/endpoint_v2/tests/destination-connectors/test_destination_connector_bigquery.py
@@ -0,0 +1,254 @@
+import json
+import os
+from unittest.mock import Mock, patch
+
+from django.test import TestCase
+from workflow_manager.endpoint_v2.constants import DestinationKey
+from workflow_manager.endpoint_v2.destination import DestinationConnector
+
+from unstract.connectors.databases.bigquery import BigQuery
+
+
+class TestDestinationConnectorBigQuery(TestCase):
+ """Integration test for insert_into_db method with real BigQuery connector."""
+
+ def setUp(self) -> None:
+ """Set up test data and real BigQuery configuration."""
+
+ # BigQuery connection settings for testing
+ service_account_json = os.getenv("SERVICE_ACCOUNT_JSON")
+ if not service_account_json:
+ self.skipTest("SERVICE_ACCOUNT_JSON environment variable not set")
+
+ # Parse the service account JSON
+ try:
+ service_account_dict = json.loads(service_account_json)
+ project_id = service_account_dict.get("project_id")
+ if not project_id:
+ self.skipTest("project_id not found in service account JSON")
+ except json.JSONDecodeError:
+ self.skipTest("Invalid JSON in SERVICE_ACCOUNT_JSON environment variable")
+
+ self.bigquery_config = {"json_credentials": service_account_json}
+
+ # Test data that will be inserted into the database
+ self.test_data = {"key": "value", "result": "test_result", "status": "success"}
+ self.test_metadata = {
+ "file_execution_id": "test-file-exec-id",
+ "extracted_text": "This is sample extracted text",
+ "processing_time": 1.5,
+ }
+ self.input_file_path = "/path/to/test/file.pdf"
+
+ # BigQuery table format: project.dataset.table
+ dataset_name = "temp_dataset" # Use your dataset name
+ self.test_table_name = f"{project_id}.{dataset_name}.output_1"
+
+ # Create real BigQuery connector instance
+ self.bigquery_connector = BigQuery(settings=self.bigquery_config)
+
+ # Expected columns based on configuration (BigQuery data types)
+ self.expected_columns = {
+ "id": "STRING",
+ "file_path": "STRING",
+ "execution_id": "STRING",
+ "data": "JSON",
+ "created_at": "TIMESTAMP",
+ "created_by": "STRING",
+ "metadata": "JSON",
+ "error_message": "STRING",
+ "status": "STRING",
+ "user_field_1": "BOOL",
+ "user_field_2": "INT64",
+ "user_field_3": "STRING",
+ }
+
+ def verify_table_columns(self, table_name: str) -> None:
+ """Verify that all expected columns exist in the table with correct types."""
+ table_info = self.bigquery_connector.get_information_schema(
+ table_name=table_name
+ )
+
+ print(f"🔍 Actual table structure for '{table_name}': {table_info}")
+
+ # Check that key columns exist (some columns may be created on-demand)
+ required_columns = {
+ "id": "STRING",
+ "file_path": "STRING",
+ "execution_id": "STRING",
+ # "data": "STRING", # Legacy column remains STRING after migration
+ # "data_v2": "JSON", # New JSON column added during migration
+ "metadata": "JSON", # New JSON column added during migration
+ "created_at": "TIMESTAMP",
+ "created_by": "STRING",
+ }
+
+ for expected_column, expected_type in required_columns.items():
+ self.assertIn(
+ expected_column,
+ table_info,
+ f"Expected column '{expected_column}' not found in table '{table_name}'",
+ )
+
+ actual_type = table_info[expected_column]
+ # BigQuery may return lowercase types, so normalize for comparison
+ self.assertEqual(
+ actual_type.upper(),
+ expected_type.upper(),
+ f"Column '{expected_column}' has type '{actual_type}', expected '{expected_type}'",
+ )
+
+ print(f"✅ All required columns verified in table '{table_name}'")
+
+ def create_mock_workflow(self) -> Mock:
+ """Create a mock workflow object."""
+ workflow = Mock()
+ workflow.id = "test-workflow-id"
+ return workflow
+
+ def create_mock_workflow_log(self) -> Mock:
+ """Create a mock workflow log object."""
+ return Mock()
+
+ def create_real_connector_instance(self) -> Mock:
+ """Create connector instance using real BigQuery connector."""
+ connector = Mock()
+ connector.connector_id = BigQuery.get_id()
+ connector.connector_metadata = self.bigquery_config
+ # Store reference to real connector for potential future use
+ connector._real_bigquery_connector = self.bigquery_connector
+ return connector
+
+ def create_mock_endpoint(self, mock_connector_instance: Mock) -> Mock:
+ """Create a mock endpoint with real database configuration."""
+ endpoint = Mock()
+ endpoint.connector_instance = mock_connector_instance
+ endpoint.configuration = {
+ DestinationKey.TABLE: self.test_table_name,
+ DestinationKey.INCLUDE_AGENT: True,
+ DestinationKey.INCLUDE_TIMESTAMP: True,
+ DestinationKey.AGENT_NAME: "Unstract/DBWriter",
+ DestinationKey.COLUMN_MODE: "Write JSON to a single column",
+ DestinationKey.SINGLE_COLUMN_NAME: "data",
+ DestinationKey.FILE_PATH: "file_path",
+ DestinationKey.EXECUTION_ID: "execution_id",
+ }
+ return endpoint
+
+ def create_destination_connector(
+ self, mock_workflow: Mock, mock_workflow_log: Mock, mock_endpoint: Mock
+ ) -> DestinationConnector:
+ """Create a destination connector with mocked Django dependencies but real DB."""
+ with patch(
+ "workflow_manager.endpoint_v2.destination.UserContext"
+ ) as mock_context:
+ mock_context.get_organization_identifier.return_value = "test-org"
+
+ with patch.object(
+ DestinationConnector,
+ "_get_endpoint_for_workflow",
+ return_value=mock_endpoint,
+ ):
+ with patch.object(
+ DestinationConnector,
+ "_get_source_endpoint_for_workflow",
+ return_value=mock_endpoint,
+ ):
+ connector = DestinationConnector(
+ workflow=mock_workflow,
+ execution_id="test-execution-id",
+ workflow_log=mock_workflow_log,
+ use_file_history=False,
+ )
+ return connector
+
+ def test_insert_into_db_happy_path_bigquery(self) -> None:
+ """Test successful insertion into real BigQuery database."""
+ # Create mock objects for Django models
+ mock_workflow = self.create_mock_workflow()
+ mock_workflow_log = self.create_mock_workflow_log()
+ mock_connector_instance = self.create_real_connector_instance()
+ mock_endpoint = self.create_mock_endpoint(mock_connector_instance)
+
+ # Create destination connector with real database connection
+ destination_connector = self.create_destination_connector(
+ mock_workflow, mock_workflow_log, mock_endpoint
+ )
+
+ # Mock only the methods that get data, let database operations be real
+ with patch.object(
+ destination_connector,
+ "get_tool_execution_result",
+ return_value=self.test_data,
+ ):
+ with patch.object(
+ destination_connector,
+ "get_combined_metadata",
+ return_value=self.test_metadata,
+ ):
+ # This will execute real database operations using BigQuery connector
+ destination_connector.insert_into_db(
+ input_file_path=self.input_file_path, error=None
+ )
+
+ # Verify that all expected columns were created
+ self.verify_table_columns(self.test_table_name)
+
+ print(
+ f"✅ Successfully inserted test data into BigQuery table: {self.test_table_name}"
+ )
+
+ def test_insert_into_db_with_error_bigquery(self) -> None:
+ """Test insertion with error parameter into real BigQuery database."""
+ # Create mock objects
+ mock_workflow = self.create_mock_workflow()
+ mock_workflow_log = self.create_mock_workflow_log()
+ mock_connector_instance = self.create_real_connector_instance()
+ mock_endpoint = self.create_mock_endpoint(mock_connector_instance)
+
+ # Create destination connector
+ destination_connector = self.create_destination_connector(
+ mock_workflow, mock_workflow_log, mock_endpoint
+ )
+
+ error_message = "Test processing error occurred"
+
+ # Mock the methods that get data
+ with patch.object(
+ destination_connector,
+ "get_tool_execution_result",
+ return_value=self.test_data,
+ ):
+ with patch.object(
+ destination_connector,
+ "get_combined_metadata",
+ return_value=self.test_metadata,
+ ):
+ # Execute with error parameter
+ destination_connector.insert_into_db(
+ input_file_path=self.input_file_path, error=error_message
+ )
+
+ # Verify that all expected columns were created
+ self.verify_table_columns(self.test_table_name)
+
+ print(
+ f"✅ Successfully inserted error data into BigQuery table: {self.test_table_name}"
+ )
+
+ def test_bigquery_connector_connection(self) -> None:
+ """Test that the BigQuery connector can establish a connection."""
+ # Test the real BigQuery connector directly
+ try:
+ engine = self.bigquery_connector.get_engine()
+ self.assertIsNotNone(engine)
+ print("✅ BigQuery connector successfully established connection")
+
+ # Test a simple query to verify connection works
+ test_query = "SELECT 1 as test_column"
+ result = self.bigquery_connector.execute(test_query)
+ self.assertIsNotNone(result)
+ print("✅ BigQuery connector successfully executed test query")
+
+ except Exception as e:
+ self.fail(f"BigQuery connector failed to connect: {str(e)}")
diff --git a/backend/workflow_manager/endpoint_v2/tests/destination-connectors/test_destination_connector_mariadb.py b/backend/workflow_manager/endpoint_v2/tests/destination-connectors/test_destination_connector_mariadb.py
new file mode 100644
index 000000000..0f2bd39ab
--- /dev/null
+++ b/backend/workflow_manager/endpoint_v2/tests/destination-connectors/test_destination_connector_mariadb.py
@@ -0,0 +1,675 @@
+import json
+import os
+from unittest.mock import Mock, patch
+
+from django.test import TestCase
+from workflow_manager.endpoint_v2.constants import DestinationKey
+from workflow_manager.endpoint_v2.destination import DestinationConnector
+
+from unstract.connectors.databases.mariadb import MariaDB
+
+
+class TestDestinationConnectorMariaDB(TestCase):
+ """Integration test for insert_into_db method with real MariaDB connector."""
+
+ def setUp(self) -> None:
+ """Set up test data and real MariaDB configuration."""
+
+ # MariaDB connection settings for testing
+ required_env_vars = [
+ "MARIADB_HOST",
+ "MARIADB_PORT",
+ "MARIADB_DATABASE",
+ "MARIADB_USER",
+ "MARIADB_PASSWORD",
+ ]
+
+ # Check if all required environment variables are set
+ missing_vars = [var for var in required_env_vars if not os.getenv(var)]
+ if missing_vars:
+ self.skipTest(
+ f"Required MariaDB environment variables not set: {', '.join(missing_vars)}"
+ )
+
+ self.mariadb_config = {
+ "host": os.getenv("MARIADB_HOST"),
+ "port": os.getenv("MARIADB_PORT"),
+ "database": os.getenv("MARIADB_DATABASE"),
+ "user": os.getenv("MARIADB_USER"),
+ "password": os.getenv("MARIADB_PASSWORD"),
+ }
+
+ # Test data that will be inserted into the database
+ self.test_data = {"key": "value", "result": "test_result", "status": "success"}
+ self.test_metadata = {
+ "file_execution_id": "test-file-exec-id",
+ "extracted_text": "This is sample extracted text",
+ "processing_time": 1.5,
+ }
+ self.input_file_path = "/path/to/test/file.pdf"
+ self.test_table_name = "MIGRATION"
+
+ # Create real MariaDB connector instance
+ self.mariadb_connector = MariaDB(settings=self.mariadb_config)
+
+ # Expected columns based on configuration (MariaDB/MySQL data types)
+ self.expected_columns = {
+ "id": "longtext",
+ "file_path": "longtext",
+ "execution_id": "longtext",
+ "data": "longtext",
+ "created_at": "timestamp",
+ "created_by": "longtext",
+ "metadata": "json",
+ "error_message": "longtext",
+ "status": "enum",
+ "user_field_1": "tinyint", # BOOLEAN maps to TINYINT in MariaDB
+ "user_field_2": "bigint",
+ "user_field_3": "longtext",
+ }
+
+ def verify_table_columns(self, table_name: str) -> None:
+ """Verify that all expected columns exist in the table with correct types."""
+ table_info = self.mariadb_connector.get_information_schema(
+ table_name=table_name
+ )
+
+ print(f"🔍 Actual table structure for '{table_name}': {table_info}")
+
+ # Check that key columns exist (some columns may be created on-demand)
+ required_columns = {
+ "id": "longtext",
+ "file_path": "longtext",
+ "execution_id": "longtext",
+ "data": "longtext",
+ "created_at": "timestamp",
+ "created_by": "longtext",
+ }
+
+ for expected_column, expected_type in required_columns.items():
+ self.assertIn(
+ expected_column,
+ table_info,
+ f"Expected column '{expected_column}' not found in table '{table_name}'",
+ )
+
+ actual_type = table_info[expected_column].lower()
+ expected_type_lower = expected_type.lower()
+
+ # MariaDB may return slightly different type names, so we do flexible matching
+ if expected_type_lower == "timestamp":
+ self.assertIn(
+ actual_type,
+ ["timestamp", "datetime", "timestamp(0)"],
+ f"Column '{expected_column}' has type '{actual_type}', expected timestamp-like type",
+ )
+ elif expected_type_lower == "tinyint":
+ self.assertIn(
+ actual_type,
+ ["tinyint", "tinyint(1)", "boolean", "bool"],
+ f"Column '{expected_column}' has type '{actual_type}', expected boolean/tinyint type",
+ )
+ else:
+ self.assertEqual(
+ actual_type,
+ expected_type_lower,
+ f"Column '{expected_column}' has type '{actual_type}', expected '{expected_type_lower}'",
+ )
+
+ print(f"✅ All required columns verified in table '{table_name}'")
+
+ def create_mock_workflow(self) -> Mock:
+ """Create a mock workflow object."""
+ workflow = Mock()
+ workflow.id = "test-workflow-id"
+ return workflow
+
+ def create_mock_workflow_log(self) -> Mock:
+ """Create a mock workflow log object."""
+ return Mock()
+
+ def create_real_connector_instance(self) -> Mock:
+ """Create connector instance using real MariaDB connector."""
+ connector = Mock()
+ connector.connector_id = MariaDB.get_id()
+ connector.connector_metadata = self.mariadb_config
+ # Store reference to real connector for potential future use
+ connector._real_mariadb_connector = self.mariadb_connector
+ return connector
+
+ def create_mock_endpoint(self, mock_connector_instance: Mock) -> Mock:
+ """Create a mock endpoint with real database configuration."""
+ endpoint = Mock()
+ endpoint.connector_instance = mock_connector_instance
+ endpoint.configuration = {
+ DestinationKey.TABLE: self.test_table_name,
+ DestinationKey.INCLUDE_AGENT: True,
+ DestinationKey.INCLUDE_TIMESTAMP: True,
+ DestinationKey.AGENT_NAME: "Unstract/DBWriter",
+ DestinationKey.COLUMN_MODE: "Write JSON to a single column",
+ DestinationKey.SINGLE_COLUMN_NAME: "data",
+ DestinationKey.FILE_PATH: "file_path",
+ DestinationKey.EXECUTION_ID: "execution_id",
+ }
+ return endpoint
+
+ def create_destination_connector(
+ self, mock_workflow: Mock, mock_workflow_log: Mock, mock_endpoint: Mock
+ ) -> DestinationConnector:
+ """Create a destination connector with mocked Django dependencies but real DB."""
+ with patch(
+ "workflow_manager.endpoint_v2.destination.UserContext"
+ ) as mock_context:
+ mock_context.get_organization_identifier.return_value = "test-org"
+
+ with patch.object(
+ DestinationConnector,
+ "_get_endpoint_for_workflow",
+ return_value=mock_endpoint,
+ ):
+ with patch.object(
+ DestinationConnector,
+ "_get_source_endpoint_for_workflow",
+ return_value=mock_endpoint,
+ ):
+ connector = DestinationConnector(
+ workflow=mock_workflow,
+ execution_id="test-execution-id",
+ workflow_log=mock_workflow_log,
+ use_file_history=False,
+ )
+ return connector
+
+ def test_insert_into_db_happy_path_mariadb(self) -> None:
+ """Test successful insertion into real MariaDB database."""
+ # Create mock objects for Django models
+ mock_workflow = self.create_mock_workflow()
+ mock_workflow_log = self.create_mock_workflow_log()
+ mock_connector_instance = self.create_real_connector_instance()
+ mock_endpoint = self.create_mock_endpoint(mock_connector_instance)
+
+ # Create destination connector with real database connection
+ destination_connector = self.create_destination_connector(
+ mock_workflow, mock_workflow_log, mock_endpoint
+ )
+
+ # Mock only the methods that get data, let database operations be real
+ with patch.object(
+ destination_connector,
+ "get_tool_execution_result",
+ return_value=self.test_data,
+ ):
+ with patch.object(
+ destination_connector,
+ "get_combined_metadata",
+ return_value=self.test_metadata,
+ ):
+ # This will execute real database operations using MariaDB connector
+ destination_connector.insert_into_db(
+ input_file_path=self.input_file_path, error=None
+ )
+
+ # Verify that all expected columns were created
+ self.verify_table_columns(self.test_table_name)
+
+ print(
+ f"✅ Successfully inserted test data into MariaDB table: {self.test_table_name}"
+ )
+
+ def test_insert_into_db_with_error_mariadb(self) -> None:
+ """Test insertion with error parameter into real MariaDB database."""
+ # Create mock objects
+ mock_workflow = self.create_mock_workflow()
+ mock_workflow_log = self.create_mock_workflow_log()
+ mock_connector_instance = self.create_real_connector_instance()
+ mock_endpoint = self.create_mock_endpoint(mock_connector_instance)
+
+ # Create destination connector
+ destination_connector = self.create_destination_connector(
+ mock_workflow, mock_workflow_log, mock_endpoint
+ )
+
+ error_message = "Test processing error occurred"
+
+ # Mock the methods that get data
+ with patch.object(
+ destination_connector,
+ "get_tool_execution_result",
+ return_value=self.test_data,
+ ):
+ with patch.object(
+ destination_connector,
+ "get_combined_metadata",
+ return_value=self.test_metadata,
+ ):
+ # Execute with error parameter
+ destination_connector.insert_into_db(
+ input_file_path=self.input_file_path, error=error_message
+ )
+
+ # Verify that all expected columns were created
+ self.verify_table_columns(self.test_table_name)
+
+ print(
+ f"✅ Successfully inserted error data into MariaDB table: {self.test_table_name}"
+ )
+
+ def test_mariadb_connector_connection(self) -> None:
+ """Test that the MariaDB connector can establish a connection."""
+ # Test the real MariaDB connector directly
+ try:
+ engine = self.mariadb_connector.get_engine()
+ self.assertIsNotNone(engine)
+ print("✅ MariaDB connector successfully established connection")
+
+ # Test a simple query to verify connection works
+ cursor = engine.cursor()
+ cursor.execute("SELECT 1 as test_column")
+ result = cursor.fetchall()
+ self.assertIsNotNone(result)
+ self.assertEqual(result[0][0], 1)
+ cursor.close()
+ print("✅ MariaDB connector successfully executed test query")
+
+ # Clean up
+ if hasattr(engine, "close"):
+ engine.close()
+ except Exception as e:
+ self.fail(f"MariaDB connector failed to connect: {str(e)}")
+
+ def test_comprehensive_column_data_verification(self) -> None:
+ """Test that data is properly inserted into all columns with correct values."""
+ # Create mock objects
+ mock_workflow = self.create_mock_workflow()
+ mock_workflow_log = self.create_mock_workflow_log()
+ mock_connector_instance = self.create_real_connector_instance()
+ mock_endpoint = self.create_mock_endpoint(mock_connector_instance)
+
+ # Create destination connector
+ destination_connector = self.create_destination_connector(
+ mock_workflow, mock_workflow_log, mock_endpoint
+ )
+
+ # Mock the methods that get data
+ with patch.object(
+ destination_connector,
+ "get_tool_execution_result",
+ return_value=self.test_data,
+ ):
+ with patch.object(
+ destination_connector,
+ "get_combined_metadata",
+ return_value=self.test_metadata,
+ ):
+ # Execute database insertion
+ destination_connector.insert_into_db(
+ input_file_path=self.input_file_path, error=None
+ )
+
+ # Verify table structure
+ self.verify_table_columns(self.test_table_name)
+
+ # Verify data was inserted correctly by querying the table
+ engine = self.mariadb_connector.get_engine()
+ try:
+ cursor = engine.cursor()
+ cursor.execute(
+ f"SELECT created_by, file_path, execution_id, created_at, data, metadata FROM {self.test_table_name} ORDER BY created_at DESC LIMIT 1"
+ )
+ row = cursor.fetchone()
+
+ self.assertIsNotNone(row, "No data found in table after insertion")
+
+ # Map results to column names
+ created_by, file_path, execution_id, created_at, data, metadata = row
+
+ # Verify specific column values
+ self.assertEqual(
+ created_by,
+ "Unstract/DBWriter",
+ f"Expected created_by to be 'Unstract/DBWriter', got '{created_by}'",
+ )
+
+ self.assertEqual(
+ file_path,
+ self.input_file_path,
+ f"Expected file_path to be '{self.input_file_path}', got '{file_path}'",
+ )
+
+ self.assertEqual(
+ execution_id,
+ "test-execution-id",
+ f"Expected execution_id to be 'test-execution-id', got '{execution_id}'",
+ )
+
+ self.assertIsNotNone(created_at, "Expected created_at timestamp to be set")
+
+ self.assertIsNotNone(data, "Expected data column to contain inserted data")
+
+ self.assertIsNotNone(
+ metadata, "Expected metadata column to contain metadata"
+ )
+
+ cursor.close()
+ print("✅ All column data values verified successfully")
+
+ finally:
+ if hasattr(engine, "close"):
+ engine.close()
+
+ def test_mariadb_specific_data_types(self) -> None:
+ """Test MariaDB-specific data types and constraints."""
+ # Test that status column accepts ENUM values correctly
+ engine = self.mariadb_connector.get_engine()
+ try:
+ cursor = engine.cursor()
+
+ # Create the table first if it doesn't exist
+ create_table_query = self.mariadb_connector.get_create_table_base_query(
+ self.test_table_name
+ )
+ # Add the required columns for the test
+ create_table_query += (
+ "file_path LONGTEXT, " "execution_id LONGTEXT, " "data LONGTEXT)"
+ )
+
+ cursor.execute(create_table_query)
+
+ # Test inserting valid ENUM values
+ cursor.execute(
+ f"INSERT INTO {self.test_table_name} (id, status, created_at) VALUES (%s, %s, NOW())",
+ ("test-enum-1", "SUCCESS"),
+ )
+
+ cursor.execute(
+ f"INSERT INTO {self.test_table_name} (id, status, created_at) VALUES (%s, %s, NOW())",
+ ("test-enum-2", "ERROR"),
+ )
+
+ # Verify the data was inserted
+ cursor.execute(
+ f"SELECT id, status FROM {self.test_table_name} WHERE id IN (%s, %s)",
+ ("test-enum-1", "test-enum-2"),
+ )
+ results = cursor.fetchall()
+
+ self.assertEqual(len(results), 2, "Expected 2 rows to be inserted")
+
+ # Verify ENUM values
+ statuses = [row[1] for row in results]
+ self.assertIn("SUCCESS", statuses)
+ self.assertIn("ERROR", statuses)
+
+ cursor.close()
+ print("✅ MariaDB ENUM data type verification successful")
+
+ finally:
+ if hasattr(engine, "close"):
+ engine.close()
+
+ def test_mariadb_legacy_to_v2_migration(self) -> None:
+ """Test complete legacy table to v2 migration workflow."""
+ legacy_table_name = "output_1"
+
+ # # Step 1: Create a legacy table manually (simulating old table structure)
+ # print("🏗️ Step 1: Creating legacy table...")
+ # self._create_legacy_table(legacy_table_name)
+
+ # Step 2: Verify table is detected as legacy
+ print("🔍 Step 2: Verifying table is detected as legacy...")
+ is_legacy = self._verify_table_is_legacy(legacy_table_name)
+ self.assertTrue(is_legacy, "Table should be detected as legacy")
+
+ # Step 3: Trigger migration using the real migration flow
+ print("⚡ Step 3: Initiating migration...")
+ self._initiate_migration_via_workflow(legacy_table_name)
+
+ # Step 4: Verify migration was successful
+ print("✅ Step 4: Verifying migration success...")
+ self._verify_migration_success(legacy_table_name)
+
+ # Step 5: Test dual column writing (legacy + v2)
+ print("📝 Step 5: Testing dual column writing...")
+ self._test_dual_column_writing(legacy_table_name)
+
+ print("🎉 Migration test completed successfully!")
+
+ def _create_legacy_table(self, table_name: str) -> None:
+ """Create a legacy table structure (data column as LONGTEXT, not JSON)."""
+ engine = self.mariadb_connector.get_engine()
+ try:
+ cursor = engine.cursor()
+
+ # FORCE DROP AND RECREATE to ensure clean legacy table
+ cursor.execute(f"DROP TABLE IF EXISTS {table_name}")
+ print(f"🗑️ Dropped existing table '{table_name}' if it existed")
+
+ # Create fresh legacy table with LONGTEXT data column (not JSON)
+ legacy_create_query = f"""
+ CREATE TABLE {table_name} (
+ id LONGTEXT,
+ created_by LONGTEXT,
+ created_at TIMESTAMP,
+ data LONGTEXT,
+ file_path LONGTEXT,
+ execution_id LONGTEXT
+ )
+ """
+
+ cursor.execute(legacy_create_query)
+ print(
+ f"📋 Created fresh legacy table '{table_name}' with LONGTEXT data column"
+ )
+
+ # Insert some legacy data to make it realistic
+ cursor.execute(
+ f"INSERT INTO {table_name} (id, created_by, created_at, data, file_path, execution_id) VALUES (%s, %s, NOW(), %s, %s, %s)",
+ (
+ "legacy-1",
+ "Legacy/DBWriter",
+ '{"old": "legacy_data"}',
+ "/legacy/file.pdf",
+ "legacy-exec-1",
+ ),
+ )
+ print(f"📝 Inserted legacy test data into '{table_name}'")
+
+ cursor.close()
+ print(f"✅ Legacy table '{table_name}' setup completed")
+
+ finally:
+ if hasattr(engine, "close"):
+ engine.close()
+
+ def _verify_table_is_legacy(self, table_name: str) -> bool:
+ """Verify that table is detected as legacy using the real detection logic."""
+ try:
+ # Use the actual detection logic from the codebase
+ is_string_column = self.mariadb_connector.is_string_column(
+ table_info=self.mariadb_connector.get_information_schema(table_name),
+ column_name="data",
+ )
+
+ if is_string_column:
+ print(
+ f"✅ Table '{table_name}' correctly detected as legacy (data column is string type)"
+ )
+ return True
+ else:
+ print(f"❌ Table '{table_name}' not detected as legacy")
+ return False
+
+ except Exception as e:
+ print(f"❌ Error checking legacy status: {e}")
+ return False
+
+ def _initiate_migration_via_workflow(self, table_name: str) -> None:
+ """Initiate migration using the real workflow destination connector logic."""
+ # Create mock objects for the workflow system
+ mock_workflow = self.create_mock_workflow()
+ mock_workflow_log = self.create_mock_workflow_log()
+ mock_connector_instance = self.create_real_connector_instance()
+
+ # Create endpoint with the legacy table name
+ mock_endpoint = Mock()
+ mock_endpoint.connector_instance = mock_connector_instance
+ mock_endpoint.configuration = {
+ DestinationKey.TABLE: table_name, # Use legacy table
+ DestinationKey.INCLUDE_AGENT: True,
+ DestinationKey.INCLUDE_TIMESTAMP: True,
+ DestinationKey.AGENT_NAME: "Unstract/DBWriter",
+ DestinationKey.COLUMN_MODE: "Write JSON to a single column",
+ DestinationKey.SINGLE_COLUMN_NAME: "data",
+ DestinationKey.FILE_PATH: "file_path",
+ DestinationKey.EXECUTION_ID: "execution_id",
+ }
+
+ # Create destination connector
+ destination_connector = self.create_destination_connector(
+ mock_workflow, mock_workflow_log, mock_endpoint
+ )
+
+ # Mock the data methods to provide test data
+ test_migration_data = {"migration_test": "v2_data", "status": "migrated"}
+ test_migration_metadata = {
+ "file_execution_id": "migration-test-exec-id",
+ "migration_test": True,
+ "extracted_text": "Migration test extracted text",
+ }
+
+ with patch.object(
+ destination_connector,
+ "get_tool_execution_result",
+ return_value=test_migration_data,
+ ):
+ with patch.object(
+ destination_connector,
+ "get_combined_metadata",
+ return_value=test_migration_metadata,
+ ):
+ # This should trigger the migration logic automatically
+ destination_connector.insert_into_db(
+ input_file_path="/migration/test/file.pdf", error=None
+ )
+
+ print(f"✅ Migration initiated for table '{table_name}'")
+
+ def _verify_migration_success(self, table_name: str) -> None:
+ """Verify that migration was successful by checking for v2 columns."""
+ table_info = self.mariadb_connector.get_information_schema(table_name)
+
+ # Check for v2 columns that should have been added during migration
+ expected_v2_columns = {
+ "data_v2": "json",
+ "metadata": "json",
+ "user_field_1": "tinyint",
+ "user_field_2": "bigint",
+ "user_field_3": "longtext",
+ "status": "enum",
+ "error_message": "longtext",
+ }
+
+ for column_name, expected_type in expected_v2_columns.items():
+ self.assertIn(
+ column_name,
+ table_info,
+ f"Migration failed: Column '{column_name}' not found after migration",
+ )
+
+ actual_type = table_info[column_name].lower()
+ if expected_type == "json":
+ # MariaDB might return different representations for JSON
+ self.assertIn(
+ actual_type,
+ ["json", "longtext"], # MariaDB sometimes shows JSON as longtext
+ f"Column '{column_name}' has type '{actual_type}', expected JSON-compatible type",
+ )
+ elif expected_type == "enum":
+ self.assertIn(
+ actual_type,
+ ["enum"],
+ f"Column '{column_name}' has type '{actual_type}', expected enum type",
+ )
+ else:
+ self.assertEqual(
+ actual_type,
+ expected_type,
+ f"Column '{column_name}' has type '{actual_type}', expected '{expected_type}'",
+ )
+
+ print(
+ f"✅ Migration verification successful - all v2 columns present in '{table_name}'"
+ )
+
+ def _test_dual_column_writing(self, table_name: str) -> None:
+ """Test that data is written to both legacy and v2 columns after migration."""
+ engine = self.mariadb_connector.get_engine()
+ try:
+ cursor = engine.cursor()
+
+ # Query the latest inserted row to verify dual writing
+ cursor.execute(
+ f"""
+ SELECT data, data_v2, metadata, status, error_message
+ FROM {table_name}
+ WHERE file_path = '/migration/test/file.pdf'
+ ORDER BY created_at DESC
+ LIMIT 1
+ """
+ )
+ row = cursor.fetchone()
+
+ self.assertIsNotNone(row, "No migrated data found in table")
+
+ data_legacy, data_v2, metadata, status, error_message = row
+
+ # Verify legacy column has data
+ self.assertIsNotNone(data_legacy, "Legacy data column should not be empty")
+
+ # Verify v2 column has data
+ self.assertIsNotNone(data_v2, "V2 data column should not be empty")
+
+ # Verify metadata column has data
+ self.assertIsNotNone(metadata, "Metadata column should not be empty")
+
+ # Verify status is set correctly
+ self.assertEqual(
+ status, "SUCCESS", "Status should be SUCCESS for successful migration"
+ )
+
+ # Parse and verify JSON data (if stored as JSON string in legacy)
+ if isinstance(data_legacy, str):
+ try:
+ parsed_legacy = json.loads(data_legacy)
+ self.assertIn(
+ "migration_test",
+ parsed_legacy,
+ "Legacy data should contain migration test data",
+ )
+ except json.JSONDecodeError:
+ # If not JSON, just verify it contains our test data somehow
+ self.assertIn("migration_test", str(data_legacy))
+
+ cursor.close()
+ print(f"✅ Dual column writing verification successful for '{table_name}'")
+
+ finally:
+ if hasattr(engine, "close"):
+ engine.close()
+
+ def tearDown(self) -> None:
+ """Clean up test resources."""
+ # Optionally clean up test tables
+ # Note: Be careful with cleanup in production environments
+ # TEMPORARILY DISABLED FOR DEBUGGING - Uncomment to clean up tables
+ # try:
+ # engine = self.mariadb_connector.get_engine()
+ # cursor = engine.cursor()
+ # # Clean up both regular test table and migration test table
+ # cursor.execute(f"DROP TABLE IF EXISTS {self.test_table_name}")
+ # cursor.execute(f"DROP TABLE IF EXISTS LEGACY_OUTPUT_TEST")
+ # cursor.close()
+ # engine.close()
+ # except Exception:
+ # # Ignore cleanup errors
+ pass
diff --git a/backend/workflow_manager/endpoint_v2/tests/destination-connectors/test_destination_connector_mssql.py b/backend/workflow_manager/endpoint_v2/tests/destination-connectors/test_destination_connector_mssql.py
new file mode 100644
index 000000000..e9bafbc39
--- /dev/null
+++ b/backend/workflow_manager/endpoint_v2/tests/destination-connectors/test_destination_connector_mssql.py
@@ -0,0 +1,302 @@
+import os
+from unittest.mock import Mock, patch
+
+from django.test import TestCase
+from workflow_manager.endpoint_v2.constants import DestinationKey
+from workflow_manager.endpoint_v2.destination import DestinationConnector
+
+from unstract.connectors.databases.mssql import MSSQL
+
+
+class TestDestinationConnectorMSSQL(TestCase):
+ """Integration test for insert_into_db method with real MSSQL connector."""
+
+ def setUp(self) -> None:
+ """Set up test data and real MSSQL configuration."""
+
+ # MSSQL connection settings for testing
+ required_env_vars = [
+ "MSSQL_SERVER",
+ "MSSQL_USER",
+ "MSSQL_PASSWORD",
+ "MSSQL_DATABASE",
+ ]
+
+ # Check if all required environment variables are set
+ missing_vars = [var for var in required_env_vars if not os.getenv(var)]
+ if missing_vars:
+ self.skipTest(
+ f"Required MSSQL environment variables not set: {', '.join(missing_vars)}"
+ )
+
+ self.mssql_config = {
+ "server": os.getenv("MSSQL_SERVER"),
+ "port": os.getenv("MSSQL_PORT", "1433"),
+ "user": os.getenv("MSSQL_USER"),
+ "password": os.getenv("MSSQL_PASSWORD"),
+ "database": os.getenv("MSSQL_DATABASE"),
+ }
+
+ # Test data that will be inserted into the database
+ self.test_data = {"key": "value", "result": "test_result", "status": "success"}
+ self.test_metadata = {
+ "file_execution_id": "test-file-exec-id",
+ "extracted_text": "This is sample extracted text",
+ "processing_time": 1.5,
+ }
+ self.input_file_path = "/path/to/test/file.pdf"
+
+ # MSSQL table naming (use schema.table format)
+ self.test_table_name = "test_schema_1.MIGRATION"
+
+ # Create real MSSQL connector instance
+ self.mssql_connector = MSSQL(settings=self.mssql_config)
+
+ # Expected columns based on configuration (MSSQL data types)
+ self.expected_columns = {
+ "id": "NVARCHAR",
+ "file_path": "NVARCHAR",
+ "execution_id": "NVARCHAR",
+ "data": "NVARCHAR", # Legacy column remains NVARCHAR after migration
+ "data_v2": "NVARCHAR", # New NVARCHAR column added during migration
+ "metadata": "NVARCHAR", # New NVARCHAR column added during migration
+ "created_at": "DATETIMEOFFSET",
+ "created_by": "NVARCHAR",
+ "status": "NVARCHAR",
+ "error_message": "NVARCHAR",
+ "user_field_1": "BIT",
+ "user_field_2": "INT",
+ "user_field_3": "NVARCHAR",
+ }
+
+ def verify_table_columns(self, table_name: str) -> None:
+ """Verify that the table has expected columns with correct data types."""
+ table_info = self.mssql_connector.get_information_schema(table_name=table_name)
+
+ print(f"🔍 Actual table structure for '{table_name}': {table_info}")
+
+ # Check that key columns exist (some columns may be created on-demand)
+ required_columns = {
+ "id": ["NVARCHAR", "TEXT"], # MSSQL uses NVARCHAR
+ "file_path": ["NVARCHAR", "TEXT"], # MSSQL uses NVARCHAR
+ "execution_id": ["NVARCHAR", "TEXT"], # MSSQL uses NVARCHAR
+ "data": ["NVARCHAR", "TEXT"], # Can be NVARCHAR, migration might convert
+ "metadata": [
+ "NVARCHAR",
+ "TEXT",
+ ], # New NVARCHAR column added during migration
+ "created_at": "DATETIMEOFFSET",
+ "created_by": ["NVARCHAR", "TEXT"], # MSSQL uses NVARCHAR
+ }
+
+ # Optional columns that may exist in migration scenarios
+ optional_columns = {
+ "data_v2": ["NVARCHAR"], # Only exists in migration scenarios
+ }
+
+ for expected_column, expected_type in required_columns.items():
+ self.assertIn(
+ expected_column,
+ table_info,
+ f"Expected column '{expected_column}' not found in table '{table_name}'",
+ )
+
+ actual_type = table_info[expected_column]
+ # Handle both single expected types and lists of acceptable types
+ if isinstance(expected_type, list):
+ self.assertIn(
+ actual_type.upper(),
+ [t.upper() for t in expected_type],
+ f"Column '{expected_column}' has type '{actual_type}', expected one of {expected_type}",
+ )
+ else:
+ # MSSQL may return different type names, normalize for comparison
+ self.assertEqual(
+ actual_type.upper(),
+ expected_type.upper(),
+ f"Column '{expected_column}' has type '{actual_type}', expected '{expected_type}'",
+ )
+
+ # Check optional columns if they exist
+ for optional_column, expected_type in optional_columns.items():
+ if optional_column in table_info:
+ actual_type = table_info[optional_column]
+ if isinstance(expected_type, list):
+ self.assertIn(
+ actual_type.upper(),
+ [t.upper() for t in expected_type],
+ f"Optional column '{optional_column}' has type '{actual_type}', expected one of {expected_type}",
+ )
+ else:
+ self.assertEqual(
+ actual_type.upper(),
+ expected_type.upper(),
+ f"Optional column '{optional_column}' has type '{actual_type}', expected '{expected_type}'",
+ )
+
+ def create_mock_workflow(self) -> Mock:
+ """Create a mock workflow object."""
+ workflow = Mock()
+ workflow.id = "test-workflow-id"
+ return workflow
+
+ def create_mock_workflow_log(self) -> Mock:
+ """Create a mock workflow log object."""
+ return Mock()
+
+ def create_real_connector_instance(self) -> Mock:
+ """Create connector instance using real MSSQL connector."""
+ connector = Mock()
+ connector.connector_id = MSSQL.get_id()
+ connector.connector_metadata = self.mssql_config
+ # Store reference to real connector for potential future use
+ connector._real_mssql_connector = self.mssql_connector
+ return connector
+
+ def create_mock_endpoint(self, mock_connector_instance: Mock) -> Mock:
+ """Create a mock endpoint with real database configuration."""
+ endpoint = Mock()
+ endpoint.connector_instance = mock_connector_instance
+ endpoint.configuration = {
+ DestinationKey.TABLE: self.test_table_name,
+ DestinationKey.INCLUDE_AGENT: True,
+ DestinationKey.INCLUDE_TIMESTAMP: True,
+ DestinationKey.AGENT_NAME: "Unstract/DBWriter",
+ DestinationKey.COLUMN_MODE: "Write JSON to a single column",
+ DestinationKey.SINGLE_COLUMN_NAME: "data",
+ DestinationKey.FILE_PATH: "file_path",
+ DestinationKey.EXECUTION_ID: "execution_id",
+ }
+ return endpoint
+
+ def create_destination_connector(
+ self, mock_workflow: Mock, mock_workflow_log: Mock, mock_endpoint: Mock
+ ) -> DestinationConnector:
+ """Create a destination connector with mocked Django dependencies but real DB."""
+ with patch(
+ "workflow_manager.endpoint_v2.destination.UserContext"
+ ) as mock_context:
+ mock_context.get_organization_identifier.return_value = "test-org"
+
+ with patch.object(
+ DestinationConnector,
+ "_get_endpoint_for_workflow",
+ return_value=mock_endpoint,
+ ):
+ with patch.object(
+ DestinationConnector,
+ "_get_source_endpoint_for_workflow",
+ return_value=mock_endpoint,
+ ):
+ connector = DestinationConnector(
+ workflow=mock_workflow,
+ execution_id="test-execution-id",
+ workflow_log=mock_workflow_log,
+ use_file_history=False,
+ )
+ return connector
+
+ def test_insert_into_db_happy_path_mssql(self) -> None:
+ """Test successful insertion into real MSSQL database."""
+ # Create mock objects for Django models
+ mock_workflow = self.create_mock_workflow()
+ mock_workflow_log = self.create_mock_workflow_log()
+ mock_connector_instance = self.create_real_connector_instance()
+ mock_endpoint = self.create_mock_endpoint(mock_connector_instance)
+
+ # Create destination connector with real database connection
+ destination_connector = self.create_destination_connector(
+ mock_workflow, mock_workflow_log, mock_endpoint
+ )
+
+ # Test the insert_into_db method with real MSSQL
+ try:
+ # Mock only the methods that get data, let database operations be real
+ with patch.object(
+ destination_connector,
+ "get_tool_execution_result",
+ return_value=self.test_data,
+ ):
+ with patch.object(
+ destination_connector,
+ "get_combined_metadata",
+ return_value=self.test_metadata,
+ ):
+ # This will execute real database operations using MSSQL connector
+ destination_connector.insert_into_db(
+ input_file_path=self.input_file_path, error=None
+ )
+
+ # Verify the table structure
+ self.verify_table_columns(self.test_table_name)
+
+ print("✅ MSSQL integration test completed successfully!")
+
+ except Exception as e:
+ print(f"❌ MSSQL integration test failed: {str(e)}")
+ raise
+
+ def test_insert_into_db_with_error_mssql(self) -> None:
+ """Test insertion with error parameter into real MSSQL database."""
+ # Create mock objects
+ mock_workflow = self.create_mock_workflow()
+ mock_workflow_log = self.create_mock_workflow_log()
+ mock_connector_instance = self.create_real_connector_instance()
+ mock_endpoint = self.create_mock_endpoint(mock_connector_instance)
+
+ # Create destination connector
+ destination_connector = self.create_destination_connector(
+ mock_workflow, mock_workflow_log, mock_endpoint
+ )
+
+ error_message = "Test processing error occurred"
+
+ # Mock the methods that get data
+ with patch.object(
+ destination_connector,
+ "get_tool_execution_result",
+ return_value=self.test_data,
+ ):
+ with patch.object(
+ destination_connector,
+ "get_combined_metadata",
+ return_value=self.test_metadata,
+ ):
+ # Execute with error parameter
+ destination_connector.insert_into_db(
+ input_file_path=self.input_file_path, error=error_message
+ )
+
+ # Verify the table structure
+ self.verify_table_columns(self.test_table_name)
+
+ print(
+ f"✅ Successfully inserted error data into MSSQL table: {self.test_table_name}"
+ )
+
+ def test_mssql_connector_connection(self) -> None:
+ """Test that the MSSQL connector can establish a connection."""
+ # Test the real MSSQL connector directly
+ try:
+ engine = self.mssql_connector.get_engine()
+ self.assertIsNotNone(engine)
+ print("✅ MSSQL connector successfully established connection")
+
+ # Test a simple query to verify connection works
+ with engine.cursor() as cursor:
+ cursor.execute("SELECT 1 as test_column")
+ result = cursor.fetchall()
+ self.assertIsNotNone(result)
+ print("✅ MSSQL connector successfully executed test query")
+
+ # Clean up
+ if hasattr(engine, "close"):
+ engine.close()
+ except Exception as e:
+ self.fail(f"MSSQL connector failed to connect: {str(e)}")
+
+ def tearDown(self) -> None:
+ """Clean up test resources."""
+ # Optionally clean up test table
+ # Note: Be careful with cleanup in production environments
+ pass
diff --git a/backend/workflow_manager/endpoint_v2/tests/destination-connectors/test_destination_connector_mysql.py b/backend/workflow_manager/endpoint_v2/tests/destination-connectors/test_destination_connector_mysql.py
new file mode 100644
index 000000000..2e418647d
--- /dev/null
+++ b/backend/workflow_manager/endpoint_v2/tests/destination-connectors/test_destination_connector_mysql.py
@@ -0,0 +1,644 @@
+import json
+import os
+from unittest.mock import Mock, patch
+
+from django.test import TestCase
+from workflow_manager.endpoint_v2.constants import DestinationKey
+from workflow_manager.endpoint_v2.destination import DestinationConnector
+
+from unstract.connectors.databases.mysql import MySQL
+
+
+class TestDestinationConnectorMySQL(TestCase):
+ """Integration test for insert_into_db method with real MySQL connector."""
+
+ def setUp(self) -> None:
+ """Set up test data and real MySQL configuration."""
+
+ # MySQL connection settings for testing
+ required_env_vars = [
+ "MYSQL_HOST",
+ "MYSQL_PORT",
+ "MYSQL_DATABASE",
+ "MYSQL_USER",
+ "MYSQL_PASSWORD",
+ ]
+
+ # Check if all required environment variables are set
+ missing_vars = [var for var in required_env_vars if not os.getenv(var)]
+ if missing_vars:
+ self.skipTest(
+ f"Required MySQL environment variables not set: {', '.join(missing_vars)}"
+ )
+
+ self.mysql_config = {
+ "host": os.getenv("MYSQL_HOST"),
+ "port": os.getenv("MYSQL_PORT", 3306),
+ "database": os.getenv("MYSQL_DATABASE"),
+ "user": os.getenv("MYSQL_USER"),
+ "password": os.getenv("MYSQL_PASSWORD"),
+ }
+
+ # Test data that will be inserted into the database
+ self.test_data = {"key": "value", "result": "test_result", "status": "success"}
+ self.test_metadata = {
+ "file_execution_id": "test-file-exec-id",
+ "extracted_text": "This is sample extracted text",
+ "processing_time": 1.5,
+ }
+ self.input_file_path = "/path/to/test/file.pdf"
+ self.test_table_name = "MIGRATION"
+
+ # Create real MySQL connector instance
+ self.mysql_connector = MySQL(settings=self.mysql_config)
+
+ # Expected columns based on configuration (MySQL data types)
+ self.expected_columns = {
+ "id": "longtext",
+ "file_path": "longtext",
+ "execution_id": "longtext",
+ "data": "json",
+ "created_at": "timestamp",
+ "created_by": "longtext",
+ "metadata": "json",
+ "error_message": "longtext",
+ "status": "enum",
+ "user_field_1": "boolean", # BOOLEAN maps to TINYINT in MySQL
+ "user_field_2": "bigint",
+ "user_field_3": "longtext",
+ }
+
+ def verify_table_columns(self, table_name: str) -> None:
+ """Verify that all expected columns exist in the table with correct types."""
+ table_info = self.mysql_connector.get_information_schema(table_name=table_name)
+
+ print(f"🔍 Actual table structure for '{table_name}': {table_info}")
+
+ # Check that key columns exist (some columns may be created on-demand)
+ required_columns = {
+ "id": "longtext",
+ "file_path": "longtext",
+ "execution_id": "longtext",
+ "data": "json",
+ "created_at": "timestamp",
+ "created_by": "longtext",
+ }
+
+ for expected_column, expected_type in required_columns.items():
+ self.assertIn(
+ expected_column,
+ table_info,
+ f"Expected column '{expected_column}' not found in table '{table_name}'",
+ )
+
+ actual_type = table_info[expected_column].lower()
+ expected_type_lower = expected_type.lower()
+
+ # MySQL may return slightly different type names, so we do flexible matching
+ if expected_type_lower == "timestamp":
+ self.assertIn(
+ actual_type,
+ ["timestamp", "datetime", "timestamp(0)"],
+ f"Column '{expected_column}' has type '{actual_type}', expected timestamp-like type",
+ )
+ elif expected_type_lower == "tinyint":
+ self.assertIn(
+ actual_type,
+ ["tinyint", "tinyint(1)", "boolean", "bool"],
+ f"Column '{expected_column}' has type '{actual_type}', expected boolean/tinyint type",
+ )
+ else:
+ self.assertEqual(
+ actual_type,
+ expected_type_lower,
+ f"Column '{expected_column}' has type '{actual_type}', expected '{expected_type_lower}'",
+ )
+
+ print(f"✅ All required columns verified in table '{table_name}'")
+
+ def create_mock_workflow(self) -> Mock:
+ """Create a mock workflow object."""
+ workflow = Mock()
+ workflow.id = "test-workflow-id"
+ return workflow
+
+ def create_mock_workflow_log(self) -> Mock:
+ """Create a mock workflow log object."""
+ return Mock()
+
+ def create_real_connector_instance(self) -> Mock:
+ """Create connector instance using real MySQL connector."""
+ connector = Mock()
+ connector.connector_id = MySQL.get_id()
+ connector.connector_metadata = self.mysql_config
+ # Store reference to real connector for potential future use
+ connector._real_mysql_connector = self.mysql_connector
+ return connector
+
+ def create_mock_endpoint(self, mock_connector_instance: Mock) -> Mock:
+ """Create a mock endpoint with real database configuration."""
+ endpoint = Mock()
+ endpoint.connector_instance = mock_connector_instance
+ endpoint.configuration = {
+ DestinationKey.TABLE: self.test_table_name,
+ DestinationKey.INCLUDE_AGENT: True,
+ DestinationKey.INCLUDE_TIMESTAMP: True,
+ DestinationKey.AGENT_NAME: "Unstract/DBWriter",
+ DestinationKey.COLUMN_MODE: "Write JSON to a single column",
+ DestinationKey.SINGLE_COLUMN_NAME: "data",
+ DestinationKey.FILE_PATH: "file_path",
+ DestinationKey.EXECUTION_ID: "execution_id",
+ }
+ return endpoint
+
+ def create_destination_connector(
+ self, mock_workflow: Mock, mock_workflow_log: Mock, mock_endpoint: Mock
+ ) -> DestinationConnector:
+ """Create a destination connector with mocked Django dependencies but real DB."""
+ with patch(
+ "workflow_manager.endpoint_v2.destination.UserContext"
+ ) as mock_context:
+ mock_context.get_organization_identifier.return_value = "test-org"
+
+ with patch.object(
+ DestinationConnector,
+ "_get_endpoint_for_workflow",
+ return_value=mock_endpoint,
+ ):
+ with patch.object(
+ DestinationConnector,
+ "_get_source_endpoint_for_workflow",
+ return_value=mock_endpoint,
+ ):
+ connector = DestinationConnector(
+ workflow=mock_workflow,
+ execution_id="test-execution-id",
+ workflow_log=mock_workflow_log,
+ use_file_history=False,
+ )
+ return connector
+
+ def test_insert_into_db_happy_path_mysql(self) -> None:
+ """Test successful insertion into real MySQL database."""
+ # Create mock objects for Django models
+ mock_workflow = self.create_mock_workflow()
+ mock_workflow_log = self.create_mock_workflow_log()
+ mock_connector_instance = self.create_real_connector_instance()
+ mock_endpoint = self.create_mock_endpoint(mock_connector_instance)
+
+ # Create destination connector with real database connection
+ destination_connector = self.create_destination_connector(
+ mock_workflow, mock_workflow_log, mock_endpoint
+ )
+
+ # Mock only the methods that get data, let database operations be real
+ with patch.object(
+ destination_connector,
+ "get_tool_execution_result",
+ return_value=self.test_data,
+ ):
+ with patch.object(
+ destination_connector,
+ "get_combined_metadata",
+ return_value=self.test_metadata,
+ ):
+ # This will execute real database operations using MySQL connector
+ destination_connector.insert_into_db(
+ input_file_path=self.input_file_path, error=None
+ )
+
+ # Verify that all expected columns were created
+ self.verify_table_columns(self.test_table_name)
+
+ print(
+ f"✅ Successfully inserted test data into MySQL table: {self.test_table_name}"
+ )
+
+ def test_insert_into_db_with_error_mysql(self) -> None:
+ """Test insertion with error parameter into real MySQL database."""
+ # Create mock objects
+ mock_workflow = self.create_mock_workflow()
+ mock_workflow_log = self.create_mock_workflow_log()
+ mock_connector_instance = self.create_real_connector_instance()
+ mock_endpoint = self.create_mock_endpoint(mock_connector_instance)
+
+ # Create destination connector
+ destination_connector = self.create_destination_connector(
+ mock_workflow, mock_workflow_log, mock_endpoint
+ )
+
+ error_message = "Test processing error occurred"
+
+ # Mock the methods that get data
+ with patch.object(
+ destination_connector,
+ "get_tool_execution_result",
+ return_value=self.test_data,
+ ):
+ with patch.object(
+ destination_connector,
+ "get_combined_metadata",
+ return_value=self.test_metadata,
+ ):
+ # Execute with error parameter
+ destination_connector.insert_into_db(
+ input_file_path=self.input_file_path, error=error_message
+ )
+
+ # Verify that all expected columns were created
+ self.verify_table_columns(self.test_table_name)
+
+ print(
+ f"✅ Successfully inserted error data into MySQL table: {self.test_table_name}"
+ )
+
+ def test_mysql_connector_connection(self) -> None:
+ """Test that the MySQL connector can establish a connection."""
+ # Test the real MySQL connector directly
+ try:
+ engine = self.mysql_connector.get_engine()
+ self.assertIsNotNone(engine)
+ print("✅ MySQL connector successfully established connection")
+
+ # Test a simple query to verify connection works
+ cursor = engine.cursor()
+ cursor.execute("SELECT 1 as test_column")
+ result = cursor.fetchall()
+ self.assertIsNotNone(result)
+ self.assertEqual(result[0][0], 1)
+ cursor.close()
+ print("✅ MySQL connector successfully executed test query")
+
+ # Clean up
+ if hasattr(engine, "close"):
+ engine.close()
+ except Exception as e:
+ self.fail(f"MySQL connector failed to connect: {str(e)}")
+
+ def test_comprehensive_column_data_verification(self) -> None:
+ """Test that data is properly inserted into all columns with correct values."""
+ # Create mock objects
+ mock_workflow = self.create_mock_workflow()
+ mock_workflow_log = self.create_mock_workflow_log()
+ mock_connector_instance = self.create_real_connector_instance()
+ mock_endpoint = self.create_mock_endpoint(mock_connector_instance)
+
+ # Create destination connector
+ destination_connector = self.create_destination_connector(
+ mock_workflow, mock_workflow_log, mock_endpoint
+ )
+
+ # Mock the methods that get data
+ with patch.object(
+ destination_connector,
+ "get_tool_execution_result",
+ return_value=self.test_data,
+ ):
+ with patch.object(
+ destination_connector,
+ "get_combined_metadata",
+ return_value=self.test_metadata,
+ ):
+ # Execute database insertion
+ destination_connector.insert_into_db(
+ input_file_path=self.input_file_path, error=None
+ )
+
+ # Verify table structure
+ self.verify_table_columns(self.test_table_name)
+
+ # Verify data was inserted correctly by querying the table
+ engine = self.mysql_connector.get_engine()
+ try:
+ cursor = engine.cursor()
+ cursor.execute(
+ f"SELECT created_by, file_path, execution_id, created_at, data, metadata FROM {self.test_table_name} ORDER BY created_at DESC LIMIT 1"
+ )
+ row = cursor.fetchone()
+
+ self.assertIsNotNone(row, "No data found in table after insertion")
+
+ # Map results to column names
+ created_by, file_path, execution_id, created_at, data, metadata = row
+
+ # Verify specific column values
+ self.assertEqual(
+ created_by,
+ "Unstract/DBWriter",
+ f"Expected created_by to be 'Unstract/DBWriter', got '{created_by}'",
+ )
+
+ self.assertEqual(
+ file_path,
+ self.input_file_path,
+ f"Expected file_path to be '{self.input_file_path}', got '{file_path}'",
+ )
+
+ self.assertEqual(
+ execution_id,
+ "test-execution-id",
+ f"Expected execution_id to be 'test-execution-id', got '{execution_id}'",
+ )
+
+ self.assertIsNotNone(created_at, "Expected created_at timestamp to be set")
+
+ self.assertIsNotNone(data, "Expected data column to contain inserted data")
+
+ self.assertIsNotNone(
+ metadata, "Expected metadata column to contain metadata"
+ )
+
+ cursor.close()
+ print("✅ All column data values verified successfully")
+
+ finally:
+ if hasattr(engine, "close"):
+ engine.close()
+
+ def test_mysql_specific_data_types(self) -> None:
+ """Test MySQL-specific data types and constraints."""
+ # Test that status column accepts ENUM values correctly
+ engine = self.mysql_connector.get_engine()
+ try:
+ cursor = engine.cursor()
+
+ # Create the table first if it doesn't exist
+ create_table_query = self.mysql_connector.get_create_table_base_query(
+ self.test_table_name
+ )
+ # Add the required columns for the test
+ create_table_query += (
+ "file_path LONGTEXT, " "execution_id LONGTEXT, " "data LONGTEXT)"
+ )
+
+ cursor.execute(create_table_query)
+
+ # Test inserting valid ENUM values
+ cursor.execute(
+ f"INSERT INTO {self.test_table_name} (id, status, created_at) VALUES (%s, %s, NOW())",
+ ("test-enum-1", "SUCCESS"),
+ )
+
+ cursor.execute(
+ f"INSERT INTO {self.test_table_name} (id, status, created_at) VALUES (%s, %s, NOW())",
+ ("test-enum-2", "ERROR"),
+ )
+
+ # Verify the data was inserted
+ cursor.execute(
+ f"SELECT id, status FROM {self.test_table_name} WHERE id IN (%s, %s)",
+ ("test-enum-1", "test-enum-2"),
+ )
+ results = cursor.fetchall()
+
+ self.assertEqual(len(results), 2, "Expected 2 rows to be inserted")
+
+ # Verify ENUM values
+ statuses = [row[1] for row in results]
+ self.assertIn("SUCCESS", statuses)
+ self.assertIn("ERROR", statuses)
+
+ cursor.close()
+ print("✅ MySQL ENUM data type verification successful")
+
+ finally:
+ if hasattr(engine, "close"):
+ engine.close()
+
+ def _create_legacy_table(self, table_name: str) -> None:
+ """Create a legacy table structure (data column as LONGTEXT, not JSON)."""
+ engine = self.mysql_connector.get_engine()
+ try:
+ cursor = engine.cursor()
+
+ # FORCE DROP AND RECREATE to ensure clean legacy table
+ cursor.execute(f"DROP TABLE IF EXISTS {table_name}")
+ print(f"🗑️ Dropped existing table '{table_name}' if it existed")
+
+ # Create fresh legacy table with LONGTEXT data column (not JSON)
+ legacy_create_query = f"""
+ CREATE TABLE {table_name} (
+ id LONGTEXT,
+ created_by LONGTEXT,
+ created_at TIMESTAMP,
+ data LONGTEXT,
+ file_path LONGTEXT,
+ execution_id LONGTEXT
+ )
+ """
+
+ cursor.execute(legacy_create_query)
+ print(
+ f"📋 Created fresh legacy table '{table_name}' with LONGTEXT data column"
+ )
+
+ # Insert some legacy data to make it realistic
+ cursor.execute(
+ f"INSERT INTO {table_name} (id, created_by, created_at, data, file_path, execution_id) VALUES (%s, %s, NOW(), %s, %s, %s)",
+ (
+ "legacy-1",
+ "Legacy/DBWriter",
+ '{"old": "legacy_data"}',
+ "/legacy/file.pdf",
+ "legacy-exec-1",
+ ),
+ )
+ print(f"📝 Inserted legacy test data into '{table_name}'")
+
+ cursor.close()
+ print(f"✅ Legacy table '{table_name}' setup completed")
+
+ finally:
+ if hasattr(engine, "close"):
+ engine.close()
+
+ def _verify_table_is_legacy(self, table_name: str) -> bool:
+ """Verify that table is detected as legacy using the real detection logic."""
+ try:
+ # Use the actual detection logic from the codebase
+ is_string_column = self.mysql_connector.is_string_column(
+ table_info=self.mysql_connector.get_information_schema(table_name),
+ column_name="data",
+ )
+
+ if is_string_column:
+ print(
+ f"✅ Table '{table_name}' correctly detected as legacy (data column is string type)"
+ )
+ return True
+ else:
+ print(f"❌ Table '{table_name}' not detected as legacy")
+ return False
+
+ except Exception as e:
+ print(f"❌ Error checking legacy status: {e}")
+ return False
+
+ def _initiate_migration_via_workflow(self, table_name: str) -> None:
+ """Initiate migration using the real workflow destination connector logic."""
+ # Create mock objects for the workflow system
+ mock_workflow = self.create_mock_workflow()
+ mock_workflow_log = self.create_mock_workflow_log()
+ mock_connector_instance = self.create_real_connector_instance()
+
+ # Create endpoint with the legacy table name
+ mock_endpoint = Mock()
+ mock_endpoint.connector_instance = mock_connector_instance
+ mock_endpoint.configuration = {
+ DestinationKey.TABLE: table_name, # Use legacy table
+ DestinationKey.INCLUDE_AGENT: True,
+ DestinationKey.INCLUDE_TIMESTAMP: True,
+ DestinationKey.AGENT_NAME: "Unstract/DBWriter",
+ DestinationKey.COLUMN_MODE: "Write JSON to a single column",
+ DestinationKey.SINGLE_COLUMN_NAME: "data",
+ DestinationKey.FILE_PATH: "file_path",
+ DestinationKey.EXECUTION_ID: "execution_id",
+ }
+
+ # Create destination connector
+ destination_connector = self.create_destination_connector(
+ mock_workflow, mock_workflow_log, mock_endpoint
+ )
+
+ # Mock the data methods to provide test data
+ test_migration_data = {"migration_test": "v2_data", "status": "migrated"}
+ test_migration_metadata = {
+ "file_execution_id": "migration-test-exec-id",
+ "migration_test": True,
+ "extracted_text": "Migration test extracted text",
+ }
+
+ with patch.object(
+ destination_connector,
+ "get_tool_execution_result",
+ return_value=test_migration_data,
+ ):
+ with patch.object(
+ destination_connector,
+ "get_combined_metadata",
+ return_value=test_migration_metadata,
+ ):
+ # This should trigger the migration logic automatically
+ destination_connector.insert_into_db(
+ input_file_path="/migration/test/file.pdf", error=None
+ )
+
+ print(f"✅ Migration initiated for table '{table_name}'")
+
+ def _verify_migration_success(self, table_name: str) -> None:
+ """Verify that migration was successful by checking for v2 columns."""
+ table_info = self.mysql_connector.get_information_schema(table_name)
+
+ # Check for v2 columns that should have been added during migration
+ expected_v2_columns = {
+ "data_v2": "json",
+ "metadata": "json",
+ "user_field_1": "boolean",
+ "user_field_2": "bigint",
+ "user_field_3": "longtext",
+ "status": "enum",
+ "error_message": "longtext",
+ }
+
+ for column_name, expected_type in expected_v2_columns.items():
+ self.assertIn(
+ column_name,
+ table_info,
+ f"Migration failed: Column '{column_name}' not found after migration",
+ )
+
+ actual_type = table_info[column_name].lower()
+ if expected_type == "json":
+ # MySQL might return different representations for JSON
+ self.assertIn(
+ actual_type,
+ ["json", "longtext"], # MySQL sometimes shows JSON as longtext
+ f"Column '{column_name}' has type '{actual_type}', expected JSON-compatible type",
+ )
+ elif expected_type == "enum":
+ self.assertIn(
+ actual_type,
+ ["enum"],
+ f"Column '{column_name}' has type '{actual_type}', expected enum type",
+ )
+ else:
+ self.assertEqual(
+ actual_type,
+ expected_type,
+ f"Column '{column_name}' has type '{actual_type}', expected '{expected_type}'",
+ )
+
+ print(
+ f"✅ Migration verification successful - all v2 columns present in '{table_name}'"
+ )
+
+ def _test_dual_column_writing(self, table_name: str) -> None:
+ """Test that data is written to both legacy and v2 columns after migration."""
+ engine = self.mysql_connector.get_engine()
+ try:
+ cursor = engine.cursor()
+
+ # Query the latest inserted row to verify dual writing
+ cursor.execute(
+ f"""
+ SELECT data, data_v2, metadata, status, error_message
+ FROM {table_name}
+ WHERE file_path = '/migration/test/file.pdf'
+ ORDER BY created_at DESC
+ LIMIT 1
+ """
+ )
+ row = cursor.fetchone()
+
+ self.assertIsNotNone(row, "No migrated data found in table")
+
+ data_legacy, data_v2, metadata, status, error_message = row
+
+ # Verify legacy column has data
+ self.assertIsNotNone(data_legacy, "Legacy data column should not be empty")
+
+ # Verify v2 column has data
+ self.assertIsNotNone(data_v2, "V2 data column should not be empty")
+
+ # Verify metadata column has data
+ self.assertIsNotNone(metadata, "Metadata column should not be empty")
+
+ # Verify status is set correctly
+ self.assertEqual(
+ status, "SUCCESS", "Status should be SUCCESS for successful migration"
+ )
+
+ # Parse and verify JSON data (if stored as JSON string in legacy)
+ if isinstance(data_legacy, str):
+ try:
+ parsed_legacy = json.loads(data_legacy)
+ self.assertIn(
+ "migration_test",
+ parsed_legacy,
+ "Legacy data should contain migration test data",
+ )
+ except json.JSONDecodeError:
+ # If not JSON, just verify it contains our test data somehow
+ self.assertIn("migration_test", str(data_legacy))
+
+ cursor.close()
+ print(f"✅ Dual column writing verification successful for '{table_name}'")
+
+ finally:
+ if hasattr(engine, "close"):
+ engine.close()
+
+ def tearDown(self) -> None:
+ """Clean up test resources."""
+ # # Clean up test tables
+ # try:
+ # engine = self.mysql_connector.get_engine()
+ # cursor = engine.cursor()
+ # # Clean up both regular test table and migration test table
+ # cursor.execute(f"DROP TABLE IF EXISTS {self.test_table_name}")
+ # cursor.execute(f"DROP TABLE IF EXISTS LEGACY_OUTPUT_MYSQL")
+ # cursor.close()
+ # engine.close()
+ # except Exception:
+ # Ignore cleanup errors
+ pass
diff --git a/backend/workflow_manager/endpoint_v2/tests/destination-connectors/test_destination_connector_oracle.py b/backend/workflow_manager/endpoint_v2/tests/destination-connectors/test_destination_connector_oracle.py
new file mode 100644
index 000000000..5698e722a
--- /dev/null
+++ b/backend/workflow_manager/endpoint_v2/tests/destination-connectors/test_destination_connector_oracle.py
@@ -0,0 +1,313 @@
+import os
+from unittest.mock import Mock, patch
+
+from django.test import TestCase
+from workflow_manager.endpoint_v2.constants import DestinationKey
+from workflow_manager.endpoint_v2.destination import DestinationConnector
+
+from unstract.connectors.databases.oracle_db import OracleDB
+
+
+class TestDestinationConnectorOracle(TestCase):
+ """Integration test for insert_into_db method with real Oracle DB connector."""
+
+ def setUp(self) -> None:
+ """Set up test data and real Oracle DB configuration."""
+
+ # Oracle DB connection settings for testing
+ required_env_vars = [
+ "ORACLE_CONFIG_DIR",
+ "ORACLE_USER",
+ "ORACLE_PASSWORD",
+ "ORACLE_DSN",
+ "ORACLE_WALLET_LOCATION",
+ "ORACLE_WALLET_PASSWORD",
+ ]
+
+ # Check if all required environment variables are set
+ missing_vars = [var for var in required_env_vars if not os.getenv(var)]
+ if missing_vars:
+ self.skipTest(
+ f"Required Oracle DB environment variables not set: {', '.join(missing_vars)}"
+ )
+
+ self.oracle_config = {
+ "config_dir": os.getenv("ORACLE_CONFIG_DIR"),
+ "user": os.getenv("ORACLE_USER"),
+ "password": os.getenv("ORACLE_PASSWORD"),
+ "dsn": os.getenv("ORACLE_DSN"),
+ "wallet_location": os.getenv("ORACLE_WALLET_LOCATION"),
+ "wallet_password": os.getenv("ORACLE_WALLET_PASSWORD"),
+ }
+
+ # Test data that will be inserted into the database
+ self.test_data = {"key": "value", "result": "test_result", "status": "success"}
+ self.test_metadata = {
+ "file_execution_id": "test-file-exec-id",
+ "extracted_text": "This is sample extracted text",
+ "processing_time": 1.5,
+ }
+ self.input_file_path = "/path/to/test/file.pdf"
+
+ # Oracle table naming (typically uppercase)
+ self.test_table_name = "MIGRATION"
+
+ # Create real Oracle DB connector instance
+ self.oracle_connector = OracleDB(settings=self.oracle_config)
+
+ # Expected columns based on configuration (Oracle data types)
+ self.expected_columns = {
+ "ID": "VARCHAR2",
+ "FILE_PATH": "VARCHAR2",
+ "EXECUTION_ID": "VARCHAR2",
+ "DATA": "CLOB", # Legacy column remains CLOB after migration
+ "DATA_V2": "CLOB", # New CLOB column added during migration
+ "METADATA": "CLOB", # New CLOB column added during migration
+ "CREATED_AT": "TIMESTAMP",
+ "CREATED_BY": "VARCHAR2",
+ "STATUS": "VARCHAR2",
+ "ERROR_MESSAGE": "VARCHAR2",
+ "USER_FIELD_1": "NUMBER",
+ "USER_FIELD_2": "NUMBER",
+ "USER_FIELD_3": "VARCHAR2",
+ }
+
+ def verify_table_columns(self, table_name: str) -> None:
+ """Verify that the table has expected columns with correct data types."""
+ table_info = self.oracle_connector.get_information_schema(table_name=table_name)
+
+ print(f"🔍 Actual table structure for '{table_name}': {table_info}")
+
+ # Check that key columns exist (some columns may be created on-demand)
+ required_columns = {
+ "ID": ["VARCHAR2"], # Oracle uses VARCHAR2
+ "FILE_PATH": ["CLOB", "VARCHAR2"], # Oracle can use either CLOB or VARCHAR2
+ "EXECUTION_ID": [
+ "CLOB",
+ "VARCHAR2",
+ ], # Oracle can use either CLOB or VARCHAR2
+ "DATA": [
+ "CLOB",
+ "VARCHAR2",
+ ], # Can be CLOB or VARCHAR2, depends on migration scenario
+ "METADATA": [
+ "CLOB",
+ ], # New CLOB column added during migration for JSON data
+ "CREATED_AT": ["TIMESTAMP", "TIMESTAMP(6)"], # Oracle timestamp types
+ "CREATED_BY": ["VARCHAR2"], # Oracle uses VARCHAR2
+ }
+
+ # Optional columns that may exist in migration scenarios
+ optional_columns = {
+ "DATA_V2": [
+ "CLOB",
+ "VARCHAR2",
+ ], # Can be CLOB or VARCHAR2 in migration scenarios
+ }
+
+ for expected_column, expected_type in required_columns.items():
+ self.assertIn(
+ expected_column,
+ table_info,
+ f"Expected column '{expected_column}' not found in table '{table_name}'",
+ )
+
+ actual_type = table_info[expected_column]
+ # Handle both single expected types and lists of acceptable types
+ if isinstance(expected_type, list):
+ self.assertIn(
+ actual_type.upper(),
+ [t.upper() for t in expected_type],
+ f"Column '{expected_column}' has type '{actual_type}', expected one of {expected_type}",
+ )
+ else:
+ # Oracle may return different type names, normalize for comparison
+ self.assertEqual(
+ actual_type.upper(),
+ expected_type.upper(),
+ f"Column '{expected_column}' has type '{actual_type}', expected '{expected_type}'",
+ )
+
+ # Check optional columns if they exist
+ for optional_column, expected_type in optional_columns.items():
+ if optional_column in table_info:
+ actual_type = table_info[optional_column]
+ if isinstance(expected_type, list):
+ self.assertIn(
+ actual_type.upper(),
+ [t.upper() for t in expected_type],
+ f"Optional column '{optional_column}' has type '{actual_type}', expected one of {expected_type}",
+ )
+ else:
+ self.assertEqual(
+ actual_type.upper(),
+ expected_type.upper(),
+ f"Optional column '{optional_column}' has type '{actual_type}', expected '{expected_type}'",
+ )
+
+ def create_mock_workflow(self) -> Mock:
+ """Create a mock workflow object."""
+ workflow = Mock()
+ workflow.id = "test-workflow-id"
+ return workflow
+
+ def create_mock_workflow_log(self) -> Mock:
+ """Create a mock workflow log object."""
+ return Mock()
+
+ def create_real_connector_instance(self) -> Mock:
+ """Create connector instance using real Oracle DB connector."""
+ connector = Mock()
+ connector.connector_id = OracleDB.get_id()
+ connector.connector_metadata = self.oracle_config
+ # Store reference to real connector for potential future use
+ connector._real_oracle_connector = self.oracle_connector
+ return connector
+
+ def create_mock_endpoint(self, mock_connector_instance: Mock) -> Mock:
+ """Create a mock endpoint with real database configuration."""
+ endpoint = Mock()
+ endpoint.connector_instance = mock_connector_instance
+ endpoint.configuration = {
+ DestinationKey.TABLE: self.test_table_name,
+ DestinationKey.INCLUDE_AGENT: True,
+ DestinationKey.INCLUDE_TIMESTAMP: True,
+ DestinationKey.AGENT_NAME: "Unstract/DBWriter",
+ DestinationKey.COLUMN_MODE: "Write JSON to a single column",
+ DestinationKey.SINGLE_COLUMN_NAME: "data",
+ DestinationKey.FILE_PATH: "file_path",
+ DestinationKey.EXECUTION_ID: "execution_id",
+ }
+ return endpoint
+
+ def create_destination_connector(
+ self, mock_workflow: Mock, mock_workflow_log: Mock, mock_endpoint: Mock
+ ) -> DestinationConnector:
+ """Create a destination connector with mocked Django dependencies but real DB."""
+ with patch(
+ "workflow_manager.endpoint_v2.destination.UserContext"
+ ) as mock_context:
+ mock_context.get_organization_identifier.return_value = "test-org"
+
+ with patch.object(
+ DestinationConnector,
+ "_get_endpoint_for_workflow",
+ return_value=mock_endpoint,
+ ):
+ with patch.object(
+ DestinationConnector,
+ "_get_source_endpoint_for_workflow",
+ return_value=mock_endpoint,
+ ):
+ connector = DestinationConnector(
+ workflow=mock_workflow,
+ execution_id="test-execution-id",
+ workflow_log=mock_workflow_log,
+ use_file_history=False,
+ )
+ return connector
+
+ def test_insert_into_db_happy_path_oracle(self) -> None:
+ """Test successful insertion into real Oracle DB database."""
+ # Create mock objects for Django models
+ mock_workflow = self.create_mock_workflow()
+ mock_workflow_log = self.create_mock_workflow_log()
+ mock_connector_instance = self.create_real_connector_instance()
+ mock_endpoint = self.create_mock_endpoint(mock_connector_instance)
+
+ # Create destination connector with real database connection
+ destination_connector = self.create_destination_connector(
+ mock_workflow, mock_workflow_log, mock_endpoint
+ )
+
+ # Test the insert_into_db method with real Oracle DB
+ try:
+ # Mock only the methods that get data, let database operations be real
+ with patch.object(
+ destination_connector,
+ "get_tool_execution_result",
+ return_value=self.test_data,
+ ):
+ with patch.object(
+ destination_connector,
+ "get_combined_metadata",
+ return_value=self.test_metadata,
+ ):
+ # This will execute real database operations using Oracle DB connector
+ destination_connector.insert_into_db(
+ input_file_path=self.input_file_path, error=None
+ )
+
+ # Verify the table structure
+ self.verify_table_columns(self.test_table_name)
+
+ print("✅ Oracle DB integration test completed successfully!")
+
+ except Exception as e:
+ print(f"❌ Oracle DB integration test failed: {str(e)}")
+ raise
+
+ def test_insert_into_db_with_error_oracle(self) -> None:
+ """Test insertion with error parameter into real Oracle DB database."""
+ # Create mock objects
+ mock_workflow = self.create_mock_workflow()
+ mock_workflow_log = self.create_mock_workflow_log()
+ mock_connector_instance = self.create_real_connector_instance()
+ mock_endpoint = self.create_mock_endpoint(mock_connector_instance)
+
+ # Create destination connector
+ destination_connector = self.create_destination_connector(
+ mock_workflow, mock_workflow_log, mock_endpoint
+ )
+
+ error_message = "Test processing error occurred"
+
+ # Mock the methods that get data
+ with patch.object(
+ destination_connector,
+ "get_tool_execution_result",
+ return_value=self.test_data,
+ ):
+ with patch.object(
+ destination_connector,
+ "get_combined_metadata",
+ return_value=self.test_metadata,
+ ):
+ # Execute with error parameter
+ destination_connector.insert_into_db(
+ input_file_path=self.input_file_path, error=error_message
+ )
+
+ # Verify the table structure
+ self.verify_table_columns(self.test_table_name)
+
+ print(
+ f"✅ Successfully inserted error data into Oracle DB table: {self.test_table_name}"
+ )
+
+ def test_oracle_connector_connection(self) -> None:
+ """Test that the Oracle DB connector can establish a connection."""
+ # Test the real Oracle DB connector directly
+ try:
+ engine = self.oracle_connector.get_engine()
+ self.assertIsNotNone(engine)
+ print("✅ Oracle DB connector successfully established connection")
+
+ # Test a simple query to verify connection works
+ with engine.cursor() as cursor:
+ cursor.execute("SELECT 1 as test_column FROM DUAL")
+ result = cursor.fetchall()
+ self.assertIsNotNone(result)
+ print("✅ Oracle DB connector successfully executed test query")
+
+ # Clean up
+ if hasattr(engine, "close"):
+ engine.close()
+ except Exception as e:
+ self.fail(f"Oracle DB connector failed to connect: {str(e)}")
+
+ def tearDown(self) -> None:
+ """Clean up test resources."""
+ # Optionally clean up test table
+ # Note: Be careful with cleanup in production environments
+ pass
diff --git a/backend/workflow_manager/endpoint_v2/tests/destination-connectors/test_destination_connector_postgres.py b/backend/workflow_manager/endpoint_v2/tests/destination-connectors/test_destination_connector_postgres.py
new file mode 100644
index 000000000..33f31f055
--- /dev/null
+++ b/backend/workflow_manager/endpoint_v2/tests/destination-connectors/test_destination_connector_postgres.py
@@ -0,0 +1,326 @@
+import os
+from unittest.mock import Mock, patch
+
+from django.test import TestCase
+from workflow_manager.endpoint_v2.constants import DestinationKey
+from workflow_manager.endpoint_v2.destination import DestinationConnector
+
+from unstract.connectors.databases.postgresql import PostgreSQL
+
+
+class TestDestinationConnectorPostgreSQL(TestCase):
+ """Integration test for insert_into_db method with real PostgreSQL connector."""
+
+ def setUp(self) -> None:
+ """Set up test data and real PostgreSQL configuration."""
+
+ # Real PostgreSQL connection settings for testing
+ self.postgres_config = {
+ "host": os.getenv("DB_HOST", "localhost"),
+ "port": os.getenv("DB_PORT", "5432"),
+ "database": os.getenv("DB_NAME", "test_unstract"),
+ "user": os.getenv("DB_USER", "postgres"),
+ "password": os.getenv("DB_PASSWORD", "password"),
+ "schema": "test", # Add schema to fix PostgreSQL issue
+ }
+
+ # Test data that will be inserted into the database
+ self.test_data = {"key": "value", "result": "test_result", "status": "success"}
+ self.test_metadata = {
+ "file_execution_id": "test-file-exec-id",
+ "extracted_text": "This is sample extracted text",
+ "processing_time": 1.5,
+ }
+ self.input_file_path = "/path/to/test/file.pdf"
+ self.test_table_name = "OUTPUT_3"
+
+ # Create real PostgreSQL connector instance
+ self.postgres_connector = PostgreSQL(settings=self.postgres_config)
+
+ # Expected columns based on configuration
+ self.expected_columns = {
+ "id": "text",
+ "file_path": "text",
+ "execution_id": "text",
+ "data": "jsonb",
+ "created_at": "timestamp without time zone",
+ "created_by": "text",
+ "metadata": "jsonb",
+ "error_message": "text",
+ "status": "text",
+ "user_field_1": "boolean",
+ "user_field_2": "integer",
+ "user_field_3": "text",
+ }
+
+ def verify_table_columns(self, table_name: str) -> None:
+ """Verify that all expected columns exist in the table with correct types."""
+ table_info = self.postgres_connector.get_information_schema(
+ table_name=table_name
+ )
+
+ print(f"🔍 Actual table structure for '{table_name}': {table_info}")
+
+ # Check that key columns exist (some columns may be created on-demand)
+ required_columns = {
+ "id": "text",
+ "file_path": "text",
+ "execution_id": "text",
+ "data": ["text", "jsonb"], # Can be either, migration might convert
+ "created_at": "timestamp without time zone",
+ "created_by": "text",
+ }
+
+ for expected_column, expected_types in required_columns.items():
+ self.assertIn(
+ expected_column,
+ table_info,
+ f"Expected column '{expected_column}' not found in table '{table_name}'",
+ )
+
+ actual_type = table_info[expected_column]
+ if isinstance(expected_types, list):
+ self.assertIn(
+ actual_type,
+ expected_types,
+ f"Column '{expected_column}' has type '{actual_type}', expected one of {expected_types}",
+ )
+ else:
+ self.assertEqual(
+ actual_type,
+ expected_types,
+ f"Column '{expected_column}' has type '{actual_type}', expected '{expected_types}'",
+ )
+
+ print(f"✅ All required columns verified in table '{table_name}'")
+
+ def create_mock_workflow(self) -> Mock:
+ """Create a mock workflow object."""
+ workflow = Mock()
+ workflow.id = "test-workflow-id"
+ return workflow
+
+ def create_mock_workflow_log(self) -> Mock:
+ """Create a mock workflow log object."""
+ return Mock()
+
+ def create_real_connector_instance(self) -> Mock:
+ """Create connector instance using real PostgreSQL connector."""
+ connector = Mock()
+ connector.connector_id = "postgresql|6db35f45-be11-4fd5-80c5-85c48183afbb"
+ connector.connector_metadata = self.postgres_config
+ # Store reference to real connector for potential future use
+ connector._real_postgresql_connector = self.postgres_connector
+ return connector
+
+ def create_mock_endpoint(self, mock_connector_instance: Mock) -> Mock:
+ """Create a mock endpoint with real database configuration."""
+ endpoint = Mock()
+ endpoint.connector_instance = mock_connector_instance
+ endpoint.configuration = {
+ DestinationKey.TABLE: self.test_table_name,
+ DestinationKey.INCLUDE_AGENT: True,
+ DestinationKey.INCLUDE_TIMESTAMP: True,
+ DestinationKey.AGENT_NAME: "Unstract/DBWriter",
+ DestinationKey.COLUMN_MODE: "Write JSON to a single column",
+ DestinationKey.SINGLE_COLUMN_NAME: "data",
+ DestinationKey.FILE_PATH: "file_path",
+ DestinationKey.EXECUTION_ID: "execution_id",
+ }
+ return endpoint
+
+ def create_destination_connector(
+ self, mock_workflow: Mock, mock_workflow_log: Mock, mock_endpoint: Mock
+ ) -> DestinationConnector:
+ """Create a destination connector with mocked Django dependencies but real DB."""
+ with patch(
+ "workflow_manager.endpoint_v2.destination.UserContext"
+ ) as mock_context:
+ mock_context.get_organization_identifier.return_value = "test-org"
+
+ with patch.object(
+ DestinationConnector,
+ "_get_endpoint_for_workflow",
+ return_value=mock_endpoint,
+ ):
+ with patch.object(
+ DestinationConnector,
+ "_get_source_endpoint_for_workflow",
+ return_value=mock_endpoint,
+ ):
+ connector = DestinationConnector(
+ workflow=mock_workflow,
+ execution_id="test-execution-id",
+ workflow_log=mock_workflow_log,
+ use_file_history=False,
+ )
+ return connector
+
+ def test_insert_into_db_happy_path_postgresql(self) -> None:
+ """Test successful insertion into real PostgreSQL database."""
+ # Create mock objects for Django models
+ mock_workflow = self.create_mock_workflow()
+ mock_workflow_log = self.create_mock_workflow_log()
+ mock_connector_instance = self.create_real_connector_instance()
+ mock_endpoint = self.create_mock_endpoint(mock_connector_instance)
+
+ # Create destination connector with real database connection
+ destination_connector = self.create_destination_connector(
+ mock_workflow, mock_workflow_log, mock_endpoint
+ )
+
+ # Mock only the methods that get data, let database operations be real
+ with patch.object(
+ destination_connector,
+ "get_tool_execution_result",
+ return_value=self.test_data,
+ ):
+ with patch.object(
+ destination_connector,
+ "get_combined_metadata",
+ return_value=self.test_metadata,
+ ):
+ # This will execute real database operations using PostgreSQL connector
+ destination_connector.insert_into_db(
+ input_file_path=self.input_file_path, error=None
+ )
+
+ # Verify that all expected columns were created
+ self.verify_table_columns(self.test_table_name)
+
+ print(
+ f"✅ Successfully inserted test data into PostgreSQL table: {self.test_table_name}"
+ )
+
+ def test_insert_into_db_with_error_postgresql(self) -> None:
+ """Test insertion with error parameter into real PostgreSQL database."""
+ # Create mock objects
+ mock_workflow = self.create_mock_workflow()
+ mock_workflow_log = self.create_mock_workflow_log()
+ mock_connector_instance = self.create_real_connector_instance()
+ mock_endpoint = self.create_mock_endpoint(mock_connector_instance)
+
+ # Create destination connector
+ destination_connector = self.create_destination_connector(
+ mock_workflow, mock_workflow_log, mock_endpoint
+ )
+
+ error_message = "Test processing error occurred"
+
+ # Mock the methods that get data
+ with patch.object(
+ destination_connector,
+ "get_tool_execution_result",
+ return_value=self.test_data,
+ ):
+ with patch.object(
+ destination_connector,
+ "get_combined_metadata",
+ return_value=self.test_metadata,
+ ):
+ # Execute with error parameter
+ destination_connector.insert_into_db(
+ input_file_path=self.input_file_path, error=error_message
+ )
+
+ # Verify that all expected columns were created
+ self.verify_table_columns(self.test_table_name)
+
+ print(
+ f"✅ Successfully inserted error data into PostgreSQL table: {self.test_table_name}"
+ )
+
+ def test_postgresql_connector_connection(self) -> None:
+ """Test that the PostgreSQL connector can establish a connection."""
+ # Test the real PostgreSQL connector directly
+ try:
+ engine = self.postgres_connector.get_engine()
+ self.assertIsNotNone(engine)
+ print("✅ PostgreSQL connector successfully established connection")
+
+ # Clean up
+ if hasattr(engine, "close"):
+ engine.close()
+ except Exception as e:
+ self.fail(f"PostgreSQL connector failed to connect: {str(e)}")
+
+ def test_comprehensive_column_data_verification(self) -> None:
+ """Test that data is properly inserted into all columns with correct values."""
+ # Create mock objects
+ mock_workflow = self.create_mock_workflow()
+ mock_workflow_log = self.create_mock_workflow_log()
+ mock_connector_instance = self.create_real_connector_instance()
+ mock_endpoint = self.create_mock_endpoint(mock_connector_instance)
+
+ # Create destination connector
+ destination_connector = self.create_destination_connector(
+ mock_workflow, mock_workflow_log, mock_endpoint
+ )
+
+ # Mock the methods that get data
+ with patch.object(
+ destination_connector,
+ "get_tool_execution_result",
+ return_value=self.test_data,
+ ):
+ with patch.object(
+ destination_connector,
+ "get_combined_metadata",
+ return_value=self.test_metadata,
+ ):
+ # Execute database insertion
+ destination_connector.insert_into_db(
+ input_file_path=self.input_file_path, error=None
+ )
+
+ # Verify table structure
+ self.verify_table_columns(self.test_table_name)
+
+ # Verify data was inserted correctly by querying the table
+ engine = self.postgres_connector.get_engine()
+ try:
+ # Use cursor for raw psycopg2 connection
+ cursor = engine.cursor()
+ cursor.execute(
+ f"SELECT created_by, file_path, execution_id, created_at, data, metadata FROM {self.test_table_name} ORDER BY created_at DESC LIMIT 1"
+ )
+ row = cursor.fetchone()
+
+ self.assertIsNotNone(row, "No data found in table after insertion")
+
+ # Map results to column names
+ created_by, file_path, execution_id, created_at, data, metadata = row
+
+ # Verify specific column values
+ self.assertEqual(
+ created_by,
+ "Unstract/DBWriter",
+ f"Expected created_by to be 'Unstract/DBWriter', got '{created_by}'",
+ )
+
+ self.assertEqual(
+ file_path,
+ self.input_file_path,
+ f"Expected file_path to be '{self.input_file_path}', got '{file_path}'",
+ )
+
+ self.assertEqual(
+ execution_id,
+ "test-execution-id",
+ f"Expected execution_id to be 'test-execution-id', got '{execution_id}'",
+ )
+
+ self.assertIsNotNone(created_at, "Expected created_at timestamp to be set")
+
+ self.assertIsNotNone(data, "Expected data column to contain inserted data")
+
+ self.assertIsNotNone(
+ metadata, "Expected metadata column to contain metadata"
+ )
+
+ cursor.close()
+ print("✅ All column data values verified successfully")
+
+ finally:
+ if hasattr(engine, "close"):
+ engine.close()
diff --git a/backend/workflow_manager/endpoint_v2/tests/destination-connectors/test_destination_connector_redshift.py b/backend/workflow_manager/endpoint_v2/tests/destination-connectors/test_destination_connector_redshift.py
new file mode 100644
index 000000000..0dee96694
--- /dev/null
+++ b/backend/workflow_manager/endpoint_v2/tests/destination-connectors/test_destination_connector_redshift.py
@@ -0,0 +1,326 @@
+import os
+from unittest.mock import Mock, patch
+
+from django.test import TestCase
+from workflow_manager.endpoint_v2.constants import DestinationKey
+from workflow_manager.endpoint_v2.destination import DestinationConnector
+
+from unstract.connectors.databases.redshift import Redshift
+
+
+class TestDestinationConnectorRedshift(TestCase):
+ """Integration test for insert_into_db method with real Redshift connector."""
+
+ def setUp(self) -> None:
+ """Set up test data and real Redshift configuration."""
+
+ # Redshift connection settings for testing
+ required_env_vars = [
+ "REDSHIFT_HOST",
+ "REDSHIFT_USER",
+ "REDSHIFT_PASSWORD",
+ "REDSHIFT_DATABASE",
+ ]
+
+ # Check if all required environment variables are set
+ missing_vars = [var for var in required_env_vars if not os.getenv(var)]
+ if missing_vars:
+ self.skipTest(
+ f"Required Redshift environment variables not set: {', '.join(missing_vars)}"
+ )
+
+ self.redshift_config = {
+ "host": os.getenv("REDSHIFT_HOST"),
+ "port": os.getenv("REDSHIFT_PORT", "5439"),
+ "user": os.getenv("REDSHIFT_USER"),
+ "password": os.getenv("REDSHIFT_PASSWORD"),
+ "database": os.getenv("REDSHIFT_DATABASE"),
+ "schema": os.getenv("REDSHIFT_SCHEMA", "public"),
+ }
+
+ # Test data that will be inserted into the database
+ self.test_data = {"key": "value", "result": "test_result", "status": "success"}
+ self.test_metadata = {
+ "file_execution_id": "test-file-exec-id",
+ "extracted_text": "This is sample extracted text",
+ "processing_time": 1.5,
+ }
+ self.input_file_path = "/path/to/test/file.pdf"
+
+ # Redshift table naming (use schema.table format)
+ self.test_table_name = "MIGRATION"
+
+ # Create real Redshift connector instance
+ self.redshift_connector = Redshift(settings=self.redshift_config)
+
+ # Expected columns based on configuration (Redshift data types)
+ self.expected_columns = {
+ "id": "CHARACTER VARYING",
+ "file_path": "CHARACTER VARYING",
+ "execution_id": "CHARACTER VARYING",
+ "data": "CHARACTER VARYING", # Legacy column remains VARCHAR after migration
+ "data_v2": "CHARACTER VARYING", # New VARCHAR column added during migration
+ "metadata": "CHARACTER VARYING", # New VARCHAR column added during migration
+ "created_at": "TIMESTAMP WITHOUT TIME ZONE",
+ "created_by": "CHARACTER VARYING",
+ "status": "CHARACTER VARYING",
+ "error_message": "CHARACTER VARYING",
+ "user_field_1": "BOOLEAN",
+ "user_field_2": "INTEGER",
+ "user_field_3": "CHARACTER VARYING",
+ }
+
+ def verify_table_columns(self, table_name: str) -> None:
+ """Verify that the table has expected columns with correct data types."""
+ table_info = self.redshift_connector.get_information_schema(
+ table_name=table_name
+ )
+
+ print(f"🔍 Actual table structure for '{table_name}': {table_info}")
+
+ # Check that key columns exist (some columns may be created on-demand)
+ required_columns = {
+ "id": [
+ "CHARACTER VARYING",
+ "VARCHAR",
+ ], # Redshift uses CHARACTER VARYING or VARCHAR
+ "file_path": [
+ "CHARACTER VARYING",
+ "VARCHAR",
+ ], # Redshift uses CHARACTER VARYING or VARCHAR
+ "execution_id": [
+ "CHARACTER VARYING",
+ "VARCHAR",
+ ], # Redshift uses CHARACTER VARYING or VARCHAR
+ "data": [
+ "CHARACTER VARYING",
+ "VARCHAR",
+ "SUPER",
+ ], # Can be VARCHAR, migration might convert
+ "metadata": [
+ "CHARACTER VARYING",
+ "VARCHAR",
+ "SUPER",
+ ], # New VARCHAR column added during migration, or SUPER for JSON
+ "created_at": ["TIMESTAMP WITHOUT TIME ZONE", "TIMESTAMP"],
+ "created_by": [
+ "CHARACTER VARYING",
+ "VARCHAR",
+ ], # Redshift uses CHARACTER VARYING or VARCHAR
+ }
+
+ # Optional columns that may exist in migration scenarios
+ optional_columns = {
+ "data_v2": [
+ "CHARACTER VARYING",
+ "VARCHAR",
+ "SUPER",
+ ], # Only exists in migration scenarios
+ }
+
+ for expected_column, expected_type in required_columns.items():
+ self.assertIn(
+ expected_column,
+ table_info,
+ f"Expected column '{expected_column}' not found in table '{table_name}'",
+ )
+
+ actual_type = table_info[expected_column]
+ # Handle both single expected types and lists of acceptable types
+ if isinstance(expected_type, list):
+ self.assertIn(
+ actual_type.upper(),
+ [t.upper() for t in expected_type],
+ f"Column '{expected_column}' has type '{actual_type}', expected one of {expected_type}",
+ )
+ else:
+ # Redshift may return different type names, normalize for comparison
+ self.assertEqual(
+ actual_type.upper(),
+ expected_type.upper(),
+ f"Column '{expected_column}' has type '{actual_type}', expected '{expected_type}'",
+ )
+
+ # Check optional columns if they exist
+ for optional_column, expected_type in optional_columns.items():
+ if optional_column in table_info:
+ actual_type = table_info[optional_column]
+ if isinstance(expected_type, list):
+ self.assertIn(
+ actual_type.upper(),
+ [t.upper() for t in expected_type],
+ f"Optional column '{optional_column}' has type '{actual_type}', expected one of {expected_type}",
+ )
+ else:
+ self.assertEqual(
+ actual_type.upper(),
+ expected_type.upper(),
+ f"Optional column '{optional_column}' has type '{actual_type}', expected '{expected_type}'",
+ )
+
+ def create_mock_workflow(self) -> Mock:
+ """Create a mock workflow object."""
+ workflow = Mock()
+ workflow.id = "test-workflow-id"
+ return workflow
+
+ def create_mock_workflow_log(self) -> Mock:
+ """Create a mock workflow log object."""
+ return Mock()
+
+ def create_real_connector_instance(self) -> Mock:
+ """Create connector instance using real Redshift connector."""
+ connector = Mock()
+ connector.connector_id = Redshift.get_id()
+ connector.connector_metadata = self.redshift_config
+ # Store reference to real connector for potential future use
+ connector._real_redshift_connector = self.redshift_connector
+ return connector
+
+ def create_mock_endpoint(self, mock_connector_instance: Mock) -> Mock:
+ """Create a mock endpoint with real database configuration."""
+ endpoint = Mock()
+ endpoint.connector_instance = mock_connector_instance
+ endpoint.configuration = {
+ DestinationKey.TABLE: self.test_table_name,
+ DestinationKey.INCLUDE_AGENT: True,
+ DestinationKey.INCLUDE_TIMESTAMP: True,
+ DestinationKey.AGENT_NAME: "Unstract/DBWriter",
+ DestinationKey.COLUMN_MODE: "Write JSON to a single column",
+ DestinationKey.SINGLE_COLUMN_NAME: "data",
+ DestinationKey.FILE_PATH: "file_path",
+ DestinationKey.EXECUTION_ID: "execution_id",
+ }
+ return endpoint
+
+ def create_destination_connector(
+ self, mock_workflow: Mock, mock_workflow_log: Mock, mock_endpoint: Mock
+ ) -> DestinationConnector:
+ """Create a destination connector with mocked Django dependencies but real DB."""
+ with patch(
+ "workflow_manager.endpoint_v2.destination.UserContext"
+ ) as mock_context:
+ mock_context.get_organization_identifier.return_value = "test-org"
+
+ with patch.object(
+ DestinationConnector,
+ "_get_endpoint_for_workflow",
+ return_value=mock_endpoint,
+ ):
+ with patch.object(
+ DestinationConnector,
+ "_get_source_endpoint_for_workflow",
+ return_value=mock_endpoint,
+ ):
+ connector = DestinationConnector(
+ workflow=mock_workflow,
+ execution_id="test-execution-id",
+ workflow_log=mock_workflow_log,
+ use_file_history=False,
+ )
+ return connector
+
+ def test_insert_into_db_happy_path_redshift(self) -> None:
+ """Test successful insertion into real Redshift database."""
+ # Create mock objects for Django models
+ mock_workflow = self.create_mock_workflow()
+ mock_workflow_log = self.create_mock_workflow_log()
+ mock_connector_instance = self.create_real_connector_instance()
+ mock_endpoint = self.create_mock_endpoint(mock_connector_instance)
+
+ # Create destination connector with real database connection
+ destination_connector = self.create_destination_connector(
+ mock_workflow, mock_workflow_log, mock_endpoint
+ )
+
+ # Test the insert_into_db method with real Redshift
+ try:
+ # Mock only the methods that get data, let database operations be real
+ with patch.object(
+ destination_connector,
+ "get_tool_execution_result",
+ return_value=self.test_data,
+ ):
+ with patch.object(
+ destination_connector,
+ "get_combined_metadata",
+ return_value=self.test_metadata,
+ ):
+ # This will execute real database operations using Redshift connector
+ destination_connector.insert_into_db(
+ input_file_path=self.input_file_path, error=None
+ )
+
+ # Verify the table structure
+ self.verify_table_columns(self.test_table_name)
+
+ print("✅ Redshift integration test completed successfully!")
+
+ except Exception as e:
+ print(f"❌ Redshift integration test failed: {str(e)}")
+ raise
+
+ def test_insert_into_db_with_error_redshift(self) -> None:
+ """Test insertion with error parameter into real Redshift database."""
+ # Create mock objects
+ mock_workflow = self.create_mock_workflow()
+ mock_workflow_log = self.create_mock_workflow_log()
+ mock_connector_instance = self.create_real_connector_instance()
+ mock_endpoint = self.create_mock_endpoint(mock_connector_instance)
+
+ # Create destination connector
+ destination_connector = self.create_destination_connector(
+ mock_workflow, mock_workflow_log, mock_endpoint
+ )
+
+ error_message = "Test processing error occurred"
+
+ # Mock the methods that get data
+ with patch.object(
+ destination_connector,
+ "get_tool_execution_result",
+ return_value=self.test_data,
+ ):
+ with patch.object(
+ destination_connector,
+ "get_combined_metadata",
+ return_value=self.test_metadata,
+ ):
+ # Execute with error parameter
+ destination_connector.insert_into_db(
+ input_file_path=self.input_file_path, error=error_message
+ )
+
+ # Verify the table structure
+ self.verify_table_columns(self.test_table_name)
+
+ print(
+ f"✅ Successfully inserted error data into Redshift table: {self.test_table_name}"
+ )
+
+ def test_redshift_connector_connection(self) -> None:
+ """Test that the Redshift connector can establish a connection."""
+ # Test the real Redshift connector directly
+ try:
+ engine = self.redshift_connector.get_engine()
+ self.assertIsNotNone(engine)
+ print("✅ Redshift connector successfully established connection")
+
+ # Test a simple query to verify connection works
+ with engine.cursor() as cursor:
+ cursor.execute("SELECT 1 as test_column")
+ result = cursor.fetchall()
+ self.assertIsNotNone(result)
+ print("✅ Redshift connector successfully executed test query")
+
+ # Clean up
+ if hasattr(engine, "close"):
+ engine.close()
+ except Exception as e:
+ self.fail(f"Redshift connector failed to connect: {str(e)}")
+
+ def tearDown(self) -> None:
+ """Clean up test resources."""
+ # Optionally clean up test table
+ # Note: Be careful with cleanup in production environments
+ pass
diff --git a/backend/workflow_manager/endpoint_v2/tests/destination-connectors/test_destination_connector_snowflake.py b/backend/workflow_manager/endpoint_v2/tests/destination-connectors/test_destination_connector_snowflake.py
new file mode 100644
index 000000000..d54ef8763
--- /dev/null
+++ b/backend/workflow_manager/endpoint_v2/tests/destination-connectors/test_destination_connector_snowflake.py
@@ -0,0 +1,308 @@
+import os
+from unittest.mock import Mock, patch
+
+from django.test import TestCase
+from workflow_manager.endpoint_v2.constants import DestinationKey
+from workflow_manager.endpoint_v2.destination import DestinationConnector
+
+from unstract.connectors.databases.snowflake import SnowflakeDB
+
+
+class TestDestinationConnectorSnowflake(TestCase):
+ """Integration test for insert_into_db method with real Snowflake connector."""
+
+ def setUp(self) -> None:
+ """Set up test data and real Snowflake configuration."""
+
+ # Snowflake connection settings for testing
+ required_env_vars = [
+ "SNOWFLAKE_ACCOUNT",
+ "SNOWFLAKE_USER",
+ "SNOWFLAKE_PASSWORD",
+ "SNOWFLAKE_DATABASE",
+ "SNOWFLAKE_SCHEMA",
+ "SNOWFLAKE_WAREHOUSE",
+ "SNOWFLAKE_ROLE",
+ ]
+
+ # Check if all required environment variables are set
+ missing_vars = [var for var in required_env_vars if not os.getenv(var)]
+ if missing_vars:
+ self.skipTest(
+ f"Required Snowflake environment variables not set: {', '.join(missing_vars)}"
+ )
+
+ self.snowflake_config = {
+ "account": os.getenv("SNOWFLAKE_ACCOUNT"),
+ "user": os.getenv("SNOWFLAKE_USER"),
+ "password": os.getenv("SNOWFLAKE_PASSWORD"),
+ "database": os.getenv("SNOWFLAKE_DATABASE"),
+ "schema": os.getenv("SNOWFLAKE_SCHEMA"),
+ "warehouse": os.getenv("SNOWFLAKE_WAREHOUSE"),
+ "role": os.getenv("SNOWFLAKE_ROLE"),
+ }
+
+ # Test data that will be inserted into the database
+ self.test_data = {"key": "value", "result": "test_result", "status": "success"}
+ self.test_metadata = {
+ "file_execution_id": "test-file-exec-id",
+ "extracted_text": "This is sample extracted text",
+ "processing_time": 1.5,
+ }
+ self.input_file_path = "/path/to/test/file.pdf"
+
+ # Snowflake table naming (use schema.table format)
+ self.test_table_name = f"{self.snowflake_config['schema']}.migration"
+
+ # Create real Snowflake connector instance
+ self.snowflake_connector = SnowflakeDB(settings=self.snowflake_config)
+
+ # Expected columns based on configuration (Snowflake data types)
+ self.expected_columns = {
+ "id": "TEXT",
+ "file_path": "TEXT",
+ "execution_id": "TEXT",
+ "data": "TEXT", # Legacy column remains TEXT after migration
+ "data_v2": "VARIANT", # New VARIANT column added during migration
+ "metadata": "VARIANT", # New VARIANT column added during migration
+ "created_at": "TIMESTAMP_NTZ",
+ "created_by": "TEXT",
+ "status": "TEXT",
+ "error_message": "TEXT",
+ "user_field_1": "BOOLEAN",
+ "user_field_2": "NUMBER",
+ "user_field_3": "TEXT",
+ }
+
+ def verify_table_columns(self, table_name: str) -> None:
+ """Verify that the table has expected columns with correct data types."""
+ table_info = self.snowflake_connector.get_information_schema(
+ table_name=table_name
+ )
+
+ print(f"🔍 Actual table structure for '{table_name}': {table_info}")
+
+ # Check that key columns exist (some columns may be created on-demand)
+ required_columns = {
+ "id": "VARCHAR", # Snowflake uses VARCHAR instead of TEXT
+ "file_path": "VARCHAR", # Snowflake uses VARCHAR instead of TEXT
+ "execution_id": "VARCHAR", # Snowflake uses VARCHAR instead of TEXT
+ "data": ["VARCHAR", "VARIANT"], # Can be either, migration might convert
+ "metadata": "VARIANT", # New VARIANT column added during migration
+ "created_at": "TIMESTAMP_NTZ",
+ "created_by": "VARCHAR", # Snowflake uses VARCHAR instead of TEXT
+ }
+
+ # Optional columns that may exist in migration scenarios
+ optional_columns = {
+ "data_v2": ["VARCHAR", "VARIANT"], # Only exists in migration scenarios
+ }
+
+ for expected_column, expected_type in required_columns.items():
+ self.assertIn(
+ expected_column,
+ table_info,
+ f"Expected column '{expected_column}' not found in table '{table_name}'",
+ )
+
+ actual_type = table_info[expected_column]
+ # Handle both single expected types and lists of acceptable types
+ if isinstance(expected_type, list):
+ self.assertIn(
+ actual_type.upper(),
+ [t.upper() for t in expected_type],
+ f"Column '{expected_column}' has type '{actual_type}', expected one of {expected_type}",
+ )
+ else:
+ # Snowflake may return different type names, normalize for comparison
+ self.assertEqual(
+ actual_type.upper(),
+ expected_type.upper(),
+ f"Column '{expected_column}' has type '{actual_type}', expected '{expected_type}'",
+ )
+
+ # Check optional columns if they exist
+ for optional_column, expected_type in optional_columns.items():
+ if optional_column in table_info:
+ actual_type = table_info[optional_column]
+ if isinstance(expected_type, list):
+ self.assertIn(
+ actual_type.upper(),
+ [t.upper() for t in expected_type],
+ f"Optional column '{optional_column}' has type '{actual_type}', expected one of {expected_type}",
+ )
+ else:
+ self.assertEqual(
+ actual_type.upper(),
+ expected_type.upper(),
+ f"Optional column '{optional_column}' has type '{actual_type}', expected '{expected_type}'",
+ )
+
+ def create_mock_workflow(self) -> Mock:
+ """Create a mock workflow object."""
+ workflow = Mock()
+ workflow.id = "test-workflow-id"
+ return workflow
+
+ def create_mock_workflow_log(self) -> Mock:
+ """Create a mock workflow log object."""
+ return Mock()
+
+ def create_real_connector_instance(self) -> Mock:
+ """Create connector instance using real Snowflake connector."""
+ connector = Mock()
+ connector.connector_id = SnowflakeDB.get_id()
+ connector.connector_metadata = self.snowflake_config
+ # Store reference to real connector for potential future use
+ connector._real_snowflake_connector = self.snowflake_connector
+ return connector
+
+ def create_mock_endpoint(self, mock_connector_instance: Mock) -> Mock:
+ """Create a mock endpoint with real database configuration."""
+ endpoint = Mock()
+ endpoint.connector_instance = mock_connector_instance
+ endpoint.configuration = {
+ DestinationKey.TABLE: self.test_table_name.split(".")[
+ -1
+ ], # Just table name
+ DestinationKey.INCLUDE_AGENT: True,
+ DestinationKey.INCLUDE_TIMESTAMP: True,
+ DestinationKey.AGENT_NAME: "Unstract/DBWriter",
+ DestinationKey.COLUMN_MODE: "Write JSON to a single column",
+ DestinationKey.SINGLE_COLUMN_NAME: "data",
+ DestinationKey.FILE_PATH: "file_path",
+ DestinationKey.EXECUTION_ID: "execution_id",
+ }
+ return endpoint
+
+ def create_destination_connector(
+ self, mock_workflow: Mock, mock_workflow_log: Mock, mock_endpoint: Mock
+ ) -> DestinationConnector:
+ """Create a destination connector with mocked Django dependencies but real DB."""
+ with patch(
+ "workflow_manager.endpoint_v2.destination.UserContext"
+ ) as mock_context:
+ mock_context.get_organization_identifier.return_value = "test-org"
+
+ with patch.object(
+ DestinationConnector,
+ "_get_endpoint_for_workflow",
+ return_value=mock_endpoint,
+ ):
+ with patch.object(
+ DestinationConnector,
+ "_get_source_endpoint_for_workflow",
+ return_value=mock_endpoint,
+ ):
+ connector = DestinationConnector(
+ workflow=mock_workflow,
+ execution_id="test-execution-id",
+ workflow_log=mock_workflow_log,
+ use_file_history=False,
+ )
+ return connector
+
+ def test_insert_into_db_happy_path_snowflake(self) -> None:
+ """Test successful insertion into real Snowflake database."""
+ # Create mock objects for Django models
+ mock_workflow = self.create_mock_workflow()
+ mock_workflow_log = self.create_mock_workflow_log()
+ mock_connector_instance = self.create_real_connector_instance()
+ mock_endpoint = self.create_mock_endpoint(mock_connector_instance)
+
+ # Create destination connector with real database connection
+ destination_connector = self.create_destination_connector(
+ mock_workflow, mock_workflow_log, mock_endpoint
+ )
+
+ # Test the insert_into_db method with real Snowflake
+ try:
+ # Mock only the methods that get data, let database operations be real
+ with patch.object(
+ destination_connector,
+ "get_tool_execution_result",
+ return_value=self.test_data,
+ ):
+ with patch.object(
+ destination_connector,
+ "get_combined_metadata",
+ return_value=self.test_metadata,
+ ):
+ # This will execute real database operations using Snowflake connector
+ destination_connector.insert_into_db(
+ input_file_path=self.input_file_path, error=None
+ )
+
+ # Verify the table structure
+ self.verify_table_columns(self.test_table_name)
+
+ print("✅ Snowflake integration test completed successfully!")
+
+ except Exception as e:
+ print(f"❌ Snowflake integration test failed: {str(e)}")
+ raise
+
+ def test_insert_into_db_with_error_snowflake(self) -> None:
+ """Test insertion with error parameter into real Snowflake database."""
+ # Create mock objects
+ mock_workflow = self.create_mock_workflow()
+ mock_workflow_log = self.create_mock_workflow_log()
+ mock_connector_instance = self.create_real_connector_instance()
+ mock_endpoint = self.create_mock_endpoint(mock_connector_instance)
+
+ # Create destination connector
+ destination_connector = self.create_destination_connector(
+ mock_workflow, mock_workflow_log, mock_endpoint
+ )
+
+ error_message = "Test processing error occurred"
+
+ # Mock the methods that get data
+ with patch.object(
+ destination_connector,
+ "get_tool_execution_result",
+ return_value=self.test_data,
+ ):
+ with patch.object(
+ destination_connector,
+ "get_combined_metadata",
+ return_value=self.test_metadata,
+ ):
+ # Execute with error parameter
+ destination_connector.insert_into_db(
+ input_file_path=self.input_file_path, error=error_message
+ )
+
+ # Verify the table structure
+ self.verify_table_columns(self.test_table_name)
+
+ print(
+ f"✅ Successfully inserted error data into Snowflake table: {self.test_table_name}"
+ )
+
+ def test_snowflake_connector_connection(self) -> None:
+ """Test that the Snowflake connector can establish a connection."""
+ # Test the real Snowflake connector directly
+ try:
+ engine = self.snowflake_connector.get_engine()
+ self.assertIsNotNone(engine)
+ print("✅ Snowflake connector successfully established connection")
+
+ # Test a simple query to verify connection works
+ with engine.cursor() as cursor:
+ cursor.execute("SELECT 1 as test_column")
+ result = cursor.fetchall()
+ self.assertIsNotNone(result)
+ print("✅ Snowflake connector successfully executed test query")
+
+ # Clean up
+ if hasattr(engine, "close"):
+ engine.close()
+ except Exception as e:
+ self.fail(f"Snowflake connector failed to connect: {str(e)}")
+
+ def tearDown(self) -> None:
+ """Clean up test resources."""
+ # Optionally clean up test table
+ # Note: Be careful with cleanup in production environments
+ pass
diff --git a/backend/workflow_manager/endpoint_v2/tests/test_database_utils/__init__.py b/backend/workflow_manager/endpoint_v2/tests/test_database_utils/__init__.py
index e69de29bb..fca2b2401 100644
--- a/backend/workflow_manager/endpoint_v2/tests/test_database_utils/__init__.py
+++ b/backend/workflow_manager/endpoint_v2/tests/test_database_utils/__init__.py
@@ -0,0 +1,3 @@
+from backend.celery import app as celery_app
+
+__all__ = ["celery_app"]
diff --git a/backend/workflow_manager/endpoint_v2/tests/test_database_utils/base_test_db.py b/backend/workflow_manager/endpoint_v2/tests/test_database_utils/base_test_db.py
deleted file mode 100644
index d83fc3a59..000000000
--- a/backend/workflow_manager/endpoint_v2/tests/test_database_utils/base_test_db.py
+++ /dev/null
@@ -1,160 +0,0 @@
-import datetime
-import json
-import os
-from typing import Any
-
-import pytest # type: ignore
-from dotenv import load_dotenv
-
-from unstract.connectors.databases.bigquery import BigQuery
-from unstract.connectors.databases.mariadb import MariaDB
-from unstract.connectors.databases.mssql import MSSQL
-from unstract.connectors.databases.mysql import MySQL
-from unstract.connectors.databases.postgresql import PostgreSQL
-from unstract.connectors.databases.redshift import Redshift
-from unstract.connectors.databases.snowflake import SnowflakeDB
-from unstract.connectors.databases.unstract_db import UnstractDB
-
-load_dotenv("test.env")
-
-
-class BaseTestDB:
- @pytest.fixture(autouse=True)
- def base_setup(self) -> None:
- self.postgres_creds = {
- "user": os.getenv("DB_USER"),
- "password": os.getenv("DB_PASSWORD"),
- "host": os.getenv("DB_HOST"),
- "port": os.getenv("DB_PORT"),
- "database": os.getenv("DB_NAME"),
- }
- self.redshift_creds = {
- "user": os.getenv("REDSHIFT_USER"),
- "password": os.getenv("REDSHIFT_PASSWORD"),
- "host": os.getenv("REDSHIFT_HOST"),
- "port": os.getenv("REDSHIFT_PORT"),
- "database": os.getenv("REDSHIFT_DB"),
- }
- self.snowflake_creds = {
- "user": os.getenv("SNOWFLAKE_USER"),
- "password": os.getenv("SNOWFLAKE_PASSWORD"),
- "account": os.getenv("SNOWFLAKE_ACCOUNT"),
- "role": os.getenv("SNOWFLAKE_ROLE"),
- "database": os.getenv("SNOWFLAKE_DB"),
- "schema": os.getenv("SNOWFLAKE_SCHEMA"),
- "warehouse": os.getenv("SNOWFLAKE_WAREHOUSE"),
- }
- self.mssql_creds = {
- "user": os.getenv("MSSQL_USER"),
- "password": os.getenv("MSSQL_PASSWORD"),
- "server": os.getenv("MSSQL_SERVER"),
- "port": os.getenv("MSSQL_PORT"),
- "database": os.getenv("MSSQL_DB"),
- }
- self.mysql_creds = {
- "user": os.getenv("MYSQL_USER"),
- "password": os.getenv("MYSQL_PASSWORD"),
- "host": os.getenv("MYSQL_SERVER"),
- "port": os.getenv("MYSQL_PORT"),
- "database": os.getenv("MYSQL_DB"),
- }
- self.mariadb_creds = {
- "user": os.getenv("MARIADB_USER"),
- "password": os.getenv("MARIADB_PASSWORD"),
- "host": os.getenv("MARIADB_SERVER"),
- "port": os.getenv("MARIADB_PORT"),
- "database": os.getenv("MARIADB_DB"),
- }
- self.database_entry = {
- "created_by": "Unstract/DBWriter",
- "created_at": datetime.datetime(2024, 5, 20, 7, 46, 57, 307998),
- "data": '{"input_file": "simple.pdf", "result": "report"}',
- }
- valid_schema_name = "public"
- invalid_schema_name = "public_1"
- self.valid_postgres_creds = {**self.postgres_creds, "schema": valid_schema_name}
- self.invalid_postgres_creds = {
- **self.postgres_creds,
- "schema": invalid_schema_name,
- }
- self.valid_redshift_creds = {**self.redshift_creds, "schema": valid_schema_name}
- self.invalid_redshift_creds = {
- **self.redshift_creds,
- "schema": invalid_schema_name,
- }
- self.invalid_syntax_table_name = "invalid-syntax.name.test_output"
- self.invalid_wrong_table_name = "database.schema.test_output"
- self.valid_table_name = "test_output"
- bigquery_json_str = os.getenv("BIGQUERY_CREDS", "{}")
- self.bigquery_settings = json.loads(bigquery_json_str)
- self.bigquery_settings["json_credentials"] = bigquery_json_str
- self.valid_bigquery_table_name = "unstract.bigquery_test.bigquery_output"
- self.invalid_snowflake_db = {**self.snowflake_creds, "database": "invalid"}
- self.invalid_snowflake_schema = {**self.snowflake_creds, "schema": "invalid"}
- self.invalid_snowflake_warehouse = {
- **self.snowflake_creds,
- "warehouse": "invalid",
- }
-
- # Gets all valid db instances except
- # Bigquery (table name needs to be writted separately for bigquery)
- @pytest.fixture(
- params=[
- ("valid_postgres_creds", PostgreSQL),
- ("snowflake_creds", SnowflakeDB),
- ("mssql_creds", MSSQL),
- ("mysql_creds", MySQL),
- ("mariadb_creds", MariaDB),
- ("valid_redshift_creds", Redshift),
- ]
- )
- def valid_dbs_instance(self, request: Any) -> Any:
- return self.get_db_instance(request=request)
-
- # Gets all valid db instances except:
- # Bigquery (table name needs to be writted separately for bigquery)
- # Redshift (can't process more than 64KB character type)
- @pytest.fixture(
- params=[
- ("valid_postgres_creds", PostgreSQL),
- ("snowflake_creds", SnowflakeDB),
- ("mssql_creds", MSSQL),
- ("mysql_creds", MySQL),
- ("mariadb_creds", MariaDB),
- ]
- )
- def valid_dbs_instance_to_handle_large_doc(self, request: Any) -> Any:
- return self.get_db_instance(request=request)
-
- def get_db_instance(self, request: Any) -> UnstractDB:
- creds_name, db_class = request.param
- creds = getattr(self, creds_name)
- if not creds:
- pytest.fail(f"Unknown credentials: {creds_name}")
- db_instance = db_class(settings=creds)
- return db_instance
-
- # Gets all invalid-db instances for postgres, redshift:
- @pytest.fixture(
- params=[
- ("invalid_postgres_creds", PostgreSQL),
- ("invalid_redshift_creds", Redshift),
- ]
- )
- def invalid_dbs_instance(self, request: Any) -> Any:
- return self.get_db_instance(request=request)
-
- @pytest.fixture
- def valid_bigquery_db_instance(self) -> Any:
- return BigQuery(settings=self.bigquery_settings)
-
- # Gets all invalid-db instances for snowflake:
- @pytest.fixture(
- params=[
- ("invalid_snowflake_db", SnowflakeDB),
- ("invalid_snowflake_schema", SnowflakeDB),
- ("invalid_snowflake_warehouse", SnowflakeDB),
- ]
- )
- def invalid_snowflake_db_instance(self, request: Any) -> Any:
- return self.get_db_instance(request=request)
diff --git a/backend/workflow_manager/endpoint_v2/tests/test_database_utils/test_create_table_if_not_exists.py b/backend/workflow_manager/endpoint_v2/tests/test_database_utils/test_create_table_if_not_exists.py
deleted file mode 100644
index cbf4f1619..000000000
--- a/backend/workflow_manager/endpoint_v2/tests/test_database_utils/test_create_table_if_not_exists.py
+++ /dev/null
@@ -1,97 +0,0 @@
-import pytest # type: ignore
-from workflow_manager.endpoint_v2.database_utils import DatabaseUtils
-from workflow_manager.endpoint_v2.exceptions import UnstractDBException
-
-from unstract.connectors.databases.unstract_db import UnstractDB
-
-from .base_test_db import BaseTestDB
-
-
-class TestCreateTableIfNotExists(BaseTestDB):
- def test_create_table_if_not_exists_valid(
- self, valid_dbs_instance: UnstractDB
- ) -> None:
- engine = valid_dbs_instance.get_engine()
- result = DatabaseUtils.create_table_if_not_exists(
- db_class=valid_dbs_instance,
- engine=engine,
- table_name=self.valid_table_name,
- database_entry=self.database_entry,
- )
- assert result is None
-
- def test_create_table_if_not_exists_bigquery_valid(
- self, valid_bigquery_db_instance: UnstractDB
- ) -> None:
- engine = valid_bigquery_db_instance.get_engine()
- result = DatabaseUtils.create_table_if_not_exists(
- db_class=valid_bigquery_db_instance,
- engine=engine,
- table_name=self.valid_bigquery_table_name,
- database_entry=self.database_entry,
- )
- assert result is None
-
- def test_create_table_if_not_exists_invalid_schema(
- self, invalid_dbs_instance: UnstractDB
- ) -> None:
- engine = invalid_dbs_instance.get_engine()
- with pytest.raises(UnstractDBException):
- DatabaseUtils.create_table_if_not_exists(
- db_class=invalid_dbs_instance,
- engine=engine,
- table_name=self.valid_table_name,
- database_entry=self.database_entry,
- )
-
- def test_create_table_if_not_exists_invalid_syntax(
- self, valid_dbs_instance: UnstractDB
- ) -> None:
- engine = valid_dbs_instance.get_engine()
- with pytest.raises(UnstractDBException):
- DatabaseUtils.create_table_if_not_exists(
- db_class=valid_dbs_instance,
- engine=engine,
- table_name=self.invalid_syntax_table_name,
- database_entry=self.database_entry,
- )
-
- def test_create_table_if_not_exists_wrong_table_name(
- self, valid_dbs_instance: UnstractDB
- ) -> None:
- engine = valid_dbs_instance.get_engine()
- with pytest.raises(UnstractDBException):
- DatabaseUtils.create_table_if_not_exists(
- db_class=valid_dbs_instance,
- engine=engine,
- table_name=self.invalid_wrong_table_name,
- database_entry=self.database_entry,
- )
-
- def test_create_table_if_not_exists_feature_not_supported(
- self, invalid_dbs_instance: UnstractDB
- ) -> None:
- engine = invalid_dbs_instance.get_engine()
- with pytest.raises(UnstractDBException):
- DatabaseUtils.create_table_if_not_exists(
- db_class=invalid_dbs_instance,
- engine=engine,
- table_name=self.invalid_wrong_table_name,
- database_entry=self.database_entry,
- )
-
- def test_create_table_if_not_exists_invalid_snowflake_db(
- self, invalid_snowflake_db_instance: UnstractDB
- ) -> None:
- engine = invalid_snowflake_db_instance.get_engine()
- with pytest.raises(UnstractDBException):
- DatabaseUtils.create_table_if_not_exists(
- db_class=invalid_snowflake_db_instance,
- engine=engine,
- table_name=self.invalid_wrong_table_name,
- database_entry=self.database_entry,
- )
-
-
-if __name__ == "__main__":
- pytest.main()
diff --git a/backend/workflow_manager/endpoint_v2/tests/test_database_utils/test_execute_write_query.py b/backend/workflow_manager/endpoint_v2/tests/test_database_utils/test_execute_write_query.py
deleted file mode 100644
index cb3686e4d..000000000
--- a/backend/workflow_manager/endpoint_v2/tests/test_database_utils/test_execute_write_query.py
+++ /dev/null
@@ -1,167 +0,0 @@
-import os
-import uuid
-from typing import Any
-
-import pytest # type: ignore
-from workflow_manager.endpoint_v2.database_utils import DatabaseUtils
-from workflow_manager.endpoint_v2.exceptions import UnstractDBException
-
-from unstract.connectors.databases.redshift import Redshift
-from unstract.connectors.databases.unstract_db import UnstractDB
-
-from .base_test_db import BaseTestDB
-
-
-class TestExecuteWriteQuery(BaseTestDB):
- @pytest.fixture(autouse=True)
- def setup(self, base_setup: Any) -> None:
- self.sql_columns_and_values = {
- "created_by": "Unstract/DBWriter",
- "created_at": "2024-05-20 10:36:25.362609",
- "data": '{"input_file": "simple.pdf", "result": "report"}',
- "id": str(uuid.uuid4()),
- }
-
- def test_execute_write_query_valid(self, valid_dbs_instance: Any) -> None:
- engine = valid_dbs_instance.get_engine()
- result = DatabaseUtils.execute_write_query(
- db_class=valid_dbs_instance,
- engine=engine,
- table_name=self.valid_table_name,
- sql_keys=list(self.sql_columns_and_values.keys()),
- sql_values=list(self.sql_columns_and_values.values()),
- )
- assert result is None
-
- def test_execute_write_query_invalid_schema(self, invalid_dbs_instance: Any) -> None:
- engine = invalid_dbs_instance.get_engine()
- with pytest.raises(UnstractDBException):
- DatabaseUtils.execute_write_query(
- db_class=invalid_dbs_instance,
- engine=engine,
- table_name=self.valid_table_name,
- sql_keys=list(self.sql_columns_and_values.keys()),
- sql_values=list(self.sql_columns_and_values.values()),
- )
-
- def test_execute_write_query_invalid_syntax(self, valid_dbs_instance: Any) -> None:
- engine = valid_dbs_instance.get_engine()
- with pytest.raises(UnstractDBException):
- DatabaseUtils.execute_write_query(
- db_class=valid_dbs_instance,
- engine=engine,
- table_name=self.invalid_syntax_table_name,
- sql_keys=list(self.sql_columns_and_values.keys()),
- sql_values=list(self.sql_columns_and_values.values()),
- )
-
- def test_execute_write_query_feature_not_supported(
- self, invalid_dbs_instance: Any
- ) -> None:
- engine = invalid_dbs_instance.get_engine()
- with pytest.raises(UnstractDBException):
- DatabaseUtils.execute_write_query(
- db_class=invalid_dbs_instance,
- engine=engine,
- table_name=self.invalid_wrong_table_name,
- sql_keys=list(self.sql_columns_and_values.keys()),
- sql_values=list(self.sql_columns_and_values.values()),
- )
-
- def load_text_to_sql_values(self) -> dict[str, Any]:
- file_path = os.path.join(os.path.dirname(__file__), "static", "large_doc.txt")
- with open(file_path, encoding="utf-8") as file:
- content = file.read()
- sql_columns_and_values = self.sql_columns_and_values.copy()
- sql_columns_and_values["data"] = content
- return sql_columns_and_values
-
- @pytest.fixture
- def valid_redshift_db_instance(self) -> Any:
- return Redshift(self.valid_redshift_creds)
-
- def test_execute_write_query_datatype_too_large_redshift(
- self, valid_redshift_db_instance: Any
- ) -> None:
- engine = valid_redshift_db_instance.get_engine()
- sql_columns_and_values = self.load_text_to_sql_values()
- with pytest.raises(UnstractDBException):
- DatabaseUtils.execute_write_query(
- db_class=valid_redshift_db_instance,
- engine=engine,
- table_name=self.valid_table_name,
- sql_keys=list(sql_columns_and_values.keys()),
- sql_values=list(sql_columns_and_values.values()),
- )
-
- def test_execute_write_query_bigquery_valid(
- self, valid_bigquery_db_instance: Any
- ) -> None:
- engine = valid_bigquery_db_instance.get_engine()
- result = DatabaseUtils.execute_write_query(
- db_class=valid_bigquery_db_instance,
- engine=engine,
- table_name=self.valid_bigquery_table_name,
- sql_keys=list(self.sql_columns_and_values.keys()),
- sql_values=list(self.sql_columns_and_values.values()),
- )
- assert result is None
-
- def test_execute_write_query_wrong_table_name(
- self, valid_dbs_instance: UnstractDB
- ) -> None:
- engine = valid_dbs_instance.get_engine()
- with pytest.raises(UnstractDBException):
- DatabaseUtils.execute_write_query(
- db_class=valid_dbs_instance,
- engine=engine,
- table_name=self.invalid_wrong_table_name,
- sql_keys=list(self.sql_columns_and_values.keys()),
- sql_values=list(self.sql_columns_and_values.values()),
- )
-
- def test_execute_write_query_bigquery_large_doc(
- self, valid_bigquery_db_instance: Any
- ) -> None:
- engine = valid_bigquery_db_instance.get_engine()
- sql_columns_and_values = self.load_text_to_sql_values()
- result = DatabaseUtils.execute_write_query(
- db_class=valid_bigquery_db_instance,
- engine=engine,
- table_name=self.valid_bigquery_table_name,
- sql_keys=list(sql_columns_and_values.keys()),
- sql_values=list(sql_columns_and_values.values()),
- )
- assert result is None
-
- def test_execute_write_query_invalid_snowflake_db(
- self, invalid_snowflake_db_instance: UnstractDB
- ) -> None:
- engine = invalid_snowflake_db_instance.get_engine()
- with pytest.raises(UnstractDBException):
- DatabaseUtils.execute_write_query(
- db_class=invalid_snowflake_db_instance,
- engine=engine,
- table_name=self.invalid_wrong_table_name,
- sql_keys=list(self.sql_columns_and_values.keys()),
- sql_values=list(self.sql_columns_and_values.values()),
- )
-
- # Make this function at last to cover all large doc
- def test_execute_write_query_large_doc(
- self, valid_dbs_instance_to_handle_large_doc: Any
- ) -> None:
- engine = valid_dbs_instance_to_handle_large_doc.get_engine()
- sql_columns_and_values = self.load_text_to_sql_values()
- result = DatabaseUtils.execute_write_query(
- db_class=valid_dbs_instance_to_handle_large_doc,
- engine=engine,
- table_name=self.valid_table_name,
- sql_keys=list(sql_columns_and_values.keys()),
- sql_values=list(sql_columns_and_values.values()),
- )
- assert result is None
-
-
-if __name__ == "__main__":
- pytest.main()
diff --git a/unstract/connectors/src/unstract/connectors/databases/bigquery/bigquery.py b/unstract/connectors/src/unstract/connectors/databases/bigquery/bigquery.py
index 49d185cf7..2f42cd333 100644
--- a/unstract/connectors/src/unstract/connectors/databases/bigquery/bigquery.py
+++ b/unstract/connectors/src/unstract/connectors/databases/bigquery/bigquery.py
@@ -2,6 +2,8 @@ import datetime
import json
import logging
import os
+import uuid
+from enum import Enum
from typing import Any
import google.api_core.exceptions
@@ -59,9 +61,6 @@ class BigQuery(UnstractDB):
def can_read() -> bool:
return True
- def get_string_type(self) -> str:
- return "string"
-
def get_engine(self) -> Client:
return bigquery.Client.from_service_account_info( # type: ignore
info=self.json_credentials
@@ -121,8 +120,8 @@ class BigQuery(UnstractDB):
f"user_field_1 BOOL DEFAULT FALSE, "
f"user_field_2 INT64 DEFAULT 0, "
f"user_field_3 STRING DEFAULT NULL, "
- f"status STRING CHECK (status IN ('ERROR', 'SUCCESS')), "
- f"error_message STRING"
+ f"status STRING, "
+ f"error_message STRING, "
)
return sql_query
@@ -131,25 +130,29 @@ class BigQuery(UnstractDB):
f"ALTER TABLE {table_name} "
f"ADD COLUMN {column_name}_v2 JSON, "
f"ADD COLUMN metadata JSON, "
- f"ADD COLUMN user_field_1 BOOL DEFAULT FALSE, "
- f"ADD COLUMN user_field_2 INT64 DEFAULT 0, "
- f"ADD COLUMN user_field_3 STRING DEFAULT NULL, "
- f"ADD COLUMN status STRING CHECK (status IN ('ERROR', 'SUCCESS')), "
+ f"ADD COLUMN user_field_1 BOOL, "
+ f"ADD COLUMN user_field_2 INT64, "
+ f"ADD COLUMN user_field_3 STRING, "
+ f"ADD COLUMN status STRING, "
f"ADD COLUMN error_message STRING"
)
return sql_query
@staticmethod
- def get_sql_insert_query(table_name: str, sql_keys: list[str]) -> str:
+ def get_sql_insert_query(
+ table_name: str, sql_keys: list[str], sql_values: list[str] = None
+ ) -> str:
"""Function to generate parameterised insert sql query.
Args:
table_name (str): db-connector table name
sql_keys (list[str]): column names
+ sql_values (list[str], optional): SQL values for database-specific handling
Returns:
str: returns a string with parameterised insert sql query
"""
+ # BigQuery uses @ parameterization, ignore sql_values for now
keys_str = ",".join(sql_keys)
values_placeholder = ",".join(["@" + key for key in sql_keys])
return f"INSERT INTO {table_name} ({keys_str}) VALUES ({values_placeholder})"
@@ -172,15 +175,44 @@ class BigQuery(UnstractDB):
table_name = kwargs.get("table_name", None)
if table_name is None:
raise ValueError("Please enter a valid table_name to to create/insert table")
+
sql_keys = list(kwargs.get("sql_keys", []))
+ column_types = self.get_information_schema(table_name=table_name)
+
try:
if sql_values:
- query_parameters = [
- bigquery.ScalarQueryParameter(key, "STRING", value)
- for key, value in zip(sql_keys, sql_values, strict=False)
- ]
+ query_parameters = []
+ # Modify SQL query to use PARSE_JSON for JSON columns
+ modified_sql = sql_query
+
+ for key, value in zip(sql_keys, sql_values, strict=False):
+ column_type = column_types.get(key.lower(), "").upper()
+
+ if isinstance(value, (dict, list)) and column_type == "JSON":
+ # For JSON objects in JSON columns, convert to string and use PARSE_JSON
+ json_str = json.dumps(value) if value else None
+ if json_str:
+ # Replace @key with PARSE_JSON(@key) in the SQL query
+ modified_sql = modified_sql.replace(
+ f"@{key}", f"PARSE_JSON(@{key})"
+ )
+ query_parameters.append(
+ bigquery.ScalarQueryParameter(key, "STRING", json_str)
+ )
+ elif isinstance(value, (dict, list)):
+ # For dict/list values in STRING columns, serialize to JSON string
+ json_str = json.dumps(value) if value else None
+ query_parameters.append(
+ bigquery.ScalarQueryParameter(key, "STRING", json_str)
+ )
+ else:
+ # For other values, use STRING as before
+ query_parameters.append(
+ bigquery.ScalarQueryParameter(key, "STRING", value)
+ )
+
query_params = bigquery.QueryJobConfig(query_parameters=query_parameters)
- query_job = engine.query(sql_query, job_config=query_params)
+ query_job = engine.query(modified_sql, job_config=query_params)
else:
query_job = engine.query(sql_query)
query_job.result()
@@ -232,7 +264,52 @@ class BigQuery(UnstractDB):
f"table_name = '{table}'"
)
results = self.execute(query=query)
+
+ # If table doesn't exist, execute returns None
+ if results is None:
+ logger.info(f"Table {table_name} does not exist, returning empty schema")
+ return {}
+
+ # Process the schema results
column_types: dict[str, str] = self.get_db_column_types(
columns_with_types=results
)
return column_types
+
+ def get_sql_values_for_query(
+ self, values: dict[str, Any], column_types: dict[str, str]
+ ) -> dict[str, str]:
+ """Prepare SQL values for BigQuery queries with JSON column support.
+
+ Args:
+ values (dict[str, Any]): dictionary of columns and values
+ column_types (dict[str, str]): types of columns from database schema
+
+ Returns:
+ dict[str, str]: Dictionary of column names to SQL values for parameterized queries
+ """
+ sql_values: dict[str, Any] = {}
+ for column in values:
+ value = values[column]
+ if isinstance(value, (dict, list)):
+ # For BigQuery, keep dict/list objects as-is for JSON columns
+ sql_values[column] = value
+ elif isinstance(value, str):
+ # Try to parse JSON strings back to objects for BigQuery
+ try:
+ parsed_value = json.loads(value)
+ sql_values[column] = parsed_value
+ except (TypeError, ValueError, json.JSONDecodeError):
+ # Not a JSON string, keep as string
+ sql_values[column] = f"{value}"
+ elif isinstance(value, Enum):
+ sql_values[column] = value.value
+ else:
+ sql_values[column] = f"{value}"
+
+ # If table has a column 'id', unstract inserts a unique value to it
+ if any(key in column_types for key in ["id", "ID"]):
+ uuid_id = str(uuid.uuid4())
+ sql_values["id"] = f"{uuid_id}"
+
+ return sql_values
diff --git a/unstract/connectors/src/unstract/connectors/databases/mariadb/mariadb.py b/unstract/connectors/src/unstract/connectors/databases/mariadb/mariadb.py
index 753336d54..06e2421fa 100644
--- a/unstract/connectors/src/unstract/connectors/databases/mariadb/mariadb.py
+++ b/unstract/connectors/src/unstract/connectors/databases/mariadb/mariadb.py
@@ -59,7 +59,7 @@ class MariaDB(UnstractDB, MysqlHandler):
)
return con
- def sql_to_db_mapping(self, value: str) -> str:
+ def sql_to_db_mapping(self, value: Any) -> str:
return str(MysqlHandler.sql_to_db_mapping(value=value))
def execute_query(
@@ -74,3 +74,51 @@ class MariaDB(UnstractDB, MysqlHandler):
host=self.host,
table_name=table_name,
)
+
+ def get_create_table_base_query(self, table: str) -> str:
+ """Function to create a base create table sql query with MySQL specific types.
+
+ Args:
+ table (str): db-connector table name
+
+ Returns:
+ str: generates a create sql base query with the constant columns
+ """
+ sql_query = (
+ f"CREATE TABLE IF NOT EXISTS {table} "
+ f"(id LONGTEXT, "
+ f"created_by LONGTEXT, created_at TIMESTAMP, "
+ f"metadata JSON, "
+ f"user_field_1 BOOLEAN DEFAULT FALSE, "
+ f"user_field_2 BIGINT DEFAULT 0, "
+ f"user_field_3 LONGTEXT DEFAULT NULL, "
+ f"status ENUM('ERROR', 'SUCCESS'), "
+ f"error_message LONGTEXT, "
+ )
+ return sql_query
+
+ def get_information_schema(self, table_name: str) -> dict[str, str]:
+ """Get information schema for MySQL database."""
+ query = (
+ "SELECT column_name, data_type FROM "
+ "information_schema.columns WHERE "
+ f"UPPER(table_name) = UPPER('{table_name}') AND table_schema = '{self.database}'"
+ )
+ results = self.execute(query=query)
+ column_types: dict[str, str] = self.get_db_column_types(
+ columns_with_types=results
+ )
+ return column_types
+
+ def prepare_multi_column_migration(self, table_name: str, column_name: str) -> str:
+ sql_query = (
+ f"ALTER TABLE {table_name} "
+ f"ADD COLUMN {column_name}_v2 JSON, "
+ f"ADD COLUMN metadata JSON, "
+ f"ADD COLUMN user_field_1 BOOLEAN DEFAULT FALSE, "
+ f"ADD COLUMN user_field_2 BIGINT DEFAULT 0, "
+ f"ADD COLUMN user_field_3 LONGTEXT DEFAULT NULL, "
+ f"ADD COLUMN status ENUM('ERROR', 'SUCCESS'), "
+ f"ADD COLUMN error_message LONGTEXT"
+ )
+ return sql_query
diff --git a/unstract/connectors/src/unstract/connectors/databases/mssql/mssql.py b/unstract/connectors/src/unstract/connectors/databases/mssql/mssql.py
index 9dc426d62..c4ec0b1ab 100644
--- a/unstract/connectors/src/unstract/connectors/databases/mssql/mssql.py
+++ b/unstract/connectors/src/unstract/connectors/databases/mssql/mssql.py
@@ -58,9 +58,6 @@ class MSSQL(UnstractDB):
def can_read() -> bool:
return True
- def get_string_type(self) -> str:
- return "text"
-
def sql_to_db_mapping(self, value: str) -> str:
"""Gets the python datatype of value and converts python datatype to
corresponding DB datatype.
@@ -131,22 +128,28 @@ class MSSQL(UnstractDB):
f"user_field_2 INT DEFAULT 0, "
f"user_field_3 NVARCHAR(MAX) DEFAULT NULL, "
f"status NVARCHAR(10) CHECK (status IN ('ERROR', 'SUCCESS')), "
- f"error_message NVARCHAR(MAX))"
+ f"error_message NVARCHAR(MAX), "
)
return sql_query
- def prepare_multi_column_migration(self, table_name: str, column_name: str) -> str:
- sql_query = (
- f"ALTER TABLE {table_name} "
- f"ADD {column_name}_v2 NVARCHAR(MAX), "
- + "metadata NVARCHAR(MAX), "
- + "user_field_1 BIT DEFAULT 0, "
- + "user_field_2 INT DEFAULT 0, "
- + "user_field_3 NVARCHAR(MAX) DEFAULT NULL, "
- + "status NVARCHAR(10) CHECK (status IN ('ERROR', 'SUCCESS')), "
- + "error_message NVARCHAR(MAX)"
- )
- return sql_query
+ def prepare_multi_column_migration(
+ self, table_name: str, column_name: str
+ ) -> list[str]:
+ """Returns a list of ALTER TABLE statements for MSSQL column migration.
+
+ MSSQL doesn't support adding multiple columns in a single ALTER TABLE statement,
+ so we return a list of individual statements like Snowflake.
+ """
+ sql_statements = [
+ f"ALTER TABLE {table_name} ADD {column_name}_v2 NVARCHAR(MAX)",
+ f"ALTER TABLE {table_name} ADD metadata NVARCHAR(MAX)",
+ f"ALTER TABLE {table_name} ADD user_field_1 BIT DEFAULT 0",
+ f"ALTER TABLE {table_name} ADD user_field_2 INT DEFAULT 0",
+ f"ALTER TABLE {table_name} ADD user_field_3 NVARCHAR(MAX) DEFAULT NULL",
+ f"ALTER TABLE {table_name} ADD status NVARCHAR(10)",
+ f"ALTER TABLE {table_name} ADD error_message NVARCHAR(MAX)",
+ ]
+ return sql_statements
def execute_query(
self, engine: Any, sql_query: str, sql_values: Any, **kwargs: Any
diff --git a/unstract/connectors/src/unstract/connectors/databases/mysql/mysql.py b/unstract/connectors/src/unstract/connectors/databases/mysql/mysql.py
index deb666c11..4fcd2e8f9 100644
--- a/unstract/connectors/src/unstract/connectors/databases/mysql/mysql.py
+++ b/unstract/connectors/src/unstract/connectors/databases/mysql/mysql.py
@@ -52,6 +52,20 @@ class MySQL(UnstractDB, MysqlHandler):
def get_string_type(self) -> str:
return "longtext"
+ def get_information_schema(self, table_name: str) -> dict[str, str]:
+ """Get information schema for MySQL database."""
+ # Try case-insensitive search since MySQL table names can be case-sensitive
+ query = (
+ "SELECT column_name, data_type FROM "
+ "information_schema.columns WHERE "
+ f"UPPER(table_name) = UPPER('{table_name}') AND table_schema = '{self.database}'"
+ )
+ results = self.execute(query=query)
+ column_types: dict[str, str] = self.get_db_column_types(
+ columns_with_types=results
+ )
+ return column_types
+
def get_engine(self) -> Connection: # type: ignore[type-arg]
con = pymysql.connect(
host=self.host,
@@ -62,7 +76,8 @@ class MySQL(UnstractDB, MysqlHandler):
)
return con
- def sql_to_db_mapping(self, value: str) -> str:
+ def sql_to_db_mapping(self, value: Any) -> str:
+ """Override to handle JSON columns correctly for MySQL."""
return str(MysqlHandler.sql_to_db_mapping(value=value))
def get_create_table_base_query(self, table: str) -> str:
diff --git a/unstract/connectors/src/unstract/connectors/databases/oracle_db/oracle_db.py b/unstract/connectors/src/unstract/connectors/databases/oracle_db/oracle_db.py
index 6c1592995..f20dc2fa7 100644
--- a/unstract/connectors/src/unstract/connectors/databases/oracle_db/oracle_db.py
+++ b/unstract/connectors/src/unstract/connectors/databases/oracle_db/oracle_db.py
@@ -1,4 +1,5 @@
import datetime
+import logging
import os
from typing import Any
@@ -7,6 +8,8 @@ from oracledb.connection import Connection
from unstract.connectors.databases.unstract_db import UnstractDB
+logger = logging.getLogger(__name__)
+
class OracleDB(UnstractDB):
def __init__(self, settings: dict[str, Any]):
@@ -59,9 +62,6 @@ class OracleDB(UnstractDB):
def can_read() -> bool:
return True
- def get_string_type(self) -> str:
- return "clob"
-
def get_engine(self) -> Connection:
con = oracledb.connect(
config_dir=self.config_dir,
@@ -85,14 +85,14 @@ class OracleDB(UnstractDB):
"""
python_type = type(value)
mapping = {
- str: "CLOB",
+ str: "VARCHAR2(32767)",
int: "NUMBER",
float: "LONG",
datetime.datetime: "TIMESTAMP",
- dict: "CLOB CHECK (IS JSON)",
- list: "CLOB CHECK (IS JSON)",
+ dict: "CLOB",
+ list: "CLOB",
}
- return mapping.get(python_type, "CLOB")
+ return mapping.get(python_type, "VARCHAR2(32767)")
def get_create_table_base_query(self, table: str) -> str:
"""Function to create a base create table sql query.
@@ -104,38 +104,53 @@ class OracleDB(UnstractDB):
str: generates a create sql base query with the constant columns
"""
sql_query = (
- f"CREATE TABLE IF NOT EXISTS {table} "
+ f"CREATE TABLE {table} "
f"(id VARCHAR2(32767) , "
f"created_by VARCHAR2(32767), created_at TIMESTAMP, "
- f"metadata CLOB CHECK (metadata IS JSON), "
+ f"metadata CLOB, "
f"user_field_1 NUMBER(1) DEFAULT 0, "
f"user_field_2 NUMBER DEFAULT 0, "
f"user_field_3 VARCHAR2(32767) DEFAULT NULL, "
- f"status VARCHAR2(10) CHECK (status IN ('ERROR', 'SUCCESS')), "
+ f"status VARCHAR2(10), "
f"error_message VARCHAR2(32767), "
)
return sql_query
- def prepare_multi_column_migration(self, table_name: str, column_name: str) -> str:
- sql_query = (
- f"ALTER TABLE {table_name} "
- f"ADD {column_name}_v2 CLOB CHECK ({column_name}_v2 IS JSON), "
- f"ADD metadata CLOB CHECK (metadata IS JSON), "
- f"ADD user_field_1 NUMBER(1) DEFAULT 0, "
- f"ADD user_field_2 NUMBER DEFAULT 0, "
- f"ADD user_field_3 VARCHAR2(32767) DEFAULT NULL, "
- f"ADD status VARCHAR2(10) CHECK (status IN ('ERROR', 'SUCCESS')), "
- f"ADD error_message VARCHAR2(32767)"
- )
- return sql_query
+ def prepare_multi_column_migration(self, table_name: str, column_name: str) -> list:
+ """Prepare ALTER TABLE statements for adding new columns to an existing table.
+
+ Args:
+ table_name (str): The name of the table to alter
+ column_name (str): The base name of the column to add a _v2 version for
+
+ Returns:
+ list: List of ALTER TABLE statements, one per column addition
+
+ Note:
+ Oracle does not support multiple ADD clauses in a single ALTER TABLE statement.
+ Each column addition requires a separate ALTER TABLE statement.
+ """
+ # Return one ALTER statement per column for Oracle compatibility
+ return [
+ f"ALTER TABLE {table_name} ADD {column_name}_v2 CLOB",
+ f"ALTER TABLE {table_name} ADD metadata CLOB",
+ f"ALTER TABLE {table_name} ADD user_field_1 NUMBER(1) DEFAULT 0",
+ f"ALTER TABLE {table_name} ADD user_field_2 NUMBER DEFAULT 0",
+ f"ALTER TABLE {table_name} ADD user_field_3 VARCHAR2(32767) DEFAULT NULL",
+ f"ALTER TABLE {table_name} ADD status VARCHAR2(10)",
+ f"ALTER TABLE {table_name} ADD error_message VARCHAR2(32767)",
+ ]
@staticmethod
- def get_sql_insert_query(table_name: str, sql_keys: list[str]) -> str:
+ def get_sql_insert_query(
+ table_name: str, sql_keys: list[str], sql_values: list[str] | None = None
+ ) -> str:
"""Function to generate parameterised insert sql query.
Args:
table_name (str): db-connector table name
sql_keys (list[str]): column names
+ sql_values (list[str], optional): SQL values for database-specific handling (ignored for Oracle)
Returns:
str: returns a string with parameterised insert sql query
@@ -152,7 +167,7 @@ class OracleDB(UnstractDB):
def execute_query(
self, engine: Any, sql_query: str, sql_values: Any, **kwargs: Any
) -> None:
- """Executes create/insert query.
+ """Executes create/insert query with Oracle-specific error handling.
Args:
engine (Any): oracle db client engine
@@ -161,12 +176,19 @@ class OracleDB(UnstractDB):
"""
sql_keys = list(kwargs.get("sql_keys", []))
with engine.cursor() as cursor:
- if sql_values:
- params = dict(zip(sql_keys, sql_values, strict=False))
- cursor.execute(sql_query, params)
- else:
- cursor.execute(sql_query)
- engine.commit()
+ try:
+ if sql_values:
+ params = dict(zip(sql_keys, sql_values, strict=False))
+ cursor.execute(sql_query, params)
+ else:
+ cursor.execute(sql_query)
+ engine.commit()
+ except oracledb.DatabaseError as e:
+ logger.debug(f"Oracle database error occurred: {str(e)}")
+ if "ORA-00955" in str(e):
+ logger.info("Table already exists (ORA-00955) for oracle-db")
+ else:
+ raise
def get_information_schema(self, table_name: str) -> dict[str, str]:
"""Function to generate information schema of the big query table.
diff --git a/unstract/connectors/src/unstract/connectors/databases/postgresql/postgresql.py b/unstract/connectors/src/unstract/connectors/databases/postgresql/postgresql.py
index aba40f70e..21bee6a20 100644
--- a/unstract/connectors/src/unstract/connectors/databases/postgresql/postgresql.py
+++ b/unstract/connectors/src/unstract/connectors/databases/postgresql/postgresql.py
@@ -67,9 +67,6 @@ class PostgreSQL(UnstractDB, PsycoPgHandler):
def can_read() -> bool:
return True
- def get_string_type(self) -> str:
- return "text"
-
def sql_to_db_mapping(self, value: str) -> str:
"""Gets the python datatype of value and converts python datatype to
corresponding DB datatype.
diff --git a/unstract/connectors/src/unstract/connectors/databases/redshift/redshift.py b/unstract/connectors/src/unstract/connectors/databases/redshift/redshift.py
index 13dc31af7..d50764d3f 100644
--- a/unstract/connectors/src/unstract/connectors/databases/redshift/redshift.py
+++ b/unstract/connectors/src/unstract/connectors/databases/redshift/redshift.py
@@ -53,9 +53,6 @@ class Redshift(UnstractDB, PsycoPgHandler):
def can_read() -> bool:
return True
- def get_string_type(self) -> str:
- return "character varying"
-
def get_engine(self) -> connection:
return psycopg2.connect(
host=self.host,
@@ -69,14 +66,14 @@ class Redshift(UnstractDB, PsycoPgHandler):
def sql_to_db_mapping(self, value: str) -> str:
python_type = type(value)
mapping = {
- str: "VARCHAR(MAX)",
+ str: "VARCHAR(65535)",
int: "BIGINT",
float: "DOUBLE PRECISION",
datetime.datetime: "TIMESTAMP",
- dict: "VARCHAR(MAX)",
- list: "VARCHAR(MAX)",
+ dict: "SUPER",
+ list: "SUPER",
}
- return mapping.get(python_type, "VARCHAR(MAX)")
+ return mapping.get(python_type, "VARCHAR(65535)")
def get_create_table_base_query(self, table: str) -> str:
sql_query = (
@@ -87,7 +84,7 @@ class Redshift(UnstractDB, PsycoPgHandler):
f"user_field_1 BOOLEAN DEFAULT FALSE, "
f"user_field_2 INTEGER DEFAULT 0, "
f"user_field_3 VARCHAR(65535) DEFAULT NULL, "
- f"status VARCHAR(10) CHECK (status IN ('ERROR', 'SUCCESS')), "
+ f"status VARCHAR(256), "
f"error_message VARCHAR(65535), "
)
return sql_query
@@ -109,12 +106,12 @@ class Redshift(UnstractDB, PsycoPgHandler):
"""
# Return one ALTER statement per column for Redshift compatibility
return [
- f"ALTER TABLE {table_name} ADD COLUMN {column_name}_v2 VARCHAR(65535);",
- f"ALTER TABLE {table_name} ADD COLUMN metadata VARCHAR(65535);",
+ f"ALTER TABLE {table_name} ADD COLUMN {column_name}_v2 SUPER;",
+ f"ALTER TABLE {table_name} ADD COLUMN metadata SUPER;",
f"ALTER TABLE {table_name} ADD COLUMN user_field_1 BOOLEAN DEFAULT FALSE;",
f"ALTER TABLE {table_name} ADD COLUMN user_field_2 INTEGER DEFAULT 0;",
f"ALTER TABLE {table_name} ADD COLUMN user_field_3 VARCHAR(65535) DEFAULT NULL;",
- f"ALTER TABLE {table_name} ADD COLUMN status VARCHAR(10) CHECK (status IN ('ERROR', 'SUCCESS'));",
+ f"ALTER TABLE {table_name} ADD COLUMN status VARCHAR(256);",
f"ALTER TABLE {table_name} ADD COLUMN error_message VARCHAR(65535);",
]
diff --git a/unstract/connectors/src/unstract/connectors/databases/snowflake/snowflake.py b/unstract/connectors/src/unstract/connectors/databases/snowflake/snowflake.py
index 49dc42932..62167bdc7 100644
--- a/unstract/connectors/src/unstract/connectors/databases/snowflake/snowflake.py
+++ b/unstract/connectors/src/unstract/connectors/databases/snowflake/snowflake.py
@@ -1,6 +1,9 @@
import datetime
+import json
import logging
import os
+import uuid
+from enum import Enum
from typing import Any
import snowflake.connector
@@ -9,6 +12,7 @@ from snowflake.connector.connection import SnowflakeConnection
from unstract.connectors.databases.exceptions import SnowflakeProgrammingException
from unstract.connectors.databases.unstract_db import UnstractDB
+from unstract.connectors.exceptions import ConnectorError
logger = logging.getLogger(__name__)
@@ -56,9 +60,6 @@ class SnowflakeDB(UnstractDB):
def can_read() -> bool:
return True
- def get_string_type(self) -> str:
- return "text"
-
def sql_to_db_mapping(self, value: str) -> str:
"""Gets the python datatype of value and converts python datatype to
corresponding DB datatype.
@@ -95,49 +96,67 @@ class SnowflakeDB(UnstractDB):
def get_create_table_base_query(self, table: str) -> str:
sql_query = (
- f"CREATE TABLE {table} IF NOT EXISTS "
+ f"CREATE TABLE IF NOT EXISTS {table} "
f"(id TEXT ,"
f"created_by TEXT, created_at TIMESTAMP, "
f"metadata VARIANT, "
f"user_field_1 BOOLEAN DEFAULT FALSE, "
f"user_field_2 INT DEFAULT 0, "
f"user_field_3 TEXT DEFAULT NULL, "
- f"status VARCHAR CHECK (status IN ('ERROR', 'SUCCESS')), "
+ f"status VARCHAR, "
f"error_message VARCHAR, "
)
return sql_query
- def prepare_multi_column_migration(self, table_name: str, column_name: str) -> str:
- sql_query = (
- f"ALTER TABLE {table_name} "
- f"ADD COLUMN {column_name}_v2 VARIANT, "
- f"ADD COLUMN metadata VARIANT, "
- f"ADD COLUMN user_field_1 BOOLEAN DEFAULT FALSE, "
- f"ADD COLUMN user_field_2 INT DEFAULT 0, "
- f"ADD COLUMN user_field_3 TEXT DEFAULT NULL, "
- f"ADD COLUMN status VARCHAR CHECK (status IN ('ERROR', 'SUCCESS')), "
- f"ADD COLUMN error_message VARCHAR"
- )
- return sql_query
+ def prepare_multi_column_migration(self, table_name: str, column_name: str) -> list:
+ """Returns a list of separate ALTER TABLE statements for Snowflake.
+
+ Snowflake doesn't support multiple ADD COLUMN clauses in a single statement,
+ so we return a list of individual ALTER TABLE statements.
+ """
+ sql_statements = [
+ f"ALTER TABLE {table_name} ADD COLUMN {column_name}_v2 VARIANT",
+ f"ALTER TABLE {table_name} ADD COLUMN metadata VARIANT",
+ f"ALTER TABLE {table_name} ADD COLUMN user_field_1 BOOLEAN DEFAULT FALSE",
+ f"ALTER TABLE {table_name} ADD COLUMN user_field_2 INT DEFAULT 0",
+ f"ALTER TABLE {table_name} ADD COLUMN user_field_3 TEXT DEFAULT NULL",
+ f"ALTER TABLE {table_name} ADD COLUMN status VARCHAR",
+ f"ALTER TABLE {table_name} ADD COLUMN error_message VARCHAR",
+ ]
+ return sql_statements
def execute_query(
self, engine: Any, sql_query: str, sql_values: Any, **kwargs: Any
) -> None:
table_name = kwargs.get("table_name", None)
+ logger.debug(f"Snowflake execute_query called with sql_query: {sql_query}")
+ logger.debug(f"sql_values: {sql_values}")
+
try:
with engine.cursor() as cursor:
- if sql_values:
+ # Check if the SQL query is already complete (contains SELECT)
+ if "SELECT" in sql_query.upper():
+ # Complete SQL query - execute directly
+ logger.debug("Executing complete SQL query")
+ cursor.execute(sql_query)
+ elif sql_values:
+ # Parameterized query - execute with values
+ logger.debug("Executing parameterized query")
cursor.execute(sql_query, sql_values)
else:
+ # Direct SQL execution (used for DDL statements)
+ logger.debug("Executing SQL directly (no parameters)")
cursor.execute(sql_query)
engine.commit()
except SnowflakeError.ProgrammingError as e:
logger.error(
- f"snowflake programming error in crearing/inserting table: "
+ f"snowflake programming error in creating/inserting table: "
f"{e.msg} {e.errno}"
)
+ logger.error(f"SQL Query: {sql_query}")
+ logger.error(f"SQL Values: {sql_values}")
raise SnowflakeProgrammingException(
- detail=e.msg,
+ detail=f"{e.msg} | SQL: {sql_query} | Values: {sql_values}",
database=self.database,
schema=self.schema,
table_name=table_name,
@@ -146,7 +165,176 @@ class SnowflakeDB(UnstractDB):
def get_information_schema(self, table_name: str) -> dict[str, str]:
query = f"describe table {table_name}"
column_types: dict[str, str] = {}
- results = self.execute(query=query)
- for column in results:
- column_types[column[0].lower()] = column[1].split("(")[0]
+ try:
+ results = self.execute(query=query)
+ if results:
+ for column in results:
+ column_types[column[0].lower()] = column[1].split("(")[0]
+ except (SnowflakeError.ProgrammingError, ConnectorError) as e:
+ # Handle table not found errors gracefully
+ error_str = str(e).lower()
+ if (
+ "002003" in error_str
+ or "does not exist" in error_str
+ or "not authorized" in error_str
+ ):
+ logger.info(f"Table {table_name} does not exist or not authorized: {e}")
+ return {}
+ else:
+ # Re-raise other programming/connector errors
+ raise
return column_types
+
+ def get_sql_values_for_query(
+ self, values: dict[str, Any], column_types: dict[str, str]
+ ) -> dict[str, str]:
+ """Prepare SQL values for Snowflake queries with VARIANT column support.
+
+ Args:
+ values (dict[str, Any]): dictionary of columns and values
+ column_types (dict[str, str]): types of columns from database schema
+
+ Returns:
+ dict[str, str]: Dictionary of column names to SQL values or SQL fragments
+
+ Note:
+ For VARIANT columns, this returns SQL fragments like PARSE_JSON('...')
+ instead of parameterized values, since Snowflake needs special handling
+ for JSON data in VARIANT columns.
+ """
+ sql_values: dict[str, Any] = {}
+ has_variant_columns = any(
+ column_types.get(col.lower(), "").upper() == "VARIANT"
+ for col in values.keys()
+ )
+
+ for column in values:
+ value = values[column]
+ col = column.lower()
+ type_x = column_types.get(col, "")
+
+ if isinstance(value, Enum):
+ if has_variant_columns and type_x.upper() == "VARIANT":
+ # For VARIANT Enum values, create SQL fragment
+ escaped_value = str(value.value).replace("'", "''")
+ sql_values[column] = f"'{escaped_value}'"
+ else:
+ # For non-VARIANT Enum values, create SQL fragment when we have VARIANT columns
+ if has_variant_columns:
+ escaped_value = str(value.value).replace("'", "''")
+ sql_values[column] = f"'{escaped_value}'"
+ else:
+ sql_values[column] = value.value
+ elif isinstance(value, (dict, list)):
+ # For dict/list values, check if this is a VARIANT column
+ try:
+ json_str = json.dumps(value)
+ if has_variant_columns and type_x.upper() == "VARIANT":
+ # For VARIANT columns, return SQL fragment with PARSE_JSON
+ escaped_value = json_str.replace("'", "''")
+ sql_values[column] = f"PARSE_JSON('{escaped_value}')"
+ else:
+ # For non-VARIANT columns with VARIANT columns present, create quoted SQL fragment
+ if has_variant_columns:
+ escaped_value = json_str.replace("'", "''")
+ sql_values[column] = f"'{escaped_value}'"
+ else:
+ # No VARIANT columns, use regular parameterization
+ sql_values[column] = json_str
+ except (TypeError, ValueError) as e:
+ logger.error(
+ f"Failed to serialize value to JSON for column {column}: {e}"
+ )
+ fallback_value = {
+ "error": "JSON serialization failed",
+ "error_type": e.__class__.__name__,
+ "error_message": str(e),
+ "data_type": str(type(value)),
+ "data_description": f"column_{column}",
+ "timestamp": datetime.datetime.now().isoformat(),
+ }
+ fallback_json = json.dumps(fallback_value)
+ if has_variant_columns and type_x.upper() == "VARIANT":
+ escaped_value = fallback_json.replace("'", "''")
+ sql_values[column] = f"PARSE_JSON('{escaped_value}')"
+ else:
+ # For non-VARIANT columns with VARIANT columns present, create quoted SQL fragment
+ if has_variant_columns:
+ escaped_value = fallback_json.replace("'", "''")
+ sql_values[column] = f"'{escaped_value}'"
+ else:
+ # No VARIANT columns, use regular parameterization
+ sql_values[column] = fallback_json
+ elif type_x.upper() == "VARIANT" and isinstance(value, str):
+ if has_variant_columns:
+ try:
+ # For VARIANT columns with string values, validate if it's already valid JSON
+ json.loads(value)
+ escaped_value = value.replace("'", "''")
+ sql_values[column] = f"PARSE_JSON('{escaped_value}')"
+ except (json.JSONDecodeError, TypeError):
+ # Not JSON, convert to JSON string and create SQL fragment
+ json_str = json.dumps(value)
+ escaped_value = json_str.replace("'", "''")
+ sql_values[column] = f"PARSE_JSON('{escaped_value}')"
+ else:
+ # No VARIANT columns, use regular parameterization
+ try:
+ json.loads(value)
+ sql_values[column] = value
+ except (json.JSONDecodeError, TypeError):
+ sql_values[column] = json.dumps(value)
+ else:
+ # All other values
+ if has_variant_columns:
+ # When we have VARIANT columns, create SQL fragments for consistency
+ if value is None:
+ sql_values[column] = "NULL"
+ else:
+ escaped_value = str(value).replace("'", "''")
+ sql_values[column] = f"'{escaped_value}'"
+ else:
+ # No VARIANT columns, use regular parameterization
+ sql_values[column] = f"{value}"
+
+ # If table has a column 'id', unstract inserts a unique value to it
+ if any(key in column_types for key in ["id", "ID"]):
+ uuid_id = str(uuid.uuid4())
+ if has_variant_columns:
+ sql_values["id"] = f"'{uuid_id}'"
+ else:
+ sql_values["id"] = uuid_id
+
+ return sql_values
+
+ def get_sql_insert_query(
+ self, table_name: str, sql_keys: list[str], sql_values: list[str] = None
+ ) -> str:
+ """Generate SQL insert query for Snowflake with special handling for VARIANT columns.
+
+ Args:
+ table_name (str): db-connector table name
+ sql_keys (list[str]): column names
+ sql_values (list[str], optional): SQL values, may contain fragments like PARSE_JSON(...)
+
+ Returns:
+ str: Complete SQL insert query with VARIANT columns handled appropriately
+ """
+ keys_str = ",".join(sql_keys)
+
+ if sql_values:
+ # Check if we have SQL fragments that need SELECT format
+ has_sql_fragments = any(
+ isinstance(v, str)
+ and ("PARSE_JSON(" in v or "NULL" == v or v.startswith("'"))
+ for v in sql_values
+ )
+
+ if has_sql_fragments:
+ # Build complete SQL with SELECT format for VARIANT columns
+ values_str = ",".join(str(v) for v in sql_values)
+ return f"INSERT INTO {table_name} ({keys_str}) SELECT {values_str}"
+
+ # Fall back to parameterized format for standard queries
+ values_placeholder = ",".join(["%s" for _ in sql_keys])
+ return f"INSERT INTO {table_name} ({keys_str}) VALUES ({values_placeholder})"
diff --git a/unstract/connectors/src/unstract/connectors/databases/unstract_db.py b/unstract/connectors/src/unstract/connectors/databases/unstract_db.py
index 74a5ad6e6..95298d1f8 100644
--- a/unstract/connectors/src/unstract/connectors/databases/unstract_db.py
+++ b/unstract/connectors/src/unstract/connectors/databases/unstract_db.py
@@ -1,6 +1,9 @@
import datetime
+import json
import logging
+import uuid
from abc import ABC, abstractmethod
+from enum import Enum
from typing import Any
from workflow_manager.endpoint_v2.constants import TableColumns
@@ -65,28 +68,6 @@ class UnstractDB(UnstractConnector, ABC):
def python_social_auth_backend() -> str:
return ""
- @abstractmethod
- def get_string_type(self) -> str:
- """Child classes implement this to return the string type name for their DB."""
- pass
-
- @abstractmethod
- def prepare_multi_column_migration(
- self, table_name: str, column_name: str
- ) -> str | list:
- """Returns the ALTER TABLE query specific to the database.
-
- Args:
- table_name (str): The name of the table to alter
- column_name (str): The base name of the column to add a _v2 version for
-
- Returns:
- str | list: Either a single SQL ALTER TABLE statement (str) or
- a list of separate ALTER TABLE statements for databases
- that don't support multiple column additions in one statement
- """
- pass
-
@abstractmethod
def get_engine(self) -> Any:
pass
@@ -107,6 +88,7 @@ class UnstractDB(UnstractConnector, ABC):
except Exception as e:
raise ConnectorError(str(e)) from e
+ # TO DO: Remove if needed
def sql_to_db_mapping(self, value: str) -> str:
"""Gets the python datatype of value and converts python datatype
to corresponding DB datatype
@@ -125,6 +107,24 @@ class UnstractDB(UnstractConnector, ABC):
}
return mapping.get(python_type, "TEXT")
+ @abstractmethod
+ def prepare_multi_column_migration(
+ self, table_name: str, column_name: str
+ ) -> str | list:
+ """Returns the ALTER TABLE query specific to the database.
+
+ Args:
+ table_name (str): The name of the table to alter
+ column_name (str): The base name of the column to add a _v2 version for
+
+ Returns:
+ str | list: Either a single SQL ALTER TABLE statement (str) or
+ a list of separate ALTER TABLE statements for databases
+ that don't support multiple column additions in one statement
+ """
+ pass
+
+ @abstractmethod
def get_create_table_base_query(self, table: str) -> str:
"""Function to create a base create table sql query.
@@ -134,13 +134,6 @@ class UnstractDB(UnstractConnector, ABC):
Returns:
str: generates a create sql base query with the constant columns
"""
- sql_query = (
- f"CREATE TABLE IF NOT EXISTS {table} "
- f"(id TEXT , "
- f"created_by TEXT, created_at TIMESTAMP, "
- f"metadata TEXT, "
- )
- return sql_query
def create_table_query(self, table: str, database_entry: dict[str, Any]) -> Any:
"""Function to create a create table sql query.
@@ -156,6 +149,8 @@ class UnstractDB(UnstractConnector, ABC):
sql_query = ""
create_table_query = self.get_create_table_base_query(table=table)
+ logger.info(f"Create table base query {create_table_query}")
+
sql_query += create_table_query
for key, val in database_entry.items():
@@ -166,16 +161,20 @@ class UnstractDB(UnstractConnector, ABC):
return sql_query.rstrip(", ") + ")"
@staticmethod
- def get_sql_insert_query(table_name: str, sql_keys: list[str]) -> str:
+ def get_sql_insert_query(
+ table_name: str, sql_keys: list[str], sql_values: list[str] = None
+ ) -> str:
"""Function to generate parameterised insert sql query.
Args:
table_name (str): db-connector table name
sql_keys (list[str]): column names
+ sql_values (list[str], optional): SQL values for database-specific handling
Returns:
str: returns a string with parameterised insert sql query
"""
+ # Base implementation ignores sql_values and returns parameterized query
keys_str = ",".join(sql_keys)
values_placeholder = ",".join(["%s" for _ in sql_keys])
return f"INSERT INTO {table_name} ({keys_str}) VALUES ({values_placeholder})"
@@ -229,39 +228,120 @@ class UnstractDB(UnstractConnector, ABC):
column_types[column_name] = data_type
return column_types
- def is_string_column(self, table_info: dict[str, str], column_name: str) -> bool:
- """Check if the column is a string type specific to the DB connector.
+ def has_no_metadata(self, table_info: dict[str, str]) -> bool:
+ """Check if metadata field exists in table_info (case-insensitive).
Args:
- table_info (dict): column_name -> column_type
- column_name (str): name of column to check
+ table_info: Dictionary containing table field names and their types
Returns:
- bool: True if column is a string type
+ bool: False if metadata exists, True if metadata does not exist
"""
- column_type = table_info.get(column_name)
- if column_type is None:
- return False
+ logger.info("Checking if column 'metadata' exists in table_info")
+ metadata_exists = any(key.lower() == "metadata" for key in table_info.keys())
+ if metadata_exists:
+ logger.info(
+ "column 'metadata' exists for corresponding table. No migration needed"
+ )
+ return not metadata_exists
+
+ def migrate_table_to_v2(
+ self, table_name: str, column_name: str, engine: Any
+ ) -> dict[str, str]:
+ """Retruns the information schema of table after alteraring table
+ This will add column _v2 to the table and return the information schema
- # Skip migration if *_v2 column already exists
- if f"{column_name}_v2" in table_info:
- return False
+ Args:
+ table_name (str): _description_
+ column_name (str): _description_
+ engine (Any): _description_
- return column_type.lower() == self.get_string_type().lower()
+ Raises:
+ UnstractDBException: _description_
- def migrate_table_to_v2(self, table_name: str, column_name: str, engine: Any) -> None:
- sql_query = self.prepare_multi_column_migration(
+ Returns:
+ dict[str, str]: _description_
+ """
+ sql_query_or_list = self.prepare_multi_column_migration(
table_name=table_name, column_name=column_name
)
+ logger.info(
+ "Running table migration for table %s by adding following columns %s",
+ table_name,
+ sql_query_or_list,
+ )
try:
- self.execute_query(
- engine=engine,
- sql_query=sql_query,
- sql_values=None,
- table_name=table_name,
- )
+ if isinstance(sql_query_or_list, list):
+ for sql_query in sql_query_or_list:
+ self.execute_query(
+ engine=engine,
+ sql_query=sql_query,
+ sql_values=None,
+ table_name=table_name,
+ )
+ else:
+ self.execute_query(
+ engine=engine,
+ sql_query=sql_query_or_list,
+ sql_values=None,
+ table_name=table_name,
+ )
+ logger.info(
+ "successfully migrated table %s with: %s query",
+ table_name,
+ sql_query_or_list,
+ )
+ return self.get_information_schema(table_name=table_name)
except UnstractDBConnectorException as e:
raise UnstractDBException(detail=e.detail) from e
- logger.debug(f"successfully migrated table {table_name} with: {sql_query} query")
+ def get_sql_values_for_query(
+ self, values: dict[str, Any], column_types: dict[str, str]
+ ) -> dict[str, str]:
+ """Prepare SQL values for query execution.
+
+ Args:
+ values (dict[str, Any]): dictionary of columns and values
+ column_types (dict[str, str]): types of columns from database schema
+
+ Returns:
+ dict[str, str]: Dictionary of column names to SQL values for parameterized queries
+
+ Note:
+ This is the base implementation for standard databases.
+ Database-specific connectors can override this method for custom logic.
+ """
+ sql_values: dict[str, Any] = {}
+ for column in values:
+ value = values[column]
+ # Handle JSON and Enum types for standard databases
+ if isinstance(value, (dict, list)):
+ try:
+ sql_values[column] = json.dumps(value)
+ except (TypeError, ValueError) as e:
+ logger.error(
+ f"Failed to serialize value to JSON for column {column}: {e}"
+ )
+ # Create a safe fallback error object
+ fallback_value = {
+ "error": "JSON serialization failed",
+ "error_type": e.__class__.__name__,
+ "error_message": str(e),
+ "data_type": str(type(value)),
+ "data_description": f"column_{column}",
+ "timestamp": datetime.datetime.now().isoformat(),
+ }
+ sql_values[column] = json.dumps(fallback_value)
+ elif isinstance(value, Enum):
+ sql_values[column] = value.value
+ else:
+ sql_values[column] = f"{value}"
+
+ # If table has a column 'id', unstract inserts a unique value to it
+ # Oracle db has column 'ID' instead of 'id'
+ if any(key in column_types for key in ["id", "ID"]):
+ uuid_id = str(uuid.uuid4())
+ sql_values["id"] = f"{uuid_id}"
+
+ return sql_values