Skip to content

LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows

UN-2816 [FIX] Improved error handling in destination connector table migration during alter permissions (#1549)

Kirtiman Mishra64b90f45bd55
diff --git a/backend/workflow_manager/endpoint_v2/database_utils.py b/backend/workflow_manager/endpoint_v2/database_utils.py
index 2d9d54177..7f0aeb7ab 100644
--- a/backend/workflow_manager/endpoint_v2/database_utils.py
+++ b/backend/workflow_manager/endpoint_v2/database_utils.py
@@ -289,8 +289,12 @@ class DatabaseUtils:
Raises:
e: _description_
"""
- sql = db_class.create_table_query(table=table_name, database_entry=database_entry)
- logger.debug(f"creating table {table_name} with: {sql} query")
+ sql = db_class.create_table_query(
+ table=table_name,
+ database_entry=database_entry,
+ permanent_columns=TableColumns.PERMANENT_COLUMNS,
+ )
+ logger.info(f"creating table {table_name} with: {sql} query")
try:
db_class.execute_query(
@@ -298,4 +302,35 @@ class DatabaseUtils:
)
except UnstractDBConnectorException as e:
raise UnstractDBException(detail=e.detail) from e
- logger.debug(f"successfully created table {table_name} with: {sql} query")
+ logger.info(f"successfully created table {table_name} with: {sql} query")
+
+ @staticmethod
+ def migrate_table_to_v2(
+ db_class: UnstractDB,
+ engine: Any,
+ table_name: str,
+ column_name: str,
+ ) -> dict[str, str]:
+ """Migrate table to v2 by adding _v2 columns.
+
+ Args:
+ db_class (UnstractDB): DB Connection class
+ engine (Any): Database engine
+ table_name (str): Name of the table to migrate
+ column_name (str): Base column name for v2 migration
+
+ Returns:
+ dict[str, str]: Updated table information schema
+
+ Raises:
+ UnstractDBException: If migration fails
+ """
+ try:
+ result: dict[str, str] = db_class.migrate_table_to_v2(
+ table_name=table_name,
+ column_name=column_name,
+ engine=engine,
+ )
+ return result
+ except UnstractDBConnectorException as e:
+ raise UnstractDBException(detail=e.detail) from e
diff --git a/backend/workflow_manager/endpoint_v2/destination.py b/backend/workflow_manager/endpoint_v2/destination.py
index 77a7a9da9..5f3d39863 100644
--- a/backend/workflow_manager/endpoint_v2/destination.py
+++ b/backend/workflow_manager/endpoint_v2/destination.py
@@ -367,7 +367,6 @@ class DestinationConnector(BaseConnector):
)
engine = db_class.get_engine()
-
table_info = db_class.get_information_schema(table_name=table_name)
logger.info(
@@ -376,10 +375,11 @@ class DestinationConnector(BaseConnector):
if table_info:
if db_class.has_no_metadata(table_info=table_info):
- table_info = db_class.migrate_table_to_v2(
+ table_info = DatabaseUtils.migrate_table_to_v2(
+ db_class=db_class,
+ engine=engine,
table_name=table_name,
column_name=single_column_name,
- engine=engine,
)
values = DatabaseUtils.get_columns_and_values(
diff --git a/docker/connector/docker-compose-connector.yaml b/docker/connector/docker-compose-connector.yaml
index 24b1d2e73..30756c589 100644
--- a/docker/connector/docker-compose-connector.yaml
+++ b/docker/connector/docker-compose-connector.yaml
@@ -22,7 +22,7 @@ services:
- mysql_data:/var/lib/mysql
mssql:
- image: mcr.microsoft.com/mssql/server:2022-latest
+ image: mcr.microsoft.com/mssql/server:latest
container_name: unstract-mssql
user: "0:0"
env_file:
diff --git a/docker/connector/sample.env b/docker/connector/sample.env
index 05555998d..8c36310cc 100644
--- a/docker/connector/sample.env
+++ b/docker/connector/sample.env
@@ -5,6 +5,9 @@ MYSQL_PASSWORD=unstract_pass
ACCEPT_EULA=Y
MSSQL_SA_PASSWORD=unstractPassword@123
MSSQL_PID=Express
+MSSQL_LIMITED_USER=unstract_limited_user
+MSSQL_LIMITED_PASSWORD=limitedPass@123
+MSSQL_LIMITED_DATABASE=unstract_test_db
SFTP_USER=foo
SFTP_PASS=pass
SFTP_USER_ID=1001
diff --git a/unstract/connectors/src/unstract/connectors/databases/unstract_db.py b/unstract/connectors/src/unstract/connectors/databases/unstract_db.py
index ff2bbbac2..dfe604823 100644
--- a/unstract/connectors/src/unstract/connectors/databases/unstract_db.py
+++ b/unstract/connectors/src/unstract/connectors/databases/unstract_db.py
@@ -6,11 +6,7 @@ from abc import ABC, abstractmethod
from enum import Enum
from typing import Any
-from workflow_manager.endpoint_v2.constants import TableColumns
-from workflow_manager.endpoint_v2.exceptions import UnstractDBException
-
from unstract.connectors.base import UnstractConnector
-from unstract.connectors.databases.exceptions import UnstractDBConnectorException
from unstract.connectors.enums import ConnectorMode
from unstract.connectors.exceptions import ConnectorError
@@ -130,17 +126,24 @@ class UnstractDB(UnstractConnector, ABC):
str: generates a create sql base query with the constant columns
"""
- def create_table_query(self, table: str, database_entry: dict[str, Any]) -> Any:
+ def create_table_query(
+ self, table: str, database_entry: dict[str, Any], permanent_columns: list[str]
+ ) -> Any:
"""Function to create a create table sql query.
Args:
table (str): db-connector table name
database_entry (dict[str, Any]): a dictionary of column name and types
+ permanent_columns (list[str]): list of permanent column names to exclude
+ from dynamic column creation. These columns are already defined in the
+ base table schema (e.g., 'id', 'created_by', 'created_at', 'metadata',
+ 'status', 'error_message', etc.) and should not be added again during
+ table creation to avoid duplication.
Returns:
Any: generates a create sql query for all the columns
"""
- PERMANENT_COLUMNS = TableColumns.PERMANENT_COLUMNS
+ PERMANENT_COLUMNS = permanent_columns
sql_query = ""
create_table_query = self.get_create_table_base_query(table=table)
@@ -251,8 +254,6 @@ class UnstractDB(UnstractConnector, ABC):
column_name (str): _description_
engine (Any): _description_
- Raises:
- UnstractDBException: _description_
Returns:
dict[str, str]: _description_
@@ -266,30 +267,27 @@ class UnstractDB(UnstractConnector, ABC):
sql_query_or_list,
)
- try:
- if isinstance(sql_query_or_list, list):
- for sql_query in sql_query_or_list:
- self.execute_query(
- engine=engine,
- sql_query=sql_query,
- sql_values=None,
- table_name=table_name,
- )
- else:
+ if isinstance(sql_query_or_list, list):
+ for sql_query in sql_query_or_list:
self.execute_query(
engine=engine,
- sql_query=sql_query_or_list,
+ sql_query=sql_query,
sql_values=None,
table_name=table_name,
)
- logger.info(
- "successfully migrated table %s with: %s query",
- table_name,
- sql_query_or_list,
- )
- return self.get_information_schema(table_name=table_name)
- except UnstractDBConnectorException as e:
- raise UnstractDBException(detail=e.detail) from e
+ else:
+ self.execute_query(
+ engine=engine,
+ sql_query=sql_query_or_list,
+ sql_values=None,
+ table_name=table_name,
+ )
+ logger.info(
+ "successfully migrated table %s with: %s query",
+ table_name,
+ sql_query_or_list,
+ )
+ return self.get_information_schema(table_name=table_name)
def get_sql_values_for_query(
self, values: dict[str, Any], column_types: dict[str, str]