LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
UN-2816 [FIX] Improved error handling in destination connector table migration during alter permissions (#1549)
Kirtiman Mishra64b90f45bd55
diff --git a/backend/workflow_manager/endpoint_v2/database_utils.py b/backend/workflow_manager/endpoint_v2/database_utils.pyindex 2d9d54177..7f0aeb7ab 100644--- a/backend/workflow_manager/endpoint_v2/database_utils.py+++ b/backend/workflow_manager/endpoint_v2/database_utils.py@@ -289,8 +289,12 @@ class DatabaseUtils:Raises:e: _description_"""- sql = db_class.create_table_query(table=table_name, database_entry=database_entry)- logger.debug(f"creating table {table_name} with: {sql} query")+ sql = db_class.create_table_query(+ table=table_name,+ database_entry=database_entry,+ permanent_columns=TableColumns.PERMANENT_COLUMNS,+ )+ logger.info(f"creating table {table_name} with: {sql} query")try:db_class.execute_query(@@ -298,4 +302,35 @@ class DatabaseUtils:)except UnstractDBConnectorException as e:raise UnstractDBException(detail=e.detail) from e- logger.debug(f"successfully created table {table_name} with: {sql} query")+ logger.info(f"successfully created table {table_name} with: {sql} query")++ @staticmethod+ def migrate_table_to_v2(+ db_class: UnstractDB,+ engine: Any,+ table_name: str,+ column_name: str,+ ) -> dict[str, str]:+ """Migrate table to v2 by adding _v2 columns.++ Args:+ db_class (UnstractDB): DB Connection class+ engine (Any): Database engine+ table_name (str): Name of the table to migrate+ column_name (str): Base column name for v2 migration++ Returns:+ dict[str, str]: Updated table information schema++ Raises:+ UnstractDBException: If migration fails+ """+ try:+ result: dict[str, str] = db_class.migrate_table_to_v2(+ table_name=table_name,+ column_name=column_name,+ engine=engine,+ )+ return result+ except UnstractDBConnectorException as e:+ raise UnstractDBException(detail=e.detail) from ediff --git a/backend/workflow_manager/endpoint_v2/destination.py b/backend/workflow_manager/endpoint_v2/destination.pyindex 77a7a9da9..5f3d39863 100644--- a/backend/workflow_manager/endpoint_v2/destination.py+++ b/backend/workflow_manager/endpoint_v2/destination.py@@ -367,7 +367,6 @@ class DestinationConnector(BaseConnector):)engine = db_class.get_engine()-table_info = db_class.get_information_schema(table_name=table_name)logger.info(@@ -376,10 +375,11 @@ class DestinationConnector(BaseConnector):if table_info:if db_class.has_no_metadata(table_info=table_info):- table_info = db_class.migrate_table_to_v2(+ table_info = DatabaseUtils.migrate_table_to_v2(+ db_class=db_class,+ engine=engine,table_name=table_name,column_name=single_column_name,- engine=engine,)values = DatabaseUtils.get_columns_and_values(diff --git a/docker/connector/docker-compose-connector.yaml b/docker/connector/docker-compose-connector.yamlindex 24b1d2e73..30756c589 100644--- a/docker/connector/docker-compose-connector.yaml+++ b/docker/connector/docker-compose-connector.yaml@@ -22,7 +22,7 @@ services:- mysql_data:/var/lib/mysqlmssql:- image: mcr.microsoft.com/mssql/server:2022-latest+ image: mcr.microsoft.com/mssql/server:latestcontainer_name: unstract-mssqluser: "0:0"env_file:diff --git a/docker/connector/sample.env b/docker/connector/sample.envindex 05555998d..8c36310cc 100644--- a/docker/connector/sample.env+++ b/docker/connector/sample.env@@ -5,6 +5,9 @@ MYSQL_PASSWORD=unstract_passACCEPT_EULA=YMSSQL_SA_PASSWORD=unstractPassword@123MSSQL_PID=Express+MSSQL_LIMITED_USER=unstract_limited_user+MSSQL_LIMITED_PASSWORD=limitedPass@123+MSSQL_LIMITED_DATABASE=unstract_test_dbSFTP_USER=fooSFTP_PASS=passSFTP_USER_ID=1001diff --git a/unstract/connectors/src/unstract/connectors/databases/unstract_db.py b/unstract/connectors/src/unstract/connectors/databases/unstract_db.pyindex ff2bbbac2..dfe604823 100644--- a/unstract/connectors/src/unstract/connectors/databases/unstract_db.py+++ b/unstract/connectors/src/unstract/connectors/databases/unstract_db.py@@ -6,11 +6,7 @@ from abc import ABC, abstractmethodfrom enum import Enumfrom typing import Any-from workflow_manager.endpoint_v2.constants import TableColumns-from workflow_manager.endpoint_v2.exceptions import UnstractDBException-from unstract.connectors.base import UnstractConnector-from unstract.connectors.databases.exceptions import UnstractDBConnectorExceptionfrom unstract.connectors.enums import ConnectorModefrom unstract.connectors.exceptions import ConnectorError@@ -130,17 +126,24 @@ class UnstractDB(UnstractConnector, ABC):str: generates a create sql base query with the constant columns"""- def create_table_query(self, table: str, database_entry: dict[str, Any]) -> Any:+ def create_table_query(+ self, table: str, database_entry: dict[str, Any], permanent_columns: list[str]+ ) -> Any:"""Function to create a create table sql query.Args:table (str): db-connector table namedatabase_entry (dict[str, Any]): a dictionary of column name and types+ permanent_columns (list[str]): list of permanent column names to exclude+ from dynamic column creation. These columns are already defined in the+ base table schema (e.g., 'id', 'created_by', 'created_at', 'metadata',+ 'status', 'error_message', etc.) and should not be added again during+ table creation to avoid duplication.Returns:Any: generates a create sql query for all the columns"""- PERMANENT_COLUMNS = TableColumns.PERMANENT_COLUMNS+ PERMANENT_COLUMNS = permanent_columnssql_query = ""create_table_query = self.get_create_table_base_query(table=table)@@ -251,8 +254,6 @@ class UnstractDB(UnstractConnector, ABC):column_name (str): _description_engine (Any): _description_- Raises:- UnstractDBException: _description_Returns:dict[str, str]: _description_@@ -266,30 +267,27 @@ class UnstractDB(UnstractConnector, ABC):sql_query_or_list,)- try:- if isinstance(sql_query_or_list, list):- for sql_query in sql_query_or_list:- self.execute_query(- engine=engine,- sql_query=sql_query,- sql_values=None,- table_name=table_name,- )- else:+ if isinstance(sql_query_or_list, list):+ for sql_query in sql_query_or_list:self.execute_query(engine=engine,- sql_query=sql_query_or_list,+ sql_query=sql_query,sql_values=None,table_name=table_name,)- logger.info(- "successfully migrated table %s with: %s query",- table_name,- sql_query_or_list,- )- return self.get_information_schema(table_name=table_name)- except UnstractDBConnectorException as e:- raise UnstractDBException(detail=e.detail) from e+ else:+ self.execute_query(+ engine=engine,+ sql_query=sql_query_or_list,+ sql_values=None,+ table_name=table_name,+ )+ logger.info(+ "successfully migrated table %s with: %s query",+ table_name,+ sql_query_or_list,+ )+ return self.get_information_schema(table_name=table_name)def get_sql_values_for_query(self, values: dict[str, Any], column_types: dict[str, str]