LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
UN-2698 [FIX] Added MSSQL support for creating missing tables and inserting data into schema.table (#1479)
Kirtiman Mishra47afc730daf3
diff --git a/docker/connector/docker-compose-connector.yaml b/docker/connector/docker-compose-connector.yamlindex b07fa5ed9..24b1d2e73 100644--- a/docker/connector/docker-compose-connector.yaml+++ b/docker/connector/docker-compose-connector.yaml@@ -22,14 +22,21 @@ services:- mysql_data:/var/lib/mysqlmssql:- image: mcr.microsoft.com/mssql/server:2022-preview-ubuntu-22.04+ image: mcr.microsoft.com/mssql/server:2022-latestcontainer_name: unstract-mssql+ user: "0:0"env_file:- .envports:- "1433:1433"volumes:- mssql_data:/var/opt/mssql+ healthcheck:+ test: ["CMD-SHELL", "/opt/mssql-tools18/bin/sqlcmd -S localhost -U SA -P $$MSSQL_SA_PASSWORD -C -Q 'SELECT 1'"]+ interval: 10s+ retries: 10+ start_period: 10s+ timeout: 3ssftp:image: atmoz/sftp:debiandiff --git a/docker/connector/sample.env b/docker/connector/sample.envindex d6b160d41..05555998d 100644--- a/docker/connector/sample.env+++ b/docker/connector/sample.env@@ -4,6 +4,7 @@ MYSQL_USER=unstract_devMYSQL_PASSWORD=unstract_passACCEPT_EULA=YMSSQL_SA_PASSWORD=unstractPassword@123+MSSQL_PID=ExpressSFTP_USER=fooSFTP_PASS=passSFTP_USER_ID=1001diff --git a/unstract/connectors/src/unstract/connectors/databases/exceptions.py b/unstract/connectors/src/unstract/connectors/databases/exceptions.pyindex 4e9c8551d..5aaf37843 100644--- a/unstract/connectors/src/unstract/connectors/databases/exceptions.py+++ b/unstract/connectors/src/unstract/connectors/databases/exceptions.py@@ -21,10 +21,13 @@ class UnstractDBConnectorException(ConnectorBaseException):class InvalidSyntaxException(UnstractDBConnectorException):def __init__(self, detail: Any, database: Any) -> None:default_detail = (- f"Error creating/writing to {database}. Syntax incorrect. "+ f"Error creating/writing to `{database}`. Syntax incorrect. "f"Please check your table-name or schema. ")- super().__init__(detail=default_detail)+ final_detail = (+ f"{default_detail}\nDetails: {detail}" if detail else default_detail+ )+ super().__init__(detail=final_detail)class InvalidSchemaException(UnstractDBConnectorException):diff --git a/unstract/connectors/src/unstract/connectors/databases/exceptions_helper.py b/unstract/connectors/src/unstract/connectors/databases/exceptions_helper.pyindex 59611f0b6..367993783 100644--- a/unstract/connectors/src/unstract/connectors/databases/exceptions_helper.py+++ b/unstract/connectors/src/unstract/connectors/databases/exceptions_helper.py@@ -1,20 +1,17 @@-from typing import Any--class ExceptionHelper:@staticmethod- def extract_byte_exception(e: Exception) -> Any:- """_summary_- Extract error details from byte_exception.- Used by mssql+ def extract_byte_exception(e: Exception) -> str:+ """Extract error details from byte_exception.+ Used by mssql and mysql connectors.+Args:- e (Exception): _description_+ e (Exception): Database exception to extract details fromReturns:- Any: _description_+ str: Extracted and stripped error details as string"""error_message = str(e)error_code, error_details = eval(error_message)if isinstance(error_details, bytes):error_details = error_details.decode("utf-8")- return error_details+ return str(error_details).strip()diff --git a/unstract/connectors/src/unstract/connectors/databases/mssql/mssql.py b/unstract/connectors/src/unstract/connectors/databases/mssql/mssql.pyindex 3ff943f4d..512e9ea89 100644--- a/unstract/connectors/src/unstract/connectors/databases/mssql/mssql.py+++ b/unstract/connectors/src/unstract/connectors/databases/mssql/mssql.py@@ -67,19 +67,38 @@ class MSSQL(UnstractDB):)def get_create_table_base_query(self, table: str) -> str:- """Function to create a base create table sql query.+ """Function to create a base create table sql query with proper schema support.Args:- table (str): db-connector table name+ table (str): db-connector table name (supports schema.table format)Returns:str: generates a create sql base query with the constant columns"""+ # Parse schema and table name for existence check+ if "." in table:+ # Handle schema.table format like "[schema].[table]"+ parts = table.rsplit(".", 1)+ schema_name, table_name = parts[0], parts[1]+ existence_check = (+ f"IF NOT EXISTS ("+ f"SELECT * FROM INFORMATION_SCHEMA.TABLES "+ f"WHERE TABLE_SCHEMA = '{schema_name}' AND TABLE_NAME = '{table_name}'"+ f")"+ )+ else:+ # Handle unqualified table names (default to dbo schema)+ existence_check = (+ f"IF NOT EXISTS ("+ f"SELECT * FROM INFORMATION_SCHEMA.TABLES "+ f"WHERE TABLE_SCHEMA = 'dbo' AND TABLE_NAME = '{table}'"+ f")"+ )+sql_query = (- f"IF NOT EXISTS ("- f"SELECT * FROM sysobjects WHERE name='{table}' and xtype='U')"- f" CREATE TABLE {table} "- f"(id TEXT ,"+ f"{existence_check} "+ f"CREATE TABLE {table} "+ f"(id TEXT, "f"created_by TEXT, created_at DATETIMEOFFSET, ")return sql_query@@ -123,3 +142,37 @@ class MSSQL(UnstractDB):database=self.database,table_name=table_name,) from e++ def get_information_schema(self, table_name: str) -> dict[str, str]:+ """Function to generate information schema with proper schema and database support.++ Args:+ table_name (str): db-connector table name (supports schema.table format)++ Returns:+ dict[str, str]: a dictionary contains db column name and+ db column types of corresponding table+ """+ table_name = str.lower(table_name)+ if "." in table_name:+ # Handle schema.table format+ parts = table_name.rsplit(".", 1)+ schema_name, table_only = parts[0], parts[1]+ query = (+ f"SELECT column_name, data_type FROM "+ f"information_schema.columns WHERE "+ f"table_schema = '{schema_name}' AND table_name = '{table_only}'"+ )+ else:+ # Handle unqualified table names (default to dbo)+ query = (+ f"SELECT column_name, data_type FROM "+ f"information_schema.columns WHERE "+ f"table_schema = 'dbo' AND table_name = '{table_name}'"+ )++ results = self.execute(query=query)+ column_types: dict[str, str] = self.get_db_column_types(+ columns_with_types=results+ )+ return column_types