Skip to content

LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows

UN-2698 [FIX] Added MSSQL support for creating missing tables and inserting data into schema.table (#1479)

Kirtiman Mishra47afc730daf3
diff --git a/docker/connector/docker-compose-connector.yaml b/docker/connector/docker-compose-connector.yaml
index b07fa5ed9..24b1d2e73 100644
--- a/docker/connector/docker-compose-connector.yaml
+++ b/docker/connector/docker-compose-connector.yaml
@@ -22,14 +22,21 @@ services:
- mysql_data:/var/lib/mysql
mssql:
- image: mcr.microsoft.com/mssql/server:2022-preview-ubuntu-22.04
+ image: mcr.microsoft.com/mssql/server:2022-latest
container_name: unstract-mssql
+ user: "0:0"
env_file:
- .env
ports:
- "1433:1433"
volumes:
- mssql_data:/var/opt/mssql
+ healthcheck:
+ test: ["CMD-SHELL", "/opt/mssql-tools18/bin/sqlcmd -S localhost -U SA -P $$MSSQL_SA_PASSWORD -C -Q 'SELECT 1'"]
+ interval: 10s
+ retries: 10
+ start_period: 10s
+ timeout: 3s
sftp:
image: atmoz/sftp:debian
diff --git a/docker/connector/sample.env b/docker/connector/sample.env
index d6b160d41..05555998d 100644
--- a/docker/connector/sample.env
+++ b/docker/connector/sample.env
@@ -4,6 +4,7 @@ MYSQL_USER=unstract_dev
MYSQL_PASSWORD=unstract_pass
ACCEPT_EULA=Y
MSSQL_SA_PASSWORD=unstractPassword@123
+MSSQL_PID=Express
SFTP_USER=foo
SFTP_PASS=pass
SFTP_USER_ID=1001
diff --git a/unstract/connectors/src/unstract/connectors/databases/exceptions.py b/unstract/connectors/src/unstract/connectors/databases/exceptions.py
index 4e9c8551d..5aaf37843 100644
--- a/unstract/connectors/src/unstract/connectors/databases/exceptions.py
+++ b/unstract/connectors/src/unstract/connectors/databases/exceptions.py
@@ -21,10 +21,13 @@ class UnstractDBConnectorException(ConnectorBaseException):
class InvalidSyntaxException(UnstractDBConnectorException):
def __init__(self, detail: Any, database: Any) -> None:
default_detail = (
- f"Error creating/writing to {database}. Syntax incorrect. "
+ f"Error creating/writing to `{database}`. Syntax incorrect. "
f"Please check your table-name or schema. "
)
- super().__init__(detail=default_detail)
+ final_detail = (
+ f"{default_detail}\nDetails: {detail}" if detail else default_detail
+ )
+ super().__init__(detail=final_detail)
class InvalidSchemaException(UnstractDBConnectorException):
diff --git a/unstract/connectors/src/unstract/connectors/databases/exceptions_helper.py b/unstract/connectors/src/unstract/connectors/databases/exceptions_helper.py
index 59611f0b6..367993783 100644
--- a/unstract/connectors/src/unstract/connectors/databases/exceptions_helper.py
+++ b/unstract/connectors/src/unstract/connectors/databases/exceptions_helper.py
@@ -1,20 +1,17 @@
-from typing import Any
-
-
class ExceptionHelper:
@staticmethod
- def extract_byte_exception(e: Exception) -> Any:
- """_summary_
- Extract error details from byte_exception.
- Used by mssql
+ def extract_byte_exception(e: Exception) -> str:
+ """Extract error details from byte_exception.
+ Used by mssql and mysql connectors.
+
Args:
- e (Exception): _description_
+ e (Exception): Database exception to extract details from
Returns:
- Any: _description_
+ str: Extracted and stripped error details as string
"""
error_message = str(e)
error_code, error_details = eval(error_message)
if isinstance(error_details, bytes):
error_details = error_details.decode("utf-8")
- return error_details
+ return str(error_details).strip()
diff --git a/unstract/connectors/src/unstract/connectors/databases/mssql/mssql.py b/unstract/connectors/src/unstract/connectors/databases/mssql/mssql.py
index 3ff943f4d..512e9ea89 100644
--- a/unstract/connectors/src/unstract/connectors/databases/mssql/mssql.py
+++ b/unstract/connectors/src/unstract/connectors/databases/mssql/mssql.py
@@ -67,19 +67,38 @@ class MSSQL(UnstractDB):
)
def get_create_table_base_query(self, table: str) -> str:
- """Function to create a base create table sql query.
+ """Function to create a base create table sql query with proper schema support.
Args:
- table (str): db-connector table name
+ table (str): db-connector table name (supports schema.table format)
Returns:
str: generates a create sql base query with the constant columns
"""
+ # Parse schema and table name for existence check
+ if "." in table:
+ # Handle schema.table format like "[schema].[table]"
+ parts = table.rsplit(".", 1)
+ schema_name, table_name = parts[0], parts[1]
+ existence_check = (
+ f"IF NOT EXISTS ("
+ f"SELECT * FROM INFORMATION_SCHEMA.TABLES "
+ f"WHERE TABLE_SCHEMA = '{schema_name}' AND TABLE_NAME = '{table_name}'"
+ f")"
+ )
+ else:
+ # Handle unqualified table names (default to dbo schema)
+ existence_check = (
+ f"IF NOT EXISTS ("
+ f"SELECT * FROM INFORMATION_SCHEMA.TABLES "
+ f"WHERE TABLE_SCHEMA = 'dbo' AND TABLE_NAME = '{table}'"
+ f")"
+ )
+
sql_query = (
- f"IF NOT EXISTS ("
- f"SELECT * FROM sysobjects WHERE name='{table}' and xtype='U')"
- f" CREATE TABLE {table} "
- f"(id TEXT ,"
+ f"{existence_check} "
+ f"CREATE TABLE {table} "
+ f"(id TEXT, "
f"created_by TEXT, created_at DATETIMEOFFSET, "
)
return sql_query
@@ -123,3 +142,37 @@ class MSSQL(UnstractDB):
database=self.database,
table_name=table_name,
) from e
+
+ def get_information_schema(self, table_name: str) -> dict[str, str]:
+ """Function to generate information schema with proper schema and database support.
+
+ Args:
+ table_name (str): db-connector table name (supports schema.table format)
+
+ Returns:
+ dict[str, str]: a dictionary contains db column name and
+ db column types of corresponding table
+ """
+ table_name = str.lower(table_name)
+ if "." in table_name:
+ # Handle schema.table format
+ parts = table_name.rsplit(".", 1)
+ schema_name, table_only = parts[0], parts[1]
+ query = (
+ f"SELECT column_name, data_type FROM "
+ f"information_schema.columns WHERE "
+ f"table_schema = '{schema_name}' AND table_name = '{table_only}'"
+ )
+ else:
+ # Handle unqualified table names (default to dbo)
+ query = (
+ f"SELECT column_name, data_type FROM "
+ f"information_schema.columns WHERE "
+ f"table_schema = 'dbo' AND table_name = '{table_name}'"
+ )
+
+ results = self.execute(query=query)
+ column_types: dict[str, str] = self.get_db_column_types(
+ columns_with_types=results
+ )
+ return column_types