Skip to content

LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows

Fix: enable all db connectors (#213)

Kirtiman Mishra851e661d329b
diff --git a/backend/workflow_manager/endpoint/database_utils.py b/backend/workflow_manager/endpoint/database_utils.py
index 8adcc989a..6402b0697 100644
--- a/backend/workflow_manager/endpoint/database_utils.py
+++ b/backend/workflow_manager/endpoint/database_utils.py
@@ -304,14 +304,14 @@ class DatabaseUtils:
raise e
@staticmethod
- def get_db_engine(
+ def get_db_class(
connector_id: str, connector_settings: dict[str, Any]
- ) -> Any:
+ ) -> UnstractDB:
connector = db_connectors[connector_id][Common.METADATA][
Common.CONNECTOR
]
connector_class: UnstractDB = connector(connector_settings)
- return connector_class.get_engine()
+ return connector_class
@staticmethod
def execute_and_fetch_data(
@@ -325,11 +325,15 @@ class DatabaseUtils:
@staticmethod
def create_table_if_not_exists(
- engine: Any, table_name: str, database_entry: dict[str, Any]
+ db_class: UnstractDB,
+ engine: Any,
+ table_name: str,
+ database_entry: dict[str, Any],
) -> None:
"""Creates table if not exists.
Args:
+ class_name (UnstractDB): Type of Unstract DB connector
engine (Any): _description_
table_name (str): _description_
database_entry (dict[str, Any]): _description_
@@ -337,9 +341,8 @@ class DatabaseUtils:
Raises:
e: _description_
"""
- class_name = engine.__class__.__name__
sql = DBConnectorQueryHelper.create_table_query(
- conn_cls=class_name, table=table_name, database_entry=database_entry
+ conn_cls=db_class, table=table_name, database_entry=database_entry
)
logger.debug(f"creating table with: {sql} query")
try:
@@ -359,14 +362,14 @@ class DBConnectorQueryHelper:
@staticmethod
def create_table_query(
- conn_cls: str, table: str, database_entry: dict[str, Any]
+ conn_cls: UnstractDB, table: str, database_entry: dict[str, Any]
) -> Any:
sql_query = ""
"""Generate a SQL query to create a table, based on the provided
database entry.
Args:
- conn_cls (str): The database connection class.
+ conn_cls (str): The database connector class.
Should be one of 'BIGQUERY', 'SNOWFLAKE', or other.
table (str): The name of the table to be created.
database_entry (dict[str, Any]):
@@ -381,76 +384,22 @@ class DBConnectorQueryHelper:
and column definitions.
Note:
- For 'BIGQUERY', 'SNOWFLAKE', or other database connection classes,
- the appropriate data types will be mapped based
- on the Python types of the values in the database_entry dictionary.
-
- Permanent columns, will always be present in table creation.
+ - Each conn_cls have it's implementation for SQL create table query
+ Based on the implementation, a base SQL create table query will be
+ created containing Permanent columns
+ - Each conn_cls also has a mapping to convert python datatype to
+ corresponding column type (string, VARCHAR etc)
+ - keys in database_entry will be converted to column type, and
+ column values will be the valus in database_entry
+ - base SQL create table will be appended based column type and
+ values, and generates a complete SQL create table query
"""
-
- if conn_cls == DBConnectionClass.BIGQUERY:
- sql_query += (
- f"CREATE TABLE IF NOT EXISTS {table} "
- f"(id string,"
- f"created_by string, created_at TIMESTAMP, "
- )
- elif conn_cls == DBConnectionClass.SNOWFLAKE:
- sql_query += (
- f"CREATE TABLE {table} IF NOT EXISTS "
- f"(id TEXT ,"
- f"created_by TEXT, created_at TIMESTAMP, "
- )
- else:
- sql_query += (
- f"CREATE TABLE IF NOT EXISTS {table} "
- f"(id TEXT , "
- f"created_by TEXT, created_at TIMESTAMP, "
- )
+ create_table_query = conn_cls.get_create_table_query(table=table)
+ sql_query += create_table_query
for key, val in database_entry.items():
if key not in TableColumns.PERMANENT_COLUMNS:
- python_type = type(val)
- if conn_cls == DBConnectionClass.BIGQUERY:
- sql_type = (
- DBConnectorTypeConverter.python_to_bigquery_mapping(
- python_type
- )
- )
- else:
- sql_type = DBConnectorTypeConverter.python_to_sql_mapping(
- python_type
- )
+ sql_type = conn_cls.sql_to_db_mapping(val)
sql_query += f"{key} {sql_type}, "
return sql_query.rstrip(", ") + ");"
-
-
-class DBConnectorTypeConverter:
- """_summary_
-
- Class to convert python data type to corresponding connector
- database data type
- """
-
- @staticmethod
- def python_to_sql_mapping(python_type: Any) -> Optional[str]:
- """Method used to convert python to SQL datatype Used by Postgres,
- Redshift, Snowflake."""
- mapping = {
- str: "TEXT",
- int: "INT",
- float: "FLOAT",
- datetime.datetime: "TIMESTAMP",
- }
- return mapping.get(python_type, "TEXT")
-
- @staticmethod
- def python_to_bigquery_mapping(python_type: Any) -> Optional[str]:
- """Method used to convert python to bigquery datatype."""
- mapping = {
- str: "string",
- int: "INT64",
- float: "FLOAT64",
- datetime.datetime: "TIMESTAMP",
- }
- return mapping.get(python_type, "string")
diff --git a/backend/workflow_manager/endpoint/destination.py b/backend/workflow_manager/endpoint/destination.py
index a9e203d7b..f76bd6eda 100644
--- a/backend/workflow_manager/endpoint/destination.py
+++ b/backend/workflow_manager/endpoint/destination.py
@@ -216,15 +216,19 @@ class DestinationConnector(BaseConnector):
agent_name=agent_name,
single_column_name=single_column_name,
)
- engine = DatabaseUtils.get_db_engine(
+ db_class = DatabaseUtils.get_db_class(
connector_id=connector_instance.connector_id,
connector_settings=connector_settings,
)
+ engine = db_class.get_engine()
# If data is None, don't execute CREATE or INSERT query
if data is None:
return
DatabaseUtils.create_table_if_not_exists(
- engine=engine, table_name=table_name, database_entry=values
+ db_class=db_class,
+ engine=engine,
+ table_name=table_name,
+ database_entry=values,
)
sql_columns_and_values = (
DatabaseUtils.get_sql_columns_and_values_for_query(
diff --git a/frontend/src/components/agency/ds-settings-card/DsSettingsCard.jsx b/frontend/src/components/agency/ds-settings-card/DsSettingsCard.jsx
index 0e1de04e5..e63dffbc4 100644
--- a/frontend/src/components/agency/ds-settings-card/DsSettingsCard.jsx
+++ b/frontend/src/components/agency/ds-settings-card/DsSettingsCard.jsx
@@ -55,11 +55,6 @@ const disabledIdsByType = {
"azure_cloud_storage|1476a54a-ed17-4a01-9f8f-cb7e4cf91c8a",
"http|6fdea346-86e4-4383-9a21-132db7c9a576",
],
- DATABASE: [
- "mariadb|146b0124-b9fc-466f-8e68-098ff60703e8",
- "mysql|db709852-fa51-4aa6-9b91-afc45f111bec",
- "mssql|6c6af35c-9498-4bd6-9258-23b5337e068b",
- ],
};
const needToRemove = {
diff --git a/unstract/connectors/src/unstract/connectors/databases/bigquery/bigquery.py b/unstract/connectors/src/unstract/connectors/databases/bigquery/bigquery.py
index 741836829..5661f32ff 100644
--- a/unstract/connectors/src/unstract/connectors/databases/bigquery/bigquery.py
+++ b/unstract/connectors/src/unstract/connectors/databases/bigquery/bigquery.py
@@ -1,3 +1,4 @@
+import datetime
import json
import os
from typing import Any
@@ -58,3 +59,33 @@ class BigQuery(UnstractDB):
return query_job.result()
except Exception as e:
raise ConnectorError(str(e))
+
+ @staticmethod
+ def sql_to_db_mapping(value: str) -> str:
+ """
+ Gets the python datatype of value and converts python datatype
+ to corresponding DB datatype
+ Args:
+ value (str): _description_
+
+ Returns:
+ str: _description_
+ """
+ python_type = type(value)
+
+ mapping = {
+ str: "string",
+ int: "INT64",
+ float: "FLOAT64",
+ datetime.datetime: "TIMESTAMP",
+ }
+ return mapping.get(python_type, "string")
+
+ @staticmethod
+ def get_create_table_query(table: str) -> str:
+ sql_query = (
+ f"CREATE TABLE IF NOT EXISTS {table} "
+ f"(id string,"
+ f"created_by string, created_at TIMESTAMP, "
+ )
+ return sql_query
diff --git a/unstract/connectors/src/unstract/connectors/databases/mariadb/mariadb.py b/unstract/connectors/src/unstract/connectors/databases/mariadb/mariadb.py
index cd1997eac..e926cd41f 100644
--- a/unstract/connectors/src/unstract/connectors/databases/mariadb/mariadb.py
+++ b/unstract/connectors/src/unstract/connectors/databases/mariadb/mariadb.py
@@ -4,6 +4,7 @@ from typing import Any
import pymysql
from pymysql.connections import Connection
+from unstract.connectors.databases.mysql import MySQL
from unstract.connectors.databases.unstract_db import UnstractDB
@@ -57,3 +58,19 @@ class MariaDB(UnstractDB):
password=self.password,
)
return con
+
+ @staticmethod
+ def sql_to_db_mapping(value: str) -> str:
+ """
+ Gets the python datatype of value and converts python datatype
+ to corresponding DB datatype
+ Args:
+ value (str): _description_
+
+ Returns:
+ str: returns Mysql.sql_to_db_mapping
+
+ Note:
+ Mysql and Mariadb share same SQL column type
+ """
+ return str(MySQL.sql_to_db_mapping(value=value))
diff --git a/unstract/connectors/src/unstract/connectors/databases/mssql/mssql.py b/unstract/connectors/src/unstract/connectors/databases/mssql/mssql.py
index 3f134238b..d0109b849 100644
--- a/unstract/connectors/src/unstract/connectors/databases/mssql/mssql.py
+++ b/unstract/connectors/src/unstract/connectors/databases/mssql/mssql.py
@@ -63,3 +63,14 @@ class MSSQL(UnstractDB):
password=self.password,
database=self.database,
)
+
+ @staticmethod
+ def get_create_table_query(table: str) -> str:
+ sql_query = (
+ f"IF NOT EXISTS ("
+ f"SELECT * FROM sysobjects WHERE name='{table}' and xtype='U')"
+ f" CREATE TABLE {table} "
+ f"(id TEXT ,"
+ f"created_by TEXT, created_at DATETIMEOFFSET, "
+ )
+ return sql_query
diff --git a/unstract/connectors/src/unstract/connectors/databases/mssql/static/json_schema.json b/unstract/connectors/src/unstract/connectors/databases/mssql/static/json_schema.json
index 2492b70db..da2920795 100644
--- a/unstract/connectors/src/unstract/connectors/databases/mssql/static/json_schema.json
+++ b/unstract/connectors/src/unstract/connectors/databases/mssql/static/json_schema.json
@@ -5,6 +5,7 @@
"required": [
"connectorName",
"server",
+ "port",
"database",
"user",
"password"
@@ -17,8 +18,7 @@
},
"server": {
"type": "string",
- "title": "Server",
- "default": "mydbserver.mycompany.com"
+ "title": "Server"
},
"port": {
"type": "string",
@@ -33,8 +33,7 @@
},
"user": {
"type": "string",
- "title": "User",
- "default": "sa"
+ "title": "User"
},
"password": {
"type": "string",
diff --git a/unstract/connectors/src/unstract/connectors/databases/mysql/mysql.py b/unstract/connectors/src/unstract/connectors/databases/mysql/mysql.py
index f2fb369d4..72315a67a 100644
--- a/unstract/connectors/src/unstract/connectors/databases/mysql/mysql.py
+++ b/unstract/connectors/src/unstract/connectors/databases/mysql/mysql.py
@@ -1,3 +1,4 @@
+import datetime
import os
from typing import Any
@@ -57,3 +58,23 @@ class MySQL(UnstractDB):
password=self.password,
)
return con
+
+ @staticmethod
+ def sql_to_db_mapping(value: str) -> str:
+ """
+ Gets the python datatype of value and converts python datatype
+ to corresponding DB datatype
+ Args:
+ value (str): _description_
+
+ Returns:
+ str: _description_
+ """
+ python_type = type(value)
+ mapping = {
+ str: "LONGTEXT",
+ int: "BIGINT",
+ float: "FLOAT",
+ datetime.datetime: "TIMESTAMP",
+ }
+ return mapping.get(python_type, "LONGTEXT")
diff --git a/unstract/connectors/src/unstract/connectors/databases/redshift/redshift.py b/unstract/connectors/src/unstract/connectors/databases/redshift/redshift.py
index ca7c3564f..a56d614ce 100644
--- a/unstract/connectors/src/unstract/connectors/databases/redshift/redshift.py
+++ b/unstract/connectors/src/unstract/connectors/databases/redshift/redshift.py
@@ -1,3 +1,4 @@
+import datetime
import os
from typing import Any
@@ -60,3 +61,33 @@ class Redshift(UnstractDB):
password=self.password,
options=f"-c search_path={self.schema}",
)
+
+ @staticmethod
+ def sql_to_db_mapping(value: str) -> str:
+ """
+ Gets the python datatype of value and converts python datatype
+ to corresponding DB datatype
+ Args:
+ value (str): _description_
+
+ Returns:
+ str: _description_
+ """
+ python_type = type(value)
+
+ mapping = {
+ str: "VARCHAR(65535)",
+ int: "BIGINT",
+ float: "DOUBLE PRECISION",
+ datetime.datetime: "TIMESTAMP",
+ }
+ return mapping.get(python_type, "VARCHAR(65535)")
+
+ @staticmethod
+ def get_create_table_query(table: str) -> str:
+ sql_query = (
+ f"CREATE TABLE IF NOT EXISTS {table} "
+ f"(id VARCHAR(65535) ,"
+ f"created_by VARCHAR(65535), created_at TIMESTAMP, "
+ )
+ return sql_query
diff --git a/unstract/connectors/src/unstract/connectors/databases/snowflake/snowflake.py b/unstract/connectors/src/unstract/connectors/databases/snowflake/snowflake.py
index 8c8feb473..ad00ea1a8 100644
--- a/unstract/connectors/src/unstract/connectors/databases/snowflake/snowflake.py
+++ b/unstract/connectors/src/unstract/connectors/databases/snowflake/snowflake.py
@@ -61,3 +61,12 @@ class SnowflakeDB(UnstractDB):
role=self.role,
)
return con
+
+ @staticmethod
+ def get_create_table_query(table: str) -> str:
+ sql_query = (
+ f"CREATE TABLE {table} IF NOT EXISTS "
+ f"(id TEXT ,"
+ f"created_by TEXT, created_at TIMESTAMP, "
+ )
+ return sql_query
diff --git a/unstract/connectors/src/unstract/connectors/databases/unstract_db.py b/unstract/connectors/src/unstract/connectors/databases/unstract_db.py
index af3435d8a..a9304fce2 100644
--- a/unstract/connectors/src/unstract/connectors/databases/unstract_db.py
+++ b/unstract/connectors/src/unstract/connectors/databases/unstract_db.py
@@ -1,3 +1,4 @@
+import datetime
import logging
from abc import ABC, abstractmethod
from typing import Any
@@ -77,3 +78,32 @@ class UnstractDB(UnstractConnector, ABC):
return cursor.fetchall()
except Exception as e:
raise ConnectorError(str(e))
+
+ @staticmethod
+ def sql_to_db_mapping(value: str) -> str:
+ """
+ Gets the python datatype of value and converts python datatype
+ to corresponding DB datatype
+ Args:
+ value (str): _description_
+
+ Returns:
+ str: _description_
+ """
+ python_type = type(value)
+ mapping = {
+ str: "TEXT",
+ int: "INT",
+ float: "FLOAT",
+ datetime.datetime: "TIMESTAMP",
+ }
+ return mapping.get(python_type, "TEXT")
+
+ @staticmethod
+ def get_create_table_query(table: str) -> str:
+ sql_query = (
+ f"CREATE TABLE IF NOT EXISTS {table} "
+ f"(id TEXT , "
+ f"created_by TEXT, created_at TIMESTAMP, "
+ )
+ return sql_query