LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
Fix: enable all db connectors (#213)
Kirtiman Mishra851e661d329b
diff --git a/backend/workflow_manager/endpoint/database_utils.py b/backend/workflow_manager/endpoint/database_utils.pyindex 8adcc989a..6402b0697 100644--- a/backend/workflow_manager/endpoint/database_utils.py+++ b/backend/workflow_manager/endpoint/database_utils.py@@ -304,14 +304,14 @@ class DatabaseUtils:raise e@staticmethod- def get_db_engine(+ def get_db_class(connector_id: str, connector_settings: dict[str, Any]- ) -> Any:+ ) -> UnstractDB:connector = db_connectors[connector_id][Common.METADATA][Common.CONNECTOR]connector_class: UnstractDB = connector(connector_settings)- return connector_class.get_engine()+ return connector_class@staticmethoddef execute_and_fetch_data(@@ -325,11 +325,15 @@ class DatabaseUtils:@staticmethoddef create_table_if_not_exists(- engine: Any, table_name: str, database_entry: dict[str, Any]+ db_class: UnstractDB,+ engine: Any,+ table_name: str,+ database_entry: dict[str, Any],) -> None:"""Creates table if not exists.Args:+ class_name (UnstractDB): Type of Unstract DB connectorengine (Any): _description_table_name (str): _description_database_entry (dict[str, Any]): _description_@@ -337,9 +341,8 @@ class DatabaseUtils:Raises:e: _description_"""- class_name = engine.__class__.__name__sql = DBConnectorQueryHelper.create_table_query(- conn_cls=class_name, table=table_name, database_entry=database_entry+ conn_cls=db_class, table=table_name, database_entry=database_entry)logger.debug(f"creating table with: {sql} query")try:@@ -359,14 +362,14 @@ class DBConnectorQueryHelper:@staticmethoddef create_table_query(- conn_cls: str, table: str, database_entry: dict[str, Any]+ conn_cls: UnstractDB, table: str, database_entry: dict[str, Any]) -> Any:sql_query = """""Generate a SQL query to create a table, based on the provideddatabase entry.Args:- conn_cls (str): The database connection class.+ conn_cls (str): The database connector class.Should be one of 'BIGQUERY', 'SNOWFLAKE', or other.table (str): The name of the table to be created.database_entry (dict[str, Any]):@@ -381,76 +384,22 @@ class DBConnectorQueryHelper:and column definitions.Note:- For 'BIGQUERY', 'SNOWFLAKE', or other database connection classes,- the appropriate data types will be mapped based- on the Python types of the values in the database_entry dictionary.-- Permanent columns, will always be present in table creation.+ - Each conn_cls have it's implementation for SQL create table query+ Based on the implementation, a base SQL create table query will be+ created containing Permanent columns+ - Each conn_cls also has a mapping to convert python datatype to+ corresponding column type (string, VARCHAR etc)+ - keys in database_entry will be converted to column type, and+ column values will be the valus in database_entry+ - base SQL create table will be appended based column type and+ values, and generates a complete SQL create table query"""-- if conn_cls == DBConnectionClass.BIGQUERY:- sql_query += (- f"CREATE TABLE IF NOT EXISTS {table} "- f"(id string,"- f"created_by string, created_at TIMESTAMP, "- )- elif conn_cls == DBConnectionClass.SNOWFLAKE:- sql_query += (- f"CREATE TABLE {table} IF NOT EXISTS "- f"(id TEXT ,"- f"created_by TEXT, created_at TIMESTAMP, "- )- else:- sql_query += (- f"CREATE TABLE IF NOT EXISTS {table} "- f"(id TEXT , "- f"created_by TEXT, created_at TIMESTAMP, "- )+ create_table_query = conn_cls.get_create_table_query(table=table)+ sql_query += create_table_queryfor key, val in database_entry.items():if key not in TableColumns.PERMANENT_COLUMNS:- python_type = type(val)- if conn_cls == DBConnectionClass.BIGQUERY:- sql_type = (- DBConnectorTypeConverter.python_to_bigquery_mapping(- python_type- )- )- else:- sql_type = DBConnectorTypeConverter.python_to_sql_mapping(- python_type- )+ sql_type = conn_cls.sql_to_db_mapping(val)sql_query += f"{key} {sql_type}, "return sql_query.rstrip(", ") + ");"---class DBConnectorTypeConverter:- """_summary_-- Class to convert python data type to corresponding connector- database data type- """-- @staticmethod- def python_to_sql_mapping(python_type: Any) -> Optional[str]:- """Method used to convert python to SQL datatype Used by Postgres,- Redshift, Snowflake."""- mapping = {- str: "TEXT",- int: "INT",- float: "FLOAT",- datetime.datetime: "TIMESTAMP",- }- return mapping.get(python_type, "TEXT")-- @staticmethod- def python_to_bigquery_mapping(python_type: Any) -> Optional[str]:- """Method used to convert python to bigquery datatype."""- mapping = {- str: "string",- int: "INT64",- float: "FLOAT64",- datetime.datetime: "TIMESTAMP",- }- return mapping.get(python_type, "string")diff --git a/backend/workflow_manager/endpoint/destination.py b/backend/workflow_manager/endpoint/destination.pyindex a9e203d7b..f76bd6eda 100644--- a/backend/workflow_manager/endpoint/destination.py+++ b/backend/workflow_manager/endpoint/destination.py@@ -216,15 +216,19 @@ class DestinationConnector(BaseConnector):agent_name=agent_name,single_column_name=single_column_name,)- engine = DatabaseUtils.get_db_engine(+ db_class = DatabaseUtils.get_db_class(connector_id=connector_instance.connector_id,connector_settings=connector_settings,)+ engine = db_class.get_engine()# If data is None, don't execute CREATE or INSERT queryif data is None:returnDatabaseUtils.create_table_if_not_exists(- engine=engine, table_name=table_name, database_entry=values+ db_class=db_class,+ engine=engine,+ table_name=table_name,+ database_entry=values,)sql_columns_and_values = (DatabaseUtils.get_sql_columns_and_values_for_query(diff --git a/frontend/src/components/agency/ds-settings-card/DsSettingsCard.jsx b/frontend/src/components/agency/ds-settings-card/DsSettingsCard.jsxindex 0e1de04e5..e63dffbc4 100644--- a/frontend/src/components/agency/ds-settings-card/DsSettingsCard.jsx+++ b/frontend/src/components/agency/ds-settings-card/DsSettingsCard.jsx@@ -55,11 +55,6 @@ const disabledIdsByType = {"azure_cloud_storage|1476a54a-ed17-4a01-9f8f-cb7e4cf91c8a","http|6fdea346-86e4-4383-9a21-132db7c9a576",],- DATABASE: [- "mariadb|146b0124-b9fc-466f-8e68-098ff60703e8",- "mysql|db709852-fa51-4aa6-9b91-afc45f111bec",- "mssql|6c6af35c-9498-4bd6-9258-23b5337e068b",- ],};const needToRemove = {diff --git a/unstract/connectors/src/unstract/connectors/databases/bigquery/bigquery.py b/unstract/connectors/src/unstract/connectors/databases/bigquery/bigquery.pyindex 741836829..5661f32ff 100644--- a/unstract/connectors/src/unstract/connectors/databases/bigquery/bigquery.py+++ b/unstract/connectors/src/unstract/connectors/databases/bigquery/bigquery.py@@ -1,3 +1,4 @@+import datetimeimport jsonimport osfrom typing import Any@@ -58,3 +59,33 @@ class BigQuery(UnstractDB):return query_job.result()except Exception as e:raise ConnectorError(str(e))++ @staticmethod+ def sql_to_db_mapping(value: str) -> str:+ """+ Gets the python datatype of value and converts python datatype+ to corresponding DB datatype+ Args:+ value (str): _description_++ Returns:+ str: _description_+ """+ python_type = type(value)++ mapping = {+ str: "string",+ int: "INT64",+ float: "FLOAT64",+ datetime.datetime: "TIMESTAMP",+ }+ return mapping.get(python_type, "string")++ @staticmethod+ def get_create_table_query(table: str) -> str:+ sql_query = (+ f"CREATE TABLE IF NOT EXISTS {table} "+ f"(id string,"+ f"created_by string, created_at TIMESTAMP, "+ )+ return sql_querydiff --git a/unstract/connectors/src/unstract/connectors/databases/mariadb/mariadb.py b/unstract/connectors/src/unstract/connectors/databases/mariadb/mariadb.pyindex cd1997eac..e926cd41f 100644--- a/unstract/connectors/src/unstract/connectors/databases/mariadb/mariadb.py+++ b/unstract/connectors/src/unstract/connectors/databases/mariadb/mariadb.py@@ -4,6 +4,7 @@ from typing import Anyimport pymysqlfrom pymysql.connections import Connection+from unstract.connectors.databases.mysql import MySQLfrom unstract.connectors.databases.unstract_db import UnstractDB@@ -57,3 +58,19 @@ class MariaDB(UnstractDB):password=self.password,)return con++ @staticmethod+ def sql_to_db_mapping(value: str) -> str:+ """+ Gets the python datatype of value and converts python datatype+ to corresponding DB datatype+ Args:+ value (str): _description_++ Returns:+ str: returns Mysql.sql_to_db_mapping++ Note:+ Mysql and Mariadb share same SQL column type+ """+ return str(MySQL.sql_to_db_mapping(value=value))diff --git a/unstract/connectors/src/unstract/connectors/databases/mssql/mssql.py b/unstract/connectors/src/unstract/connectors/databases/mssql/mssql.pyindex 3f134238b..d0109b849 100644--- a/unstract/connectors/src/unstract/connectors/databases/mssql/mssql.py+++ b/unstract/connectors/src/unstract/connectors/databases/mssql/mssql.py@@ -63,3 +63,14 @@ class MSSQL(UnstractDB):password=self.password,database=self.database,)++ @staticmethod+ def get_create_table_query(table: str) -> str:+ sql_query = (+ f"IF NOT EXISTS ("+ f"SELECT * FROM sysobjects WHERE name='{table}' and xtype='U')"+ f" CREATE TABLE {table} "+ f"(id TEXT ,"+ f"created_by TEXT, created_at DATETIMEOFFSET, "+ )+ return sql_querydiff --git a/unstract/connectors/src/unstract/connectors/databases/mssql/static/json_schema.json b/unstract/connectors/src/unstract/connectors/databases/mssql/static/json_schema.jsonindex 2492b70db..da2920795 100644--- a/unstract/connectors/src/unstract/connectors/databases/mssql/static/json_schema.json+++ b/unstract/connectors/src/unstract/connectors/databases/mssql/static/json_schema.json@@ -5,6 +5,7 @@"required": ["connectorName","server",+ "port","database","user","password"@@ -17,8 +18,7 @@},"server": {"type": "string",- "title": "Server",- "default": "mydbserver.mycompany.com"+ "title": "Server"},"port": {"type": "string",@@ -33,8 +33,7 @@},"user": {"type": "string",- "title": "User",- "default": "sa"+ "title": "User"},"password": {"type": "string",diff --git a/unstract/connectors/src/unstract/connectors/databases/mysql/mysql.py b/unstract/connectors/src/unstract/connectors/databases/mysql/mysql.pyindex f2fb369d4..72315a67a 100644--- a/unstract/connectors/src/unstract/connectors/databases/mysql/mysql.py+++ b/unstract/connectors/src/unstract/connectors/databases/mysql/mysql.py@@ -1,3 +1,4 @@+import datetimeimport osfrom typing import Any@@ -57,3 +58,23 @@ class MySQL(UnstractDB):password=self.password,)return con++ @staticmethod+ def sql_to_db_mapping(value: str) -> str:+ """+ Gets the python datatype of value and converts python datatype+ to corresponding DB datatype+ Args:+ value (str): _description_++ Returns:+ str: _description_+ """+ python_type = type(value)+ mapping = {+ str: "LONGTEXT",+ int: "BIGINT",+ float: "FLOAT",+ datetime.datetime: "TIMESTAMP",+ }+ return mapping.get(python_type, "LONGTEXT")diff --git a/unstract/connectors/src/unstract/connectors/databases/redshift/redshift.py b/unstract/connectors/src/unstract/connectors/databases/redshift/redshift.pyindex ca7c3564f..a56d614ce 100644--- a/unstract/connectors/src/unstract/connectors/databases/redshift/redshift.py+++ b/unstract/connectors/src/unstract/connectors/databases/redshift/redshift.py@@ -1,3 +1,4 @@+import datetimeimport osfrom typing import Any@@ -60,3 +61,33 @@ class Redshift(UnstractDB):password=self.password,options=f"-c search_path={self.schema}",)++ @staticmethod+ def sql_to_db_mapping(value: str) -> str:+ """+ Gets the python datatype of value and converts python datatype+ to corresponding DB datatype+ Args:+ value (str): _description_++ Returns:+ str: _description_+ """+ python_type = type(value)++ mapping = {+ str: "VARCHAR(65535)",+ int: "BIGINT",+ float: "DOUBLE PRECISION",+ datetime.datetime: "TIMESTAMP",+ }+ return mapping.get(python_type, "VARCHAR(65535)")++ @staticmethod+ def get_create_table_query(table: str) -> str:+ sql_query = (+ f"CREATE TABLE IF NOT EXISTS {table} "+ f"(id VARCHAR(65535) ,"+ f"created_by VARCHAR(65535), created_at TIMESTAMP, "+ )+ return sql_querydiff --git a/unstract/connectors/src/unstract/connectors/databases/snowflake/snowflake.py b/unstract/connectors/src/unstract/connectors/databases/snowflake/snowflake.pyindex 8c8feb473..ad00ea1a8 100644--- a/unstract/connectors/src/unstract/connectors/databases/snowflake/snowflake.py+++ b/unstract/connectors/src/unstract/connectors/databases/snowflake/snowflake.py@@ -61,3 +61,12 @@ class SnowflakeDB(UnstractDB):role=self.role,)return con++ @staticmethod+ def get_create_table_query(table: str) -> str:+ sql_query = (+ f"CREATE TABLE {table} IF NOT EXISTS "+ f"(id TEXT ,"+ f"created_by TEXT, created_at TIMESTAMP, "+ )+ return sql_querydiff --git a/unstract/connectors/src/unstract/connectors/databases/unstract_db.py b/unstract/connectors/src/unstract/connectors/databases/unstract_db.pyindex af3435d8a..a9304fce2 100644--- a/unstract/connectors/src/unstract/connectors/databases/unstract_db.py+++ b/unstract/connectors/src/unstract/connectors/databases/unstract_db.py@@ -1,3 +1,4 @@+import datetimeimport loggingfrom abc import ABC, abstractmethodfrom typing import Any@@ -77,3 +78,32 @@ class UnstractDB(UnstractConnector, ABC):return cursor.fetchall()except Exception as e:raise ConnectorError(str(e))++ @staticmethod+ def sql_to_db_mapping(value: str) -> str:+ """+ Gets the python datatype of value and converts python datatype+ to corresponding DB datatype+ Args:+ value (str): _description_++ Returns:+ str: _description_+ """+ python_type = type(value)+ mapping = {+ str: "TEXT",+ int: "INT",+ float: "FLOAT",+ datetime.datetime: "TIMESTAMP",+ }+ return mapping.get(python_type, "TEXT")++ @staticmethod+ def get_create_table_query(table: str) -> str:+ sql_query = (+ f"CREATE TABLE IF NOT EXISTS {table} "+ f"(id TEXT , "+ f"created_by TEXT, created_at TIMESTAMP, "+ )+ return sql_query