LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
Fix/db connectors table create (#149)
Kirtiman Mishra867cfbf84ae5
diff --git a/backend/workflow_manager/endpoint/constants.py b/backend/workflow_manager/endpoint/constants.pyindex 061dc7a56..cec35aa5a 100644--- a/backend/workflow_manager/endpoint/constants.py+++ b/backend/workflow_manager/endpoint/constants.py@@ -1,6 +1,7 @@class TableColumns:CREATED_BY = "created_by"CREATED_AT = "created_at"+ PERMANENT_COLUMNS = ["created_by", "created_at"]class DBConnectionClass:@@ -94,3 +95,9 @@ class BigQuery:"""TABLE_NAME_SIZE = 3+ COLUMN_TYPES = [+ "DATE",+ "DATETIME",+ "TIME",+ "TIMESTAMP",+ ]diff --git a/backend/workflow_manager/endpoint/database_utils.py b/backend/workflow_manager/endpoint/database_utils.pyindex ad59188c2..8adcc989a 100644--- a/backend/workflow_manager/endpoint/database_utils.py+++ b/backend/workflow_manager/endpoint/database_utils.py@@ -1,6 +1,7 @@import datetimeimport jsonimport logging+import uuidfrom typing import Any, Optionalfrom utils.constants import Common@@ -23,8 +24,8 @@ class DatabaseUtils:@staticmethoddef make_sql_values_for_query(values: dict[str, Any], column_types: dict[str, str], cls: Any = None- ) -> list[str]:- """Making Sql Values for Query.+ ) -> dict[str, str]:+ """Making Sql Columns and Values for Query.Args:values (dict[str, Any]): dictionary of columns and values@@ -42,24 +43,40 @@ class DatabaseUtils:Default SQL database and makes values accordingly.- If `cls` is provided and matches DBConnectionClass.SNOWFLAKE,the function makes values using Snowflake-specific syntax.++ - Unstract creates id by default if table not exists.+ If there is column 'id' in db table, it will insert+ 'id' as uuid into the db table.+ Else it will GET table details from INFORMATION SCHEMA and+ insert into the table accordingly"""- sql_values: list[str] = []+ sql_values: dict[str, Any] = {}+for column in values:if cls == DBConnectionClass.SNOWFLAKE:col = column.lower()type_x = column_types[col]if type_x in Snowflake.COLUMN_TYPES:- sql_values.append(f"'{values[column]}'")+ sql_values[column] = f"'{values[column]}'"elif type_x == "VARIANT":values[column] = values[column].replace("'", "\\'")- sql_values.append(f"parse_json($${values[column]}$$)")+ sql_values[column] = f"parse_json($${values[column]}$$)"else:- sql_values.append(f"{values[column]}")+ sql_values[column] = f"{values[column]}"+ elif cls == DBConnectionClass.BIGQUERY:+ col = column.lower()+ type_x = column_types[col]+ if type_x in BigQuery.COLUMN_TYPES:+ sql_values[column] = f"{type_x}('{values[column]}')"+ else:+ sql_values[column] = f"'{values[column]}'"else:# Default to Other SQL DBs# TODO: Handle numeric types with no quotes- sql_values.append(f"'{values[column]}'")-+ sql_values[column] = f"'{values[column]}'"+ if column_types.get("id"):+ uuid_id = str(uuid.uuid4())+ sql_values["id"] = f"'{uuid_id}'"return sql_values@staticmethod@@ -208,15 +225,15 @@ class DatabaseUtils:return values@staticmethod- def get_sql_values_for_query(+ def get_sql_columns_and_values_for_query(engine: Any,connector_id: str,connector_settings: dict[str, Any],table_name: str,values: dict[str, Any],- ) -> list[str]:- """Generate SQL values for an insert query based on the provided values- and table schema.+ ) -> dict[str, Any]:+ """Generate SQL columns and values for an insert query based on the+ provided values and table schema.Args:engine (Any): The database engine.@@ -244,17 +261,12 @@ class DatabaseUtils:connector_id=connector_id,connector_settings=connector_settings,)- if class_name == DBConnectionClass.SNOWFLAKE:- sql_values = DatabaseUtils.make_sql_values_for_query(- values=values,- column_types=column_types,- cls=DBConnectionClass.SNOWFLAKE,- )- else:- sql_values = DatabaseUtils.make_sql_values_for_query(- values=values, column_types=column_types- )- return sql_values+ sql_columns_and_values = DatabaseUtils.make_sql_values_for_query(+ values=values,+ column_types=column_types,+ cls=class_name,+ )+ return sql_columns_and_values@staticmethoddef execute_write_query(@@ -279,6 +291,7 @@ class DatabaseUtils:f"INSERT INTO {table_name} ({','.join(sql_keys)}) "f"SELECT {','.join(sql_values)}")+ logger.debug(f"insertng into table with: {sql} query")try:if hasattr(engine, "cursor"):with engine.cursor() as cursor:@@ -309,3 +322,135 @@ class DatabaseUtils:]connector_class: UnstractDB = connector(connector_settings)return connector_class.execute(query=query)++ @staticmethod+ def create_table_if_not_exists(+ engine: Any, table_name: str, database_entry: dict[str, Any]+ ) -> None:+ """Creates table if not exists.++ Args:+ engine (Any): _description_+ table_name (str): _description_+ database_entry (dict[str, Any]): _description_++ Raises:+ e: _description_+ """+ class_name = engine.__class__.__name__+ sql = DBConnectorQueryHelper.create_table_query(+ conn_cls=class_name, table=table_name, database_entry=database_entry+ )+ logger.debug(f"creating table with: {sql} query")+ try:+ if hasattr(engine, "cursor"):+ with engine.cursor() as cursor:+ cursor.execute(sql)+ engine.commit()+ else:+ engine.query(sql)+ except Exception as e:+ logger.error(f"Error while creating table: {str(e)}")+ raise e+++class DBConnectorQueryHelper:+ """A class that helps to generate query for connector table operations."""++ @staticmethod+ def create_table_query(+ conn_cls: str, table: str, database_entry: dict[str, Any]+ ) -> Any:+ sql_query = ""+ """Generate a SQL query to create a table, based on the provided+ database entry.++ Args:+ conn_cls (str): The database connection class.+ Should be one of 'BIGQUERY', 'SNOWFLAKE', or other.+ table (str): The name of the table to be created.+ database_entry (dict[str, Any]):+ A dictionary containing column names as keys+ and their corresponding values.++ These values are used to determine the data types,+ for the columns in the table.++ Returns:+ str: A SQL query string to create a table with the specified name,+ and column definitions.++ Note:+ For 'BIGQUERY', 'SNOWFLAKE', or other database connection classes,+ the appropriate data types will be mapped based+ on the Python types of the values in the database_entry dictionary.++ Permanent columns, will always be present in table creation.+ """++ if conn_cls == DBConnectionClass.BIGQUERY:+ sql_query += (+ f"CREATE TABLE IF NOT EXISTS {table} "+ f"(id string,"+ f"created_by string, created_at TIMESTAMP, "+ )+ elif conn_cls == DBConnectionClass.SNOWFLAKE:+ sql_query += (+ f"CREATE TABLE {table} IF NOT EXISTS "+ f"(id TEXT ,"+ f"created_by TEXT, created_at TIMESTAMP, "+ )+ else:+ sql_query += (+ f"CREATE TABLE IF NOT EXISTS {table} "+ f"(id TEXT , "+ f"created_by TEXT, created_at TIMESTAMP, "+ )++ for key, val in database_entry.items():+ if key not in TableColumns.PERMANENT_COLUMNS:+ python_type = type(val)+ if conn_cls == DBConnectionClass.BIGQUERY:+ sql_type = (+ DBConnectorTypeConverter.python_to_bigquery_mapping(+ python_type+ )+ )+ else:+ sql_type = DBConnectorTypeConverter.python_to_sql_mapping(+ python_type+ )+ sql_query += f"{key} {sql_type}, "++ return sql_query.rstrip(", ") + ");"+++class DBConnectorTypeConverter:+ """_summary_++ Class to convert python data type to corresponding connector+ database data type+ """++ @staticmethod+ def python_to_sql_mapping(python_type: Any) -> Optional[str]:+ """Method used to convert python to SQL datatype Used by Postgres,+ Redshift, Snowflake."""+ mapping = {+ str: "TEXT",+ int: "INT",+ float: "FLOAT",+ datetime.datetime: "TIMESTAMP",+ }+ return mapping.get(python_type, "TEXT")++ @staticmethod+ def python_to_bigquery_mapping(python_type: Any) -> Optional[str]:+ """Method used to convert python to bigquery datatype."""+ mapping = {+ str: "string",+ int: "INT64",+ float: "FLOAT64",+ datetime.datetime: "TIMESTAMP",+ }+ return mapping.get(python_type, "string")diff --git a/backend/workflow_manager/endpoint/destination.py b/backend/workflow_manager/endpoint/destination.pyindex dcbe52342..a9e203d7b 100644--- a/backend/workflow_manager/endpoint/destination.py+++ b/backend/workflow_manager/endpoint/destination.py@@ -208,7 +208,6 @@ class DestinationConnector(BaseConnector):)data = self.get_result(file_history)-values = DatabaseUtils.get_columns_and_values(column_mode_str=column_mode,data=data,@@ -217,24 +216,30 @@ class DestinationConnector(BaseConnector):agent_name=agent_name,single_column_name=single_column_name,)-engine = DatabaseUtils.get_db_engine(connector_id=connector_instance.connector_id,connector_settings=connector_settings,)- sql_values = DatabaseUtils.get_sql_values_for_query(- engine=engine,- connector_id=connector_instance.connector_id,- connector_settings=connector_settings,- table_name=table_name,- values=values,+ # If data is None, don't execute CREATE or INSERT query+ if data is None:+ return+ DatabaseUtils.create_table_if_not_exists(+ engine=engine, table_name=table_name, database_entry=values+ )+ sql_columns_and_values = (+ DatabaseUtils.get_sql_columns_and_values_for_query(+ engine=engine,+ connector_id=connector_instance.connector_id,+ connector_settings=connector_settings,+ table_name=table_name,+ values=values,+ ))-DatabaseUtils.execute_write_query(engine=engine,table_name=table_name,- sql_keys=list(values.keys()),- sql_values=sql_values,+ sql_keys=list(sql_columns_and_values.keys()),+ sql_values=list(sql_columns_and_values.values()),)def _handle_api_result(@@ -317,7 +322,6 @@ class DestinationConnector(BaseConnector):"""if file_history and file_history.result:return self.parse_string(file_history.result)-output_file = os.path.join(self.execution_dir, WorkflowFileType.INFILE)metadata = self.get_workflow_metadata()output_type = self.get_output_type(metadata)diff --git a/backend/workflow_manager/endpoint/static/dest/db.json b/backend/workflow_manager/endpoint/static/dest/db.jsonindex fea9028bb..fbe919000 100644--- a/backend/workflow_manager/endpoint/static/dest/db.json+++ b/backend/workflow_manager/endpoint/static/dest/db.json@@ -40,7 +40,6 @@"type": "string","title": "Select how you want to write the output","enum": [- "Split JSON into columns","Write JSON to a single column"],"default": "Split JSON into columns"