Skip to content

LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows

Fix/db connectors table create (#149)

Kirtiman Mishra867cfbf84ae5
diff --git a/backend/workflow_manager/endpoint/constants.py b/backend/workflow_manager/endpoint/constants.py
index 061dc7a56..cec35aa5a 100644
--- a/backend/workflow_manager/endpoint/constants.py
+++ b/backend/workflow_manager/endpoint/constants.py
@@ -1,6 +1,7 @@
class TableColumns:
CREATED_BY = "created_by"
CREATED_AT = "created_at"
+ PERMANENT_COLUMNS = ["created_by", "created_at"]
class DBConnectionClass:
@@ -94,3 +95,9 @@ class BigQuery:
"""
TABLE_NAME_SIZE = 3
+ COLUMN_TYPES = [
+ "DATE",
+ "DATETIME",
+ "TIME",
+ "TIMESTAMP",
+ ]
diff --git a/backend/workflow_manager/endpoint/database_utils.py b/backend/workflow_manager/endpoint/database_utils.py
index ad59188c2..8adcc989a 100644
--- a/backend/workflow_manager/endpoint/database_utils.py
+++ b/backend/workflow_manager/endpoint/database_utils.py
@@ -1,6 +1,7 @@
import datetime
import json
import logging
+import uuid
from typing import Any, Optional
from utils.constants import Common
@@ -23,8 +24,8 @@ class DatabaseUtils:
@staticmethod
def make_sql_values_for_query(
values: dict[str, Any], column_types: dict[str, str], cls: Any = None
- ) -> list[str]:
- """Making Sql Values for Query.
+ ) -> dict[str, str]:
+ """Making Sql Columns and Values for Query.
Args:
values (dict[str, Any]): dictionary of columns and values
@@ -42,24 +43,40 @@ class DatabaseUtils:
Default SQL database and makes values accordingly.
- If `cls` is provided and matches DBConnectionClass.SNOWFLAKE,
the function makes values using Snowflake-specific syntax.
+
+ - Unstract creates id by default if table not exists.
+ If there is column 'id' in db table, it will insert
+ 'id' as uuid into the db table.
+ Else it will GET table details from INFORMATION SCHEMA and
+ insert into the table accordingly
"""
- sql_values: list[str] = []
+ sql_values: dict[str, Any] = {}
+
for column in values:
if cls == DBConnectionClass.SNOWFLAKE:
col = column.lower()
type_x = column_types[col]
if type_x in Snowflake.COLUMN_TYPES:
- sql_values.append(f"'{values[column]}'")
+ sql_values[column] = f"'{values[column]}'"
elif type_x == "VARIANT":
values[column] = values[column].replace("'", "\\'")
- sql_values.append(f"parse_json($${values[column]}$$)")
+ sql_values[column] = f"parse_json($${values[column]}$$)"
else:
- sql_values.append(f"{values[column]}")
+ sql_values[column] = f"{values[column]}"
+ elif cls == DBConnectionClass.BIGQUERY:
+ col = column.lower()
+ type_x = column_types[col]
+ if type_x in BigQuery.COLUMN_TYPES:
+ sql_values[column] = f"{type_x}('{values[column]}')"
+ else:
+ sql_values[column] = f"'{values[column]}'"
else:
# Default to Other SQL DBs
# TODO: Handle numeric types with no quotes
- sql_values.append(f"'{values[column]}'")
-
+ sql_values[column] = f"'{values[column]}'"
+ if column_types.get("id"):
+ uuid_id = str(uuid.uuid4())
+ sql_values["id"] = f"'{uuid_id}'"
return sql_values
@staticmethod
@@ -208,15 +225,15 @@ class DatabaseUtils:
return values
@staticmethod
- def get_sql_values_for_query(
+ def get_sql_columns_and_values_for_query(
engine: Any,
connector_id: str,
connector_settings: dict[str, Any],
table_name: str,
values: dict[str, Any],
- ) -> list[str]:
- """Generate SQL values for an insert query based on the provided values
- and table schema.
+ ) -> dict[str, Any]:
+ """Generate SQL columns and values for an insert query based on the
+ provided values and table schema.
Args:
engine (Any): The database engine.
@@ -244,17 +261,12 @@ class DatabaseUtils:
connector_id=connector_id,
connector_settings=connector_settings,
)
- if class_name == DBConnectionClass.SNOWFLAKE:
- sql_values = DatabaseUtils.make_sql_values_for_query(
- values=values,
- column_types=column_types,
- cls=DBConnectionClass.SNOWFLAKE,
- )
- else:
- sql_values = DatabaseUtils.make_sql_values_for_query(
- values=values, column_types=column_types
- )
- return sql_values
+ sql_columns_and_values = DatabaseUtils.make_sql_values_for_query(
+ values=values,
+ column_types=column_types,
+ cls=class_name,
+ )
+ return sql_columns_and_values
@staticmethod
def execute_write_query(
@@ -279,6 +291,7 @@ class DatabaseUtils:
f"INSERT INTO {table_name} ({','.join(sql_keys)}) "
f"SELECT {','.join(sql_values)}"
)
+ logger.debug(f"insertng into table with: {sql} query")
try:
if hasattr(engine, "cursor"):
with engine.cursor() as cursor:
@@ -309,3 +322,135 @@ class DatabaseUtils:
]
connector_class: UnstractDB = connector(connector_settings)
return connector_class.execute(query=query)
+
+ @staticmethod
+ def create_table_if_not_exists(
+ engine: Any, table_name: str, database_entry: dict[str, Any]
+ ) -> None:
+ """Creates table if not exists.
+
+ Args:
+ engine (Any): _description_
+ table_name (str): _description_
+ database_entry (dict[str, Any]): _description_
+
+ Raises:
+ e: _description_
+ """
+ class_name = engine.__class__.__name__
+ sql = DBConnectorQueryHelper.create_table_query(
+ conn_cls=class_name, table=table_name, database_entry=database_entry
+ )
+ logger.debug(f"creating table with: {sql} query")
+ try:
+ if hasattr(engine, "cursor"):
+ with engine.cursor() as cursor:
+ cursor.execute(sql)
+ engine.commit()
+ else:
+ engine.query(sql)
+ except Exception as e:
+ logger.error(f"Error while creating table: {str(e)}")
+ raise e
+
+
+class DBConnectorQueryHelper:
+ """A class that helps to generate query for connector table operations."""
+
+ @staticmethod
+ def create_table_query(
+ conn_cls: str, table: str, database_entry: dict[str, Any]
+ ) -> Any:
+ sql_query = ""
+ """Generate a SQL query to create a table, based on the provided
+ database entry.
+
+ Args:
+ conn_cls (str): The database connection class.
+ Should be one of 'BIGQUERY', 'SNOWFLAKE', or other.
+ table (str): The name of the table to be created.
+ database_entry (dict[str, Any]):
+ A dictionary containing column names as keys
+ and their corresponding values.
+
+ These values are used to determine the data types,
+ for the columns in the table.
+
+ Returns:
+ str: A SQL query string to create a table with the specified name,
+ and column definitions.
+
+ Note:
+ For 'BIGQUERY', 'SNOWFLAKE', or other database connection classes,
+ the appropriate data types will be mapped based
+ on the Python types of the values in the database_entry dictionary.
+
+ Permanent columns, will always be present in table creation.
+ """
+
+ if conn_cls == DBConnectionClass.BIGQUERY:
+ sql_query += (
+ f"CREATE TABLE IF NOT EXISTS {table} "
+ f"(id string,"
+ f"created_by string, created_at TIMESTAMP, "
+ )
+ elif conn_cls == DBConnectionClass.SNOWFLAKE:
+ sql_query += (
+ f"CREATE TABLE {table} IF NOT EXISTS "
+ f"(id TEXT ,"
+ f"created_by TEXT, created_at TIMESTAMP, "
+ )
+ else:
+ sql_query += (
+ f"CREATE TABLE IF NOT EXISTS {table} "
+ f"(id TEXT , "
+ f"created_by TEXT, created_at TIMESTAMP, "
+ )
+
+ for key, val in database_entry.items():
+ if key not in TableColumns.PERMANENT_COLUMNS:
+ python_type = type(val)
+ if conn_cls == DBConnectionClass.BIGQUERY:
+ sql_type = (
+ DBConnectorTypeConverter.python_to_bigquery_mapping(
+ python_type
+ )
+ )
+ else:
+ sql_type = DBConnectorTypeConverter.python_to_sql_mapping(
+ python_type
+ )
+ sql_query += f"{key} {sql_type}, "
+
+ return sql_query.rstrip(", ") + ");"
+
+
+class DBConnectorTypeConverter:
+ """_summary_
+
+ Class to convert python data type to corresponding connector
+ database data type
+ """
+
+ @staticmethod
+ def python_to_sql_mapping(python_type: Any) -> Optional[str]:
+ """Method used to convert python to SQL datatype Used by Postgres,
+ Redshift, Snowflake."""
+ mapping = {
+ str: "TEXT",
+ int: "INT",
+ float: "FLOAT",
+ datetime.datetime: "TIMESTAMP",
+ }
+ return mapping.get(python_type, "TEXT")
+
+ @staticmethod
+ def python_to_bigquery_mapping(python_type: Any) -> Optional[str]:
+ """Method used to convert python to bigquery datatype."""
+ mapping = {
+ str: "string",
+ int: "INT64",
+ float: "FLOAT64",
+ datetime.datetime: "TIMESTAMP",
+ }
+ return mapping.get(python_type, "string")
diff --git a/backend/workflow_manager/endpoint/destination.py b/backend/workflow_manager/endpoint/destination.py
index dcbe52342..a9e203d7b 100644
--- a/backend/workflow_manager/endpoint/destination.py
+++ b/backend/workflow_manager/endpoint/destination.py
@@ -208,7 +208,6 @@ class DestinationConnector(BaseConnector):
)
data = self.get_result(file_history)
-
values = DatabaseUtils.get_columns_and_values(
column_mode_str=column_mode,
data=data,
@@ -217,24 +216,30 @@ class DestinationConnector(BaseConnector):
agent_name=agent_name,
single_column_name=single_column_name,
)
-
engine = DatabaseUtils.get_db_engine(
connector_id=connector_instance.connector_id,
connector_settings=connector_settings,
)
- sql_values = DatabaseUtils.get_sql_values_for_query(
- engine=engine,
- connector_id=connector_instance.connector_id,
- connector_settings=connector_settings,
- table_name=table_name,
- values=values,
+ # If data is None, don't execute CREATE or INSERT query
+ if data is None:
+ return
+ DatabaseUtils.create_table_if_not_exists(
+ engine=engine, table_name=table_name, database_entry=values
+ )
+ sql_columns_and_values = (
+ DatabaseUtils.get_sql_columns_and_values_for_query(
+ engine=engine,
+ connector_id=connector_instance.connector_id,
+ connector_settings=connector_settings,
+ table_name=table_name,
+ values=values,
+ )
)
-
DatabaseUtils.execute_write_query(
engine=engine,
table_name=table_name,
- sql_keys=list(values.keys()),
- sql_values=sql_values,
+ sql_keys=list(sql_columns_and_values.keys()),
+ sql_values=list(sql_columns_and_values.values()),
)
def _handle_api_result(
@@ -317,7 +322,6 @@ class DestinationConnector(BaseConnector):
"""
if file_history and file_history.result:
return self.parse_string(file_history.result)
-
output_file = os.path.join(self.execution_dir, WorkflowFileType.INFILE)
metadata = self.get_workflow_metadata()
output_type = self.get_output_type(metadata)
diff --git a/backend/workflow_manager/endpoint/static/dest/db.json b/backend/workflow_manager/endpoint/static/dest/db.json
index fea9028bb..fbe919000 100644
--- a/backend/workflow_manager/endpoint/static/dest/db.json
+++ b/backend/workflow_manager/endpoint/static/dest/db.json
@@ -40,7 +40,6 @@
"type": "string",
"title": "Select how you want to write the output",
"enum": [
- "Split JSON into columns",
"Write JSON to a single column"
],
"default": "Split JSON into columns"