Skip to content

LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows

fix/persisting-unique-id-on-db-connector (#506)

Kirtiman Mishraf6fce93fae2a
diff --git a/backend/workflow_manager/endpoint/constants.py b/backend/workflow_manager/endpoint/constants.py
index ca84b5d15..63a121343 100644
--- a/backend/workflow_manager/endpoint/constants.py
+++ b/backend/workflow_manager/endpoint/constants.py
@@ -57,6 +57,8 @@ class DestinationKey:
PATH = "path"
OUTPUT_FOLDER = "outputFolder"
OVERWRITE_OUTPUT_DOCUMENT = "overwriteOutput"
+ FILE_PATH = "filePath"
+ EXECUTION_ID = "executionId"
class OutputJsonKey:
diff --git a/backend/workflow_manager/endpoint/database_utils.py b/backend/workflow_manager/endpoint/database_utils.py
index f79f87c6d..ae1664784 100644
--- a/backend/workflow_manager/endpoint/database_utils.py
+++ b/backend/workflow_manager/endpoint/database_utils.py
@@ -59,7 +59,7 @@ class DatabaseUtils:
for column in values:
if cls_name == DBConnectionClass.SNOWFLAKE:
col = column.lower()
- type_x = column_types[col]
+ type_x = column_types.get(col, "")
if type_x == "VARIANT":
values[column] = values[column].replace("'", "\\'")
sql_values[column] = f"parse_json($${values[column]}$$)"
@@ -162,6 +162,10 @@ class DatabaseUtils:
def get_columns_and_values(
column_mode_str: str,
data: Any,
+ file_path: str,
+ execution_id: str,
+ file_path_name: str = "file_path",
+ execution_id_name: str = "execution_id",
include_timestamp: bool = False,
include_agent: bool = False,
agent_name: Optional[str] = AgentName.UNSTRACT_DBWRITER.value,
@@ -214,7 +218,8 @@ class DatabaseUtils:
values[single_column_name] = data
else:
values[single_column_name] = json.dumps(data)
-
+ values[file_path_name] = file_path
+ values[execution_id_name] = execution_id
return values
@staticmethod
diff --git a/backend/workflow_manager/endpoint/destination.py b/backend/workflow_manager/endpoint/destination.py
index 53cf3637b..ca55d8526 100644
--- a/backend/workflow_manager/endpoint/destination.py
+++ b/backend/workflow_manager/endpoint/destination.py
@@ -143,7 +143,9 @@ class DestinationConnector(BaseConnector):
if connection_type == WorkflowEndpoint.ConnectionType.FILESYSTEM:
self.copy_output_to_output_directory()
elif connection_type == WorkflowEndpoint.ConnectionType.DATABASE:
- self.insert_into_db(file_history)
+ self.insert_into_db(
+ file_history=file_history, input_file_path=input_file_path
+ )
elif connection_type == WorkflowEndpoint.ConnectionType.API:
result = self.get_result(file_history)
meta_data = self.get_metadata(file_history)
@@ -216,7 +218,9 @@ class DestinationConnector(BaseConnector):
with open(source_path, "rb") as source_file:
destination_fsspec.write_bytes(normalized_path, source_file.read())
- def insert_into_db(self, file_history: Optional[FileHistory]) -> None:
+ def insert_into_db(
+ self, file_history: Optional[FileHistory], input_file_path: str
+ ) -> None:
"""Insert data into the database."""
connector_instance: ConnectorInstance = self.endpoint.connector_instance
connector_settings: dict[str, Any] = connector_instance.metadata
@@ -233,6 +237,12 @@ class DestinationConnector(BaseConnector):
single_column_name = str(
destination_configurations.get(DestinationKey.SINGLE_COLUMN_NAME, "data")
)
+ file_path_name = str(
+ destination_configurations.get(DestinationKey.FILE_PATH, "file_path")
+ )
+ execution_id_name = str(
+ destination_configurations.get(DestinationKey.EXECUTION_ID, "execution_id")
+ )
data = self.get_result(file_history)
values = DatabaseUtils.get_columns_and_values(
@@ -242,6 +252,10 @@ class DestinationConnector(BaseConnector):
include_agent=include_agent,
agent_name=agent_name,
single_column_name=single_column_name,
+ file_path_name=file_path_name,
+ execution_id_name=execution_id_name,
+ file_path=input_file_path,
+ execution_id=self.execution_id,
)
db_class = DatabaseUtils.get_db_class(
connector_id=connector_instance.connector_id,
diff --git a/backend/workflow_manager/endpoint/static/dest/db.json b/backend/workflow_manager/endpoint/static/dest/db.json
index 70286c704..bdbf99a16 100644
--- a/backend/workflow_manager/endpoint/static/dest/db.json
+++ b/backend/workflow_manager/endpoint/static/dest/db.json
@@ -36,6 +36,18 @@
"default": false,
"description": "Include the 'created_at' in the output row"
},
+ "filePath": {
+ "type": "string",
+ "title": "File Path Column Name",
+ "default": "file_path",
+ "description": "Name of the column to store the absolute path to the file"
+ },
+ "executionId": {
+ "type": "string",
+ "title": "Execution ID Column Name",
+ "default": "execution_id",
+ "description": "Name of the column that stores the execution ID which identifies each individual run of a workflow"
+ },
"columnMode": {
"type": "string",
"title": "Select how you want to write the output",
diff --git a/unstract/connectors/src/unstract/connectors/databases/bigquery/bigquery.py b/unstract/connectors/src/unstract/connectors/databases/bigquery/bigquery.py
index 53cd9dea5..7daffa8f8 100644
--- a/unstract/connectors/src/unstract/connectors/databases/bigquery/bigquery.py
+++ b/unstract/connectors/src/unstract/connectors/databases/bigquery/bigquery.py
@@ -11,6 +11,7 @@ from google.cloud.bigquery import Client
from unstract.connectors.databases.exceptions import (
BigQueryForbiddenException,
BigQueryNotFoundException,
+ ColumnMissingException,
)
from unstract.connectors.databases.unstract_db import UnstractDB
from unstract.connectors.exceptions import ConnectorError
@@ -117,3 +118,12 @@ class BigQuery(UnstractDB):
raise BigQueryNotFoundException(
detail=e.message, table_name=table_name
) from e
+ except google.api_core.exceptions.BadRequest as e:
+ logger.error(f"Column missing in inserting data: {str(e)}")
+ db, schema, table = table_name.split(".")
+ raise ColumnMissingException(
+ detail=e.message,
+ database=db,
+ schema=schema,
+ table_name=table,
+ ) from e
diff --git a/unstract/connectors/src/unstract/connectors/databases/exceptions.py b/unstract/connectors/src/unstract/connectors/databases/exceptions.py
index 8a1393775..a9bc4da2b 100644
--- a/unstract/connectors/src/unstract/connectors/databases/exceptions.py
+++ b/unstract/connectors/src/unstract/connectors/databases/exceptions.py
@@ -1,4 +1,4 @@
-from typing import Any
+from typing import Any, Optional
from unstract.connectors.exceptions import ConnectorBaseException
@@ -25,13 +25,13 @@ class InvalidSyntaxException(UnstractDBConnectorException):
f"Error creating/writing to {database}. Syntax incorrect. "
f"Please check your table-name or schema. "
)
- super().__init__(detail=default_detail + detail)
+ super().__init__(detail=default_detail)
class InvalidSchemaException(UnstractDBConnectorException):
def __init__(self, detail: Any, database: str) -> None:
default_detail = f"Error creating/writing to {database}. Schema not valid. "
- super().__init__(detail=default_detail + detail)
+ super().__init__(detail=default_detail)
class UnderfinedTableException(UnstractDBConnectorException):
@@ -40,7 +40,7 @@ class UnderfinedTableException(UnstractDBConnectorException):
f"Error creating/writing to {database}. Undefined table. "
f"Please check your table-name or schema. "
)
- super().__init__(detail=default_detail + detail)
+ super().__init__(detail=default_detail)
class ValueTooLongException(UnstractDBConnectorException):
@@ -49,7 +49,7 @@ class ValueTooLongException(UnstractDBConnectorException):
f"Error creating/writing to {database}. "
f"Size of the inserted data exceeds the limit provided by the database. "
)
- super().__init__(detail=default_detail + detail)
+ super().__init__(detail=default_detail)
class FeatureNotSupportedException(UnstractDBConnectorException):
@@ -59,17 +59,20 @@ class FeatureNotSupportedException(UnstractDBConnectorException):
f"Error creating/writing to {database}. "
f"Feature not supported sql error. "
)
- super().__init__(detail=default_detail + detail)
+ super().__init__(detail=default_detail)
class SnowflakeProgrammingException(UnstractDBConnectorException):
- def __init__(self, detail: Any, database: str) -> None:
+ def __init__(
+ self, detail: Any, database: str, table_name: str, schema: str
+ ) -> None:
default_detail = (
- f"Error creating/writing to {database}. "
- f"Please check your snowflake credentials. "
+ f"Error creating/writing to `{database}.{schema}.{table_name}' \n"
+ f"Please make sure all the columns exist in your table as per destination "
+ f"DB configuration \n and snowflake credentials are correct.\n"
)
- super().__init__(default_detail + detail)
+ super().__init__(default_detail)
class BigQueryForbiddenException(UnstractDBConnectorException):
@@ -79,7 +82,7 @@ class BigQueryForbiddenException(UnstractDBConnectorException):
f"Error creating/writing to {table_name}. "
f"Access forbidden in bigquery. Please check your permissions. "
)
- super().__init__(detail=default_detail + detail)
+ super().__init__(detail=default_detail)
class BigQueryNotFoundException(UnstractDBConnectorException):
@@ -89,4 +92,22 @@ class BigQueryNotFoundException(UnstractDBConnectorException):
f"Error creating/writing to {table_name}. "
f"The requested resource was not found. "
)
- super().__init__(detail=default_detail + detail)
+ super().__init__(detail=default_detail)
+
+
+class ColumnMissingException(UnstractDBConnectorException):
+
+ def __init__(
+ self,
+ detail: Any,
+ database: Any,
+ table_name: str,
+ schema: Optional[str] = None,
+ ) -> None:
+ schema_part = f".{schema}" if schema else ""
+ default_detail = (
+ f"Error writing to '{database}{schema_part}.{table_name}'. \n"
+ f"Please make sure all the columns exist in your table "
+ f"as per the destination DB configuration.\n"
+ )
+ super().__init__(detail=default_detail)
diff --git a/unstract/connectors/src/unstract/connectors/databases/mariadb/mariadb.py b/unstract/connectors/src/unstract/connectors/databases/mariadb/mariadb.py
index e44f4ba90..6270c87d4 100644
--- a/unstract/connectors/src/unstract/connectors/databases/mariadb/mariadb.py
+++ b/unstract/connectors/src/unstract/connectors/databases/mariadb/mariadb.py
@@ -79,9 +79,12 @@ class MariaDB(UnstractDB, MysqlHandler):
def execute_query(
self, engine: Any, sql_query: str, sql_values: Any, **kwargs: Any
) -> None:
+ table_name = kwargs.get("table_name", None)
MysqlHandler.execute_query(
engine=engine,
sql_query=sql_query,
sql_values=sql_values,
database=self.database,
+ host=self.host,
+ table_name=table_name,
)
diff --git a/unstract/connectors/src/unstract/connectors/databases/mssql/mssql.py b/unstract/connectors/src/unstract/connectors/databases/mssql/mssql.py
index 76740eb1c..e5cbf74c1 100644
--- a/unstract/connectors/src/unstract/connectors/databases/mssql/mssql.py
+++ b/unstract/connectors/src/unstract/connectors/databases/mssql/mssql.py
@@ -6,7 +6,10 @@ import pymssql
import pymssql._pymssql as PyMssql
from pymssql import Connection # type: ignore
-from unstract.connectors.databases.exceptions import InvalidSyntaxException
+from unstract.connectors.databases.exceptions import (
+ ColumnMissingException,
+ InvalidSyntaxException,
+)
from unstract.connectors.databases.exceptions_helper import ExceptionHelper
from unstract.connectors.databases.unstract_db import UnstractDB
@@ -77,6 +80,7 @@ class MSSQL(UnstractDB):
def execute_query(
self, engine: Any, sql_query: str, sql_values: Any, **kwargs: Any
) -> None:
+ table_name = kwargs.get("table_name", None)
try:
with engine.cursor() as cursor:
if sql_values:
@@ -84,7 +88,7 @@ class MSSQL(UnstractDB):
else:
cursor.execute(sql_query)
engine.commit()
- except (PyMssql.ProgrammingError, PyMssql.OperationalError) as e:
+ except PyMssql.OperationalError as e:
error_details = ExceptionHelper.extract_byte_exception(e=e)
logger.error(
f"Invalid syntax in creating/inserting mssql data: {error_details}"
@@ -92,3 +96,11 @@ class MSSQL(UnstractDB):
raise InvalidSyntaxException(
detail=error_details, database=self.database
) from e
+ except PyMssql.ProgrammingError as e:
+ error_details = ExceptionHelper.extract_byte_exception(e=e)
+ logger.error(f"Column missing in inserting data: {error_details}")
+ raise ColumnMissingException(
+ detail=error_details,
+ database=self.database,
+ table_name=table_name,
+ ) from e
diff --git a/unstract/connectors/src/unstract/connectors/databases/mysql/mysql.py b/unstract/connectors/src/unstract/connectors/databases/mysql/mysql.py
index 5bd083103..eec7b79ca 100644
--- a/unstract/connectors/src/unstract/connectors/databases/mysql/mysql.py
+++ b/unstract/connectors/src/unstract/connectors/databases/mysql/mysql.py
@@ -83,9 +83,12 @@ class MySQL(UnstractDB, MysqlHandler):
def execute_query(
self, engine: Any, sql_query: str, sql_values: Any, **kwargs: Any
) -> None:
+ table_name = kwargs.get("table_name", None)
MysqlHandler.execute_query(
engine=engine,
sql_query=sql_query,
sql_values=sql_values,
database=self.database,
+ host=self.host,
+ table_name=table_name,
)
diff --git a/unstract/connectors/src/unstract/connectors/databases/mysql_handler.py b/unstract/connectors/src/unstract/connectors/databases/mysql_handler.py
index 6ec29afb8..713c42bc0 100644
--- a/unstract/connectors/src/unstract/connectors/databases/mysql_handler.py
+++ b/unstract/connectors/src/unstract/connectors/databases/mysql_handler.py
@@ -3,7 +3,10 @@ from typing import Any
import pymysql.err as MysqlError
-from unstract.connectors.databases.exceptions import InvalidSyntaxException
+from unstract.connectors.databases.exceptions import (
+ ColumnMissingException,
+ InvalidSyntaxException,
+)
from unstract.connectors.databases.exceptions_helper import ExceptionHelper
logger = logging.getLogger(__name__)
@@ -12,7 +15,12 @@ logger = logging.getLogger(__name__)
class MysqlHandler:
@staticmethod
def execute_query(
- engine: Any, sql_query: str, sql_values: Any, database: Any
+ engine: Any,
+ sql_query: str,
+ sql_values: Any,
+ database: Any,
+ host: Any,
+ table_name: str,
) -> None:
try:
with engine.cursor() as cursor:
@@ -27,3 +35,11 @@ class MysqlHandler:
f"Invalid syntax in creating/inserting mysql data: {error_details}"
)
raise InvalidSyntaxException(detail=error_details, database=database) from e
+ except MysqlError.OperationalError as e:
+ error_details = ExceptionHelper.extract_byte_exception(e=e)
+ logger.error(f"Column missing in inserting data: {error_details}")
+ raise ColumnMissingException(
+ detail=error_details,
+ database=database,
+ table_name=table_name,
+ ) from e
diff --git a/unstract/connectors/src/unstract/connectors/databases/postgresql/postgresql.py b/unstract/connectors/src/unstract/connectors/databases/postgresql/postgresql.py
index d5186cfd4..3c2b5c4bd 100644
--- a/unstract/connectors/src/unstract/connectors/databases/postgresql/postgresql.py
+++ b/unstract/connectors/src/unstract/connectors/databases/postgresql/postgresql.py
@@ -76,9 +76,12 @@ class PostgreSQL(UnstractDB, PsycoPgHandler):
def execute_query(
self, engine: Any, sql_query: str, sql_values: Any, **kwargs: Any
) -> None:
+ table_name = kwargs.get("table_name", None)
PsycoPgHandler.execute_query(
engine=engine,
sql_query=sql_query,
sql_values=sql_values,
database=self.database,
+ schema=self.schema,
+ table_name=table_name,
)
diff --git a/unstract/connectors/src/unstract/connectors/databases/psycopg_handler.py b/unstract/connectors/src/unstract/connectors/databases/psycopg_handler.py
index 47c911a6b..34b089cd3 100644
--- a/unstract/connectors/src/unstract/connectors/databases/psycopg_handler.py
+++ b/unstract/connectors/src/unstract/connectors/databases/psycopg_handler.py
@@ -4,6 +4,7 @@ from typing import Any
from psycopg2 import errors as PsycopgError
from unstract.connectors.databases.exceptions import (
+ ColumnMissingException,
FeatureNotSupportedException,
InvalidSchemaException,
InvalidSyntaxException,
@@ -17,7 +18,12 @@ logger = logging.getLogger(__name__)
class PsycoPgHandler:
@staticmethod
def execute_query(
- engine: Any, sql_query: str, sql_values: Any, database: Any
+ engine: Any,
+ sql_query: str,
+ sql_values: Any,
+ database: Any,
+ schema: str,
+ table_name: str,
) -> None:
try:
with engine.cursor() as cursor:
@@ -48,3 +54,11 @@ class PsycoPgHandler:
) as e:
logger.error(f"value too long for datatype: {e.pgerror}")
raise ValueTooLongException(detail=e.pgerror, database=database) from e
+ except PsycopgError.UndefinedColumn as e:
+ logger.error(f"Column missing in inserting data: {e.pgerror}")
+ raise ColumnMissingException(
+ detail=e.pgerror,
+ database=database,
+ schema=schema,
+ table_name=table_name,
+ ) from e
diff --git a/unstract/connectors/src/unstract/connectors/databases/redshift/redshift.py b/unstract/connectors/src/unstract/connectors/databases/redshift/redshift.py
index 50ab35a55..08afce936 100644
--- a/unstract/connectors/src/unstract/connectors/databases/redshift/redshift.py
+++ b/unstract/connectors/src/unstract/connectors/databases/redshift/redshift.py
@@ -95,9 +95,12 @@ class Redshift(UnstractDB, PsycoPgHandler):
def execute_query(
self, engine: Any, sql_query: str, sql_values: Any, **kwargs: Any
) -> None:
+ table_name = kwargs.get("table_name", None)
PsycoPgHandler.execute_query(
engine=engine,
sql_query=sql_query,
sql_values=sql_values,
database=self.database,
+ schema=self.schema,
+ table_name=table_name,
)
diff --git a/unstract/connectors/src/unstract/connectors/databases/snowflake/snowflake.py b/unstract/connectors/src/unstract/connectors/databases/snowflake/snowflake.py
index a2ae457d0..4d8dc8e27 100644
--- a/unstract/connectors/src/unstract/connectors/databases/snowflake/snowflake.py
+++ b/unstract/connectors/src/unstract/connectors/databases/snowflake/snowflake.py
@@ -79,6 +79,7 @@ class SnowflakeDB(UnstractDB):
def execute_query(
self, engine: Any, sql_query: str, sql_values: Any, **kwargs: Any
) -> None:
+ table_name = kwargs.get("table_name", None)
try:
with engine.cursor() as cursor:
if sql_values:
@@ -92,5 +93,8 @@ class SnowflakeDB(UnstractDB):
f"{e.msg} {e.errno}"
)
raise SnowflakeProgrammingException(
- detail=e.msg, database=self.database
+ detail=e.msg,
+ database=self.database,
+ schema=self.schema,
+ table_name=table_name,
) from e