LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
fix/persisting-unique-id-on-db-connector (#506)
Kirtiman Mishraf6fce93fae2a
diff --git a/backend/workflow_manager/endpoint/constants.py b/backend/workflow_manager/endpoint/constants.pyindex ca84b5d15..63a121343 100644--- a/backend/workflow_manager/endpoint/constants.py+++ b/backend/workflow_manager/endpoint/constants.py@@ -57,6 +57,8 @@ class DestinationKey:PATH = "path"OUTPUT_FOLDER = "outputFolder"OVERWRITE_OUTPUT_DOCUMENT = "overwriteOutput"+ FILE_PATH = "filePath"+ EXECUTION_ID = "executionId"class OutputJsonKey:diff --git a/backend/workflow_manager/endpoint/database_utils.py b/backend/workflow_manager/endpoint/database_utils.pyindex f79f87c6d..ae1664784 100644--- a/backend/workflow_manager/endpoint/database_utils.py+++ b/backend/workflow_manager/endpoint/database_utils.py@@ -59,7 +59,7 @@ class DatabaseUtils:for column in values:if cls_name == DBConnectionClass.SNOWFLAKE:col = column.lower()- type_x = column_types[col]+ type_x = column_types.get(col, "")if type_x == "VARIANT":values[column] = values[column].replace("'", "\\'")sql_values[column] = f"parse_json($${values[column]}$$)"@@ -162,6 +162,10 @@ class DatabaseUtils:def get_columns_and_values(column_mode_str: str,data: Any,+ file_path: str,+ execution_id: str,+ file_path_name: str = "file_path",+ execution_id_name: str = "execution_id",include_timestamp: bool = False,include_agent: bool = False,agent_name: Optional[str] = AgentName.UNSTRACT_DBWRITER.value,@@ -214,7 +218,8 @@ class DatabaseUtils:values[single_column_name] = dataelse:values[single_column_name] = json.dumps(data)-+ values[file_path_name] = file_path+ values[execution_id_name] = execution_idreturn values@staticmethoddiff --git a/backend/workflow_manager/endpoint/destination.py b/backend/workflow_manager/endpoint/destination.pyindex 53cf3637b..ca55d8526 100644--- a/backend/workflow_manager/endpoint/destination.py+++ b/backend/workflow_manager/endpoint/destination.py@@ -143,7 +143,9 @@ class DestinationConnector(BaseConnector):if connection_type == WorkflowEndpoint.ConnectionType.FILESYSTEM:self.copy_output_to_output_directory()elif connection_type == WorkflowEndpoint.ConnectionType.DATABASE:- self.insert_into_db(file_history)+ self.insert_into_db(+ file_history=file_history, input_file_path=input_file_path+ )elif connection_type == WorkflowEndpoint.ConnectionType.API:result = self.get_result(file_history)meta_data = self.get_metadata(file_history)@@ -216,7 +218,9 @@ class DestinationConnector(BaseConnector):with open(source_path, "rb") as source_file:destination_fsspec.write_bytes(normalized_path, source_file.read())- def insert_into_db(self, file_history: Optional[FileHistory]) -> None:+ def insert_into_db(+ self, file_history: Optional[FileHistory], input_file_path: str+ ) -> None:"""Insert data into the database."""connector_instance: ConnectorInstance = self.endpoint.connector_instanceconnector_settings: dict[str, Any] = connector_instance.metadata@@ -233,6 +237,12 @@ class DestinationConnector(BaseConnector):single_column_name = str(destination_configurations.get(DestinationKey.SINGLE_COLUMN_NAME, "data"))+ file_path_name = str(+ destination_configurations.get(DestinationKey.FILE_PATH, "file_path")+ )+ execution_id_name = str(+ destination_configurations.get(DestinationKey.EXECUTION_ID, "execution_id")+ )data = self.get_result(file_history)values = DatabaseUtils.get_columns_and_values(@@ -242,6 +252,10 @@ class DestinationConnector(BaseConnector):include_agent=include_agent,agent_name=agent_name,single_column_name=single_column_name,+ file_path_name=file_path_name,+ execution_id_name=execution_id_name,+ file_path=input_file_path,+ execution_id=self.execution_id,)db_class = DatabaseUtils.get_db_class(connector_id=connector_instance.connector_id,diff --git a/backend/workflow_manager/endpoint/static/dest/db.json b/backend/workflow_manager/endpoint/static/dest/db.jsonindex 70286c704..bdbf99a16 100644--- a/backend/workflow_manager/endpoint/static/dest/db.json+++ b/backend/workflow_manager/endpoint/static/dest/db.json@@ -36,6 +36,18 @@"default": false,"description": "Include the 'created_at' in the output row"},+ "filePath": {+ "type": "string",+ "title": "File Path Column Name",+ "default": "file_path",+ "description": "Name of the column to store the absolute path to the file"+ },+ "executionId": {+ "type": "string",+ "title": "Execution ID Column Name",+ "default": "execution_id",+ "description": "Name of the column that stores the execution ID which identifies each individual run of a workflow"+ },"columnMode": {"type": "string","title": "Select how you want to write the output",diff --git a/unstract/connectors/src/unstract/connectors/databases/bigquery/bigquery.py b/unstract/connectors/src/unstract/connectors/databases/bigquery/bigquery.pyindex 53cd9dea5..7daffa8f8 100644--- a/unstract/connectors/src/unstract/connectors/databases/bigquery/bigquery.py+++ b/unstract/connectors/src/unstract/connectors/databases/bigquery/bigquery.py@@ -11,6 +11,7 @@ from google.cloud.bigquery import Clientfrom unstract.connectors.databases.exceptions import (BigQueryForbiddenException,BigQueryNotFoundException,+ ColumnMissingException,)from unstract.connectors.databases.unstract_db import UnstractDBfrom unstract.connectors.exceptions import ConnectorError@@ -117,3 +118,12 @@ class BigQuery(UnstractDB):raise BigQueryNotFoundException(detail=e.message, table_name=table_name) from e+ except google.api_core.exceptions.BadRequest as e:+ logger.error(f"Column missing in inserting data: {str(e)}")+ db, schema, table = table_name.split(".")+ raise ColumnMissingException(+ detail=e.message,+ database=db,+ schema=schema,+ table_name=table,+ ) from ediff --git a/unstract/connectors/src/unstract/connectors/databases/exceptions.py b/unstract/connectors/src/unstract/connectors/databases/exceptions.pyindex 8a1393775..a9bc4da2b 100644--- a/unstract/connectors/src/unstract/connectors/databases/exceptions.py+++ b/unstract/connectors/src/unstract/connectors/databases/exceptions.py@@ -1,4 +1,4 @@-from typing import Any+from typing import Any, Optionalfrom unstract.connectors.exceptions import ConnectorBaseException@@ -25,13 +25,13 @@ class InvalidSyntaxException(UnstractDBConnectorException):f"Error creating/writing to {database}. Syntax incorrect. "f"Please check your table-name or schema. ")- super().__init__(detail=default_detail + detail)+ super().__init__(detail=default_detail)class InvalidSchemaException(UnstractDBConnectorException):def __init__(self, detail: Any, database: str) -> None:default_detail = f"Error creating/writing to {database}. Schema not valid. "- super().__init__(detail=default_detail + detail)+ super().__init__(detail=default_detail)class UnderfinedTableException(UnstractDBConnectorException):@@ -40,7 +40,7 @@ class UnderfinedTableException(UnstractDBConnectorException):f"Error creating/writing to {database}. Undefined table. "f"Please check your table-name or schema. ")- super().__init__(detail=default_detail + detail)+ super().__init__(detail=default_detail)class ValueTooLongException(UnstractDBConnectorException):@@ -49,7 +49,7 @@ class ValueTooLongException(UnstractDBConnectorException):f"Error creating/writing to {database}. "f"Size of the inserted data exceeds the limit provided by the database. ")- super().__init__(detail=default_detail + detail)+ super().__init__(detail=default_detail)class FeatureNotSupportedException(UnstractDBConnectorException):@@ -59,17 +59,20 @@ class FeatureNotSupportedException(UnstractDBConnectorException):f"Error creating/writing to {database}. "f"Feature not supported sql error. ")- super().__init__(detail=default_detail + detail)+ super().__init__(detail=default_detail)class SnowflakeProgrammingException(UnstractDBConnectorException):- def __init__(self, detail: Any, database: str) -> None:+ def __init__(+ self, detail: Any, database: str, table_name: str, schema: str+ ) -> None:default_detail = (- f"Error creating/writing to {database}. "- f"Please check your snowflake credentials. "+ f"Error creating/writing to `{database}.{schema}.{table_name}' \n"+ f"Please make sure all the columns exist in your table as per destination "+ f"DB configuration \n and snowflake credentials are correct.\n")- super().__init__(default_detail + detail)+ super().__init__(default_detail)class BigQueryForbiddenException(UnstractDBConnectorException):@@ -79,7 +82,7 @@ class BigQueryForbiddenException(UnstractDBConnectorException):f"Error creating/writing to {table_name}. "f"Access forbidden in bigquery. Please check your permissions. ")- super().__init__(detail=default_detail + detail)+ super().__init__(detail=default_detail)class BigQueryNotFoundException(UnstractDBConnectorException):@@ -89,4 +92,22 @@ class BigQueryNotFoundException(UnstractDBConnectorException):f"Error creating/writing to {table_name}. "f"The requested resource was not found. ")- super().__init__(detail=default_detail + detail)+ super().__init__(detail=default_detail)+++class ColumnMissingException(UnstractDBConnectorException):++ def __init__(+ self,+ detail: Any,+ database: Any,+ table_name: str,+ schema: Optional[str] = None,+ ) -> None:+ schema_part = f".{schema}" if schema else ""+ default_detail = (+ f"Error writing to '{database}{schema_part}.{table_name}'. \n"+ f"Please make sure all the columns exist in your table "+ f"as per the destination DB configuration.\n"+ )+ super().__init__(detail=default_detail)diff --git a/unstract/connectors/src/unstract/connectors/databases/mariadb/mariadb.py b/unstract/connectors/src/unstract/connectors/databases/mariadb/mariadb.pyindex e44f4ba90..6270c87d4 100644--- a/unstract/connectors/src/unstract/connectors/databases/mariadb/mariadb.py+++ b/unstract/connectors/src/unstract/connectors/databases/mariadb/mariadb.py@@ -79,9 +79,12 @@ class MariaDB(UnstractDB, MysqlHandler):def execute_query(self, engine: Any, sql_query: str, sql_values: Any, **kwargs: Any) -> None:+ table_name = kwargs.get("table_name", None)MysqlHandler.execute_query(engine=engine,sql_query=sql_query,sql_values=sql_values,database=self.database,+ host=self.host,+ table_name=table_name,)diff --git a/unstract/connectors/src/unstract/connectors/databases/mssql/mssql.py b/unstract/connectors/src/unstract/connectors/databases/mssql/mssql.pyindex 76740eb1c..e5cbf74c1 100644--- a/unstract/connectors/src/unstract/connectors/databases/mssql/mssql.py+++ b/unstract/connectors/src/unstract/connectors/databases/mssql/mssql.py@@ -6,7 +6,10 @@ import pymssqlimport pymssql._pymssql as PyMssqlfrom pymssql import Connection # type: ignore-from unstract.connectors.databases.exceptions import InvalidSyntaxException+from unstract.connectors.databases.exceptions import (+ ColumnMissingException,+ InvalidSyntaxException,+)from unstract.connectors.databases.exceptions_helper import ExceptionHelperfrom unstract.connectors.databases.unstract_db import UnstractDB@@ -77,6 +80,7 @@ class MSSQL(UnstractDB):def execute_query(self, engine: Any, sql_query: str, sql_values: Any, **kwargs: Any) -> None:+ table_name = kwargs.get("table_name", None)try:with engine.cursor() as cursor:if sql_values:@@ -84,7 +88,7 @@ class MSSQL(UnstractDB):else:cursor.execute(sql_query)engine.commit()- except (PyMssql.ProgrammingError, PyMssql.OperationalError) as e:+ except PyMssql.OperationalError as e:error_details = ExceptionHelper.extract_byte_exception(e=e)logger.error(f"Invalid syntax in creating/inserting mssql data: {error_details}"@@ -92,3 +96,11 @@ class MSSQL(UnstractDB):raise InvalidSyntaxException(detail=error_details, database=self.database) from e+ except PyMssql.ProgrammingError as e:+ error_details = ExceptionHelper.extract_byte_exception(e=e)+ logger.error(f"Column missing in inserting data: {error_details}")+ raise ColumnMissingException(+ detail=error_details,+ database=self.database,+ table_name=table_name,+ ) from ediff --git a/unstract/connectors/src/unstract/connectors/databases/mysql/mysql.py b/unstract/connectors/src/unstract/connectors/databases/mysql/mysql.pyindex 5bd083103..eec7b79ca 100644--- a/unstract/connectors/src/unstract/connectors/databases/mysql/mysql.py+++ b/unstract/connectors/src/unstract/connectors/databases/mysql/mysql.py@@ -83,9 +83,12 @@ class MySQL(UnstractDB, MysqlHandler):def execute_query(self, engine: Any, sql_query: str, sql_values: Any, **kwargs: Any) -> None:+ table_name = kwargs.get("table_name", None)MysqlHandler.execute_query(engine=engine,sql_query=sql_query,sql_values=sql_values,database=self.database,+ host=self.host,+ table_name=table_name,)diff --git a/unstract/connectors/src/unstract/connectors/databases/mysql_handler.py b/unstract/connectors/src/unstract/connectors/databases/mysql_handler.pyindex 6ec29afb8..713c42bc0 100644--- a/unstract/connectors/src/unstract/connectors/databases/mysql_handler.py+++ b/unstract/connectors/src/unstract/connectors/databases/mysql_handler.py@@ -3,7 +3,10 @@ from typing import Anyimport pymysql.err as MysqlError-from unstract.connectors.databases.exceptions import InvalidSyntaxException+from unstract.connectors.databases.exceptions import (+ ColumnMissingException,+ InvalidSyntaxException,+)from unstract.connectors.databases.exceptions_helper import ExceptionHelperlogger = logging.getLogger(__name__)@@ -12,7 +15,12 @@ logger = logging.getLogger(__name__)class MysqlHandler:@staticmethoddef execute_query(- engine: Any, sql_query: str, sql_values: Any, database: Any+ engine: Any,+ sql_query: str,+ sql_values: Any,+ database: Any,+ host: Any,+ table_name: str,) -> None:try:with engine.cursor() as cursor:@@ -27,3 +35,11 @@ class MysqlHandler:f"Invalid syntax in creating/inserting mysql data: {error_details}")raise InvalidSyntaxException(detail=error_details, database=database) from e+ except MysqlError.OperationalError as e:+ error_details = ExceptionHelper.extract_byte_exception(e=e)+ logger.error(f"Column missing in inserting data: {error_details}")+ raise ColumnMissingException(+ detail=error_details,+ database=database,+ table_name=table_name,+ ) from ediff --git a/unstract/connectors/src/unstract/connectors/databases/postgresql/postgresql.py b/unstract/connectors/src/unstract/connectors/databases/postgresql/postgresql.pyindex d5186cfd4..3c2b5c4bd 100644--- a/unstract/connectors/src/unstract/connectors/databases/postgresql/postgresql.py+++ b/unstract/connectors/src/unstract/connectors/databases/postgresql/postgresql.py@@ -76,9 +76,12 @@ class PostgreSQL(UnstractDB, PsycoPgHandler):def execute_query(self, engine: Any, sql_query: str, sql_values: Any, **kwargs: Any) -> None:+ table_name = kwargs.get("table_name", None)PsycoPgHandler.execute_query(engine=engine,sql_query=sql_query,sql_values=sql_values,database=self.database,+ schema=self.schema,+ table_name=table_name,)diff --git a/unstract/connectors/src/unstract/connectors/databases/psycopg_handler.py b/unstract/connectors/src/unstract/connectors/databases/psycopg_handler.pyindex 47c911a6b..34b089cd3 100644--- a/unstract/connectors/src/unstract/connectors/databases/psycopg_handler.py+++ b/unstract/connectors/src/unstract/connectors/databases/psycopg_handler.py@@ -4,6 +4,7 @@ from typing import Anyfrom psycopg2 import errors as PsycopgErrorfrom unstract.connectors.databases.exceptions import (+ ColumnMissingException,FeatureNotSupportedException,InvalidSchemaException,InvalidSyntaxException,@@ -17,7 +18,12 @@ logger = logging.getLogger(__name__)class PsycoPgHandler:@staticmethoddef execute_query(- engine: Any, sql_query: str, sql_values: Any, database: Any+ engine: Any,+ sql_query: str,+ sql_values: Any,+ database: Any,+ schema: str,+ table_name: str,) -> None:try:with engine.cursor() as cursor:@@ -48,3 +54,11 @@ class PsycoPgHandler:) as e:logger.error(f"value too long for datatype: {e.pgerror}")raise ValueTooLongException(detail=e.pgerror, database=database) from e+ except PsycopgError.UndefinedColumn as e:+ logger.error(f"Column missing in inserting data: {e.pgerror}")+ raise ColumnMissingException(+ detail=e.pgerror,+ database=database,+ schema=schema,+ table_name=table_name,+ ) from ediff --git a/unstract/connectors/src/unstract/connectors/databases/redshift/redshift.py b/unstract/connectors/src/unstract/connectors/databases/redshift/redshift.pyindex 50ab35a55..08afce936 100644--- a/unstract/connectors/src/unstract/connectors/databases/redshift/redshift.py+++ b/unstract/connectors/src/unstract/connectors/databases/redshift/redshift.py@@ -95,9 +95,12 @@ class Redshift(UnstractDB, PsycoPgHandler):def execute_query(self, engine: Any, sql_query: str, sql_values: Any, **kwargs: Any) -> None:+ table_name = kwargs.get("table_name", None)PsycoPgHandler.execute_query(engine=engine,sql_query=sql_query,sql_values=sql_values,database=self.database,+ schema=self.schema,+ table_name=table_name,)diff --git a/unstract/connectors/src/unstract/connectors/databases/snowflake/snowflake.py b/unstract/connectors/src/unstract/connectors/databases/snowflake/snowflake.pyindex a2ae457d0..4d8dc8e27 100644--- a/unstract/connectors/src/unstract/connectors/databases/snowflake/snowflake.py+++ b/unstract/connectors/src/unstract/connectors/databases/snowflake/snowflake.py@@ -79,6 +79,7 @@ class SnowflakeDB(UnstractDB):def execute_query(self, engine: Any, sql_query: str, sql_values: Any, **kwargs: Any) -> None:+ table_name = kwargs.get("table_name", None)try:with engine.cursor() as cursor:if sql_values:@@ -92,5 +93,8 @@ class SnowflakeDB(UnstractDB):f"{e.msg} {e.errno}")raise SnowflakeProgrammingException(- detail=e.msg, database=self.database+ detail=e.msg,+ database=self.database,+ schema=self.schema,+ table_name=table_name,) from e