LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
Fix/insert-into-connector-fix (#405)
Kirtiman Mishra49b9873a7a3f
diff --git a/backend/workflow_manager/endpoint/constants.py b/backend/workflow_manager/endpoint/constants.pyindex 7165242ea..1674c5725 100644--- a/backend/workflow_manager/endpoint/constants.py+++ b/backend/workflow_manager/endpoint/constants.py@@ -92,9 +92,3 @@ class BigQuery:"""TABLE_NAME_SIZE = 3- COLUMN_TYPES = [- "DATE",- "DATETIME",- "TIME",- "TIMESTAMP",- ]diff --git a/backend/workflow_manager/endpoint/database_utils.py b/backend/workflow_manager/endpoint/database_utils.pyindex 28587a427..f79f87c6d 100644--- a/backend/workflow_manager/endpoint/database_utils.py+++ b/backend/workflow_manager/endpoint/database_utils.py@@ -65,13 +65,6 @@ class DatabaseUtils:sql_values[column] = f"parse_json($${values[column]}$$)"else:sql_values[column] = f"{values[column]}"- elif cls_name == DBConnectionClass.BIGQUERY:- col = column.lower()- type_x = column_types[col]- if type_x in BigQuery.COLUMN_TYPES:- sql_values[column] = f"{type_x}({values[column]})"- else:- sql_values[column] = f"{values[column]}"else:# Default to Other SQL DBs# TODO: Handle numeric types with no quotesdiff --git a/backend/workflow_manager/endpoint/tests/__init__.py b/backend/workflow_manager/endpoint/tests/__init__.pyindex fca2b2401..6ec742384 100644--- a/backend/workflow_manager/endpoint/tests/__init__.py+++ b/backend/workflow_manager/endpoint/tests/__init__.py@@ -1,3 +1,3 @@-from backend.celery import app as celery_app+from backend.celery_service import app as celery_app # type: ignore__all__ = ["celery_app"]diff --git a/unstract/connectors/src/unstract/connectors/databases/bigquery/bigquery.py b/unstract/connectors/src/unstract/connectors/databases/bigquery/bigquery.pyindex 573fe45dc..53cd9dea5 100644--- a/unstract/connectors/src/unstract/connectors/databases/bigquery/bigquery.py+++ b/unstract/connectors/src/unstract/connectors/databases/bigquery/bigquery.py@@ -102,9 +102,10 @@ class BigQuery(UnstractDB):table_name = str(kwargs.get("table_name"))try:if sql_values:- engine.query(sql_query, job_config=sql_values)+ query_job = engine.query(sql_query, job_config=sql_values)else:- engine.query(sql_query)+ query_job = engine.query(sql_query)+ query_job.result()except google.api_core.exceptions.Forbidden as e:logger.error(f"Forbidden exception in creating/inserting data: {str(e)}")raise BigQueryForbiddenException(diff --git a/unstract/connectors/src/unstract/connectors/databases/redshift/redshift.py b/unstract/connectors/src/unstract/connectors/databases/redshift/redshift.pyindex dc83ea58b..50ab35a55 100644--- a/unstract/connectors/src/unstract/connectors/databases/redshift/redshift.py+++ b/unstract/connectors/src/unstract/connectors/databases/redshift/redshift.py@@ -76,12 +76,12 @@ class Redshift(UnstractDB, PsycoPgHandler):"""python_type = type(value)mapping = {- str: "SUPER",+ str: "VARCHAR(65535)",int: "BIGINT",float: "DOUBLE PRECISION",datetime.datetime: "TIMESTAMP",}- return mapping.get(python_type, "SUPER")+ return mapping.get(python_type, "VARCHAR(65535)")@staticmethoddef get_create_table_query(table: str) -> str: