LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
minor: Logs added to the user and our services (#1337)
Chandrasekharan M07f07b95f668
diff --git a/backend/workflow_manager/endpoint_v2/destination.py b/backend/workflow_manager/endpoint_v2/destination.pyindex 4ce3ab773..039dd1829 100644--- a/backend/workflow_manager/endpoint_v2/destination.py+++ b/backend/workflow_manager/endpoint_v2/destination.py@@ -308,6 +308,7 @@ class DestinationConnector(BaseConnector):data = self.get_tool_execution_result()# If data is None, don't execute CREATE or INSERT queryif not data:+ logger.info("No data obtained from tool to insert into destination DB.")return# Remove metadata from resultdiff --git a/backend/workflow_manager/endpoint_v2/source.py b/backend/workflow_manager/endpoint_v2/source.pyindex 5cfb8a036..0da12bbf1 100644--- a/backend/workflow_manager/endpoint_v2/source.py+++ b/backend/workflow_manager/endpoint_v2/source.py@@ -333,9 +333,9 @@ class SourceConnector(BaseConnector):) -> int:for fs_metadata in fs_metadata_list:if count >= limit:- logger.info(- f"[Matched Files] Maximum limit {limit} of files reached, count: {count}"- )+ msg = f"Maximum limit of '{limit}' files to process reached"+ self.workflow_log.publish_log(msg)+ logger.info(msg)breakfile_path: str | None = fs_metadata.get("name")@@ -358,9 +358,9 @@ class SourceConnector(BaseConnector):# Skip duplicate filesif self._is_duplicate(file_hash, unique_file_hashes):- logger.info(- f"[Matched Files] Skipping execution of duplicate file: {file_path}"- )+ msg = f"Skipping execution of duplicate file '{file_path}'"+ self.workflow_log.publish_log(msg)+ logger.info(msg)continueself._update_unique_file_hashes(file_hash, unique_file_hashes)@@ -528,10 +528,11 @@ class SourceConnector(BaseConnector):# In case of ETL pipelines, its necessary to skip files which have# already been processedif self.use_file_history and file_history and file_history.is_completed():+ msg = f"Skipping file '{file_path}' as it has already been processed."self.workflow_log.publish_log(- f"Skipping file {file_path} as it has already been processed. "- "Clear the file markers to process it again."+ msg + " Clear the file markers to process it again.")+ logger.info(msg)return Falsereturn Truediff --git a/backend/workflow_manager/workflow_v2/file_history_helper.py b/backend/workflow_manager/workflow_v2/file_history_helper.pyindex 00ffa030b..b4632a94b 100644--- a/backend/workflow_manager/workflow_v2/file_history_helper.py+++ b/backend/workflow_manager/workflow_v2/file_history_helper.py@@ -30,6 +30,10 @@ class FileHistoryHelper:Optional[FileHistory]: The matching file history record, if found."""if not cache_key and not provider_file_uuid:+ logger.warning(+ "No cache key or provider file UUID provided "+ f"while fetching file history for {workflow}"+ )return Nonetry:if not cache_key:@@ -38,6 +42,11 @@ class FileHistoryHelper:)return FileHistory.objects.get(cache_key=cache_key, workflow=workflow)except FileHistory.DoesNotExist:+ logger.debug(+ f"File history not found for cache key: {cache_key}, "+ f"provider_file_uuid: {provider_file_uuid} while fetching "+ f"file history for {workflow}"+ )return None@staticmethod