Skip to content

LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows

minor: Logs added to the user and our services (#1337)

Chandrasekharan M07f07b95f668
diff --git a/backend/workflow_manager/endpoint_v2/destination.py b/backend/workflow_manager/endpoint_v2/destination.py
index 4ce3ab773..039dd1829 100644
--- a/backend/workflow_manager/endpoint_v2/destination.py
+++ b/backend/workflow_manager/endpoint_v2/destination.py
@@ -308,6 +308,7 @@ class DestinationConnector(BaseConnector):
data = self.get_tool_execution_result()
# If data is None, don't execute CREATE or INSERT query
if not data:
+ logger.info("No data obtained from tool to insert into destination DB.")
return
# Remove metadata from result
diff --git a/backend/workflow_manager/endpoint_v2/source.py b/backend/workflow_manager/endpoint_v2/source.py
index 5cfb8a036..0da12bbf1 100644
--- a/backend/workflow_manager/endpoint_v2/source.py
+++ b/backend/workflow_manager/endpoint_v2/source.py
@@ -333,9 +333,9 @@ class SourceConnector(BaseConnector):
) -> int:
for fs_metadata in fs_metadata_list:
if count >= limit:
- logger.info(
- f"[Matched Files] Maximum limit {limit} of files reached, count: {count}"
- )
+ msg = f"Maximum limit of '{limit}' files to process reached"
+ self.workflow_log.publish_log(msg)
+ logger.info(msg)
break
file_path: str | None = fs_metadata.get("name")
@@ -358,9 +358,9 @@ class SourceConnector(BaseConnector):
# Skip duplicate files
if self._is_duplicate(file_hash, unique_file_hashes):
- logger.info(
- f"[Matched Files] Skipping execution of duplicate file: {file_path}"
- )
+ msg = f"Skipping execution of duplicate file '{file_path}'"
+ self.workflow_log.publish_log(msg)
+ logger.info(msg)
continue
self._update_unique_file_hashes(file_hash, unique_file_hashes)
@@ -528,10 +528,11 @@ class SourceConnector(BaseConnector):
# In case of ETL pipelines, its necessary to skip files which have
# already been processed
if self.use_file_history and file_history and file_history.is_completed():
+ msg = f"Skipping file '{file_path}' as it has already been processed."
self.workflow_log.publish_log(
- f"Skipping file {file_path} as it has already been processed. "
- "Clear the file markers to process it again."
+ msg + " Clear the file markers to process it again."
)
+ logger.info(msg)
return False
return True
diff --git a/backend/workflow_manager/workflow_v2/file_history_helper.py b/backend/workflow_manager/workflow_v2/file_history_helper.py
index 00ffa030b..b4632a94b 100644
--- a/backend/workflow_manager/workflow_v2/file_history_helper.py
+++ b/backend/workflow_manager/workflow_v2/file_history_helper.py
@@ -30,6 +30,10 @@ class FileHistoryHelper:
Optional[FileHistory]: The matching file history record, if found.
"""
if not cache_key and not provider_file_uuid:
+ logger.warning(
+ "No cache key or provider file UUID provided "
+ f"while fetching file history for {workflow}"
+ )
return None
try:
if not cache_key:
@@ -38,6 +42,11 @@ class FileHistoryHelper:
)
return FileHistory.objects.get(cache_key=cache_key, workflow=workflow)
except FileHistory.DoesNotExist:
+ logger.debug(
+ f"File history not found for cache key: {cache_key}, "
+ f"provider_file_uuid: {provider_file_uuid} while fetching "
+ f"file history for {workflow}"
+ )
return None
@staticmethod