Skip to content

LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows

[FIX] Removed duplicate indexing and reduced log noise during workflow tool runs for clearer logs and improved execution performance (#1356)

Chandrasekharan Mb1888697b921
diff --git a/tools/structure/src/config/properties.json b/tools/structure/src/config/properties.json
index 1bbd9cbeb..a1124eb2a 100644
--- a/tools/structure/src/config/properties.json
+++ b/tools/structure/src/config/properties.json
@@ -2,7 +2,7 @@
"schemaVersion": "0.0.1",
"displayName": "Structure Tool",
"functionName": "structure_tool",
- "toolVersion": "0.0.80",
+ "toolVersion": "0.0.81",
"description": "This is a template tool which can answer set of input prompts designed in the Prompt Studio",
"input": {
"description": "File that needs to be indexed and parsed for answers"
diff --git a/tools/structure/src/main.py b/tools/structure/src/main.py
index f15fc4a33..85c256aee 100644
--- a/tools/structure/src/main.py
+++ b/tools/structure/src/main.py
@@ -38,13 +38,15 @@ class StructureTool(BaseTool):
output_dir: str,
) -> None:
prompt_registry_id: str = settings[SettingsKeys.PROMPT_REGISTRY_ID]
- enable_challenge: bool = settings.get(SettingsKeys.ENABLE_CHALLENGE, False)
- summarize_as_source: bool = settings.get(SettingsKeys.SUMMARIZE_AS_SOURCE, False)
- single_pass_extraction_mode: bool = settings.get(
+ is_challenge_enabled: bool = settings.get(SettingsKeys.ENABLE_CHALLENGE, False)
+ is_summarization_enabled: bool = settings.get(
+ SettingsKeys.SUMMARIZE_AS_SOURCE, False
+ )
+ is_single_pass_enabled: bool = settings.get(
SettingsKeys.SINGLE_PASS_EXTRACTION_MODE, False
)
challenge_llm: str = settings.get(SettingsKeys.CHALLENGE_LLM_ADAPTER_ID, "")
- enable_highlight: bool = settings.get(SettingsKeys.ENABLE_HIGHLIGHT, False)
+ is_highlight_enabled: bool = settings.get(SettingsKeys.ENABLE_HIGHLIGHT, False)
responder: PromptTool = PromptTool(
tool=self,
prompt_port=self.get_env_or_die(SettingsKeys.PROMPT_PORT),
@@ -55,7 +57,7 @@ class StructureTool(BaseTool):
f"Fetching prompt studio exported tool with UUID '{prompt_registry_id}'"
)
try:
- platform_helper: PromptTool = PlatformHelper(
+ platform_helper: PlatformHelper = PlatformHelper(
tool=self,
platform_port=self.get_env_or_die(ToolEnv.PLATFORM_PORT),
platform_host=self.get_env_or_die(ToolEnv.PLATFORM_HOST),
@@ -97,19 +99,16 @@ class StructureTool(BaseTool):
tool_settings = tool_metadata[SettingsKeys.TOOL_SETTINGS]
outputs = tool_metadata[SettingsKeys.OUTPUTS]
tool_settings[SettingsKeys.CHALLENGE_LLM] = challenge_llm
- tool_settings[SettingsKeys.ENABLE_CHALLENGE] = enable_challenge
- tool_settings[SettingsKeys.ENABLE_SINGLE_PASS_EXTRACTION] = (
- single_pass_extraction_mode
- )
- tool_settings[SettingsKeys.SUMMARIZE_AS_SOURCE] = summarize_as_source
- tool_settings[SettingsKeys.ENABLE_HIGHLIGHT] = enable_highlight
+ tool_settings[SettingsKeys.ENABLE_CHALLENGE] = is_challenge_enabled
+ tool_settings[SettingsKeys.ENABLE_SINGLE_PASS_EXTRACTION] = is_single_pass_enabled
+ tool_settings[SettingsKeys.SUMMARIZE_AS_SOURCE] = is_summarization_enabled
+ tool_settings[SettingsKeys.ENABLE_HIGHLIGHT] = is_highlight_enabled
_, file_name = os.path.split(input_file)
- if summarize_as_source:
+ if is_summarization_enabled:
file_name = SettingsKeys.SUMMARIZE
tool_data_dir = Path(self.get_env_or_die(ToolEnv.EXECUTION_DATA_DIR))
execution_run_data_folder = Path(self.get_env_or_die(ToolEnv.EXECUTION_DATA_DIR))
- index_metrics = {}
extracted_input_file = str(execution_run_data_folder / SettingsKeys.EXTRACT)
# Resolve and pass log events ID
payload = {
@@ -130,7 +129,7 @@ class StructureTool(BaseTool):
usage_kwargs[UsageKwargs.EXECUTION_ID] = self.execution_id
extracted_text = STHelper.dynamic_extraction(
file_path=input_file,
- enable_highlight=enable_highlight,
+ enable_highlight=is_highlight_enabled,
usage_kwargs=usage_kwargs,
run_id=self.file_execution_id,
tool_settings=tool_settings,
@@ -139,9 +138,9 @@ class StructureTool(BaseTool):
execution_run_data_folder=str(execution_run_data_folder),
)
- summarize_file_hash = None
- if summarize_as_source:
- summarize_file_path, summarize_file_hash = self._summarize_and_index(
+ index_metrics = {}
+ if is_summarization_enabled:
+ summarize_file_path, summarize_file_hash = self._summarize(
tool_settings=tool_settings,
tool_data_dir=tool_data_dir,
responder=responder,
@@ -150,46 +149,54 @@ class StructureTool(BaseTool):
)
payload[SettingsKeys.FILE_HASH] = summarize_file_hash
payload[SettingsKeys.FILE_PATH] = summarize_file_path
+ elif not is_single_pass_enabled:
+ # Track seen parameter combinations to avoid duplicate indexing
+ seen_params = set()
- if tool_settings[SettingsKeys.ENABLE_SINGLE_PASS_EXTRACTION]:
- self.stream_log("Fetching response for single pass extraction...")
- # Since indexing is not involved for single pass
- index_metrics = {"time_taken(s)": 0}
- structured_output = responder.single_pass_extraction(
- payload=payload,
- )
- else:
- # To reindex even if file is already
- # indexed to get the output in required path
- reindex = True
- index_metrics = {}
for output in outputs:
- if summarize_as_source:
- # Since indexing is not involved for summary
- index_metrics[output[SettingsKeys.NAME]] = {"time_taken(s)": 0}
- break
- if (reindex or not summarize_as_source) and output[
- SettingsKeys.CHUNK_SIZE
- ] != 0:
+ # Get current parameter combination
+ chunk_size = output[SettingsKeys.CHUNK_SIZE]
+ chunk_overlap = output[SettingsKeys.CHUNK_OVERLAP]
+ vector_db = tool_settings[SettingsKeys.VECTOR_DB]
+ embedding = tool_settings[SettingsKeys.EMBEDDING]
+ x2text = tool_settings[SettingsKeys.X2TEXT_ADAPTER]
+
+ # Create a unique key for this parameter combination
+ param_key = (
+ f"chunk_size={chunk_size}_"
+ f"chunk_overlap={chunk_overlap}_"
+ f"vector_db={vector_db}_"
+ f"embedding={embedding}_"
+ f"x2text={x2text}"
+ )
+
+ # Only process if we haven't seen this combination yet and chunk_size is not zero
+ if chunk_size != 0 and param_key not in seen_params:
+ seen_params.add(param_key)
+
indexing_start_time = datetime.datetime.now()
self.stream_log(
- f"Indexing document with chunk size '{output[SettingsKeys.CHUNK_SIZE]}' and overlap '{output[SettingsKeys.CHUNK_OVERLAP]}'"
+ f"Indexing document with: chunk_size={chunk_size}, "
+ f"chunk_overlap={chunk_overlap}, vector_db={vector_db}, "
+ f"embedding={embedding}, x2text={x2text}"
)
+
STHelper.dynamic_indexing(
tool_settings=tool_settings,
run_id=self.file_execution_id,
file_path=tool_data_dir / SettingsKeys.EXTRACT,
tool=self,
execution_run_data_folder=str(execution_run_data_folder),
- chunk_overlap=output[SettingsKeys.CHUNK_OVERLAP],
- reindex=reindex,
+ chunk_overlap=chunk_overlap,
+ reindex=True,
usage_kwargs=usage_kwargs,
- enable_highlight=enable_highlight,
- chunk_size=output[SettingsKeys.CHUNK_SIZE],
+ enable_highlight=is_highlight_enabled,
+ chunk_size=chunk_size,
tool_id=tool_metadata[SettingsKeys.TOOL_ID],
file_hash=file_hash,
extracted_text=extracted_text,
)
+
index_metrics[output[SettingsKeys.NAME]] = {
SettingsKeys.INDEXING: {
"time_taken(s)": STHelper.elapsed_time(
@@ -197,8 +204,13 @@ class StructureTool(BaseTool):
)
}
}
- reindex = False
+ if is_single_pass_enabled:
+ self.stream_log("Fetching response for single pass extraction...")
+ structured_output = responder.single_pass_extraction(
+ payload=payload,
+ )
+ else:
for output in outputs:
if SettingsKeys.TABLE_SETTINGS in output:
table_settings = output[SettingsKeys.TABLE_SETTINGS]
@@ -207,6 +219,7 @@ class StructureTool(BaseTool):
)
table_settings[SettingsKeys.INPUT_FILE] = extracted_input_file
table_settings[SettingsKeys.IS_DIRECTORY_MODE] = is_directory_mode
+ self.stream_log(f"Performing table extraction with: {table_settings}")
output.update({SettingsKeys.TABLE_SETTINGS: table_settings})
self.stream_log(f"Fetching responses for '{len(outputs)}' prompt(s)...")
@@ -220,27 +233,10 @@ class StructureTool(BaseTool):
self.source_file_name
)
- if not summarize_as_source:
- metadata = structured_output[SettingsKeys.METADATA]
- # Update the dictionary with modified metadata
- structured_output[SettingsKeys.METADATA] = metadata
-
- metrics = structured_output.get(SettingsKeys.METRICS, {})
- new_metrics = {}
- if tool_settings[SettingsKeys.ENABLE_SINGLE_PASS_EXTRACTION]:
- new_metrics = {
- **metrics,
- **index_metrics,
- }
- else:
- # Merge dictionaries
- new_metrics = {
- key: {**metrics.get(key, {}), **index_metrics.get(key, {})}
- for key in set(metrics)
- | set(index_metrics) # Union of keys from both dictionaries
- }
- if new_metrics:
- structured_output[SettingsKeys.METRICS] = new_metrics
+ if merged_metrics := self._merge_metrics(
+ structured_output.get(SettingsKeys.METRICS, {}), index_metrics
+ ):
+ structured_output[SettingsKeys.METRICS] = merged_metrics
# Update GUI
output_log = (
f"## Result\n**NOTE:** In case of a deployed pipeline, the result would "
@@ -264,7 +260,42 @@ class StructureTool(BaseTool):
self.stream_error_and_exit(f"Error encoding JSON: {e}")
self.write_tool_result(data=structured_output)
- def _summarize_and_index(
+ def _merge_metrics(self, metrics1: dict, metrics2: dict) -> dict:
+ """Intelligently merge two metrics dictionaries.
+
+ For keys that exist in both dictionaries with dictionary values, merge the dictionaries.
+ For keys that exist in only one dictionary or have non-dictionary values, use the value as-is.
+
+ Args:
+ metrics1 (dict): First metrics dictionary
+ metrics2 (dict): Second metrics dictionary
+
+ Returns:
+ dict: Merged metrics dictionary
+ """
+ merged_metrics = {}
+
+ # Get all unique keys from both dictionaries
+ all_keys = set(metrics1) | set(metrics2)
+
+ for key in all_keys:
+ # If key exists in both dictionaries and both values are dictionaries, merge them
+ if (
+ key in metrics1
+ and key in metrics2
+ and isinstance(metrics1[key], dict)
+ and isinstance(metrics2[key], dict)
+ ):
+ merged_metrics[key] = {**metrics1[key], **metrics2[key]}
+ # Otherwise just take the value from whichever dictionary has it
+ elif key in metrics1:
+ merged_metrics[key] = metrics1[key]
+ else:
+ merged_metrics[key] = metrics2[key]
+
+ return merged_metrics
+
+ def _summarize(
self,
tool_settings: dict[str, Any],
tool_data_dir: Path,
@@ -272,19 +303,17 @@ class StructureTool(BaseTool):
outputs: dict[str, Any],
usage_kwargs: dict[Any, Any] = {},
) -> tuple[str, str]:
- """Summarizes the context of the file and indexes the summarized
- content.
+ """Summarizes the context of the file.
Args:
- tool_id (str): The identifier of the tool.
tool_settings (dict[str, Any]): Settings for the tool.
tool_data_dir (Path): Directory where tool data is stored.
responder (PromptTool): Instance of a tool used to generate the summary.
outputs (dict[str, Any]): Dictionary containing prompt details.
- index (Index): Instance used to index the summarized content.
+ usage_kwargs (dict[Any, Any]): Used to capture usage metrics.
Returns:
- str: The hash of the summarized file.
+ tuple[str, str]: Tuple containing the path to the summarized file and its hash.
"""
llm_adapter_instance_id: str = tool_settings[SettingsKeys.LLM]
embedding_instance_id: str = tool_settings[SettingsKeys.EMBEDDING]