LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
[FIX] Removed duplicate indexing and reduced log noise during workflow tool runs for clearer logs and improved execution performance (#1356)
Chandrasekharan Mb1888697b921
diff --git a/tools/structure/src/config/properties.json b/tools/structure/src/config/properties.jsonindex 1bbd9cbeb..a1124eb2a 100644--- a/tools/structure/src/config/properties.json+++ b/tools/structure/src/config/properties.json@@ -2,7 +2,7 @@"schemaVersion": "0.0.1","displayName": "Structure Tool","functionName": "structure_tool",- "toolVersion": "0.0.80",+ "toolVersion": "0.0.81","description": "This is a template tool which can answer set of input prompts designed in the Prompt Studio","input": {"description": "File that needs to be indexed and parsed for answers"diff --git a/tools/structure/src/main.py b/tools/structure/src/main.pyindex f15fc4a33..85c256aee 100644--- a/tools/structure/src/main.py+++ b/tools/structure/src/main.py@@ -38,13 +38,15 @@ class StructureTool(BaseTool):output_dir: str,) -> None:prompt_registry_id: str = settings[SettingsKeys.PROMPT_REGISTRY_ID]- enable_challenge: bool = settings.get(SettingsKeys.ENABLE_CHALLENGE, False)- summarize_as_source: bool = settings.get(SettingsKeys.SUMMARIZE_AS_SOURCE, False)- single_pass_extraction_mode: bool = settings.get(+ is_challenge_enabled: bool = settings.get(SettingsKeys.ENABLE_CHALLENGE, False)+ is_summarization_enabled: bool = settings.get(+ SettingsKeys.SUMMARIZE_AS_SOURCE, False+ )+ is_single_pass_enabled: bool = settings.get(SettingsKeys.SINGLE_PASS_EXTRACTION_MODE, False)challenge_llm: str = settings.get(SettingsKeys.CHALLENGE_LLM_ADAPTER_ID, "")- enable_highlight: bool = settings.get(SettingsKeys.ENABLE_HIGHLIGHT, False)+ is_highlight_enabled: bool = settings.get(SettingsKeys.ENABLE_HIGHLIGHT, False)responder: PromptTool = PromptTool(tool=self,prompt_port=self.get_env_or_die(SettingsKeys.PROMPT_PORT),@@ -55,7 +57,7 @@ class StructureTool(BaseTool):f"Fetching prompt studio exported tool with UUID '{prompt_registry_id}'")try:- platform_helper: PromptTool = PlatformHelper(+ platform_helper: PlatformHelper = PlatformHelper(tool=self,platform_port=self.get_env_or_die(ToolEnv.PLATFORM_PORT),platform_host=self.get_env_or_die(ToolEnv.PLATFORM_HOST),@@ -97,19 +99,16 @@ class StructureTool(BaseTool):tool_settings = tool_metadata[SettingsKeys.TOOL_SETTINGS]outputs = tool_metadata[SettingsKeys.OUTPUTS]tool_settings[SettingsKeys.CHALLENGE_LLM] = challenge_llm- tool_settings[SettingsKeys.ENABLE_CHALLENGE] = enable_challenge- tool_settings[SettingsKeys.ENABLE_SINGLE_PASS_EXTRACTION] = (- single_pass_extraction_mode- )- tool_settings[SettingsKeys.SUMMARIZE_AS_SOURCE] = summarize_as_source- tool_settings[SettingsKeys.ENABLE_HIGHLIGHT] = enable_highlight+ tool_settings[SettingsKeys.ENABLE_CHALLENGE] = is_challenge_enabled+ tool_settings[SettingsKeys.ENABLE_SINGLE_PASS_EXTRACTION] = is_single_pass_enabled+ tool_settings[SettingsKeys.SUMMARIZE_AS_SOURCE] = is_summarization_enabled+ tool_settings[SettingsKeys.ENABLE_HIGHLIGHT] = is_highlight_enabled_, file_name = os.path.split(input_file)- if summarize_as_source:+ if is_summarization_enabled:file_name = SettingsKeys.SUMMARIZEtool_data_dir = Path(self.get_env_or_die(ToolEnv.EXECUTION_DATA_DIR))execution_run_data_folder = Path(self.get_env_or_die(ToolEnv.EXECUTION_DATA_DIR))- index_metrics = {}extracted_input_file = str(execution_run_data_folder / SettingsKeys.EXTRACT)# Resolve and pass log events IDpayload = {@@ -130,7 +129,7 @@ class StructureTool(BaseTool):usage_kwargs[UsageKwargs.EXECUTION_ID] = self.execution_idextracted_text = STHelper.dynamic_extraction(file_path=input_file,- enable_highlight=enable_highlight,+ enable_highlight=is_highlight_enabled,usage_kwargs=usage_kwargs,run_id=self.file_execution_id,tool_settings=tool_settings,@@ -139,9 +138,9 @@ class StructureTool(BaseTool):execution_run_data_folder=str(execution_run_data_folder),)- summarize_file_hash = None- if summarize_as_source:- summarize_file_path, summarize_file_hash = self._summarize_and_index(+ index_metrics = {}+ if is_summarization_enabled:+ summarize_file_path, summarize_file_hash = self._summarize(tool_settings=tool_settings,tool_data_dir=tool_data_dir,responder=responder,@@ -150,46 +149,54 @@ class StructureTool(BaseTool):)payload[SettingsKeys.FILE_HASH] = summarize_file_hashpayload[SettingsKeys.FILE_PATH] = summarize_file_path+ elif not is_single_pass_enabled:+ # Track seen parameter combinations to avoid duplicate indexing+ seen_params = set()- if tool_settings[SettingsKeys.ENABLE_SINGLE_PASS_EXTRACTION]:- self.stream_log("Fetching response for single pass extraction...")- # Since indexing is not involved for single pass- index_metrics = {"time_taken(s)": 0}- structured_output = responder.single_pass_extraction(- payload=payload,- )- else:- # To reindex even if file is already- # indexed to get the output in required path- reindex = True- index_metrics = {}for output in outputs:- if summarize_as_source:- # Since indexing is not involved for summary- index_metrics[output[SettingsKeys.NAME]] = {"time_taken(s)": 0}- break- if (reindex or not summarize_as_source) and output[- SettingsKeys.CHUNK_SIZE- ] != 0:+ # Get current parameter combination+ chunk_size = output[SettingsKeys.CHUNK_SIZE]+ chunk_overlap = output[SettingsKeys.CHUNK_OVERLAP]+ vector_db = tool_settings[SettingsKeys.VECTOR_DB]+ embedding = tool_settings[SettingsKeys.EMBEDDING]+ x2text = tool_settings[SettingsKeys.X2TEXT_ADAPTER]++ # Create a unique key for this parameter combination+ param_key = (+ f"chunk_size={chunk_size}_"+ f"chunk_overlap={chunk_overlap}_"+ f"vector_db={vector_db}_"+ f"embedding={embedding}_"+ f"x2text={x2text}"+ )++ # Only process if we haven't seen this combination yet and chunk_size is not zero+ if chunk_size != 0 and param_key not in seen_params:+ seen_params.add(param_key)+indexing_start_time = datetime.datetime.now()self.stream_log(- f"Indexing document with chunk size '{output[SettingsKeys.CHUNK_SIZE]}' and overlap '{output[SettingsKeys.CHUNK_OVERLAP]}'"+ f"Indexing document with: chunk_size={chunk_size}, "+ f"chunk_overlap={chunk_overlap}, vector_db={vector_db}, "+ f"embedding={embedding}, x2text={x2text}")+STHelper.dynamic_indexing(tool_settings=tool_settings,run_id=self.file_execution_id,file_path=tool_data_dir / SettingsKeys.EXTRACT,tool=self,execution_run_data_folder=str(execution_run_data_folder),- chunk_overlap=output[SettingsKeys.CHUNK_OVERLAP],- reindex=reindex,+ chunk_overlap=chunk_overlap,+ reindex=True,usage_kwargs=usage_kwargs,- enable_highlight=enable_highlight,- chunk_size=output[SettingsKeys.CHUNK_SIZE],+ enable_highlight=is_highlight_enabled,+ chunk_size=chunk_size,tool_id=tool_metadata[SettingsKeys.TOOL_ID],file_hash=file_hash,extracted_text=extracted_text,)+index_metrics[output[SettingsKeys.NAME]] = {SettingsKeys.INDEXING: {"time_taken(s)": STHelper.elapsed_time(@@ -197,8 +204,13 @@ class StructureTool(BaseTool):)}}- reindex = False+ if is_single_pass_enabled:+ self.stream_log("Fetching response for single pass extraction...")+ structured_output = responder.single_pass_extraction(+ payload=payload,+ )+ else:for output in outputs:if SettingsKeys.TABLE_SETTINGS in output:table_settings = output[SettingsKeys.TABLE_SETTINGS]@@ -207,6 +219,7 @@ class StructureTool(BaseTool):)table_settings[SettingsKeys.INPUT_FILE] = extracted_input_filetable_settings[SettingsKeys.IS_DIRECTORY_MODE] = is_directory_mode+ self.stream_log(f"Performing table extraction with: {table_settings}")output.update({SettingsKeys.TABLE_SETTINGS: table_settings})self.stream_log(f"Fetching responses for '{len(outputs)}' prompt(s)...")@@ -220,27 +233,10 @@ class StructureTool(BaseTool):self.source_file_name)- if not summarize_as_source:- metadata = structured_output[SettingsKeys.METADATA]- # Update the dictionary with modified metadata- structured_output[SettingsKeys.METADATA] = metadata-- metrics = structured_output.get(SettingsKeys.METRICS, {})- new_metrics = {}- if tool_settings[SettingsKeys.ENABLE_SINGLE_PASS_EXTRACTION]:- new_metrics = {- **metrics,- **index_metrics,- }- else:- # Merge dictionaries- new_metrics = {- key: {**metrics.get(key, {}), **index_metrics.get(key, {})}- for key in set(metrics)- | set(index_metrics) # Union of keys from both dictionaries- }- if new_metrics:- structured_output[SettingsKeys.METRICS] = new_metrics+ if merged_metrics := self._merge_metrics(+ structured_output.get(SettingsKeys.METRICS, {}), index_metrics+ ):+ structured_output[SettingsKeys.METRICS] = merged_metrics# Update GUIoutput_log = (f"## Result\n**NOTE:** In case of a deployed pipeline, the result would "@@ -264,7 +260,42 @@ class StructureTool(BaseTool):self.stream_error_and_exit(f"Error encoding JSON: {e}")self.write_tool_result(data=structured_output)- def _summarize_and_index(+ def _merge_metrics(self, metrics1: dict, metrics2: dict) -> dict:+ """Intelligently merge two metrics dictionaries.++ For keys that exist in both dictionaries with dictionary values, merge the dictionaries.+ For keys that exist in only one dictionary or have non-dictionary values, use the value as-is.++ Args:+ metrics1 (dict): First metrics dictionary+ metrics2 (dict): Second metrics dictionary++ Returns:+ dict: Merged metrics dictionary+ """+ merged_metrics = {}++ # Get all unique keys from both dictionaries+ all_keys = set(metrics1) | set(metrics2)++ for key in all_keys:+ # If key exists in both dictionaries and both values are dictionaries, merge them+ if (+ key in metrics1+ and key in metrics2+ and isinstance(metrics1[key], dict)+ and isinstance(metrics2[key], dict)+ ):+ merged_metrics[key] = {**metrics1[key], **metrics2[key]}+ # Otherwise just take the value from whichever dictionary has it+ elif key in metrics1:+ merged_metrics[key] = metrics1[key]+ else:+ merged_metrics[key] = metrics2[key]++ return merged_metrics++ def _summarize(self,tool_settings: dict[str, Any],tool_data_dir: Path,@@ -272,19 +303,17 @@ class StructureTool(BaseTool):outputs: dict[str, Any],usage_kwargs: dict[Any, Any] = {},) -> tuple[str, str]:- """Summarizes the context of the file and indexes the summarized- content.+ """Summarizes the context of the file.Args:- tool_id (str): The identifier of the tool.tool_settings (dict[str, Any]): Settings for the tool.tool_data_dir (Path): Directory where tool data is stored.responder (PromptTool): Instance of a tool used to generate the summary.outputs (dict[str, Any]): Dictionary containing prompt details.- index (Index): Instance used to index the summarized content.+ usage_kwargs (dict[Any, Any]): Used to capture usage metrics.Returns:- str: The hash of the summarized file.+ tuple[str, str]: Tuple containing the path to the summarized file and its hash."""llm_adapter_instance_id: str = tool_settings[SettingsKeys.LLM]embedding_instance_id: str = tool_settings[SettingsKeys.EMBEDDING]