Skip to content

LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows

fix: Reverted indexing optimization in structure tool (#1350)

Chandrasekharan Mac7be902ba09
diff --git a/tools/structure/src/main.py b/tools/structure/src/main.py
index 2ecbcc07b..f15fc4a33 100644
--- a/tools/structure/src/main.py
+++ b/tools/structure/src/main.py
@@ -38,15 +38,13 @@ class StructureTool(BaseTool):
output_dir: str,
) -> None:
prompt_registry_id: str = settings[SettingsKeys.PROMPT_REGISTRY_ID]
- is_challenge_enabled: bool = settings.get(SettingsKeys.ENABLE_CHALLENGE, False)
- is_summarization_enabled: bool = settings.get(
- SettingsKeys.SUMMARIZE_AS_SOURCE, False
- )
- is_single_pass_enabled: bool = settings.get(
+ enable_challenge: bool = settings.get(SettingsKeys.ENABLE_CHALLENGE, False)
+ summarize_as_source: bool = settings.get(SettingsKeys.SUMMARIZE_AS_SOURCE, False)
+ single_pass_extraction_mode: bool = settings.get(
SettingsKeys.SINGLE_PASS_EXTRACTION_MODE, False
)
challenge_llm: str = settings.get(SettingsKeys.CHALLENGE_LLM_ADAPTER_ID, "")
- is_highlight_enabled: bool = settings.get(SettingsKeys.ENABLE_HIGHLIGHT, False)
+ enable_highlight: bool = settings.get(SettingsKeys.ENABLE_HIGHLIGHT, False)
responder: PromptTool = PromptTool(
tool=self,
prompt_port=self.get_env_or_die(SettingsKeys.PROMPT_PORT),
@@ -99,12 +97,14 @@ class StructureTool(BaseTool):
tool_settings = tool_metadata[SettingsKeys.TOOL_SETTINGS]
outputs = tool_metadata[SettingsKeys.OUTPUTS]
tool_settings[SettingsKeys.CHALLENGE_LLM] = challenge_llm
- tool_settings[SettingsKeys.ENABLE_CHALLENGE] = is_challenge_enabled
- tool_settings[SettingsKeys.ENABLE_SINGLE_PASS_EXTRACTION] = is_single_pass_enabled
- tool_settings[SettingsKeys.SUMMARIZE_AS_SOURCE] = is_summarization_enabled
- tool_settings[SettingsKeys.ENABLE_HIGHLIGHT] = is_highlight_enabled
+ tool_settings[SettingsKeys.ENABLE_CHALLENGE] = enable_challenge
+ tool_settings[SettingsKeys.ENABLE_SINGLE_PASS_EXTRACTION] = (
+ single_pass_extraction_mode
+ )
+ tool_settings[SettingsKeys.SUMMARIZE_AS_SOURCE] = summarize_as_source
+ tool_settings[SettingsKeys.ENABLE_HIGHLIGHT] = enable_highlight
_, file_name = os.path.split(input_file)
- if is_summarization_enabled:
+ if summarize_as_source:
file_name = SettingsKeys.SUMMARIZE
tool_data_dir = Path(self.get_env_or_die(ToolEnv.EXECUTION_DATA_DIR))
execution_run_data_folder = Path(self.get_env_or_die(ToolEnv.EXECUTION_DATA_DIR))
@@ -130,7 +130,7 @@ class StructureTool(BaseTool):
usage_kwargs[UsageKwargs.EXECUTION_ID] = self.execution_id
extracted_text = STHelper.dynamic_extraction(
file_path=input_file,
- enable_highlight=is_highlight_enabled,
+ enable_highlight=enable_highlight,
usage_kwargs=usage_kwargs,
run_id=self.file_execution_id,
tool_settings=tool_settings,
@@ -139,7 +139,8 @@ class StructureTool(BaseTool):
execution_run_data_folder=str(execution_run_data_folder),
)
- if is_summarization_enabled:
+ summarize_file_hash = None
+ if summarize_as_source:
summarize_file_path, summarize_file_hash = self._summarize_and_index(
tool_settings=tool_settings,
tool_data_dir=tool_data_dir,
@@ -149,25 +150,8 @@ class StructureTool(BaseTool):
)
payload[SettingsKeys.FILE_HASH] = summarize_file_hash
payload[SettingsKeys.FILE_PATH] = summarize_file_path
- # Since indexing is not involved for summary
- index_metrics = {"time_taken(s)": 0}
- for output in outputs:
- if SettingsKeys.TABLE_SETTINGS in output:
- table_settings = output[SettingsKeys.TABLE_SETTINGS]
- is_directory_mode: bool = table_settings.get(
- SettingsKeys.IS_DIRECTORY_MODE, False
- )
- table_settings[SettingsKeys.INPUT_FILE] = extracted_input_file
- table_settings[SettingsKeys.IS_DIRECTORY_MODE] = is_directory_mode
- self.stream_log(f"Performing table extraction with: {table_settings}")
- output.update({SettingsKeys.TABLE_SETTINGS: table_settings})
-
- self.stream_log(f"Fetching responses for '{len(outputs)}' prompt(s)...")
- structured_output = responder.answer_prompt(
- payload=payload,
- )
- elif is_single_pass_enabled:
+ if tool_settings[SettingsKeys.ENABLE_SINGLE_PASS_EXTRACTION]:
self.stream_log("Fetching response for single pass extraction...")
# Since indexing is not involved for single pass
index_metrics = {"time_taken(s)": 0}
@@ -175,53 +159,37 @@ class StructureTool(BaseTool):
payload=payload,
)
else:
- # Track seen parameter combinations to avoid duplicate indexing
- seen_params = set()
-
+ # To reindex even if file is already
+ # indexed to get the output in required path
+ reindex = True
+ index_metrics = {}
for output in outputs:
- # Get current parameter combination
- chunk_size = output[SettingsKeys.CHUNK_SIZE]
- chunk_overlap = output[SettingsKeys.CHUNK_OVERLAP]
- vector_db = tool_settings[SettingsKeys.VECTOR_DB]
- embedding = tool_settings[SettingsKeys.EMBEDDING]
- x2text = tool_settings[SettingsKeys.X2TEXT_ADAPTER]
-
- # Create a unique key for this parameter combination
- param_key = (
- f"chunk_size={chunk_size}_"
- f"chunk_overlap={chunk_overlap}_"
- f"vector_db={vector_db}_"
- f"embedding={embedding}_"
- f"x2text={x2text}"
- )
-
- # Only process if we haven't seen this combination yet and chunk_size is not zero
- if chunk_size != 0 and param_key not in seen_params:
- seen_params.add(param_key)
-
+ if summarize_as_source:
+ # Since indexing is not involved for summary
+ index_metrics[output[SettingsKeys.NAME]] = {"time_taken(s)": 0}
+ break
+ if (reindex or not summarize_as_source) and output[
+ SettingsKeys.CHUNK_SIZE
+ ] != 0:
indexing_start_time = datetime.datetime.now()
self.stream_log(
- f"Indexing document with: chunk_size={chunk_size}, "
- f"chunk_overlap={chunk_overlap}, vector_db={vector_db}, "
- f"embedding={embedding}, x2text={x2text}"
+ f"Indexing document with chunk size '{output[SettingsKeys.CHUNK_SIZE]}' and overlap '{output[SettingsKeys.CHUNK_OVERLAP]}'"
)
-
STHelper.dynamic_indexing(
tool_settings=tool_settings,
run_id=self.file_execution_id,
file_path=tool_data_dir / SettingsKeys.EXTRACT,
tool=self,
execution_run_data_folder=str(execution_run_data_folder),
- chunk_overlap=chunk_overlap,
- reindex=True,
+ chunk_overlap=output[SettingsKeys.CHUNK_OVERLAP],
+ reindex=reindex,
usage_kwargs=usage_kwargs,
- enable_highlight=is_highlight_enabled,
- chunk_size=chunk_size,
+ enable_highlight=enable_highlight,
+ chunk_size=output[SettingsKeys.CHUNK_SIZE],
tool_id=tool_metadata[SettingsKeys.TOOL_ID],
file_hash=file_hash,
extracted_text=extracted_text,
)
-
index_metrics[output[SettingsKeys.NAME]] = {
SettingsKeys.INDEXING: {
"time_taken(s)": STHelper.elapsed_time(
@@ -229,6 +197,7 @@ class StructureTool(BaseTool):
)
}
}
+ reindex = False
for output in outputs:
if SettingsKeys.TABLE_SETTINGS in output:
@@ -238,7 +207,6 @@ class StructureTool(BaseTool):
)
table_settings[SettingsKeys.INPUT_FILE] = extracted_input_file
table_settings[SettingsKeys.IS_DIRECTORY_MODE] = is_directory_mode
- self.stream_log(f"Performing table extraction with: {table_settings}")
output.update({SettingsKeys.TABLE_SETTINGS: table_settings})
self.stream_log(f"Fetching responses for '{len(outputs)}' prompt(s)...")
@@ -252,7 +220,7 @@ class StructureTool(BaseTool):
self.source_file_name
)
- if not is_summarization_enabled:
+ if not summarize_as_source:
metadata = structured_output[SettingsKeys.METADATA]
# Update the dictionary with modified metadata
structured_output[SettingsKeys.METADATA] = metadata