LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
Revert "passing file execution id from backend to tools" (#1071)
Gayathri5ffddb76fdf5
diff --git a/backend/workflow_manager/endpoint_v2/source.py b/backend/workflow_manager/endpoint_v2/source.pyindex a030d7c70..273962787 100644--- a/backend/workflow_manager/endpoint_v2/source.py+++ b/backend/workflow_manager/endpoint_v2/source.py@@ -34,7 +34,6 @@ from workflow_manager.endpoint_v2.exceptions import (SourceConnectorNotConfigured,)from workflow_manager.endpoint_v2.models import WorkflowEndpoint-from workflow_manager.file_execution.models import WorkflowFileExecutionfrom workflow_manager.workflow_v2.execution import WorkflowExecutionServiceHelperfrom workflow_manager.workflow_v2.file_history_helper import FileHistoryHelperfrom workflow_manager.workflow_v2.models.workflow import Workflow@@ -597,14 +596,11 @@ class SourceConnector(BaseConnector):# Update the seek positionseek_position += len(chunk)- def add_file_to_volume(- self, input_file_path: str, workflow_file_execution: WorkflowFileExecution- ) -> str:+ def add_file_to_volume(self, input_file_path: str, file_hash: FileHash) -> str:"""Add input file to execution directory.Args:input_file_path (str): source file- workflow_file_execution: WorkflowFileExecution modelRaises:InvalidSource: _description_@@ -618,20 +614,18 @@ class SourceConnector(BaseConnector):file_content_hash = self.add_input_from_connector_to_volume(input_file_path=input_file_path,)- if file_content_hash != workflow_file_execution.file_hash:+ if file_content_hash != file_hash.file_hash:raise FileHashMismatched()elif connection_type == WorkflowEndpoint.ConnectionType.API:self.add_input_from_api_storage_to_volume(input_file_path=input_file_path)- if file_name != workflow_file_execution.file_name:+ if file_name != file_hash.file_name:raise FileHashNotFound()- file_content_hash = workflow_file_execution.file_hash+ file_content_hash = file_hash.file_hashelse:raise InvalidSourceConnectionType()self.add_metadata_to_volume(- input_file_path=input_file_path,- file_execution_id=workflow_file_execution.id,- source_hash=file_content_hash,+ input_file_path=input_file_path, source_hash=file_content_hash)return file_namediff --git a/backend/workflow_manager/workflow_v2/workflow_helper.py b/backend/workflow_manager/workflow_v2/workflow_helper.pyindex 1afba1c03..ab72eab64 100644--- a/backend/workflow_manager/workflow_v2/workflow_helper.py+++ b/backend/workflow_manager/workflow_v2/workflow_helper.py@@ -231,14 +231,14 @@ class WorkflowHelper:workflow_file_execution: WorkflowFileExecution,) -> Optional[str]:error: Optional[str] = None- # Multiple run_ids are linked to an execution_id- # Each run_id corresponds to workflow runs for a single file- # It should e uuid of workflow_file_execution- file_execution_id = str(workflow_file_execution.id)file_name = source.add_file_to_volume(- input_file_path=input_file, workflow_file_execution=workflow_file_execution+ input_file_path=input_file, file_hash=file_hash)try:+ # Multiple run_ids are linked to an execution_id+ # Each run_id corresponds to workflow runs for a single file+ # It should e uuid of workflow_file_execution+ file_execution_id = str(workflow_file_execution.id)execution_service.file_execution_id = file_execution_idexecution_service.initiate_tool_execution(current_file_idx, total_files, file_name, single_stepdiff --git a/tools/classifier/requirements.txt b/tools/classifier/requirements.txtindex c65022c39..625789cc3 100644--- a/tools/classifier/requirements.txt+++ b/tools/classifier/requirements.txt@@ -1,6 +1,6 @@# Add your dependencies here# Required for all unstract tools-unstract-sdk~=v0.55.0rc2+unstract-sdk~=0.54.0rc12# Required for remote storage supports3fs[boto3]==2024.6.0diff --git a/tools/classifier/src/config/properties.json b/tools/classifier/src/config/properties.jsonindex 90a89371d..358cd044a 100644--- a/tools/classifier/src/config/properties.json+++ b/tools/classifier/src/config/properties.json@@ -2,7 +2,7 @@"schemaVersion": "0.0.1","displayName": "File Classifier","functionName": "classify",- "toolVersion": "0.0.45",+ "toolVersion": "0.0.44","description": "Classifies a file into a bin based on its contents","input": {"description": "File to be classified"diff --git a/tools/classifier/src/helper.py b/tools/classifier/src/helper.pyindex 139017937..52a034344 100644--- a/tools/classifier/src/helper.py+++ b/tools/classifier/src/helper.py@@ -4,7 +4,7 @@ from pathlib import Pathfrom typing import Any, Optionalfrom unstract.sdk.cache import ToolCache-from unstract.sdk.constants import LogLevel, MetadataKey, ToolEnv, UsageKwargs+from unstract.sdk.constants import LogLevel, MetadataKey, ToolEnvfrom unstract.sdk.llm import LLMfrom unstract.sdk.tool.base import BaseToolfrom unstract.sdk.utils import ToolUtils@@ -139,13 +139,7 @@ class ClassifierHelper:self.tool.stream_log(f"Creating text extraction adapter using adapter_id: {adapter_id}")- usage_kwargs: dict[Any, Any] = dict()- usage_kwargs[UsageKwargs.FILE_NAME] = self.tool.source_file_name- usage_kwargs[UsageKwargs.RUN_ID] = self.tool.file_execution_id-- x2text = X2Text(- tool=self.tool, adapter_instance_id=adapter_id, usage_kwargs=usage_kwargs- )+ x2text = X2Text(tool=self.tool, adapter_instance_id=adapter_id)self.tool.stream_log("Text extraction adapter has been created successfully.")diff --git a/tools/classifier/src/main.py b/tools/classifier/src/main.pyindex e3176420d..5d12a379e 100644--- a/tools/classifier/src/main.py+++ b/tools/classifier/src/main.py@@ -3,13 +3,7 @@ from typing import Any, Optionalfrom helper import ClassifierHelper # type: ignorefrom helper import ReservedBins-from unstract.sdk.constants import (- LogLevel,- LogState,- MetadataKey,- ToolSettingsKey,- UsageKwargs,-)+from unstract.sdk.constants import LogLevel, LogState, MetadataKey, ToolSettingsKeyfrom unstract.sdk.exceptions import SdkErrorfrom unstract.sdk.llm import LLMfrom unstract.sdk.tool.base import BaseTool@@ -86,10 +80,8 @@ class UnstractClassifier(BaseTool):bins_with_quotes = [f"'{b}'" for b in bins]usage_kwargs: dict[Any, Any] = dict()- usage_kwargs[UsageKwargs.WORKFLOW_ID] = self.workflow_id- usage_kwargs[UsageKwargs.EXECUTION_ID] = self.execution_id- usage_kwargs[UsageKwargs.FILE_NAME] = self.source_file_name- usage_kwargs[UsageKwargs.RUN_ID] = self.file_execution_id+ usage_kwargs["workflow_id"] = self.workflow_id+ usage_kwargs["execution_id"] = self.execution_idtry:llm = LLM(diff --git a/tools/structure/requirements.txt b/tools/structure/requirements.txtindex c65022c39..625789cc3 100644--- a/tools/structure/requirements.txt+++ b/tools/structure/requirements.txt@@ -1,6 +1,6 @@# Add your dependencies here# Required for all unstract tools-unstract-sdk~=v0.55.0rc2+unstract-sdk~=0.54.0rc12# Required for remote storage supports3fs[boto3]==2024.6.0diff --git a/tools/structure/src/config/properties.json b/tools/structure/src/config/properties.jsonindex fcabcede2..4186df487 100644--- a/tools/structure/src/config/properties.json+++ b/tools/structure/src/config/properties.json@@ -2,7 +2,7 @@"schemaVersion": "0.0.1","displayName": "Structure Tool","functionName": "structure_tool",- "toolVersion": "0.0.56",+ "toolVersion": "0.0.55","description": "This is a template tool which can answer set of input prompts designed in the Prompt Studio","input": {"description": "File that needs to be indexed and parsed for answers"diff --git a/tools/structure/src/main.py b/tools/structure/src/main.pyindex a420486a1..8918365f4 100644--- a/tools/structure/src/main.py+++ b/tools/structure/src/main.py@@ -6,12 +6,13 @@ from pathlib import Pathfrom typing import Any, Callable, Optionalfrom constants import SettingsKeys # type: ignore [attr-defined]-from unstract.sdk.constants import LogLevel, LogState, MetadataKey, ToolEnv, UsageKwargs+from unstract.sdk.constants import LogLevel, LogState, MetadataKey, ToolEnvfrom unstract.sdk.index import Indexfrom unstract.sdk.prompt import PromptToolfrom unstract.sdk.tool.base import BaseToolfrom unstract.sdk.tool.entrypoint import ToolEntrypointfrom unstract.sdk.utils import ToolUtils+from unstract.sdk.utils.common_utils import CommonUtilsfrom utils import json_to_markdownlogger = logging.getLogger(__name__)@@ -45,6 +46,7 @@ class StructureTool(BaseTool):)challenge_llm: str = settings.get(SettingsKeys.CHALLENGE_LLM_ADAPTER_ID, "")enable_highlight: bool = settings.get(SettingsKeys.ENABLE_HIGHLIGHT, False)+ source_file_name = self.get_exec_metadata.get(MetadataKey.SOURCE_NAME)responder: PromptTool = PromptTool(tool=self,prompt_port=self.get_env_or_die(SettingsKeys.PROMPT_PORT),@@ -72,8 +74,8 @@ class StructureTool(BaseTool):f"## Loaded '{ps_project_name}'\n{json_to_markdown(tool_metadata)}\n")output_log = (- f"## Processing '{self.source_file_name}'\nThis might take a while and "- "involve...\n- Extracting text\n- Indexing\n- Retrieving answers "+ f"## Processing '{source_file_name}'\nThis might take a while and involve"+ "...\n- Extracting text\n- Indexing\n- Retrieving answers "f"for possibly '{total_prompt_count}' prompts")self.stream_update(input_log, state=LogState.INPUT_UPDATE)@@ -101,17 +103,17 @@ class StructureTool(BaseTool):execution_run_data_folder = Path(self.get_env_or_die(SettingsKeys.EXECUTION_RUN_DATA_FOLDER))-+ run_id = CommonUtils.generate_uuid()index = Index(tool=self,- run_id=self.file_execution_id,+ run_id=run_id,capture_metrics=True,)index_metrics = {}extracted_input_file = str(execution_run_data_folder / SettingsKeys.EXTRACT)# TODO : Resolve and pass log events IDpayload = {- SettingsKeys.RUN_ID: self.file_execution_id,+ SettingsKeys.RUN_ID: run_id,SettingsKeys.TOOL_SETTINGS: tool_settings,SettingsKeys.OUTPUTS: outputs,SettingsKeys.TOOL_ID: tool_id,@@ -122,10 +124,10 @@ class StructureTool(BaseTool):}# TODO: Need to split extraction and indexing# to avoid unwanted indexing- self.stream_log(f"Indexing document '{self.source_file_name}'")+ self.stream_log(f"Indexing document '{source_file_name}'")usage_kwargs: dict[Any, Any] = dict()- usage_kwargs[UsageKwargs.RUN_ID] = self.file_execution_id- usage_kwargs[UsageKwargs.FILE_NAME] = self.source_file_name+ usage_kwargs[SettingsKeys.RUN_ID] = run_id+ usage_kwargs[SettingsKeys.FILE_NAME] = source_file_nameprocess_text: Optional[Callable[[str], str]] = Nonetry:@@ -248,7 +250,7 @@ class StructureTool(BaseTool):if SettingsKeys.METADATA in structured_output_dict:structured_output_dict[SettingsKeys.METADATA][SettingsKeys.FILE_NAME- ] = self.source_file_name+ ] = source_file_nameif not summarize_as_source:metadata = structured_output_dict[SettingsKeys.METADATA]@@ -283,7 +285,8 @@ class StructureTool(BaseTool):# Write the translated text to output filetry:self.stream_log("Writing parsed output...")- output_path = Path(output_dir) / f"{Path(self.source_file_name).stem}.json"+ source_name = self.get_exec_metadata.get(MetadataKey.SOURCE_NAME)+ output_path = Path(output_dir) / f"{Path(source_name).stem}.json"if self.workflow_filestorage:self.workflow_filestorage.json_dump(path=output_path, data=structured_output_dict@@ -326,7 +329,7 @@ class StructureTool(BaseTool):vector_db_instance_id: str = tool_settings[SettingsKeys.VECTOR_DB]x2text_instance_id: str = tool_settings[SettingsKeys.X2TEXT_ADAPTER]summarize_prompt: str = tool_settings[SettingsKeys.SUMMARIZE_PROMPT]- run_id: str = usage_kwargs.get(UsageKwargs.RUN_ID)+ run_id: str = usage_kwargs.get(SettingsKeys.RUN_ID)extract_file_path = tool_data_dir / SettingsKeys.EXTRACTsummarize_file_path = tool_data_dir / SettingsKeys.SUMMARIZEdiff --git a/tools/text_extractor/requirements.txt b/tools/text_extractor/requirements.txtindex c65022c39..625789cc3 100644--- a/tools/text_extractor/requirements.txt+++ b/tools/text_extractor/requirements.txt@@ -1,6 +1,6 @@# Add your dependencies here# Required for all unstract tools-unstract-sdk~=v0.55.0rc2+unstract-sdk~=0.54.0rc12# Required for remote storage supports3fs[boto3]==2024.6.0diff --git a/tools/text_extractor/src/config/properties.json b/tools/text_extractor/src/config/properties.jsonindex 32293c88b..09718a7ef 100644--- a/tools/text_extractor/src/config/properties.json+++ b/tools/text_extractor/src/config/properties.json@@ -2,7 +2,7 @@"schemaVersion": "0.0.1","displayName": "Text Extractor","functionName": "text_extractor",- "toolVersion": "0.0.43",+ "toolVersion": "0.0.42","description": "The Text Extractor is a powerful tool designed to convert documents to its text form or Extract texts from documents","input": {"description": "Document"diff --git a/tools/text_extractor/src/main.py b/tools/text_extractor/src/main.pyindex ab51a2198..fe43a38ff 100644--- a/tools/text_extractor/src/main.py+++ b/tools/text_extractor/src/main.py@@ -3,7 +3,7 @@ import sysfrom pathlib import Pathfrom typing import Any-from unstract.sdk.constants import LogState, UsageKwargs+from unstract.sdk.constants import LogState, MetadataKeyfrom unstract.sdk.tool.base import BaseToolfrom unstract.sdk.tool.entrypoint import ToolEntrypointfrom unstract.sdk.x2txt import TextExtractionResult, X2Text@@ -44,23 +44,18 @@ class TextExtractor(BaseTool):None"""text_extraction_adapter_id = settings["extractorId"]+ source_name = self.get_exec_metadata.get(MetadataKey.SOURCE_NAME)self.stream_log(f"Extractor ID: {text_extraction_adapter_id} ""has been retrieved from settings.")- input_log = f"Processing file: \n\n`{self.source_file_name}`"+ input_log = f"Processing file: \n\n`{source_name}`"self.stream_update(input_log, state=LogState.INPUT_UPDATE)- usage_kwargs: dict[Any, Any] = dict()- usage_kwargs[UsageKwargs.RUN_ID] = self.file_execution_id- usage_kwargs[UsageKwargs.FILE_NAME] = self.source_file_name-text_extraction_adapter = X2Text(- tool=self,- adapter_instance_id=text_extraction_adapter_id,- usage_kwargs=usage_kwargs,+ tool=self, adapter_instance_id=text_extraction_adapter_id)self.stream_log("Text extraction adapter has been created successfully.")if self.workflow_filestorage:@@ -81,10 +76,8 @@ class TextExtractor(BaseTool):try:self.stream_log("Preparing to write the extracted text.")- if self.source_file_name:- output_path = (- Path(output_dir) / f"{Path(self.source_file_name).stem}.txt"- )+ if source_name:+ output_path = Path(output_dir) / f"{Path(source_name).stem}.txt"if self.workflow_filestorage:self.workflow_filestorage.write(path=output_path, mode="w", data=extracted_text@@ -96,7 +89,7 @@ class TextExtractor(BaseTool):self.stream_log("Tool output written successfully.")else:self.stream_error_and_exit(- "Error creating/writing output file: source_file_name not found"+ "Error creating/writing output file: source_name not found")except Exception as e:self.stream_error_and_exit(f"Error creating/writing output file: {e}")diff --git a/unstract/workflow-execution/src/unstract/workflow_execution/constants.py b/unstract/workflow-execution/src/unstract/workflow_execution/constants.pyindex af767f356..e570a3f00 100644--- a/unstract/workflow-execution/src/unstract/workflow_execution/constants.py+++ b/unstract/workflow-execution/src/unstract/workflow_execution/constants.py@@ -40,7 +40,6 @@ class MetaDataKey:SOURCE_HASH = "source_hash"WORKFLOW_ID = "workflow_id"EXECUTION_ID = "execution_id"- FILE_EXECUTION_ID = "file_execution_id"ORGANIZATION_ID = "organization_id"TOOL_METADATA = "tool_metadata"diff --git a/unstract/workflow-execution/src/unstract/workflow_execution/execution_file_handler.py b/unstract/workflow-execution/src/unstract/workflow_execution/execution_file_handler.pyindex e9d565cf1..29ae9662b 100644--- a/unstract/workflow-execution/src/unstract/workflow_execution/execution_file_handler.py+++ b/unstract/workflow-execution/src/unstract/workflow_execution/execution_file_handler.py@@ -103,9 +103,7 @@ class ExecutionFileHandler:raise ToolMetadataNotFound()return tool_metadata[-1]- def add_metadata_to_volume(- self, input_file_path: str, file_execution_id: str, source_hash: str- ) -> None:+ def add_metadata_to_volume(self, input_file_path: str, source_hash: str) -> None:"""Creating metadata for workflow. This method is responsible forcreating metadata for the workflow. It takes the input file path andthe source hash as parameters. The metadata is stored in a JSON file in@@ -113,7 +111,6 @@ class ExecutionFileHandler:Parameters:input_file_path (str): The path of the input file.- file_execution_id (str): Unique execution id for the file.source_hash (str): The hash value of the source/input file.Returns:@@ -130,7 +127,6 @@ class ExecutionFileHandler:MetaDataKey.ORGANIZATION_ID: str(self.organization_id),MetaDataKey.WORKFLOW_ID: str(self.workflow_id),MetaDataKey.EXECUTION_ID: str(self.execution_id),- MetaDataKey.FILE_EXECUTION_ID: str(file_execution_id),}if check_feature_flag_status(FeatureFlag.REMOTE_FILE_STORAGE):file_system = FileSystem(FileStorageType.WORKFLOW_EXECUTION)