LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
updated the tools to support sdk filestorage feature (#892)
ali6359adbaa868
diff --git a/tools/classifier/src/config/properties.json b/tools/classifier/src/config/properties.jsonindex 8a1d05278..7f29f2d7a 100644--- a/tools/classifier/src/config/properties.json+++ b/tools/classifier/src/config/properties.json@@ -2,7 +2,7 @@"schemaVersion": "0.0.1","displayName": "File Classifier","functionName": "classify",- "toolVersion": "0.0.40",+ "toolVersion": "0.0.41","description": "Classifies a file into a bin based on its contents","input": {"description": "File to be classified"diff --git a/tools/classifier/src/helper.py b/tools/classifier/src/helper.pyindex ffb94396d..5dae9dc34 100644--- a/tools/classifier/src/helper.py+++ b/tools/classifier/src/helper.py@@ -67,7 +67,7 @@ class ClassifierHelper:"""try:output_folder_bin = Path(self.output_dir) / classification- if hasattr(self.tool, "workflow_filestorage"):+ if self.tool.workflow_filestorage:output_file = output_folder_bin / source_nameself._copy_file(source_fs=self.tool.workflow_filestorage,@@ -163,7 +163,7 @@ class ClassifierHelper:"""self.tool.stream_log("Extracting text from file")try:- if hasattr(self.tool, "workflow_filestorage"):+ if self.tool.workflow_filestorage:text = self.tool.workflow_filestorage.read(path=file, mode="rb").decode("utf-8")diff --git a/tools/structure/src/config/properties.json b/tools/structure/src/config/properties.jsonindex 4a0429e97..46e024e92 100644--- a/tools/structure/src/config/properties.json+++ b/tools/structure/src/config/properties.json@@ -2,7 +2,7 @@"schemaVersion": "0.0.1","displayName": "Structure Tool","functionName": "structure_tool",- "toolVersion": "0.0.51",+ "toolVersion": "0.0.52","description": "This is a template tool which can answer set of input prompts designed in the Prompt Studio","input": {"description": "File that needs to be indexed and parsed for answers"diff --git a/tools/structure/src/main.py b/tools/structure/src/main.pyindex c4adade11..a1736c554 100644--- a/tools/structure/src/main.py+++ b/tools/structure/src/main.py@@ -97,7 +97,7 @@ class StructureTool(BaseTool):_, file_name = os.path.split(input_file)if summarize_as_source:file_name = SettingsKeys.SUMMARIZE- if hasattr(self, "workflow_filestorage"):+ if self.workflow_filestorage:tool_data_dir = Path(self.get_env_or_die(ToolEnv.EXECUTION_DATA_DIR))else:tool_data_dir = Path(self.get_env_or_die(SettingsKeys.TOOL_DATA_DIR))@@ -131,7 +131,6 @@ class StructureTool(BaseTool):f"Function to higlight context is not found. {PAID_FEATURE_MSG}",level=LogLevel.WARN,)- workflow_filestorage = getattr(self, "workflow_filestorage", None)if tool_settings[SettingsKeys.ENABLE_SINGLE_PASS_EXTRACTION]:index.index(tool_id=tool_id,@@ -147,8 +146,8 @@ class StructureTool(BaseTool):usage_kwargs=usage_kwargs,process_text=process_text,**(- {"fs": workflow_filestorage}- if workflow_filestorage is not None+ {"fs": self.workflow_filestorage}+ if self.workflow_filestorage is not Noneelse {}),)@@ -256,7 +255,7 @@ class StructureTool(BaseTool):self.stream_log("Writing parsed output...")source_name = self.get_exec_metadata.get(MetadataKey.SOURCE_NAME)output_path = Path(output_dir) / f"{Path(source_name).stem}.json"- if hasattr(self, "workflow_filestorage"):+ if self.workflow_filestorage:self.workflow_filestorage.json_dump(path=output_path, data=structured_output_dict)@@ -303,7 +302,7 @@ class StructureTool(BaseTool):summarize_file_path = tool_data_dir / SettingsKeys.SUMMARIZEsummarized_context = ""- if hasattr(self, "workflow_filestorage"):+ if self.workflow_filestorage:if self.workflow_filestorage.exists(summarize_file_path):summarized_context = self.workflow_filestorage.read(path=summarize_file_path, mode="r"@@ -313,7 +312,7 @@ class StructureTool(BaseTool):summarized_context = f.read()if not summarized_context:context = ""- if hasattr(self, "workflow_filestorage"):+ if self.workflow_filestorage:context = self.workflow_filestorage.read(path=extract_file_path, mode="r")@@ -344,7 +343,7 @@ class StructureTool(BaseTool):structure_output = json.loads(response[SettingsKeys.STRUCTURE_OUTPUT])summarized_context = structure_output.get(SettingsKeys.DATA, "")self.stream_log("Writing summarized context to a file")- if hasattr(self, "workflow_filestorage"):+ if self.workflow_filestorage:self.workflow_filestorage.write(path=summarize_file_path, mode="w", data=summarized_context)@@ -356,7 +355,6 @@ class StructureTool(BaseTool):summarize_file_hash: str = ToolUtils.get_hash_from_file(file_path=summarize_file_path)- workflow_filestorage = getattr(self, "workflow_filestorage", None)index.index(tool_id=tool_id,embedding_instance_id=embedding_instance_id,@@ -368,7 +366,9 @@ class StructureTool(BaseTool):chunk_overlap=0,usage_kwargs=usage_kwargs,**(- {"fs": workflow_filestorage} if workflow_filestorage is not None else {}+ {"fs": self.workflow_filestorage}+ if self.workflow_filestorage is not None+ else {}),)return summarize_file_hashdiff --git a/tools/text_extractor/src/config/properties.json b/tools/text_extractor/src/config/properties.jsonindex 6fcd878f7..123d0b52e 100644--- a/tools/text_extractor/src/config/properties.json+++ b/tools/text_extractor/src/config/properties.json@@ -2,7 +2,7 @@"schemaVersion": "0.0.1","displayName": "Text Extractor","functionName": "text_extractor",- "toolVersion": "0.0.39",+ "toolVersion": "0.0.40","description": "The Text Extractor is a powerful tool designed to convert documents to its text form or Extract texts from documents","input": {"description": "Document"diff --git a/tools/text_extractor/src/main.py b/tools/text_extractor/src/main.pyindex ba69c2dce..182154834 100644--- a/tools/text_extractor/src/main.py+++ b/tools/text_extractor/src/main.py@@ -73,7 +73,7 @@ class TextExtractor(BaseTool):self.stream_log("Preparing to write the extracted text.")if source_name:output_path = Path(output_dir) / f"{Path(source_name).stem}.txt"- if hasattr(self, "workflow_filestorage"):+ if self.workflow_filestorage:self.workflow_filestorage.write(path=output_path, mode="w", data=extracted_text)