Skip to content

LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows

updated the tools to support sdk filestorage feature (#892)

ali6359adbaa868
diff --git a/tools/classifier/src/config/properties.json b/tools/classifier/src/config/properties.json
index 8a1d05278..7f29f2d7a 100644
--- a/tools/classifier/src/config/properties.json
+++ b/tools/classifier/src/config/properties.json
@@ -2,7 +2,7 @@
"schemaVersion": "0.0.1",
"displayName": "File Classifier",
"functionName": "classify",
- "toolVersion": "0.0.40",
+ "toolVersion": "0.0.41",
"description": "Classifies a file into a bin based on its contents",
"input": {
"description": "File to be classified"
diff --git a/tools/classifier/src/helper.py b/tools/classifier/src/helper.py
index ffb94396d..5dae9dc34 100644
--- a/tools/classifier/src/helper.py
+++ b/tools/classifier/src/helper.py
@@ -67,7 +67,7 @@ class ClassifierHelper:
"""
try:
output_folder_bin = Path(self.output_dir) / classification
- if hasattr(self.tool, "workflow_filestorage"):
+ if self.tool.workflow_filestorage:
output_file = output_folder_bin / source_name
self._copy_file(
source_fs=self.tool.workflow_filestorage,
@@ -163,7 +163,7 @@ class ClassifierHelper:
"""
self.tool.stream_log("Extracting text from file")
try:
- if hasattr(self.tool, "workflow_filestorage"):
+ if self.tool.workflow_filestorage:
text = self.tool.workflow_filestorage.read(path=file, mode="rb").decode(
"utf-8"
)
diff --git a/tools/structure/src/config/properties.json b/tools/structure/src/config/properties.json
index 4a0429e97..46e024e92 100644
--- a/tools/structure/src/config/properties.json
+++ b/tools/structure/src/config/properties.json
@@ -2,7 +2,7 @@
"schemaVersion": "0.0.1",
"displayName": "Structure Tool",
"functionName": "structure_tool",
- "toolVersion": "0.0.51",
+ "toolVersion": "0.0.52",
"description": "This is a template tool which can answer set of input prompts designed in the Prompt Studio",
"input": {
"description": "File that needs to be indexed and parsed for answers"
diff --git a/tools/structure/src/main.py b/tools/structure/src/main.py
index c4adade11..a1736c554 100644
--- a/tools/structure/src/main.py
+++ b/tools/structure/src/main.py
@@ -97,7 +97,7 @@ class StructureTool(BaseTool):
_, file_name = os.path.split(input_file)
if summarize_as_source:
file_name = SettingsKeys.SUMMARIZE
- if hasattr(self, "workflow_filestorage"):
+ if self.workflow_filestorage:
tool_data_dir = Path(self.get_env_or_die(ToolEnv.EXECUTION_DATA_DIR))
else:
tool_data_dir = Path(self.get_env_or_die(SettingsKeys.TOOL_DATA_DIR))
@@ -131,7 +131,6 @@ class StructureTool(BaseTool):
f"Function to higlight context is not found. {PAID_FEATURE_MSG}",
level=LogLevel.WARN,
)
- workflow_filestorage = getattr(self, "workflow_filestorage", None)
if tool_settings[SettingsKeys.ENABLE_SINGLE_PASS_EXTRACTION]:
index.index(
tool_id=tool_id,
@@ -147,8 +146,8 @@ class StructureTool(BaseTool):
usage_kwargs=usage_kwargs,
process_text=process_text,
**(
- {"fs": workflow_filestorage}
- if workflow_filestorage is not None
+ {"fs": self.workflow_filestorage}
+ if self.workflow_filestorage is not None
else {}
),
)
@@ -256,7 +255,7 @@ class StructureTool(BaseTool):
self.stream_log("Writing parsed output...")
source_name = self.get_exec_metadata.get(MetadataKey.SOURCE_NAME)
output_path = Path(output_dir) / f"{Path(source_name).stem}.json"
- if hasattr(self, "workflow_filestorage"):
+ if self.workflow_filestorage:
self.workflow_filestorage.json_dump(
path=output_path, data=structured_output_dict
)
@@ -303,7 +302,7 @@ class StructureTool(BaseTool):
summarize_file_path = tool_data_dir / SettingsKeys.SUMMARIZE
summarized_context = ""
- if hasattr(self, "workflow_filestorage"):
+ if self.workflow_filestorage:
if self.workflow_filestorage.exists(summarize_file_path):
summarized_context = self.workflow_filestorage.read(
path=summarize_file_path, mode="r"
@@ -313,7 +312,7 @@ class StructureTool(BaseTool):
summarized_context = f.read()
if not summarized_context:
context = ""
- if hasattr(self, "workflow_filestorage"):
+ if self.workflow_filestorage:
context = self.workflow_filestorage.read(
path=extract_file_path, mode="r"
)
@@ -344,7 +343,7 @@ class StructureTool(BaseTool):
structure_output = json.loads(response[SettingsKeys.STRUCTURE_OUTPUT])
summarized_context = structure_output.get(SettingsKeys.DATA, "")
self.stream_log("Writing summarized context to a file")
- if hasattr(self, "workflow_filestorage"):
+ if self.workflow_filestorage:
self.workflow_filestorage.write(
path=summarize_file_path, mode="w", data=summarized_context
)
@@ -356,7 +355,6 @@ class StructureTool(BaseTool):
summarize_file_hash: str = ToolUtils.get_hash_from_file(
file_path=summarize_file_path
)
- workflow_filestorage = getattr(self, "workflow_filestorage", None)
index.index(
tool_id=tool_id,
embedding_instance_id=embedding_instance_id,
@@ -368,7 +366,9 @@ class StructureTool(BaseTool):
chunk_overlap=0,
usage_kwargs=usage_kwargs,
**(
- {"fs": workflow_filestorage} if workflow_filestorage is not None else {}
+ {"fs": self.workflow_filestorage}
+ if self.workflow_filestorage is not None
+ else {}
),
)
return summarize_file_hash
diff --git a/tools/text_extractor/src/config/properties.json b/tools/text_extractor/src/config/properties.json
index 6fcd878f7..123d0b52e 100644
--- a/tools/text_extractor/src/config/properties.json
+++ b/tools/text_extractor/src/config/properties.json
@@ -2,7 +2,7 @@
"schemaVersion": "0.0.1",
"displayName": "Text Extractor",
"functionName": "text_extractor",
- "toolVersion": "0.0.39",
+ "toolVersion": "0.0.40",
"description": "The Text Extractor is a powerful tool designed to convert documents to its text form or Extract texts from documents",
"input": {
"description": "Document"
diff --git a/tools/text_extractor/src/main.py b/tools/text_extractor/src/main.py
index ba69c2dce..182154834 100644
--- a/tools/text_extractor/src/main.py
+++ b/tools/text_extractor/src/main.py
@@ -73,7 +73,7 @@ class TextExtractor(BaseTool):
self.stream_log("Preparing to write the extracted text.")
if source_name:
output_path = Path(output_dir) / f"{Path(source_name).stem}.txt"
- if hasattr(self, "workflow_filestorage"):
+ if self.workflow_filestorage:
self.workflow_filestorage.write(
path=output_path, mode="w", data=extracted_text
)