LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
fix: Perform checks in classifier after LLM response, bumped to 0.0.29 (#521)
Chandrasekharan Mdeb266d3e8d9
diff --git a/tools/classifier/requirements.txt b/tools/classifier/requirements.txtindex 225c3d2c0..4002066b4 100644--- a/tools/classifier/requirements.txt+++ b/tools/classifier/requirements.txt@@ -1,4 +1,4 @@# Add your dependencies here# Required for all unstract tools-unstract-sdk~=0.38.1+unstract-sdk~=0.39.0diff --git a/tools/classifier/src/config/properties.json b/tools/classifier/src/config/properties.jsonindex a46b2c91c..e268639b0 100644--- a/tools/classifier/src/config/properties.json+++ b/tools/classifier/src/config/properties.json@@ -2,7 +2,7 @@"schemaVersion": "0.0.1","displayName": "File Classifier","functionName": "classify",- "toolVersion": "0.0.28",+ "toolVersion": "0.0.29","description": "Classifies a file into a bin based on its contents","input": {"description": "File to be classified"diff --git a/tools/classifier/src/config/runtime_variables.json b/tools/classifier/src/config/runtime_variables.jsonindex 8f4ef8f50..3f72970c3 100644--- a/tools/classifier/src/config/runtime_variables.json+++ b/tools/classifier/src/config/runtime_variables.json@@ -2,39 +2,6 @@"title": "Runtime Variables","description": "Runtime Variables for classifier","type": "object",- "required": [- "OPENAI_API_KEY",- "OPENAI_API_BASE",- "OPENAI_API_MODEL",- "OPENAI_API_ENGINE",- "OPENAI_API_VERSION"- ],- "properties": {- "OPENAI_API_KEY": {- "type": "string",- "title": "OpenAI API Key",- "description": "Your OpenAI API key"- },- "OPENAI_API_BASE": {- "type": "string",- "title": "OpenAI API Base URL",- "description": "The base URL for the OpenAI API"- },- "OPENAI_API_MODEL": {- "type": "string",- "title": "OpenAI API Model",- "description": "The OpenAI model to use"- },- "OPENAI_API_ENGINE": {- "type": "string",- "title": "OpenAI API Engine",- "description": "The OpenAI engine to use"- },- "OPENAI_API_VERSION": {- "type": "string",- "title": "OpenAI API Version",- "description": "The OpenAI API version to use",- "default": "2023-05-15"- }- }+ "required": [],+ "properties": {}}diff --git a/tools/classifier/src/config/spec.json b/tools/classifier/src/config/spec.jsonindex 6f8d1619d..47d837445 100644--- a/tools/classifier/src/config/spec.json+++ b/tools/classifier/src/config/spec.json@@ -9,7 +9,7 @@"classificationBins": {"type": "array","title": "Classification bins",- "description": "Specify at least two unique classification bins.",+ "description": "Specify at least two unique classification bins. 'unknown' and '__unstract_failed' are reserved bins. 'unknown' indicates the LLM can't determine the classification and `__unstract_failed` indicates a tool run failure for the given file.","items": {"type": "string"},diff --git a/tools/classifier/src/helper.py b/tools/classifier/src/helper.pyindex 488b67b8f..ed1722b46 100644--- a/tools/classifier/src/helper.py+++ b/tools/classifier/src/helper.py@@ -1,18 +1,79 @@+import re+import shutil+from pathlib import Pathfrom typing import Any, Optionalfrom unstract.sdk.cache import ToolCache-from unstract.sdk.constants import ToolEnv+from unstract.sdk.constants import LogLevel, MetadataKey, ToolEnvfrom unstract.sdk.llm import LLMfrom unstract.sdk.tool.base import BaseToolfrom unstract.sdk.utils import ToolUtilsfrom unstract.sdk.x2txt import TextExtractionResult, X2Text+class ReservedBins:+ UNKNOWN = "unknown"+ FAILED = "__unstract_failed"++class ClassifierHelper:"""Helper functions for Classifier."""- def __init__(self, tool: BaseTool) -> None:+ def __init__(self, tool: BaseTool, output_dir: str) -> None:+ """Creates a helper class for the Classifier tool.++ Args:+ tool (BaseTool): Base tool instance+ output_dir (str): Output directory in TOOL_DATA_DIR+ """self.tool = tool+ self.output_dir = output_dir++ def stream_error_and_exit(+ self, message: str, bin_to_copy_to: str = ReservedBins.FAILED+ ) -> None:+ """Streams error logs and performs required cleanup.++ Helper which copies files to a reserved bin in case of an error.+ Args:+ message (str): Error message to log+ bin_to_copy_to (str): The folder to copy the failed source file to.+ Defaults to `__unstract_failed`.+ input_file (Optional[str], optional): Input file to copy. Defaults to None.+ output_dir (Optional[str], optional): Output directory to copy to.+ Defaults to None.+ """+ source_name = self.tool.get_exec_metadata.get(MetadataKey.SOURCE_NAME)+ self.copy_source_to_output_bin(+ classification=bin_to_copy_to,+ source_file=self.tool.get_source_file(),+ source_name=source_name,+ )++ self.tool.stream_error_and_exit(message=message)++ def copy_source_to_output_bin(+ self,+ classification: str,+ source_file: str,+ source_name: str,+ ) -> None:+ """Method to save result in output folder and the data directory.++ Args:+ classification (str): classification result+ source_file (str): Path to source file used in the workflow+ source_name (str): Name of the actual input from the source+ """+ try:+ output_folder_bin = Path(self.output_dir) / classification+ if not output_folder_bin.is_dir():+ output_folder_bin.mkdir(parents=True, exist_ok=True)++ output_file = output_folder_bin / source_name+ shutil.copyfile(source_file, output_file)+ except Exception as e:+ self.tool.stream_error_and_exit(f"Error creating output file: {e}")def extract_text(self, file: str, text_extraction_adapter_id: Optional[str]@@ -106,22 +167,20 @@ class ClassifierHelper:f"{ToolUtils.hash_str(settings_string)}:"f"{ToolUtils.hash_str(prompt)}")+ self.tool.stream_log("Trying to fetch result from cache.")classification = self.get_result_from_cache(cache_key=cache_key)+ if classification is not None:+ return classification- if classification is None:- self.tool.stream_log("No classification found in cache, calling LLM.")- classification = self.call_llm(prompt=prompt, llm=llm)- if not classification:- classification = "unknown"- classification = classification.strip().lower()- bins = [bin.lower() for bin in bins]- if classification not in bins:- self.tool.stream_error_and_exit(- f"Invalid classification done: {classification}"- )+ self.tool.stream_log("No classification found in cache, calling LLM.")+ llm_response = self.call_llm(prompt=prompt, llm=llm)+ classification = self.clean_llm_response(llm_response=llm_response, bins=bins)+ if use_cache and cache_key:+ self.tool.stream_log("Saving result to cache.")+ self.save_result_to_cache(cache_key=cache_key, result=classification)return classification- def call_llm(self, prompt: str, llm: LLM) -> Optional[str]:+ def call_llm(self, prompt: str, llm: LLM) -> str:"""Call LLM.Args:@@ -134,11 +193,54 @@ class ClassifierHelper:try:completion = llm.complete(prompt)[LLM.RESPONSE]classification: str = completion.text.strip()- self.tool.stream_log(f"LLM response: {completion}")+ self.tool.stream_log(f"LLM response: {completion}", level=LogLevel.DEBUG)return classificationexcept Exception as e:- self.tool.stream_error_and_exit(f"Error calling LLM {e}")- return None+ self.stream_error_and_exit(f"Error calling LLM: {e}")+ raise e++ def clean_llm_response(self, llm_response: str, bins: list[str]) -> str:+ """Cleans the response from the LLM.++ Performs a substring search to find the returned classification.+ Treats it as `unknown` if the classification is not clear+ from the output.++ Args:+ llm_response (str): Response from LLM to clean+ bins (list(str)): List of bins to classify the file into.++ Returns:+ str: Cleaned classification that matches one of the bins.+ """+ classification = ReservedBins.UNKNOWN+ cleaned_response = llm_response.strip().lower()+ bins = [bin.lower() for bin in bins]++ # Truncate llm_response to the first 100 words+ words = cleaned_response.split()+ truncated_response = " ".join(words[:100])++ # Count occurrences of each bin in the truncated text+ bin_counts = {+ bin: len(re.findall(r"\b" + re.escape(bin) + r"\b", truncated_response))+ for bin in bins+ }++ # Filter bins that have a count greater than 0+ matching_bins = [bin for bin, count in bin_counts.items() if count > 0]++ # Determine classification based on the number of matching bins+ if len(matching_bins) == 1:+ classification = matching_bins[0]+ else:+ self.stream_error_and_exit(+ f"Unable to deduce classified bin from possible values of "+ f"'{matching_bins}', moving file to '{ReservedBins.UNKNOWN}' "+ "folder instead.",+ bin_to_copy_to=ReservedBins.UNKNOWN,+ )+ return classificationdef get_result_from_cache(self, cache_key: str) -> Optional[str]:"""Get result from cache.diff --git a/tools/classifier/src/main.py b/tools/classifier/src/main.pyindex e22f88ef1..5d12a379e 100644--- a/tools/classifier/src/main.py+++ b/tools/classifier/src/main.py@@ -1,9 +1,8 @@-import shutilimport sys-from pathlib import Pathfrom typing import Any, Optionalfrom helper import ClassifierHelper # type: ignore+from helper import ReservedBinsfrom unstract.sdk.constants import LogLevel, LogState, MetadataKey, ToolSettingsKeyfrom unstract.sdk.exceptions import SdkErrorfrom unstract.sdk.llm import LLM@@ -14,21 +13,26 @@ from unstract.sdk.tool.entrypoint import ToolEntrypointclass UnstractClassifier(BaseTool):def __init__(self, log_level: str = LogLevel.INFO) -> None:super().__init__(log_level)- self.helper = ClassifierHelper(tool=self)def validate(self, input_file: str, settings: dict[str, Any]) -> None:bins: Optional[list[str]] = settings.get("classificationBins")llm_adapter_instance_id = settings.get(ToolSettingsKey.LLM_ADAPTER_ID)text_extraction_adapter_id = settings.get("textExtractorId")if not bins:- self.stream_error_and_exit("Classification bins are required")+ self.stream_error_and_exit("Classification bins are required.")elif len(bins) < 2:- self.stream_error_and_exit("At least two bins are required")+ self.stream_error_and_exit("At least two classification bins are required.")+ elif ReservedBins.UNKNOWN in bins:+ self.stream_error_and_exit(+ f"Classification bin '{ReservedBins.UNKNOWN}' is reserved to mark "+ "files which cannot be classified."+ )+if not llm_adapter_instance_id:- self.stream_error_and_exit("Choose an LLM to process the classifier")+ self.stream_error_and_exit("Choose an LLM to perform the classification.")if not text_extraction_adapter_id:self.stream_error_and_exit(- "Choose a text extractor to extract the documents"+ "Choose a text extractor to extract the documents.")def run(@@ -41,6 +45,7 @@ class UnstractClassifier(BaseTool):use_cache = settings["useCache"]text_extraction_adapter_id = settings["textExtractorId"]llm_adapter_instance_id = settings[ToolSettingsKey.LLM_ADAPTER_ID]+ self.helper = ClassifierHelper(tool=self, output_dir=output_dir)# Update GUIinput_log = f"### Classification bins:\n```text\n{bins}\n```\n\n"@@ -54,7 +59,7 @@ class UnstractClassifier(BaseTool):text_extraction_adapter_id=text_extraction_adapter_id,)if not text:- self.stream_error_and_exit("Unable to extract text")+ self.helper.stream_error_and_exit("Unable to extract text")returnself.stream_log(f"Text length: {len(text)}")@@ -70,8 +75,8 @@ class UnstractClassifier(BaseTool):self.stream_update(input_log, state=LogState.INPUT_UPDATE)self.stream_update(output_log, state=LogState.OUTPUT_UPDATE)- if "unknown" not in bins:- bins.append("unknown")+ if ReservedBins.UNKNOWN not in bins:+ bins.append(ReservedBins.UNKNOWN)bins_with_quotes = [f"'{b}'" for b in bins]usage_kwargs: dict[Any, Any] = dict()@@ -85,7 +90,8 @@ class UnstractClassifier(BaseTool):usage_kwargs=usage_kwargs,)except SdkError:- self.stream_error_and_exit("Unable to get llm instance")+ self.helper.stream_error_and_exit("Unable to get llm instance")+ returnmax_tokens = llm.get_max_tokens(reserved_for_output=50 + 1000)max_bytes = int(max_tokens * 1.3)@@ -103,8 +109,8 @@ class UnstractClassifier(BaseTool):f"Classify the following text into one of the following categories: {' '.join(bins_with_quotes)}.\n\n" # noqa: E501"Your categorization should be strictly exactly one of the items in the " # noqa: E501"categories given, do not provide any explanation. Find a semantic match of category if possible. " # noqa: E501- "If it does not categorize well into any of the listed categories, categorize it as 'unknown'.\n\n" # noqa: E501- f"Text:\n\n{text}\n\n\nCategory:"+ "If it does not categorize well into any of the listed categories, categorize it as 'unknown'." # noqa: E501+ f"Do not enclose the result within single quotes.\n\nText:\n\n{text}\n\n\nCategory:" # noqa: E501)settings_string = "".join(str(value) for value in settings.values())@@ -117,8 +123,7 @@ class UnstractClassifier(BaseTool):)source_name = self.get_exec_metadata.get(MetadataKey.SOURCE_NAME)- self._copy_input_to_output_bin(- output_dir=output_dir,+ self.helper.copy_source_to_output_bin(classification=classification,source_file=self.get_source_file(),source_name=source_name,@@ -134,31 +139,6 @@ class UnstractClassifier(BaseTool):}self.write_tool_result(data=classification_dict)- def _copy_input_to_output_bin(- self,- output_dir: str,- classification: str,- source_file: str,- source_name: str,- ) -> None:- """Method to save result in output folder and the data directory.-- Args:- output_dir (str): Output directory in TOOL_DATA_DIR- classification (str): classification result- source_file (str): Path to source file used in the workflow- source_name (str): Name of the actual input from the source- """- try:- output_folder_bin = Path(output_dir) / classification- if not output_folder_bin.is_dir():- output_folder_bin.mkdir(parents=True, exist_ok=True)-- output_file = output_folder_bin / source_name- shutil.copyfile(source_file, output_file)- except Exception as e:- self.stream_error_and_exit(f"Error creating output file: {e}")-if __name__ == "__main__":args = sys.argv[1:]diff --git a/tools/structure/requirements.txt b/tools/structure/requirements.txtindex 225c3d2c0..4002066b4 100644--- a/tools/structure/requirements.txt+++ b/tools/structure/requirements.txt@@ -1,4 +1,4 @@# Add your dependencies here# Required for all unstract tools-unstract-sdk~=0.38.1+unstract-sdk~=0.39.0diff --git a/tools/structure/src/config/properties.json b/tools/structure/src/config/properties.jsonindex 0e7400f0a..c45f18c62 100644--- a/tools/structure/src/config/properties.json+++ b/tools/structure/src/config/properties.json@@ -2,7 +2,7 @@"schemaVersion": "0.0.1","displayName": "Structure Tool","functionName": "structure_tool",- "toolVersion": "0.0.32",+ "toolVersion": "0.0.33","description": "This is a template tool which can answer set of input prompts designed in the Prompt Studio","input": {"description": "File that needs to be indexed and parsed for answers"diff --git a/tools/text_extractor/requirements.txt b/tools/text_extractor/requirements.txtindex 225c3d2c0..4002066b4 100644--- a/tools/text_extractor/requirements.txt+++ b/tools/text_extractor/requirements.txt@@ -1,4 +1,4 @@# Add your dependencies here# Required for all unstract tools-unstract-sdk~=0.38.1+unstract-sdk~=0.39.0diff --git a/tools/text_extractor/src/config/properties.json b/tools/text_extractor/src/config/properties.jsonindex dcca4ca14..edafbad92 100644--- a/tools/text_extractor/src/config/properties.json+++ b/tools/text_extractor/src/config/properties.json@@ -2,7 +2,7 @@"schemaVersion": "0.0.1","displayName": "Text Extractor","functionName": "text_extractor",- "toolVersion": "0.0.26",+ "toolVersion": "0.0.27","description": "The Text Extractor is a powerful tool designed to convert documents to its text form or Extract texts from documents","input": {"description": "Document"