Skip to content

LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows

updated classifier tool with latest sdk version (#152)

alid8f5124753cc
diff --git a/backend/tool_instance/tool_instance_helper.py b/backend/tool_instance/tool_instance_helper.py
index 07b59b03c..fd3ce63b5 100644
--- a/backend/tool_instance/tool_instance_helper.py
+++ b/backend/tool_instance/tool_instance_helper.py
@@ -367,7 +367,7 @@ class ToolInstanceHelper:
user: User, tool_uid: str, tool_meta: dict[str, Any]
) -> None:
tool: Tool = ToolProcessor.get_tool_by_uid(tool_uid=tool_uid)
- adapter_ids = set()
+ adapter_ids: set[str] = set()
for llm in tool.properties.adapter.language_models:
if llm.is_enabled and llm.adapter_id:
@@ -413,7 +413,7 @@ class ToolInstanceHelper:
@staticmethod
def validate_adapter_access(
user: User,
- adapter_ids: str,
+ adapter_ids: list[str],
) -> None:
adapter_instances = AdapterInstance.objects.filter(
id__in=adapter_ids
diff --git a/tools/classifier/Dockerfile b/tools/classifier/Dockerfile
index 4c95f3d4d..a8b1f90a5 100644
--- a/tools/classifier/Dockerfile
+++ b/tools/classifier/Dockerfile
@@ -4,7 +4,7 @@ LABEL maintainer="Zipstack Inc."
ENV UNSTRACT_ENTRYPOINT "python /app/src/main.py"
# Install dependencies for unstructured library's partition
-RUN apt-get update && apt-get --no-install-recommends -y install libmagic-dev poppler-utils tesseract-ocr libreoffice pandoc\
+RUN apt-get update && apt-get --no-install-recommends -y install libmagic-dev poppler-utils\
&& apt-get clean \
&& rm -rf /var/lib/apt/lists/*
diff --git a/tools/classifier/requirements.txt b/tools/classifier/requirements.txt
index 2aa4cfe0d..7ecf41515 100644
--- a/tools/classifier/requirements.txt
+++ b/tools/classifier/requirements.txt
@@ -1,6 +1,4 @@
# Add your dependencies here
-filetype==1.2.0
-unstructured[all-docs]==0.10.10
# Required for all unstract tools
-unstract-sdk~=0.10.0
+unstract-sdk~=0.15.2
diff --git a/tools/classifier/src/config/properties.json b/tools/classifier/src/config/properties.json
index 38b425fef..b1139b9ae 100644
--- a/tools/classifier/src/config/properties.json
+++ b/tools/classifier/src/config/properties.json
@@ -2,7 +2,7 @@
"schemaVersion": "0.0.1",
"displayName": "File Classifier",
"functionName": "classify",
- "toolVersion": "0.0.1",
+ "toolVersion": "0.0.2",
"description": "Classifies a file into a bin based on its contents",
"input": {
"description": "File to be classified"
@@ -33,6 +33,15 @@
{
"isEnabled": false
}
+ ],
+ "textExtractors": [
+ {
+ "isEnabled": true,
+ "adapterId": "textExtractorId",
+ "title": "Text Extraction Adapter",
+ "isRequired": false,
+ "description": "Choose extractor adapter to extract text"
+ }
]
},
"ioCompatibility": {
diff --git a/tools/classifier/src/helper.py b/tools/classifier/src/helper.py
index 6f31d881b..c75ede6f5 100644
--- a/tools/classifier/src/helper.py
+++ b/tools/classifier/src/helper.py
@@ -1,7 +1,11 @@
-import io
+from typing import Any, Optional
+from llama_index.llms import LLM
+from unstract.sdk.cache import ToolCache
+from unstract.sdk.constants import ToolEnv
from unstract.sdk.tool.base import BaseTool
-from unstructured.partition.auto import partition
+from unstract.sdk.utils import ToolUtils
+from unstract.sdk.x2txt import X2Text
class ClassifierHelper:
@@ -10,7 +14,9 @@ class ClassifierHelper:
def __init__(self, tool: BaseTool) -> None:
self.tool = tool
- def extract_text(self, file: str) -> str:
+ def extract_text(
+ self, file: str, text_extraction_adapter_id: Optional[str]
+ ) -> Optional[str]:
"""Extract text from file.
Args:
@@ -19,11 +25,159 @@ class ClassifierHelper:
Returns:
str: page content
"""
+ if not text_extraction_adapter_id:
+ return self._extract_from_file(file)
+
+ return self._extract_from_adapter(file, text_extraction_adapter_id)
+
+ def _extract_from_adapter(
+ self, file: str, adapter_id: str
+ ) -> Optional[str]:
+ """Extract text from adapter.
+
+ Args:
+ file: The path to the input file
+ adapter_id: The id of the adapter
+ Returns:
+ str: page content
+ """
+ self.tool.stream_log(
+ "Creating text extraction adapter "
+ f"using adapter_id: {adapter_id}"
+ )
+ tool_extraction = X2Text(tool=self.tool)
+ adapter = tool_extraction.get_x2text(adapter_id)
+
+ self.tool.stream_log(
+ "Text extraction adapter has been created successfully."
+ )
+ self.tool.stream_log("Adapter created")
+
+ try:
+ extracted_text: str = adapter.process(input_file_path=file)
+ return extracted_text
+ except Exception as e:
+ self.tool.stream_log(f"Adapter error: {e}")
+ return None
+
+ def _extract_from_file(self, file: str) -> Optional[str]:
+ """Extract text from file.
+
+ Args:
+ file: The path to the input file
+ Returns:
+ str: page content
+ """
+ self.tool.stream_log("Extracting text from file")
try:
- with open(file, mode="rb") as input_file_obj:
- bytes_io = io.BytesIO(input_file_obj.read())
- elements = partition(file=bytes_io)
+ with open(file, "rb") as f:
+ text = f.read().decode("utf-8")
except Exception as e:
- self.tool.stream_error_and_exit(f"Error partitioning file: {e}")
- text = "\n\n".join([str(el) for el in elements])
+ self.tool.stream_log(f"File error: {e}")
+ return None
+
+ self.tool.stream_log("Text extracted from file")
return text
+
+ def find_classification(
+ self,
+ use_cache: bool,
+ settings_string: str,
+ bins: list[str],
+ prompt: str,
+ llm: LLM,
+ ) -> Optional[str]:
+ """Find classification for text.
+ Args:
+ use_cache (bool): Whether to use cache
+ settings_string (str): hash of settings
+ prompt (str): Prompt
+ bins (list[str]): Classification Bins
+ llm (LLM): LLM
+
+ Returns:
+ Optional[str]: Classification or None if not found in cache.
+ If found in cache, returns the classification from the cache.
+ If not found in cache, calls the LLM and returns the
+ classification.
+ """
+ classification = None
+ cache_key = None
+ if use_cache:
+ cache_key = (
+ f"cache:{self.tool.workflow_id}:"
+ f"{ToolUtils.hash_str(settings_string)}:"
+ f"{ToolUtils.hash_str(prompt)}"
+ )
+ classification = self.get_result_from_cache(cache_key=cache_key)
+
+ if classification is None:
+ self.tool.stream_log(
+ "No classification found in cache, calling LLM."
+ )
+ classification = self.call_llm(prompt=prompt, llm=llm)
+ if not classification:
+ classification = "unknown"
+ classification = classification.strip().lower()
+ bins = [bin.lower() for bin in bins]
+ if classification not in bins:
+ self.tool.stream_error_and_exit(
+ f"Invalid classification done: {classification}"
+ )
+ return classification
+
+ def call_llm(self, prompt: str, llm: LLM) -> Optional[str]:
+ """Call LLM.
+
+ Args:
+ prompt (str): Prompt
+ llm (LLM): LLM
+
+ Returns:
+ str: Classification
+ """
+ try:
+ response = llm.complete(prompt, max_tokens=50, stop=["\n"])
+ classification: str = response.text.strip()
+ self.tool.stream_log(f"LLM response: {response}")
+ return classification
+ except Exception as e:
+ self.tool.stream_error_and_exit(f"Error calling LLM {e}")
+ return None
+
+ def get_result_from_cache(self, cache_key: str) -> Optional[str]:
+ """Get result from cache.
+
+ Args:
+ cache_key (str): key
+
+ Returns:
+ Optional[str]: result
+ """
+ cache = ToolCache(
+ tool=self.tool,
+ platform_host=self.tool.get_env_or_die(ToolEnv.PLATFORM_HOST),
+ platform_port=int(self.tool.get_env_or_die(ToolEnv.PLATFORM_PORT)),
+ )
+ cached_response: Optional[str] = cache.get(cache_key)
+ if cached_response is not None:
+ classification = cached_response
+ self.tool.stream_cost(cost=0.0, cost_units="cache")
+ return classification
+ return None
+
+ def save_result_to_cache(self, cache_key: str, result: Any) -> None:
+ """Save result to cache.
+
+ Args:
+ cache_key (str): key
+ result (Any): result to save in cache
+ Returns:
+ None: None
+ """
+ cache = ToolCache(
+ tool=self.tool,
+ platform_host=self.tool.get_env_or_die(ToolEnv.PLATFORM_HOST),
+ platform_port=int(self.tool.get_env_or_die(ToolEnv.PLATFORM_PORT)),
+ )
+ cache.set(cache_key, result)
diff --git a/tools/classifier/src/main.py b/tools/classifier/src/main.py
index 2eafa0d8c..77d482cef 100644
--- a/tools/classifier/src/main.py
+++ b/tools/classifier/src/main.py
@@ -1,18 +1,18 @@
import shutil
import sys
from pathlib import Path
-from typing import Any
-
-from llama_index import set_global_service_context
-from unstract.sdk.cache import ToolCache
-from unstract.sdk.constants import LogLevel, LogState, MetadataKey, ToolEnv
+from typing import Any, Optional
+
+from helper import ClassifierHelper # type: ignore
+from unstract.sdk.constants import (
+ LogLevel,
+ LogState,
+ MetadataKey,
+ ToolSettingsKey,
+)
from unstract.sdk.llm import ToolLLM
from unstract.sdk.tool.base import BaseTool
from unstract.sdk.tool.entrypoint import ToolEntrypoint
-from unstract.sdk.utils import ToolUtils
-from unstract.sdk.utils.service_context import ServiceContext
-
-from .helper import ClassifierHelper
class UnstractClassifier(BaseTool):
@@ -21,10 +21,14 @@ class UnstractClassifier(BaseTool):
self.helper = ClassifierHelper(tool=self)
def validate(self, input_file: str, settings: dict[str, Any]) -> None:
- bins = settings["classificationBins"]
-
- if len(bins) < 2:
+ bins: Optional[list[str]] = settings.get("classificationBins")
+ llm_adapter_instance_id = settings.get(ToolSettingsKey.LLM_ADAPTER_ID)
+ if not bins:
+ self.stream_error_and_exit("Classification bins are required")
+ elif len(bins) < 2:
self.stream_error_and_exit("At least two bins are required")
+ if not llm_adapter_instance_id:
+ self.stream_error_and_exit("Choose a llm to process the classifier")
def run(
self,
@@ -34,6 +38,8 @@ class UnstractClassifier(BaseTool):
) -> None:
bins = settings["classificationBins"]
use_cache = settings["useCache"]
+ text_extraction_adapter_id = settings["textExtractorId"]
+ llm_adapter_instance_id = settings[ToolSettingsKey.LLM_ADAPTER_ID]
# Update GUI
input_log = f"### Classification bins:\n```text\n{bins}\n```\n\n"
@@ -41,14 +47,20 @@ class UnstractClassifier(BaseTool):
self.stream_update(input_log, state=LogState.INPUT_UPDATE)
self.stream_update(output_log, state=LogState.OUTPUT_UPDATE)
- self.stream_log("Reading file...")
- text = self.helper.extract_text(input_file)
+ self.stream_log(f"Reading file... {input_file}")
+ text: Optional[str] = self.helper.extract_text(
+ file=input_file,
+ text_extraction_adapter_id=text_extraction_adapter_id,
+ )
+ if not text:
+ self.stream_error_and_exit("Unable to extract text")
+ return
self.stream_log(f"Text length: {len(text)}")
# Update GUI
input_text_for_log = text
if len(input_text_for_log) > 500:
- input_text_for_log = input_text_for_log[:500] + "...(truncated)"
+ input_text_for_log = input_text_for_log[:50000] + "...(truncated)"
input_log = (
f"### Classification bins:\n```text\n{bins}\n```\n\n"
f"### Input text:\n\n```text\n{input_text_for_log}\n```\n\n"
@@ -61,17 +73,10 @@ class UnstractClassifier(BaseTool):
bins.append("unknown")
bins_with_quotes = [f"'{b}'" for b in bins]
- tool_llm = ToolLLM(tool=self, tool_settings=settings)
- llm = tool_llm.get_llm()
+ tool_llm = ToolLLM(tool=self)
+ llm = tool_llm.get_llm(adapter_instance_id=llm_adapter_instance_id)
if not llm:
self.stream_error_and_exit("Unable to get llm instance")
- service_context = ServiceContext.get_service_context(
- platform_api_key=self.get_env_or_die(ToolEnv.PLATFORM_API_KEY),
- llm=llm,
- workflow_id=self.workflow_id,
- execution_id=self.execution_id,
- )
- set_global_service_context(service_context)
max_tokens = tool_llm.get_max_tokens(reserved_for_output=50 + 1000)
max_bytes = int(max_tokens * 1.3)
@@ -93,60 +98,19 @@ class UnstractClassifier(BaseTool):
prompt = (
f"Classify the following text into one of the following categories: {' '.join(bins_with_quotes)}.\n\n" # noqa: E501
f"Your categorization should be strictly exactly one of the items in the " # noqa: E501
- f"Your categorization should be strictly exactly one of the items in the " # noqa: E501
f"categories given. Find a semantic match of category if possible. If it does not categorize well " # noqa: E501
f"into any of the listed categories, categorize it as 'unknown'.\n\nText:\n\n{text}\n\n\nCategory:" # noqa: E501
)
settings_string = "".join(str(value) for value in settings.values())
- cache_key = (
- f"cache:{self.workflow_id}:{ToolUtils.hash_str(settings_string)}:"
- f"{ToolUtils.hash_str(prompt)}"
+ classification = self.helper.find_classification(
+ use_cache=use_cache,
+ settings_string=settings_string,
+ prompt=prompt,
+ bins=bins,
+ llm=llm,
)
- classification = None
- if use_cache:
- cache = ToolCache(
- tool=self,
- platform_host=self.get_env_or_die(ToolEnv.PLATFORM_HOST),
- platform_port=int(self.get_env_or_die(ToolEnv.PLATFORM_PORT)),
- )
- cached_response = cache.get(cache_key)
- if cached_response is not None:
- classification = cached_response
- self.stream_cost(cost=0.0, cost_units="cache")
-
- if classification is None:
- self.stream_log("Calling LLM")
- try:
- response = llm.complete(prompt, max_tokens=50, stop=["\n"])
- if response is None:
- self.stream_error_and_exit("Error calling LLM")
- classification = response.text.strip()
- self.stream_log(f"LLM response: {response}")
- except Exception as e:
- self.stream_error_and_exit(f"Error calling LLM: {e}")
-
- if use_cache:
- cache = ToolCache(
- tool=self,
- platform_host=self.get_env_or_die(ToolEnv.PLATFORM_HOST),
- platform_port=int(
- self.get_env_or_die(ToolEnv.PLATFORM_PORT)
- ),
- )
- cache.set(cache_key, classification)
-
- classification = classification.lower() # type: ignore
- bins = [bin.lower() for bin in bins]
-
- if classification not in bins:
- self.stream_error_and_exit(
- f"Invalid classification done: {classification}"
- )
- if not classification:
- classification = "unknown"
-
source_name = self.get_exec_metadata.get(MetadataKey.SOURCE_NAME)
self._copy_input_to_output_bin(
output_dir=output_dir,
diff --git a/unstract/tool-registry/src/unstract/tool_registry/public_tools.json b/unstract/tool-registry/src/unstract/tool_registry/public_tools.json
index 904616f00..fe562f013 100644
--- a/unstract/tool-registry/src/unstract/tool_registry/public_tools.json
+++ b/unstract/tool-registry/src/unstract/tool_registry/public_tools.json
@@ -5,7 +5,7 @@
"schemaVersion": "0.0.1",
"displayName": "File Classifier",
"functionName": "classify",
- "toolVersion": "0.0.1",
+ "toolVersion": "0.0.2",
"description": "Classifies a file into a bin based on its contents",
"input": {
"description": "File to be classified"
@@ -36,6 +36,15 @@
{
"isEnabled": false
}
+ ],
+ "textExtractors": [
+ {
+ "isEnabled": true,
+ "adapterId": "textExtractorId",
+ "title": "Text Extraction Adapter",
+ "isRequired": false,
+ "description": "Choose extractor adapter to extract text"
+ }
]
},
"ioCompatibility": {
@@ -130,9 +139,9 @@
}
},
"icon": "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n<svg\n enable-background=\"new 0 0 20 20\"\n height=\"48\"\n viewBox=\"0 0 20 20\"\n width=\"48\"\n fill=\"#000000\"\n version=\"1.1\"\n id=\"svg8109\"\n sodipodi:docname=\"folder_copy_black_48dp.svg\"\n xmlns:inkscape=\"http://www.inkscape.org/namespaces/inkscape\"\n xmlns:sodipodi=\"http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd\"\n xmlns=\"http://www.w3.org/2000/svg\"\n xmlns:svg=\"http://www.w3.org/2000/svg\">\n <defs\n id=\"defs8113\" />\n <sodipodi:namedview\n id=\"namedview8111\"\n pagecolor=\"#ffffff\"\n bordercolor=\"#000000\"\n borderopacity=\"0.25\"\n inkscape:showpageshadow=\"2\"\n inkscape:pageopacity=\"0.0\"\n inkscape:pagecheckerboard=\"0\"\n inkscape:deskcolor=\"#d1d1d1\"\n showgrid=\"false\" />\n <g\n id=\"g8099\">\n <rect\n fill=\"none\"\n height=\"20\"\n width=\"20\"\n x=\"0\"\n id=\"rect8097\"\n y=\"0\" />\n </g>\n <g\n id=\"g8107\"\n style=\"fill:#ff4d6d;fill-opacity:1\">\n <g\n id=\"g8105\"\n style=\"fill:#ff4d6d;fill-opacity:1\">\n <path\n d=\"M 2.5,5 H 1 V 15.5 C 1,16.33 1.67,17 2.5,17 H 15.68 V 15.5 H 2.5 Z\"\n id=\"path8101\"\n style=\"fill:#ff4d6d;fill-opacity:1\" />\n <path\n d=\"M 16.5,4 H 11 L 9,2 H 5.5 C 4.67,2 4,2.67 4,3.5 v 9 C 4,13.33 4.67,14 5.5,14 h 11 c 0.83,0 1.5,-0.67 1.5,-1.5 v -7 C 18,4.67 17.33,4 16.5,4 Z m 0,8.5 h -11 v -9 h 2.88 l 2,2 h 6.12 z\"\n id=\"path8103\"\n style=\"fill:#ff4d6d;fill-opacity:1\" />\n </g>\n </g>\n</svg>\n",
- "image_url": "docker:unstract/tool-classifier:0.0.1",
+ "image_url": "docker:unstract/tool-classifier:0.0.2",
"image_name": "unstract/tool-classifier",
- "image_tag": "0.0.1"
+ "image_tag": "0.0.2"
},
"document_indexer": {
"tool_uid": "document_indexer",