Skip to content

LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows

3.9 KB
Raw Blame
import ast
import sys
from pathlib import Path
from typing import Any

from unstract.sdk1.constants import LogState, UsageKwargs
from unstract.sdk1.tool.base import BaseTool
from unstract.sdk1.tool.entrypoint import ToolEntrypoint
from unstract.sdk1.x2txt import TextExtractionResult, X2Text


class TextExtractor(BaseTool):
    def validate(self, input_file: str, settings: dict[str, Any]) -> None:
        """Validate the input file and settings.

        Args:
            input_file (str): The path to the input file.
            settings (dict[str, Any]): The settings for the tool.
        """
        text_extraction_adapter_id = settings["extractorId"]
        if not text_extraction_adapter_id:
            self.stream_error_and_exit(
                "Adaptor id not found, please select adaptor for extractor tool"
            )

    def run(
        self,
        settings: dict[str, Any],
        input_file: str,
        output_dir: str,
    ) -> None:
        """Run the text extraction process.

        Args:
            settings (dict[str, Any]): The settings for the tool.
            input_file (str): The path to the input file.
            output_dir (str): The path to the output directory.

        Raises:
            Exception: If there is an error creating/writing the output file.

        Returns:
            None
        """
        text_extraction_adapter_id = settings["extractorId"]

        self.stream_log(
            f"Extractor ID: {text_extraction_adapter_id} "
            "has been retrieved from settings."
        )

        input_log = f"Processing file: \n\n`{self.source_file_name}`"
        self.stream_update(input_log, state=LogState.INPUT_UPDATE)

        usage_kwargs: dict[Any, Any] = dict()
        usage_kwargs[UsageKwargs.RUN_ID] = self.file_execution_id
        usage_kwargs[UsageKwargs.FILE_NAME] = self.source_file_name

        text_extraction_adapter = X2Text(
            tool=self,
            adapter_instance_id=text_extraction_adapter_id,
            usage_kwargs=usage_kwargs,
        )
        self.stream_log("Text extraction adapter has been created successfully.")
        extraction_result: TextExtractionResult = text_extraction_adapter.process(
            input_file_path=input_file, fs=self.workflow_filestorage, tags=self.tags
        )
        extracted_text = self.convert_to_actual_string(extraction_result.extracted_text)

        self.stream_log("Text has been extracted successfully.")

        first_5_lines = "\n\n".join(extracted_text.split("\n")[:5])
        output_log = f"### Text\n\n```text\n{first_5_lines}\n```\n\n...(truncated)"
        self.stream_update(output_log, state=LogState.OUTPUT_UPDATE)

        try:
            self.stream_log("Preparing to write the extracted text.")
            if self.source_file_name:
                output_path = Path(output_dir) / f"{Path(self.source_file_name).stem}.txt"
                self.workflow_filestorage.write(
                    path=output_path, mode="w", data=extracted_text
                )

                self.stream_log("Tool output written successfully.")
            else:
                self.stream_error_and_exit(
                    "Error creating/writing output file: source_file_name not found"
                )
        except Exception as e:
            self.stream_error_and_exit(f"Error creating/writing output file: {e}")
        self.write_tool_result(data=extracted_text)

    def convert_to_actual_string(self, text: Any) -> str:
        if isinstance(text, bytes):
            return text.decode("utf-8")
        elif isinstance(text, str):
            if text.startswith("b'") and text.endswith("'"):
                bytes_text: bytes = ast.literal_eval(text)
                return bytes_text.decode("utf-8")
            else:
                return text
        else:
            return str(text)


if __name__ == "__main__":
    args = sys.argv[1:]
    tool = TextExtractor.from_tool_args(args=args)
    ToolEntrypoint.launch(tool=tool, args=args)