LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
UN-4042 [FIX] Bill the pages LLMWhisperer extracted, not every page in the file (#2266)
Athulb0544f652ea6
diff --git a/unstract/sdk1/src/unstract/sdk1/adapters/x2text/dto.py b/unstract/sdk1/src/unstract/sdk1/adapters/x2text/dto.pyindex 23c22a583..418eb6c09 100644--- a/unstract/sdk1/src/unstract/sdk1/adapters/x2text/dto.py+++ b/unstract/sdk1/src/unstract/sdk1/adapters/x2text/dto.py@@ -18,8 +18,19 @@ class TextExtractionMetadata:@dataclassclass TextExtractionResult:+ """Outcome of a text extraction.++ Attributes:+ extracted_text: The text the extractor produced.+ extraction_metadata: Extractor-specific metadata, if any.+ page_count: Billable page count the extractor reports, or None when it+ reports none. For non-PDF inputs this may be derived from content+ length rather than pages.+ """+extracted_text: strextraction_metadata: TextExtractionMetadata | None = None+ page_count: int | None = None@dataclassdiff --git a/unstract/sdk1/src/unstract/sdk1/adapters/x2text/llm_whisperer_v2/src/helper.py b/unstract/sdk1/src/unstract/sdk1/adapters/x2text/llm_whisperer_v2/src/helper.pyindex a7701094f..7681516c3 100644--- a/unstract/sdk1/src/unstract/sdk1/adapters/x2text/llm_whisperer_v2/src/helper.py+++ b/unstract/sdk1/src/unstract/sdk1/adapters/x2text/llm_whisperer_v2/src/helper.py@@ -137,7 +137,7 @@ class LLMWhispererHelper:params: dict[str, Any] | None = None,data: BytesIO | None = None,type: str = "whisper",- ) -> Response:+ ) -> dict[str, Any]:"""Makes a request to LLMWhisperer service.Args:@@ -152,7 +152,7 @@ class LLMWhispererHelper:Defaults to "whisper".Returns:- Response: Response from the request+ dict[str, Any]: Decoded LLMWhisperer response body"""llm_whisperer_svc_url = f"{config.get(WhispererConfig.URL)}/api/v2"if not headers:@@ -318,7 +318,7 @@ class LLMWhispererHelper:config: dict[str, Any],extra_params: WhispererRequestParams,fs: FileStorage | None = None,- ) -> requests.Response:+ ) -> dict[str, Any]:if fs is None:fs = FileStorage(provider=FileStorageProvider.LOCAL)params = LLMWhispererHelper.get_whisperer_params(@@ -326,7 +326,7 @@ class LLMWhispererHelper:)# Recorded against the extraction for cross referencing in usage reportsparams[WhispererConfig.FILE_NAME] = Path(input_file_path).name- response: requests.Response+ response: dict[str, Any]try:input_file_data = BytesIO(fs.read(path=input_file_path, mode="rb"))enable_highlight = extra_params.enable_highlight@@ -379,6 +379,54 @@ class LLMWhispererHelper:)return retrieve_response+ @staticmethod+ def get_processed_page_count(response: dict[str, Any]) -> int | None:+ """Billable page count LLMWhisperer reports for this extraction.++ This is the count LLMWhisperer bills, so it honours a page range.++ Args:+ response: Decoded `/whisper-retrieve` body returned by `make_request`.++ Returns:+ The processed page count, or None if the response does not carry a+ usable one.+ """+ metadata = response.get("whisper_metadata")+ has_metadata = isinstance(metadata, dict)+ page_count = metadata.get("processed_page_count") if has_metadata else None+ # bool is a subclass of int, so it has to be rejected explicitly.+ if (+ isinstance(page_count, bool)+ or not isinstance(page_count, int)+ or page_count <= 0+ ):+ logger.info(+ "No usable processed_page_count in LLMWhisperer response; PDF "+ "billing falls back to the file's page count "+ "(whisper_hash=%s, whisper_metadata_present=%s, value=%r)",+ response.get(X2TextConstants.WHISPER_HASH_V2),+ has_metadata,+ page_count,+ )+ return None+ requested = metadata.get("requested_page_count")+ if (+ isinstance(requested, int)+ and not isinstance(requested, bool)+ and page_count < requested+ ):+ # Pages that failed to extract are dropped and not billed.+ logger.warning(+ "LLMWhisperer extracted %d of %d requested pages; billing %d "+ "(whisper_hash=%s)",+ page_count,+ requested,+ page_count,+ response.get(X2TextConstants.WHISPER_HASH_V2),+ )+ return page_count+@staticmethoddef extract_text_from_response(output_file_path: str | None,diff --git a/unstract/sdk1/src/unstract/sdk1/adapters/x2text/llm_whisperer_v2/src/llm_whisperer_v2.py b/unstract/sdk1/src/unstract/sdk1/adapters/x2text/llm_whisperer_v2/src/llm_whisperer_v2.pyindex 03261bdbc..8d9690135 100644--- a/unstract/sdk1/src/unstract/sdk1/adapters/x2text/llm_whisperer_v2/src/llm_whisperer_v2.py+++ b/unstract/sdk1/src/unstract/sdk1/adapters/x2text/llm_whisperer_v2/src/llm_whisperer_v2.py@@ -2,7 +2,7 @@ from __future__ import annotationsimport loggingimport os-from typing import TYPE_CHECKING, Any+from typing import Anyfrom unstract.sdk1.adapters.exceptions import ExtractorErrorfrom unstract.sdk1.adapters.x2text.constants import (@@ -26,9 +26,6 @@ from unstract.sdk1.adapters.x2text.llm_whisperer_v2.src.helper import LLMWhisperfrom unstract.sdk1.adapters.x2text.x2text_adapter import X2TextAdapterfrom unstract.sdk1.file_storage import FileStorage, FileStorageProvider-if TYPE_CHECKING:- import requests-logger = logging.getLogger(__name__)@@ -166,7 +163,9 @@ class LLMWhispererV2(X2TextAdapter):Defaults to None.Returns:- str: Extracted text+ TextExtractionResult: Extracted text, metadata and, when+ LLMWhisperer reports it, the processed page count (None in+ image output mode)."""if fs is None:fs = FileStorage(provider=FileStorageProvider.LOCAL)@@ -202,7 +201,7 @@ class LLMWhispererV2(X2TextAdapter):tag=kwargs.get(X2TextConstants.TAGS),enable_highlight=enable_highlight,)- response: requests.Response = LLMWhispererHelper.send_whisper_request(+ response = LLMWhispererHelper.send_whisper_request(input_file_path=input_file_path,config=self.config,fs=fs,@@ -220,4 +219,5 @@ class LLMWhispererV2(X2TextAdapter):fs=fs,),extraction_metadata=metadata,+ page_count=LLMWhispererHelper.get_processed_page_count(response),)diff --git a/unstract/sdk1/src/unstract/sdk1/x2txt.py b/unstract/sdk1/src/unstract/sdk1/x2txt.pyindex 2024f8cdb..28b7741ed 100644--- a/unstract/sdk1/src/unstract/sdk1/x2txt.py+++ b/unstract/sdk1/src/unstract/sdk1/x2txt.py@@ -114,8 +114,13 @@ class X2Text:text_extraction_result = self._x2text_instance.process(input_file_path, output_file_path, fs, **kwargs)- # The will be executed each and every time text extraction takes place- self.push_usage_details(input_file_path, mime_type, fs=fs)+ # Usage is reported after every extraction.+ self.push_usage_details(+ input_file_path,+ mime_type,+ fs=fs,+ page_count=text_extraction_result.page_count,+ )return text_extraction_resultdef push_usage_details(@@ -123,31 +128,47 @@ class X2Text:input_file_path: str,mime_type: str,fs: FileStorage | None = None,+ page_count: int | None = None,) -> None:+ """Report the pages consumed by a completed extraction.++ Args:+ input_file_path: File that was extracted.+ mime_type: MIME type of that file.+ fs: File storage to read the file through.+ page_count: Pages the extractor processed, if it reports them. Used+ for PDFs only; when missing or not a positive int, the PDF's own+ page count is used. Non-PDF inputs always bill one page.+ """if fs is None:fs = FileStorage(provider=FileStorageProvider.LOCAL)file_size = ToolUtils.get_file_size(input_file_path, fs)if mime_type == MimeType.PDF:- pdf_contents = io.BytesIO(fs.read(path=input_file_path, mode="rb"))- with pdfplumber.open(pdf_contents) as pdf:- # calculate the number of pages- page_count = len(pdf.pages)- Audit().push_page_usage_data(- platform_api_key=self._tool.get_env_or_die(ToolEnv.PLATFORM_API_KEY),- file_size=file_size,- file_type=mime_type,- page_count=page_count,- kwargs=self._usage_kwargs,+ billable_pages = (+ page_count+ # bool is a subclass of int, so it has to be rejected explicitly.+ if isinstance(page_count, int)+ and not isinstance(page_count, bool)+ and page_count > 0+ else self._get_pdf_page_count(input_file_path, fs))else:- # TODO: Calculate page usage for other file types (3000 words = 1 page)- # We are allowing certain image types,and raw texts. We will consider them- # as single page documents as there in no concept of page numbers.- Audit().push_page_usage_data(- platform_api_key=self._tool.get_env_or_die(ToolEnv.PLATFORM_API_KEY),- file_size=file_size,- file_type=mime_type,- page_count=1,- kwargs=self._usage_kwargs,- )+ # Non-PDF inputs bill one page. The extractor's count is ignored here+ # because it may be derived from content length rather than pages.+ billable_pages = 1++ Audit().push_page_usage_data(+ platform_api_key=self._tool.get_env_or_die(ToolEnv.PLATFORM_API_KEY),+ file_size=file_size,+ file_type=mime_type,+ page_count=billable_pages,+ kwargs=self._usage_kwargs,+ )++ @staticmethod+ def _get_pdf_page_count(input_file_path: str, fs: FileStorage) -> int:+ """Number of pages held by the PDF at `input_file_path`."""+ pdf_contents = io.BytesIO(fs.read(path=input_file_path, mode="rb"))+ with pdfplumber.open(pdf_contents) as pdf:+ return len(pdf.pages)diff --git a/unstract/sdk1/tests/test_llmw_v2_process_image.py b/unstract/sdk1/tests/test_llmw_v2_process_image.pyindex 9f058b34d..5715a942d 100644--- a/unstract/sdk1/tests/test_llmw_v2_process_image.py+++ b/unstract/sdk1/tests/test_llmw_v2_process_image.py@@ -49,6 +49,8 @@ class TestTextModeRegression:assert result.extracted_text == "hello text"assert result.extraction_metadata.whisper_hash == "wh1"assert result.extraction_metadata.page_images is None+ # Responses without whisper_metadata report no page count.+ assert result.page_count is Noneassert image_called["hit"] is False # image path never toucheddef test_text_mode_error_path_propagates(self, monkeypatch: MonkeyPatch) -> None:@@ -98,6 +100,8 @@ class TestImageModeBranch:assert result.extracted_text == expected_summaryassert "page_001.png" not in result.extracted_textassert result.extraction_metadata.page_images == refs+ # Image mode converts every page, so billing must count the file.+ assert result.page_count is None# The real job id is recorded, not an empty string (HITL/QueueResult# consumers read this).assert result.extraction_metadata.whisper_hash == "run-1|doc-hash"diff --git a/unstract/sdk1/tests/test_x2txt_page_usage.py b/unstract/sdk1/tests/test_x2txt_page_usage.pynew file mode 100644index 000000000..8a2f1d65a--- /dev/null+++ b/unstract/sdk1/tests/test_x2txt_page_usage.py@@ -0,0 +1,216 @@+"""Tests for page usage accounting in X2Text.++PDF page usage follows the extractor's processed page count, so a page range+is billed for the pages extracted, not the whole file.+"""++from collections.abc import Iterator+from typing import Any+from unittest.mock import MagicMock, patch++import pytest+from unstract.sdk1.adapters.x2text.dto import TextExtractionResult+from unstract.sdk1.adapters.x2text.llm_whisperer_v2.src.helper import LLMWhispererHelper+from unstract.sdk1.adapters.x2text.llm_whisperer_v2.src.llm_whisperer_v2 import (+ LLMWhispererV2,+)+from unstract.sdk1.constants import MimeType+from unstract.sdk1.x2txt import X2Text++PDF_PAGES = 199+++def x2text() -> X2Text:+ """An X2Text with no adapter configured; page usage does not need one."""+ tool = MagicMock()+ tool.get_env_or_die.return_value = "test-platform-api-key"+ return X2Text(tool=tool, usage_kwargs={"run_id": "test-run"})+++def pdf_fs() -> MagicMock:+ """A FileStorage returning placeholder PDF bytes; `audit` patches pdfplumber."""+ fs = MagicMock()+ fs.read.return_value = b"%PDF-1.4 fake"+ fs.mime_type.return_value = MimeType.PDF+ return fs+++def audit() -> Iterator[MagicMock]:+ """The Audit mock, with any PDF counted by pdfplumber as PDF_PAGES pages."""+ with (+ patch("unstract.sdk1.x2txt.Audit") as audit,+ patch("unstract.sdk1.x2txt.pdfplumber") as pdfplumber,+ patch("unstract.sdk1.x2txt.ToolUtils.get_file_size", return_value=1234),+ ):+ pdfplumber.open.return_value.__enter__.return_value.pages = [None] * PDF_PAGES+ yield audit+++def _pushed_page_count(audit: MagicMock) -> int:+ """The page_count handed to the platform by a single push."""+ audit.return_value.push_page_usage_data.assert_called_once()+ return audit.return_value.push_page_usage_data.call_args.kwargs["page_count"]+++class TestPushUsageDetails:+ """What X2Text.push_usage_details reports to the audit service."""++ def test_pdf_uses_extractor_page_count(+ self, audit: MagicMock, x2text: X2Text, pdf_fs: MagicMock+ ) -> None:+ """5 pages extracted from a 199-page PDF bills 5, not 199."""+ x2text.push_usage_details("in.pdf", MimeType.PDF, fs=pdf_fs, page_count=5)++ assert _pushed_page_count(audit) == 5++ @pytest.mark.parametrize("bad_count", [None, 0, -3, True, 2.5, "5", 5.0])+ def test_pdf_rejects_unusable_count(+ self, audit: MagicMock, bad_count: object, x2text: X2Text, pdf_fs: MagicMock+ ) -> None:+ """A count that is not a positive int falls back to the PDF's pages."""+ x2text.push_usage_details("in.pdf", MimeType.PDF, fs=pdf_fs, page_count=bad_count)++ assert _pushed_page_count(audit) == PDF_PAGES++ def test_non_pdf_ignores_extractor_count(+ self, audit: MagicMock, x2text: X2Text+ ) -> None:+ """Non-PDF billing must not change with the reported count."""+ x2text.push_usage_details("in.txt", MimeType.TEXT, fs=MagicMock(), page_count=20)++ assert _pushed_page_count(audit) == 1+++class TestGetProcessedPageCount:+ """Reading processed_page_count out of an LLMWhisperer V2 response."""++ @pytest.mark.parametrize(+ "response,expected",+ [+ ({"whisper_metadata": {"processed_page_count": 5}}, 5),+ ({"whisper_metadata": {"processed_page_count": 1}}, 1),+ ({}, None),+ ({"whisper_metadata": None}, None),+ ({"whisper_metadata": "nope"}, None),+ ({"whisper_metadata": {}}, None),+ ({"whisper_metadata": {"processed_page_count": 0}}, None),+ ({"whisper_metadata": {"processed_page_count": -1}}, None),+ ({"whisper_metadata": {"processed_page_count": "5"}}, None),+ ({"whisper_metadata": {"processed_page_count": 5.0}}, None),+ # isinstance(True, int) is True — must not bill 1 page for a bool.+ ({"whisper_metadata": {"processed_page_count": True}}, None),+ ],+ )+ def test_validates_response(+ self, response: dict[str, Any], expected: int | None+ ) -> None:+ assert LLMWhispererHelper.get_processed_page_count(response) == expected++ @pytest.mark.parametrize(+ "metadata", [{}, {"whisper_metadata": {"processed_page_count": 0}}]+ )+ def test_logs_info_when_count_unusable(+ self, metadata: dict[str, Any], caplog: pytest.LogCaptureFixture+ ) -> None:+ """INFO, not WARNING: old LLMWhisperer versions send no count."""+ with caplog.at_level("INFO"):+ LLMWhispererHelper.get_processed_page_count(+ {"whisper_hash": "wh-1", **metadata}+ )++ (record,) = caplog.records+ assert record.levelname == "INFO"+ assert "wh-1" in record.getMessage()+ assert "falls back" in record.getMessage()++ def test_warns_when_fewer_pages_processed_than_requested(+ self, caplog: pytest.LogCaptureFixture+ ) -> None:+ """Pages that failed to extract are not billed, so the gap must be logged."""+ response = {+ "whisper_hash": "wh-2",+ "whisper_metadata": {+ "processed_page_count": 159,+ "requested_page_count": 199,+ },+ }+ with caplog.at_level("WARNING"):+ count = LLMWhispererHelper.get_processed_page_count(response)++ assert count == 159+ (record,) = caplog.records+ assert record.levelname == "WARNING"+ assert "159" in record.getMessage()+ assert "199" in record.getMessage()+ assert "wh-2" in record.getMessage()++ @pytest.mark.parametrize(+ "metadata",+ [+ # Full extraction.+ {"processed_page_count": 5, "requested_page_count": 5},+ # Text input: 1 page requested, word-derived count can be higher.+ {"processed_page_count": 3, "requested_page_count": 1},+ # Older LLMWhisperer without requested_page_count.+ {"processed_page_count": 5},+ {"processed_page_count": 5, "requested_page_count": None},+ ],+ )+ def test_no_shortfall_warning(+ self, metadata: dict[str, Any], caplog: pytest.LogCaptureFixture+ ) -> None:+ with caplog.at_level("WARNING"):+ LLMWhispererHelper.get_processed_page_count({"whisper_metadata": metadata})++ assert caplog.text == ""++ def test_silent_when_count_usable(self, caplog: pytest.LogCaptureFixture) -> None:+ response = {"whisper_metadata": {"processed_page_count": 5}}+ with caplog.at_level("WARNING"):+ LLMWhispererHelper.get_processed_page_count(response)++ assert caplog.text == ""+++class TestProcessForwardsPageCount:+ """The wiring: what the adapter reports is what gets billed."""++ @pytest.mark.parametrize("reported,billed", [(5, 5), (None, PDF_PAGES)])+ def test_process_bills_the_count_the_adapter_reported(+ self,+ audit: MagicMock,+ reported: int | None,+ billed: int,+ x2text: X2Text,+ pdf_fs: MagicMock,+ ) -> None:+ """A reported count is billed; adapters reporting none fall back to the PDF."""+ x2text._x2text_instance = MagicMock()+ x2text._x2text_instance.process.return_value = TextExtractionResult(+ extracted_text="text", page_count=reported+ )++ x2text.process("in.pdf", fs=pdf_fs)++ assert _pushed_page_count(audit) == billed+++class TestLLMWhispererV2ReportsPageCount:+ """The V2 adapter lifts processed_page_count off the whisper response."""++ @patch.object(LLMWhispererHelper, "extract_text_from_response", return_value="text")+ @patch.object(LLMWhispererHelper, "send_whisper_request")+ def test_page_count_taken_from_whisper_metadata(+ self, send: MagicMock, _extract: MagicMock+ ) -> None:+ send.return_value = {+ "whisper_hash": "hash-1",+ "whisper_metadata": {"processed_page_count": 5, "total_page_count": 199},+ }++ result = LLMWhispererV2({}).process("in.pdf", fs=MagicMock())++ assert result.page_count == 5