Skip to content

LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows

UN-4042 [FIX] Bill the pages LLMWhisperer extracted, not every page in the file (#2266)

Athulb0544f652ea6
diff --git a/unstract/sdk1/src/unstract/sdk1/adapters/x2text/dto.py b/unstract/sdk1/src/unstract/sdk1/adapters/x2text/dto.py
index 23c22a583..418eb6c09 100644
--- a/unstract/sdk1/src/unstract/sdk1/adapters/x2text/dto.py
+++ b/unstract/sdk1/src/unstract/sdk1/adapters/x2text/dto.py
@@ -18,8 +18,19 @@ class TextExtractionMetadata:
@dataclass
class TextExtractionResult:
+ """Outcome of a text extraction.
+
+ Attributes:
+ extracted_text: The text the extractor produced.
+ extraction_metadata: Extractor-specific metadata, if any.
+ page_count: Billable page count the extractor reports, or None when it
+ reports none. For non-PDF inputs this may be derived from content
+ length rather than pages.
+ """
+
extracted_text: str
extraction_metadata: TextExtractionMetadata | None = None
+ page_count: int | None = None
@dataclass
diff --git a/unstract/sdk1/src/unstract/sdk1/adapters/x2text/llm_whisperer_v2/src/helper.py b/unstract/sdk1/src/unstract/sdk1/adapters/x2text/llm_whisperer_v2/src/helper.py
index a7701094f..7681516c3 100644
--- a/unstract/sdk1/src/unstract/sdk1/adapters/x2text/llm_whisperer_v2/src/helper.py
+++ b/unstract/sdk1/src/unstract/sdk1/adapters/x2text/llm_whisperer_v2/src/helper.py
@@ -137,7 +137,7 @@ class LLMWhispererHelper:
params: dict[str, Any] | None = None,
data: BytesIO | None = None,
type: str = "whisper",
- ) -> Response:
+ ) -> dict[str, Any]:
"""Makes a request to LLMWhisperer service.
Args:
@@ -152,7 +152,7 @@ class LLMWhispererHelper:
Defaults to "whisper".
Returns:
- Response: Response from the request
+ dict[str, Any]: Decoded LLMWhisperer response body
"""
llm_whisperer_svc_url = f"{config.get(WhispererConfig.URL)}/api/v2"
if not headers:
@@ -318,7 +318,7 @@ class LLMWhispererHelper:
config: dict[str, Any],
extra_params: WhispererRequestParams,
fs: FileStorage | None = None,
- ) -> requests.Response:
+ ) -> dict[str, Any]:
if fs is None:
fs = FileStorage(provider=FileStorageProvider.LOCAL)
params = LLMWhispererHelper.get_whisperer_params(
@@ -326,7 +326,7 @@ class LLMWhispererHelper:
)
# Recorded against the extraction for cross referencing in usage reports
params[WhispererConfig.FILE_NAME] = Path(input_file_path).name
- response: requests.Response
+ response: dict[str, Any]
try:
input_file_data = BytesIO(fs.read(path=input_file_path, mode="rb"))
enable_highlight = extra_params.enable_highlight
@@ -379,6 +379,54 @@ class LLMWhispererHelper:
)
return retrieve_response
+ @staticmethod
+ def get_processed_page_count(response: dict[str, Any]) -> int | None:
+ """Billable page count LLMWhisperer reports for this extraction.
+
+ This is the count LLMWhisperer bills, so it honours a page range.
+
+ Args:
+ response: Decoded `/whisper-retrieve` body returned by `make_request`.
+
+ Returns:
+ The processed page count, or None if the response does not carry a
+ usable one.
+ """
+ metadata = response.get("whisper_metadata")
+ has_metadata = isinstance(metadata, dict)
+ page_count = metadata.get("processed_page_count") if has_metadata else None
+ # bool is a subclass of int, so it has to be rejected explicitly.
+ if (
+ isinstance(page_count, bool)
+ or not isinstance(page_count, int)
+ or page_count <= 0
+ ):
+ logger.info(
+ "No usable processed_page_count in LLMWhisperer response; PDF "
+ "billing falls back to the file's page count "
+ "(whisper_hash=%s, whisper_metadata_present=%s, value=%r)",
+ response.get(X2TextConstants.WHISPER_HASH_V2),
+ has_metadata,
+ page_count,
+ )
+ return None
+ requested = metadata.get("requested_page_count")
+ if (
+ isinstance(requested, int)
+ and not isinstance(requested, bool)
+ and page_count < requested
+ ):
+ # Pages that failed to extract are dropped and not billed.
+ logger.warning(
+ "LLMWhisperer extracted %d of %d requested pages; billing %d "
+ "(whisper_hash=%s)",
+ page_count,
+ requested,
+ page_count,
+ response.get(X2TextConstants.WHISPER_HASH_V2),
+ )
+ return page_count
+
@staticmethod
def extract_text_from_response(
output_file_path: str | None,
diff --git a/unstract/sdk1/src/unstract/sdk1/adapters/x2text/llm_whisperer_v2/src/llm_whisperer_v2.py b/unstract/sdk1/src/unstract/sdk1/adapters/x2text/llm_whisperer_v2/src/llm_whisperer_v2.py
index 03261bdbc..8d9690135 100644
--- a/unstract/sdk1/src/unstract/sdk1/adapters/x2text/llm_whisperer_v2/src/llm_whisperer_v2.py
+++ b/unstract/sdk1/src/unstract/sdk1/adapters/x2text/llm_whisperer_v2/src/llm_whisperer_v2.py
@@ -2,7 +2,7 @@ from __future__ import annotations
import logging
import os
-from typing import TYPE_CHECKING, Any
+from typing import Any
from unstract.sdk1.adapters.exceptions import ExtractorError
from unstract.sdk1.adapters.x2text.constants import (
@@ -26,9 +26,6 @@ from unstract.sdk1.adapters.x2text.llm_whisperer_v2.src.helper import LLMWhisper
from unstract.sdk1.adapters.x2text.x2text_adapter import X2TextAdapter
from unstract.sdk1.file_storage import FileStorage, FileStorageProvider
-if TYPE_CHECKING:
- import requests
-
logger = logging.getLogger(__name__)
@@ -166,7 +163,9 @@ class LLMWhispererV2(X2TextAdapter):
Defaults to None.
Returns:
- str: Extracted text
+ TextExtractionResult: Extracted text, metadata and, when
+ LLMWhisperer reports it, the processed page count (None in
+ image output mode).
"""
if fs is None:
fs = FileStorage(provider=FileStorageProvider.LOCAL)
@@ -202,7 +201,7 @@ class LLMWhispererV2(X2TextAdapter):
tag=kwargs.get(X2TextConstants.TAGS),
enable_highlight=enable_highlight,
)
- response: requests.Response = LLMWhispererHelper.send_whisper_request(
+ response = LLMWhispererHelper.send_whisper_request(
input_file_path=input_file_path,
config=self.config,
fs=fs,
@@ -220,4 +219,5 @@ class LLMWhispererV2(X2TextAdapter):
fs=fs,
),
extraction_metadata=metadata,
+ page_count=LLMWhispererHelper.get_processed_page_count(response),
)
diff --git a/unstract/sdk1/src/unstract/sdk1/x2txt.py b/unstract/sdk1/src/unstract/sdk1/x2txt.py
index 2024f8cdb..28b7741ed 100644
--- a/unstract/sdk1/src/unstract/sdk1/x2txt.py
+++ b/unstract/sdk1/src/unstract/sdk1/x2txt.py
@@ -114,8 +114,13 @@ class X2Text:
text_extraction_result = self._x2text_instance.process(
input_file_path, output_file_path, fs, **kwargs
)
- # The will be executed each and every time text extraction takes place
- self.push_usage_details(input_file_path, mime_type, fs=fs)
+ # Usage is reported after every extraction.
+ self.push_usage_details(
+ input_file_path,
+ mime_type,
+ fs=fs,
+ page_count=text_extraction_result.page_count,
+ )
return text_extraction_result
def push_usage_details(
@@ -123,31 +128,47 @@ class X2Text:
input_file_path: str,
mime_type: str,
fs: FileStorage | None = None,
+ page_count: int | None = None,
) -> None:
+ """Report the pages consumed by a completed extraction.
+
+ Args:
+ input_file_path: File that was extracted.
+ mime_type: MIME type of that file.
+ fs: File storage to read the file through.
+ page_count: Pages the extractor processed, if it reports them. Used
+ for PDFs only; when missing or not a positive int, the PDF's own
+ page count is used. Non-PDF inputs always bill one page.
+ """
if fs is None:
fs = FileStorage(provider=FileStorageProvider.LOCAL)
file_size = ToolUtils.get_file_size(input_file_path, fs)
if mime_type == MimeType.PDF:
- pdf_contents = io.BytesIO(fs.read(path=input_file_path, mode="rb"))
- with pdfplumber.open(pdf_contents) as pdf:
- # calculate the number of pages
- page_count = len(pdf.pages)
- Audit().push_page_usage_data(
- platform_api_key=self._tool.get_env_or_die(ToolEnv.PLATFORM_API_KEY),
- file_size=file_size,
- file_type=mime_type,
- page_count=page_count,
- kwargs=self._usage_kwargs,
+ billable_pages = (
+ page_count
+ # bool is a subclass of int, so it has to be rejected explicitly.
+ if isinstance(page_count, int)
+ and not isinstance(page_count, bool)
+ and page_count > 0
+ else self._get_pdf_page_count(input_file_path, fs)
)
else:
- # TODO: Calculate page usage for other file types (3000 words = 1 page)
- # We are allowing certain image types,and raw texts. We will consider them
- # as single page documents as there in no concept of page numbers.
- Audit().push_page_usage_data(
- platform_api_key=self._tool.get_env_or_die(ToolEnv.PLATFORM_API_KEY),
- file_size=file_size,
- file_type=mime_type,
- page_count=1,
- kwargs=self._usage_kwargs,
- )
+ # Non-PDF inputs bill one page. The extractor's count is ignored here
+ # because it may be derived from content length rather than pages.
+ billable_pages = 1
+
+ Audit().push_page_usage_data(
+ platform_api_key=self._tool.get_env_or_die(ToolEnv.PLATFORM_API_KEY),
+ file_size=file_size,
+ file_type=mime_type,
+ page_count=billable_pages,
+ kwargs=self._usage_kwargs,
+ )
+
+ @staticmethod
+ def _get_pdf_page_count(input_file_path: str, fs: FileStorage) -> int:
+ """Number of pages held by the PDF at `input_file_path`."""
+ pdf_contents = io.BytesIO(fs.read(path=input_file_path, mode="rb"))
+ with pdfplumber.open(pdf_contents) as pdf:
+ return len(pdf.pages)
diff --git a/unstract/sdk1/tests/test_llmw_v2_process_image.py b/unstract/sdk1/tests/test_llmw_v2_process_image.py
index 9f058b34d..5715a942d 100644
--- a/unstract/sdk1/tests/test_llmw_v2_process_image.py
+++ b/unstract/sdk1/tests/test_llmw_v2_process_image.py
@@ -49,6 +49,8 @@ class TestTextModeRegression:
assert result.extracted_text == "hello text"
assert result.extraction_metadata.whisper_hash == "wh1"
assert result.extraction_metadata.page_images is None
+ # Responses without whisper_metadata report no page count.
+ assert result.page_count is None
assert image_called["hit"] is False # image path never touched
def test_text_mode_error_path_propagates(self, monkeypatch: MonkeyPatch) -> None:
@@ -98,6 +100,8 @@ class TestImageModeBranch:
assert result.extracted_text == expected_summary
assert "page_001.png" not in result.extracted_text
assert result.extraction_metadata.page_images == refs
+ # Image mode converts every page, so billing must count the file.
+ assert result.page_count is None
# The real job id is recorded, not an empty string (HITL/QueueResult
# consumers read this).
assert result.extraction_metadata.whisper_hash == "run-1|doc-hash"
diff --git a/unstract/sdk1/tests/test_x2txt_page_usage.py b/unstract/sdk1/tests/test_x2txt_page_usage.py
new file mode 100644
index 000000000..8a2f1d65a
--- /dev/null
+++ b/unstract/sdk1/tests/test_x2txt_page_usage.py
@@ -0,0 +1,216 @@
+"""Tests for page usage accounting in X2Text.
+
+PDF page usage follows the extractor's processed page count, so a page range
+is billed for the pages extracted, not the whole file.
+"""
+
+from collections.abc import Iterator
+from typing import Any
+from unittest.mock import MagicMock, patch
+
+import pytest
+from unstract.sdk1.adapters.x2text.dto import TextExtractionResult
+from unstract.sdk1.adapters.x2text.llm_whisperer_v2.src.helper import LLMWhispererHelper
+from unstract.sdk1.adapters.x2text.llm_whisperer_v2.src.llm_whisperer_v2 import (
+ LLMWhispererV2,
+)
+from unstract.sdk1.constants import MimeType
+from unstract.sdk1.x2txt import X2Text
+
+PDF_PAGES = 199
+
+
+def x2text() -> X2Text:
+ """An X2Text with no adapter configured; page usage does not need one."""
+ tool = MagicMock()
+ tool.get_env_or_die.return_value = "test-platform-api-key"
+ return X2Text(tool=tool, usage_kwargs={"run_id": "test-run"})
+
+
+def pdf_fs() -> MagicMock:
+ """A FileStorage returning placeholder PDF bytes; `audit` patches pdfplumber."""
+ fs = MagicMock()
+ fs.read.return_value = b"%PDF-1.4 fake"
+ fs.mime_type.return_value = MimeType.PDF
+ return fs
+
+
+def audit() -> Iterator[MagicMock]:
+ """The Audit mock, with any PDF counted by pdfplumber as PDF_PAGES pages."""
+ with (
+ patch("unstract.sdk1.x2txt.Audit") as audit,
+ patch("unstract.sdk1.x2txt.pdfplumber") as pdfplumber,
+ patch("unstract.sdk1.x2txt.ToolUtils.get_file_size", return_value=1234),
+ ):
+ pdfplumber.open.return_value.__enter__.return_value.pages = [None] * PDF_PAGES
+ yield audit
+
+
+def _pushed_page_count(audit: MagicMock) -> int:
+ """The page_count handed to the platform by a single push."""
+ audit.return_value.push_page_usage_data.assert_called_once()
+ return audit.return_value.push_page_usage_data.call_args.kwargs["page_count"]
+
+
+class TestPushUsageDetails:
+ """What X2Text.push_usage_details reports to the audit service."""
+
+ def test_pdf_uses_extractor_page_count(
+ self, audit: MagicMock, x2text: X2Text, pdf_fs: MagicMock
+ ) -> None:
+ """5 pages extracted from a 199-page PDF bills 5, not 199."""
+ x2text.push_usage_details("in.pdf", MimeType.PDF, fs=pdf_fs, page_count=5)
+
+ assert _pushed_page_count(audit) == 5
+
+ @pytest.mark.parametrize("bad_count", [None, 0, -3, True, 2.5, "5", 5.0])
+ def test_pdf_rejects_unusable_count(
+ self, audit: MagicMock, bad_count: object, x2text: X2Text, pdf_fs: MagicMock
+ ) -> None:
+ """A count that is not a positive int falls back to the PDF's pages."""
+ x2text.push_usage_details("in.pdf", MimeType.PDF, fs=pdf_fs, page_count=bad_count)
+
+ assert _pushed_page_count(audit) == PDF_PAGES
+
+ def test_non_pdf_ignores_extractor_count(
+ self, audit: MagicMock, x2text: X2Text
+ ) -> None:
+ """Non-PDF billing must not change with the reported count."""
+ x2text.push_usage_details("in.txt", MimeType.TEXT, fs=MagicMock(), page_count=20)
+
+ assert _pushed_page_count(audit) == 1
+
+
+class TestGetProcessedPageCount:
+ """Reading processed_page_count out of an LLMWhisperer V2 response."""
+
+ @pytest.mark.parametrize(
+ "response,expected",
+ [
+ ({"whisper_metadata": {"processed_page_count": 5}}, 5),
+ ({"whisper_metadata": {"processed_page_count": 1}}, 1),
+ ({}, None),
+ ({"whisper_metadata": None}, None),
+ ({"whisper_metadata": "nope"}, None),
+ ({"whisper_metadata": {}}, None),
+ ({"whisper_metadata": {"processed_page_count": 0}}, None),
+ ({"whisper_metadata": {"processed_page_count": -1}}, None),
+ ({"whisper_metadata": {"processed_page_count": "5"}}, None),
+ ({"whisper_metadata": {"processed_page_count": 5.0}}, None),
+ # isinstance(True, int) is True — must not bill 1 page for a bool.
+ ({"whisper_metadata": {"processed_page_count": True}}, None),
+ ],
+ )
+ def test_validates_response(
+ self, response: dict[str, Any], expected: int | None
+ ) -> None:
+ assert LLMWhispererHelper.get_processed_page_count(response) == expected
+
+ @pytest.mark.parametrize(
+ "metadata", [{}, {"whisper_metadata": {"processed_page_count": 0}}]
+ )
+ def test_logs_info_when_count_unusable(
+ self, metadata: dict[str, Any], caplog: pytest.LogCaptureFixture
+ ) -> None:
+ """INFO, not WARNING: old LLMWhisperer versions send no count."""
+ with caplog.at_level("INFO"):
+ LLMWhispererHelper.get_processed_page_count(
+ {"whisper_hash": "wh-1", **metadata}
+ )
+
+ (record,) = caplog.records
+ assert record.levelname == "INFO"
+ assert "wh-1" in record.getMessage()
+ assert "falls back" in record.getMessage()
+
+ def test_warns_when_fewer_pages_processed_than_requested(
+ self, caplog: pytest.LogCaptureFixture
+ ) -> None:
+ """Pages that failed to extract are not billed, so the gap must be logged."""
+ response = {
+ "whisper_hash": "wh-2",
+ "whisper_metadata": {
+ "processed_page_count": 159,
+ "requested_page_count": 199,
+ },
+ }
+ with caplog.at_level("WARNING"):
+ count = LLMWhispererHelper.get_processed_page_count(response)
+
+ assert count == 159
+ (record,) = caplog.records
+ assert record.levelname == "WARNING"
+ assert "159" in record.getMessage()
+ assert "199" in record.getMessage()
+ assert "wh-2" in record.getMessage()
+
+ @pytest.mark.parametrize(
+ "metadata",
+ [
+ # Full extraction.
+ {"processed_page_count": 5, "requested_page_count": 5},
+ # Text input: 1 page requested, word-derived count can be higher.
+ {"processed_page_count": 3, "requested_page_count": 1},
+ # Older LLMWhisperer without requested_page_count.
+ {"processed_page_count": 5},
+ {"processed_page_count": 5, "requested_page_count": None},
+ ],
+ )
+ def test_no_shortfall_warning(
+ self, metadata: dict[str, Any], caplog: pytest.LogCaptureFixture
+ ) -> None:
+ with caplog.at_level("WARNING"):
+ LLMWhispererHelper.get_processed_page_count({"whisper_metadata": metadata})
+
+ assert caplog.text == ""
+
+ def test_silent_when_count_usable(self, caplog: pytest.LogCaptureFixture) -> None:
+ response = {"whisper_metadata": {"processed_page_count": 5}}
+ with caplog.at_level("WARNING"):
+ LLMWhispererHelper.get_processed_page_count(response)
+
+ assert caplog.text == ""
+
+
+class TestProcessForwardsPageCount:
+ """The wiring: what the adapter reports is what gets billed."""
+
+ @pytest.mark.parametrize("reported,billed", [(5, 5), (None, PDF_PAGES)])
+ def test_process_bills_the_count_the_adapter_reported(
+ self,
+ audit: MagicMock,
+ reported: int | None,
+ billed: int,
+ x2text: X2Text,
+ pdf_fs: MagicMock,
+ ) -> None:
+ """A reported count is billed; adapters reporting none fall back to the PDF."""
+ x2text._x2text_instance = MagicMock()
+ x2text._x2text_instance.process.return_value = TextExtractionResult(
+ extracted_text="text", page_count=reported
+ )
+
+ x2text.process("in.pdf", fs=pdf_fs)
+
+ assert _pushed_page_count(audit) == billed
+
+
+class TestLLMWhispererV2ReportsPageCount:
+ """The V2 adapter lifts processed_page_count off the whisper response."""
+
+ @patch.object(LLMWhispererHelper, "extract_text_from_response", return_value="text")
+ @patch.object(LLMWhispererHelper, "send_whisper_request")
+ def test_page_count_taken_from_whisper_metadata(
+ self, send: MagicMock, _extract: MagicMock
+ ) -> None:
+ send.return_value = {
+ "whisper_hash": "hash-1",
+ "whisper_metadata": {"processed_page_count": 5, "total_page_count": 199},
+ }
+
+ result = LLMWhispererV2({}).process("in.pdf", fs=MagicMock())
+
+ assert result.page_count == 5