LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
"""Tests for the answer_prompt pipeline (Phase 2E).
Tests the _handle_answer_prompt method, AnswerPromptService,
VariableReplacementService, and type conversion logic.
All heavy dependencies (LLM, VectorDB, etc.) are mocked.
"""
from unittest.mock import MagicMock, patch
import pytest
from executor.executors.constants import (
PromptServiceConstants as PSKeys,
)
from unstract.sdk1.execution.context import ExecutionContext, Operation
# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------
def _make_prompt(
name: str = "field_a",
prompt: str = "What is the revenue?",
output_type: str = "text",
chunk_size: int = 512,
chunk_overlap: int = 128,
retrieval_strategy: str = "simple",
llm_id: str = "llm-1",
embedding_id: str = "emb-1",
vector_db_id: str = "vdb-1",
x2text_id: str = "x2t-1",
similarity_top_k: int = 5,
):
"""Build a minimal prompt definition dict."""
return {
PSKeys.NAME: name,
PSKeys.PROMPT: prompt,
PSKeys.TYPE: output_type,
PSKeys.CHUNK_SIZE: chunk_size,
PSKeys.CHUNK_OVERLAP: chunk_overlap,
PSKeys.RETRIEVAL_STRATEGY: retrieval_strategy,
PSKeys.LLM: llm_id,
PSKeys.EMBEDDING: embedding_id,
PSKeys.VECTOR_DB: vector_db_id,
PSKeys.X2TEXT_ADAPTER: x2text_id,
PSKeys.SIMILARITY_TOP_K: similarity_top_k,
}
def _make_context(
prompts=None,
tool_settings=None,
file_hash="abc123",
file_path="/data/doc.txt",
file_name="doc.txt",
execution_source="ide",
platform_api_key="pk-test",
run_id="run-1",
):
"""Build an ExecutionContext for answer_prompt."""
if prompts is None:
prompts = [_make_prompt()]
if tool_settings is None:
tool_settings = {}
params = {
PSKeys.OUTPUTS: prompts,
PSKeys.TOOL_SETTINGS: tool_settings,
PSKeys.TOOL_ID: "tool-1",
PSKeys.EXECUTION_ID: "exec-1",
PSKeys.FILE_HASH: file_hash,
PSKeys.FILE_PATH: file_path,
PSKeys.FILE_NAME: file_name,
PSKeys.LOG_EVENTS_ID: "",
PSKeys.CUSTOM_DATA: {},
PSKeys.EXECUTION_SOURCE: execution_source,
PSKeys.PLATFORM_SERVICE_API_KEY: platform_api_key,
}
return ExecutionContext(
executor_name="legacy",
operation=Operation.ANSWER_PROMPT.value,
executor_params=params,
run_id=run_id,
execution_source=execution_source,
)
def _mock_llm():
"""Create a mock LLM that returns a configurable answer."""
llm = MagicMock(name="llm")
response = MagicMock()
response.text = "test answer"
llm.complete.return_value = {
PSKeys.RESPONSE: response,
PSKeys.HIGHLIGHT_DATA: [],
PSKeys.CONFIDENCE_DATA: None,
PSKeys.WORD_CONFIDENCE_DATA: None,
PSKeys.LINE_NUMBERS: [],
PSKeys.WHISPER_HASH: "",
}
llm.get_usage_reason.return_value = "extraction"
llm.get_metrics.return_value = {"tokens": 100}
return llm
def _mock_deps(llm=None):
"""Return a tuple of mocked prompt deps matching _get_prompt_deps()."""
if llm is None:
llm = _mock_llm()
# AnswerPromptService — use the real class
from executor.executors.answer_prompt import AnswerPromptService as answer_prompt_svc_cls
retrieval_svc = MagicMock(name="RetrievalService")
retrieval_svc.run_retrieval.return_value = ["chunk1", "chunk2"]
retrieval_svc.retrieve_complete_context.return_value = ["full content"]
variable_replacement_svc = MagicMock(name="VariableReplacementService")
variable_replacement_svc.is_variables_present.return_value = False
index_cls = MagicMock(name="Index")
index_instance = MagicMock()
index_instance.generate_index_key.return_value = "doc-id-1"
index_cls.return_value = index_instance
llm_cls = MagicMock(name="LLM")
llm_cls.return_value = llm
embedding_compat_cls = MagicMock(name="EmbeddingCompat")
vector_db_cls = MagicMock(name="VectorDB")
return (
answer_prompt_svc_cls,
retrieval_svc,
variable_replacement_svc,
index_cls,
llm_cls,
embedding_compat_cls,
vector_db_cls,
)
# ---------------------------------------------------------------------------
# Fixtures
# ---------------------------------------------------------------------------
_PATCH_INDEX_UTILS = (
"unstract.sdk1.utils.indexing.IndexingUtils.generate_index_key"
)
@pytest.fixture(autouse=True)
def _mock_indexing_utils():
"""Mock IndexingUtils.generate_index_key for all answer_prompt tests.
_handle_answer_prompt calls IndexingUtils.generate_index_key(tool=shim)
which delegates to PlatformHelper.get_adapter_config() — a real HTTP
call. Since tests use a mock shim, the platform URL is invalid.
"""
with patch(_PATCH_INDEX_UTILS, return_value="doc-id-test"):
yield
# ---------------------------------------------------------------------------
# Tests — _handle_answer_prompt
# ---------------------------------------------------------------------------
class TestHandleAnswerPromptText:
"""Tests for TEXT type prompts."""
@patch(
"executor.executors.legacy_executor.LegacyExecutor._get_prompt_deps"
)
@patch("executor.executors.legacy_executor.ExecutorToolShim")
def test_text_prompt_returns_success(self, mock_shim_cls, mock_deps):
"""Simple TEXT prompt returns success with structured output."""
from executor.executors.legacy_executor import LegacyExecutor
llm = _mock_llm()
mock_deps.return_value = _mock_deps(llm)
mock_shim_cls.return_value = MagicMock()
executor = LegacyExecutor()
ctx = _make_context()
result = executor._handle_answer_prompt(ctx)
assert result.success is True
assert PSKeys.OUTPUT in result.data
assert PSKeys.METADATA in result.data
assert PSKeys.METRICS in result.data
assert "field_a" in result.data[PSKeys.OUTPUT]
@patch(
"executor.executors.legacy_executor.LegacyExecutor._get_prompt_deps"
)
@patch("executor.executors.legacy_executor.ExecutorToolShim")
def test_text_prompt_answer_stored(self, mock_shim_cls, mock_deps):
"""The LLM answer is stored in structured_output."""
from executor.executors.legacy_executor import LegacyExecutor
llm = _mock_llm()
mock_deps.return_value = _mock_deps(llm)
mock_shim_cls.return_value = MagicMock()
executor = LegacyExecutor()
ctx = _make_context()
result = executor._handle_answer_prompt(ctx)
assert result.data[PSKeys.OUTPUT]["field_a"] == "test answer"
@patch(
"executor.executors.legacy_executor.LegacyExecutor._get_prompt_deps"
)
@patch("executor.executors.legacy_executor.ExecutorToolShim")
def test_trailing_newline_stripped(self, mock_shim_cls, mock_deps):
"""Trailing newlines are stripped from text answers."""
from executor.executors.legacy_executor import LegacyExecutor
llm = _mock_llm()
response = MagicMock()
response.text = "answer with trailing\n"
llm.complete.return_value = {
PSKeys.RESPONSE: response,
PSKeys.HIGHLIGHT_DATA: [],
PSKeys.CONFIDENCE_DATA: None,
PSKeys.WORD_CONFIDENCE_DATA: None,
PSKeys.LINE_NUMBERS: [],
PSKeys.WHISPER_HASH: "",
}
mock_deps.return_value = _mock_deps(llm)
mock_shim_cls.return_value = MagicMock()
executor = LegacyExecutor()
result = executor._handle_answer_prompt(_make_context())
assert result.data[PSKeys.OUTPUT]["field_a"] == "answer with trailing"
class TestHandleAnswerPromptTypes:
"""Tests for type-specific post-processing."""
@patch(
"executor.executors.legacy_executor.LegacyExecutor._get_prompt_deps"
)
@patch("executor.executors.legacy_executor.ExecutorToolShim")
def test_number_type_converts_to_float(self, mock_shim_cls, mock_deps):
"""NUMBER type converts answer to float."""
from executor.executors.legacy_executor import LegacyExecutor
llm = _mock_llm()
# First call: main retrieval answer. Second call: number extraction.
response1 = MagicMock()
response1.text = "revenue is $42.5M"
response2 = MagicMock()
response2.text = "42500000"
llm.complete.side_effect = [
{PSKeys.RESPONSE: response1, PSKeys.HIGHLIGHT_DATA: [],
PSKeys.CONFIDENCE_DATA: None, PSKeys.WORD_CONFIDENCE_DATA: None,
PSKeys.LINE_NUMBERS: [], PSKeys.WHISPER_HASH: ""},
{PSKeys.RESPONSE: response2, PSKeys.HIGHLIGHT_DATA: [],
PSKeys.CONFIDENCE_DATA: None, PSKeys.WORD_CONFIDENCE_DATA: None,
PSKeys.LINE_NUMBERS: [], PSKeys.WHISPER_HASH: ""},
]
mock_deps.return_value = _mock_deps(llm)
mock_shim_cls.return_value = MagicMock()
executor = LegacyExecutor()
ctx = _make_context(prompts=[_make_prompt(output_type="number")])
result = executor._handle_answer_prompt(ctx)
assert result.data[PSKeys.OUTPUT]["field_a"] == pytest.approx(42500000.0)
@patch(
"executor.executors.legacy_executor.LegacyExecutor._get_prompt_deps"
)
@patch("executor.executors.legacy_executor.ExecutorToolShim")
def test_number_na_returns_none(self, mock_shim_cls, mock_deps):
"""NUMBER type with NA answer returns None."""
from executor.executors.legacy_executor import LegacyExecutor
llm = _mock_llm()
response = MagicMock()
response.text = "NA"
llm.complete.return_value = {
PSKeys.RESPONSE: response, PSKeys.HIGHLIGHT_DATA: [],
PSKeys.CONFIDENCE_DATA: None, PSKeys.WORD_CONFIDENCE_DATA: None,
PSKeys.LINE_NUMBERS: [], PSKeys.WHISPER_HASH: "",
}
mock_deps.return_value = _mock_deps(llm)
mock_shim_cls.return_value = MagicMock()
executor = LegacyExecutor()
ctx = _make_context(prompts=[_make_prompt(output_type="number")])
result = executor._handle_answer_prompt(ctx)
# NA → sanitized to None
assert result.data[PSKeys.OUTPUT]["field_a"] is None
@patch(
"executor.executors.legacy_executor.LegacyExecutor._get_prompt_deps"
)
@patch("executor.executors.legacy_executor.ExecutorToolShim")
def test_boolean_yes(self, mock_shim_cls, mock_deps):
"""BOOLEAN type converts 'yes' to True."""
from executor.executors.legacy_executor import LegacyExecutor
llm = _mock_llm()
response1 = MagicMock()
response1.text = "The document confirms it"
response2 = MagicMock()
response2.text = "yes"
llm.complete.side_effect = [
{PSKeys.RESPONSE: response1, PSKeys.HIGHLIGHT_DATA: [],
PSKeys.CONFIDENCE_DATA: None, PSKeys.WORD_CONFIDENCE_DATA: None,
PSKeys.LINE_NUMBERS: [], PSKeys.WHISPER_HASH: ""},
{PSKeys.RESPONSE: response2, PSKeys.HIGHLIGHT_DATA: [],
PSKeys.CONFIDENCE_DATA: None, PSKeys.WORD_CONFIDENCE_DATA: None,
PSKeys.LINE_NUMBERS: [], PSKeys.WHISPER_HASH: ""},
]
mock_deps.return_value = _mock_deps(llm)
mock_shim_cls.return_value = MagicMock()
executor = LegacyExecutor()
ctx = _make_context(prompts=[_make_prompt(output_type="boolean")])
result = executor._handle_answer_prompt(ctx)
assert result.data[PSKeys.OUTPUT]["field_a"] is True
@patch(
"executor.executors.legacy_executor.LegacyExecutor._get_prompt_deps"
)
@patch("executor.executors.legacy_executor.ExecutorToolShim")
def test_boolean_no(self, mock_shim_cls, mock_deps):
"""BOOLEAN type converts 'no' to False."""
from executor.executors.legacy_executor import LegacyExecutor
llm = _mock_llm()
response1 = MagicMock()
response1.text = "not confirmed"
response2 = MagicMock()
response2.text = "no"
llm.complete.side_effect = [
{PSKeys.RESPONSE: response1, PSKeys.HIGHLIGHT_DATA: [],
PSKeys.CONFIDENCE_DATA: None, PSKeys.WORD_CONFIDENCE_DATA: None,
PSKeys.LINE_NUMBERS: [], PSKeys.WHISPER_HASH: ""},
{PSKeys.RESPONSE: response2, PSKeys.HIGHLIGHT_DATA: [],
PSKeys.CONFIDENCE_DATA: None, PSKeys.WORD_CONFIDENCE_DATA: None,
PSKeys.LINE_NUMBERS: [], PSKeys.WHISPER_HASH: ""},
]
mock_deps.return_value = _mock_deps(llm)
mock_shim_cls.return_value = MagicMock()
executor = LegacyExecutor()
ctx = _make_context(prompts=[_make_prompt(output_type="boolean")])
result = executor._handle_answer_prompt(ctx)
assert result.data[PSKeys.OUTPUT]["field_a"] is False
@patch(
"executor.executors.legacy_executor.LegacyExecutor._get_prompt_deps"
)
@patch("executor.executors.legacy_executor.ExecutorToolShim")
def test_email_type(self, mock_shim_cls, mock_deps):
"""EMAIL type extracts email address."""
from executor.executors.legacy_executor import LegacyExecutor
llm = _mock_llm()
response1 = MagicMock()
response1.text = "Contact: [email protected]"
response2 = MagicMock()
response2.text = "[email protected]"
llm.complete.side_effect = [
{PSKeys.RESPONSE: response1, PSKeys.HIGHLIGHT_DATA: [],
PSKeys.CONFIDENCE_DATA: None, PSKeys.WORD_CONFIDENCE_DATA: None,
PSKeys.LINE_NUMBERS: [], PSKeys.WHISPER_HASH: ""},
{PSKeys.RESPONSE: response2, PSKeys.HIGHLIGHT_DATA: [],
PSKeys.CONFIDENCE_DATA: None, PSKeys.WORD_CONFIDENCE_DATA: None,
PSKeys.LINE_NUMBERS: [], PSKeys.WHISPER_HASH: ""},
]
mock_deps.return_value = _mock_deps(llm)
mock_shim_cls.return_value = MagicMock()
executor = LegacyExecutor()
ctx = _make_context(prompts=[_make_prompt(output_type="email")])
result = executor._handle_answer_prompt(ctx)
assert result.data[PSKeys.OUTPUT]["field_a"] == "[email protected]"
@patch(
"executor.executors.legacy_executor.LegacyExecutor._get_prompt_deps"
)
@patch("executor.executors.legacy_executor.ExecutorToolShim")
def test_email_na_returns_none(self, mock_shim_cls, mock_deps):
"""EMAIL type returns None when no email is found (not literal 'NA')."""
from executor.executors.legacy_executor import LegacyExecutor
llm = _mock_llm()
response1 = MagicMock()
response1.text = "There is no email mentioned in the document."
response2 = MagicMock()
response2.text = "NA"
llm.complete.side_effect = [
{PSKeys.RESPONSE: response1, PSKeys.HIGHLIGHT_DATA: [],
PSKeys.CONFIDENCE_DATA: None, PSKeys.WORD_CONFIDENCE_DATA: None,
PSKeys.LINE_NUMBERS: [], PSKeys.WHISPER_HASH: ""},
{PSKeys.RESPONSE: response2, PSKeys.HIGHLIGHT_DATA: [],
PSKeys.CONFIDENCE_DATA: None, PSKeys.WORD_CONFIDENCE_DATA: None,
PSKeys.LINE_NUMBERS: [], PSKeys.WHISPER_HASH: ""},
]
mock_deps.return_value = _mock_deps(llm)
mock_shim_cls.return_value = MagicMock()
executor = LegacyExecutor()
ctx = _make_context(prompts=[_make_prompt(output_type="email")])
result = executor._handle_answer_prompt(ctx)
assert result.data[PSKeys.OUTPUT]["field_a"] is None
@patch(
"executor.executors.legacy_executor.LegacyExecutor._get_prompt_deps"
)
@patch("executor.executors.legacy_executor.ExecutorToolShim")
def test_date_type(self, mock_shim_cls, mock_deps):
"""DATE type extracts date in ISO format."""
from executor.executors.legacy_executor import LegacyExecutor
llm = _mock_llm()
response1 = MagicMock()
response1.text = "The date is January 15, 2024"
response2 = MagicMock()
response2.text = "2024-01-15"
llm.complete.side_effect = [
{PSKeys.RESPONSE: response1, PSKeys.HIGHLIGHT_DATA: [],
PSKeys.CONFIDENCE_DATA: None, PSKeys.WORD_CONFIDENCE_DATA: None,
PSKeys.LINE_NUMBERS: [], PSKeys.WHISPER_HASH: ""},
{PSKeys.RESPONSE: response2, PSKeys.HIGHLIGHT_DATA: [],
PSKeys.CONFIDENCE_DATA: None, PSKeys.WORD_CONFIDENCE_DATA: None,
PSKeys.LINE_NUMBERS: [], PSKeys.WHISPER_HASH: ""},
]
mock_deps.return_value = _mock_deps(llm)
mock_shim_cls.return_value = MagicMock()
executor = LegacyExecutor()
ctx = _make_context(prompts=[_make_prompt(output_type="date")])
result = executor._handle_answer_prompt(ctx)
assert result.data[PSKeys.OUTPUT]["field_a"] == "2024-01-15"
@patch(
"executor.executors.legacy_executor.LegacyExecutor._get_prompt_deps"
)
@patch("executor.executors.legacy_executor.ExecutorToolShim")
def test_date_na_returns_none(self, mock_shim_cls, mock_deps):
"""DATE type returns None when no date is found (not literal 'NA')."""
from executor.executors.legacy_executor import LegacyExecutor
llm = _mock_llm()
response1 = MagicMock()
response1.text = "No date is mentioned in the text."
response2 = MagicMock()
response2.text = "NA"
llm.complete.side_effect = [
{PSKeys.RESPONSE: response1, PSKeys.HIGHLIGHT_DATA: [],
PSKeys.CONFIDENCE_DATA: None, PSKeys.WORD_CONFIDENCE_DATA: None,
PSKeys.LINE_NUMBERS: [], PSKeys.WHISPER_HASH: ""},
{PSKeys.RESPONSE: response2, PSKeys.HIGHLIGHT_DATA: [],
PSKeys.CONFIDENCE_DATA: None, PSKeys.WORD_CONFIDENCE_DATA: None,
PSKeys.LINE_NUMBERS: [], PSKeys.WHISPER_HASH: ""},
]
mock_deps.return_value = _mock_deps(llm)
mock_shim_cls.return_value = MagicMock()
executor = LegacyExecutor()
ctx = _make_context(prompts=[_make_prompt(output_type="date")])
result = executor._handle_answer_prompt(ctx)
assert result.data[PSKeys.OUTPUT]["field_a"] is None
class TestHandleAnswerPromptJSON:
"""Tests for JSON type handling."""
@patch(
"executor.executors.legacy_executor.LegacyExecutor._get_prompt_deps"
)
@patch("executor.executors.legacy_executor.ExecutorToolShim")
def test_json_parsed(self, mock_shim_cls, mock_deps):
"""JSON type parses valid JSON from answer."""
from executor.executors.legacy_executor import LegacyExecutor
llm = _mock_llm()
response = MagicMock()
response.text = '{"key": "value"}'
llm.complete.return_value = {
PSKeys.RESPONSE: response, PSKeys.HIGHLIGHT_DATA: [],
PSKeys.CONFIDENCE_DATA: None, PSKeys.WORD_CONFIDENCE_DATA: None,
PSKeys.LINE_NUMBERS: [], PSKeys.WHISPER_HASH: "",
}
mock_deps.return_value = _mock_deps(llm)
mock_shim_cls.return_value = MagicMock()
executor = LegacyExecutor()
ctx = _make_context(prompts=[_make_prompt(output_type="json")])
result = executor._handle_answer_prompt(ctx)
assert result.data[PSKeys.OUTPUT]["field_a"] == {"key": "value"}
@patch(
"executor.executors.legacy_executor.LegacyExecutor._get_prompt_deps"
)
@patch("executor.executors.legacy_executor.ExecutorToolShim")
def test_json_na_returns_none(self, mock_shim_cls, mock_deps):
"""JSON type with NA answer returns None."""
from executor.executors.legacy_executor import LegacyExecutor
llm = _mock_llm()
response = MagicMock()
response.text = "NA"
llm.complete.return_value = {
PSKeys.RESPONSE: response, PSKeys.HIGHLIGHT_DATA: [],
PSKeys.CONFIDENCE_DATA: None, PSKeys.WORD_CONFIDENCE_DATA: None,
PSKeys.LINE_NUMBERS: [], PSKeys.WHISPER_HASH: "",
}
mock_deps.return_value = _mock_deps(llm)
mock_shim_cls.return_value = MagicMock()
executor = LegacyExecutor()
ctx = _make_context(prompts=[_make_prompt(output_type="json")])
result = executor._handle_answer_prompt(ctx)
assert result.data[PSKeys.OUTPUT]["field_a"] is None
class TestHandleAnswerPromptRetrieval:
"""Tests for retrieval integration."""
@patch(
"executor.executors.legacy_executor.LegacyExecutor._get_prompt_deps"
)
@patch("executor.executors.legacy_executor.ExecutorToolShim")
def test_chunked_retrieval_uses_run_retrieval(
self, mock_shim_cls, mock_deps
):
"""chunk_size > 0 uses RetrievalService.run_retrieval."""
from executor.executors.legacy_executor import LegacyExecutor
llm = _mock_llm()
deps = _mock_deps(llm)
_, retrieval_svc, *_ = deps
mock_deps.return_value = deps
mock_shim_cls.return_value = MagicMock()
executor = LegacyExecutor()
ctx = _make_context(
prompts=[_make_prompt(chunk_size=512)]
)
result = executor._handle_answer_prompt(ctx)
retrieval_svc.run_retrieval.assert_called_once()
assert result.success is True
@patch(
"executor.executors.legacy_executor.LegacyExecutor._get_prompt_deps"
)
@patch("executor.executors.legacy_executor.ExecutorToolShim")
def test_complete_context_for_chunk_zero(
self, mock_shim_cls, mock_deps
):
"""chunk_size=0 uses RetrievalService.retrieve_complete_context."""
from executor.executors.legacy_executor import LegacyExecutor
llm = _mock_llm()
deps = _mock_deps(llm)
_, retrieval_svc, *_ = deps
mock_deps.return_value = deps
mock_shim_cls.return_value = MagicMock()
executor = LegacyExecutor()
ctx = _make_context(
prompts=[_make_prompt(chunk_size=0)]
)
result = executor._handle_answer_prompt(ctx)
retrieval_svc.retrieve_complete_context.assert_called_once()
assert result.success is True
@patch(
"executor.executors.legacy_executor.LegacyExecutor._get_prompt_deps"
)
@patch("executor.executors.legacy_executor.ExecutorToolShim")
def test_context_stored_in_metadata(self, mock_shim_cls, mock_deps):
"""Retrieved context is stored in metadata."""
from executor.executors.legacy_executor import LegacyExecutor
llm = _mock_llm()
mock_deps.return_value = _mock_deps(llm)
mock_shim_cls.return_value = MagicMock()
executor = LegacyExecutor()
result = executor._handle_answer_prompt(_make_context())
metadata = result.data[PSKeys.METADATA]
assert "field_a" in metadata[PSKeys.CONTEXT]
@patch(
"executor.executors.legacy_executor.LegacyExecutor._get_prompt_deps"
)
@patch("executor.executors.legacy_executor.ExecutorToolShim")
def test_invalid_strategy_skips_retrieval(
self, mock_shim_cls, mock_deps
):
"""Invalid retrieval strategy skips retrieval, answer stays NA."""
from executor.executors.legacy_executor import LegacyExecutor
llm = _mock_llm()
mock_deps.return_value = _mock_deps(llm)
mock_shim_cls.return_value = MagicMock()
executor = LegacyExecutor()
ctx = _make_context(
prompts=[_make_prompt(retrieval_strategy="nonexistent")]
)
result = executor._handle_answer_prompt(ctx)
# Answer stays "NA" which gets sanitized to None
assert result.data[PSKeys.OUTPUT]["field_a"] is None
class TestHandleAnswerPromptMultiPrompt:
"""Tests for multi-prompt processing."""
@patch(
"executor.executors.legacy_executor.LegacyExecutor._get_prompt_deps"
)
@patch("executor.executors.legacy_executor.ExecutorToolShim")
def test_multiple_prompts(self, mock_shim_cls, mock_deps):
"""Multiple prompts are all processed."""
from executor.executors.legacy_executor import LegacyExecutor
llm = _mock_llm()
mock_deps.return_value = _mock_deps(llm)
mock_shim_cls.return_value = MagicMock()
prompts = [
_make_prompt(name="revenue"),
_make_prompt(name="date_signed", output_type="text"),
]
executor = LegacyExecutor()
ctx = _make_context(prompts=prompts)
result = executor._handle_answer_prompt(ctx)
output = result.data[PSKeys.OUTPUT]
assert "revenue" in output
assert "date_signed" in output
class TestHandleAnswerPromptErrors:
"""Tests for error handling."""
@patch(
"executor.executors.legacy_executor.LegacyExecutor._get_prompt_deps"
)
@patch("executor.executors.legacy_executor.ExecutorToolShim")
def test_table_type_raises_error(self, mock_shim_cls, mock_deps):
"""TABLE type raises LegacyExecutorError (plugins not available)."""
from executor.executors.legacy_executor import LegacyExecutor
llm = _mock_llm()
mock_deps.return_value = _mock_deps(llm)
mock_shim_cls.return_value = MagicMock()
executor = LegacyExecutor()
ctx = _make_context(
prompts=[_make_prompt(output_type="table")]
)
# TABLE raises LegacyExecutorError which is caught by execute()
result = executor.execute(ctx)
assert result.success is False
assert "TABLE" in result.error
@patch(
"executor.executors.legacy_executor.LegacyExecutor._get_prompt_deps"
)
@patch("executor.executors.legacy_executor.ExecutorToolShim")
def test_line_item_type_raises_error(self, mock_shim_cls, mock_deps):
"""LINE_ITEM type raises LegacyExecutorError."""
from executor.executors.legacy_executor import LegacyExecutor
llm = _mock_llm()
mock_deps.return_value = _mock_deps(llm)
mock_shim_cls.return_value = MagicMock()
executor = LegacyExecutor()
ctx = _make_context(
prompts=[_make_prompt(output_type="line-item")]
)
result = executor.execute(ctx)
assert result.success is False
assert "LINE_ITEM" in result.error
class TestHandleAnswerPromptMetrics:
"""Tests for metrics collection."""
@patch(
"executor.executors.legacy_executor.LegacyExecutor._get_prompt_deps"
)
@patch("executor.executors.legacy_executor.ExecutorToolShim")
def test_metrics_collected(self, mock_shim_cls, mock_deps):
"""Metrics include context_retrieval and LLM metrics."""
from executor.executors.legacy_executor import LegacyExecutor
llm = _mock_llm()
mock_deps.return_value = _mock_deps(llm)
mock_shim_cls.return_value = MagicMock()
executor = LegacyExecutor()
result = executor._handle_answer_prompt(_make_context())
metrics = result.data[PSKeys.METRICS]
assert "field_a" in metrics
assert "context_retrieval" in metrics["field_a"]
assert "extraction_llm" in metrics["field_a"]
@patch(
"executor.executors.legacy_executor.LegacyExecutor._get_prompt_deps"
)
@patch("executor.executors.legacy_executor.ExecutorToolShim")
def test_vectordb_closed(self, mock_shim_cls, mock_deps):
"""VectorDB is closed after processing."""
from executor.executors.legacy_executor import LegacyExecutor
llm = _mock_llm()
deps = _mock_deps(llm)
mock_deps.return_value = deps
_, _, _, _, _, _, vector_db_cls = deps
vdb_instance = MagicMock()
vector_db_cls.return_value = vdb_instance
mock_shim_cls.return_value = MagicMock()
executor = LegacyExecutor()
executor._handle_answer_prompt(_make_context())
vdb_instance.close.assert_called_once()
class TestNullSanitization:
"""Tests for _sanitize_null_values."""
def test_na_string_becomes_none(self):
"""Top-level 'NA' string → None."""
from executor.executors.legacy_executor import LegacyExecutor
output = {"field": "NA"}
result = LegacyExecutor._sanitize_null_values(output)
assert result["field"] is None
def test_na_case_insensitive(self):
"""'na' (lowercase) → None."""
from executor.executors.legacy_executor import LegacyExecutor
output = {"field": "na"}
result = LegacyExecutor._sanitize_null_values(output)
assert result["field"] is None
def test_nested_list_na(self):
"""NA in nested list items → None."""
from executor.executors.legacy_executor import LegacyExecutor
output = {"field": ["value", "NA", "other"]}
result = LegacyExecutor._sanitize_null_values(output)
assert result["field"] == ["value", None, "other"]
def test_nested_dict_in_list_na(self):
"""NA in dicts inside lists → None."""
from executor.executors.legacy_executor import LegacyExecutor
output = {"field": [{"a": "NA", "b": "ok"}]}
result = LegacyExecutor._sanitize_null_values(output)
assert result["field"] == [{"a": None, "b": "ok"}]
def test_nested_dict_na(self):
"""NA in nested dict values → None."""
from executor.executors.legacy_executor import LegacyExecutor
output = {"field": {"a": "NA", "b": "ok"}}
result = LegacyExecutor._sanitize_null_values(output)
assert result["field"] == {"a": None, "b": "ok"}
def test_non_na_values_untouched(self):
"""Non-NA values are not modified."""
from executor.executors.legacy_executor import LegacyExecutor
output = {"field": "hello", "num": 42, "flag": True}
result = LegacyExecutor._sanitize_null_values(output)
assert result == {"field": "hello", "num": 42, "flag": True}
class TestConvertScalarAnswer:
"""Tests for _convert_scalar_answer second-pass NA handling."""
def test_first_pass_na_returns_none(self):
"""If the initial answer is 'na', return None without a second LLM call."""
from executor.executors.legacy_executor import LegacyExecutor
mock_svc = MagicMock()
result = LegacyExecutor._convert_scalar_answer(
"NA", llm=MagicMock(), answer_prompt_svc=mock_svc, prompt="ignored"
)
assert result is None
mock_svc.run_completion.assert_not_called()
def test_second_pass_na_returns_none(self):
"""If the LLM extraction also returns 'NA', return None."""
from executor.executors.legacy_executor import LegacyExecutor
mock_svc = MagicMock()
mock_svc.run_completion.return_value = "NA"
result = LegacyExecutor._convert_scalar_answer(
"I do not see an email here.",
llm=MagicMock(),
answer_prompt_svc=mock_svc,
prompt="extract email",
)
assert result is None
mock_svc.run_completion.assert_called_once()
def test_successful_extraction_returns_value(self):
"""If extraction yields a real value, return it."""
from executor.executors.legacy_executor import LegacyExecutor
mock_svc = MagicMock()
mock_svc.run_completion.return_value = "[email protected]"
result = LegacyExecutor._convert_scalar_answer(
"Contact: [email protected]",
llm=MagicMock(),
answer_prompt_svc=mock_svc,
prompt="extract email",
)
assert result == "[email protected]"
class TestAnswerPromptServiceUnit:
"""Unit tests for AnswerPromptService methods."""
def test_extract_variable_replaces_percent_vars(self):
"""Replace %var% references in prompt text."""
from executor.executors.answer_prompt import AnswerPromptService
structured = {"field_a": "42"}
output = {"prompt": "Original: %field_a%"}
result = AnswerPromptService.extract_variable(
structured, ["field_a"], output, "Value is %field_a%"
)
assert result == "Value is 42"
def test_extract_variable_missing_raises(self):
"""Missing variable raises ValueError."""
from executor.executors.answer_prompt import AnswerPromptService
output = {"prompt": "test"}
with pytest.raises(ValueError, match="not found"):
AnswerPromptService.extract_variable(
{}, ["missing_var"], output, "Value is %missing_var%"
)
def test_construct_prompt_includes_all_parts(self):
"""Constructed prompt includes preamble, prompt, postamble, context."""
from executor.executors.answer_prompt import AnswerPromptService
result = AnswerPromptService.construct_prompt(
preamble="You are a helpful assistant",
prompt="What is the revenue?",
postamble="Be precise",
grammar_list=[],
context="Revenue was $1M",
platform_postamble="",
word_confidence_postamble="",
)
assert "You are a helpful assistant" in result
assert "What is the revenue?" in result
assert "Be precise" in result
assert "Revenue was $1M" in result
assert "Answer:" in result
def test_construct_prompt_with_grammar(self):
"""Grammar list adds synonym notes."""
from executor.executors.answer_prompt import AnswerPromptService
result = AnswerPromptService.construct_prompt(
preamble="",
prompt="Find the amount",
postamble="",
grammar_list=[{"word": "amount", "synonyms": ["sum", "total"]}],
context="test",
platform_postamble="",
word_confidence_postamble="",
)
assert "amount" in result
assert "sum, total" in result
class TestVariableReplacementService:
"""Tests for the VariableReplacementService."""
def test_is_variables_present_true(self):
"""Detects {{variables}} in text."""
from executor.executors.variable_replacement import (
VariableReplacementService,
)
assert VariableReplacementService.is_variables_present(
"Hello {{name}}"
) is True
def test_is_variables_present_false(self):
"""Returns False when no variables present."""
from executor.executors.variable_replacement import (
VariableReplacementService,
)
assert VariableReplacementService.is_variables_present(
"Hello world"
) is False
def test_replace_static_variable(self):
"""Static variable {{var}} is replaced with structured output value."""
from executor.executors.variable_replacement import (
VariableReplacementHelper,
)
result = VariableReplacementHelper.replace_static_variable(
prompt="Total is {{revenue}}",
structured_output={"revenue": "$1M"},
variable="revenue",
)
assert result == "Total is $1M"
def test_custom_data_variable(self):
"""Custom data variable {{custom_data.key}} is replaced."""
from executor.executors.variable_replacement import (
VariableReplacementHelper,
)
result = VariableReplacementHelper.replace_custom_data_variable(
prompt="Company: {{custom_data.company_name}}",
variable="custom_data.company_name",
custom_data={"company_name": "Acme Inc"},
)
assert result == "Company: Acme Inc"
def test_custom_data_missing_raises(self):
"""Missing custom data key raises CustomDataError."""
from executor.executors.exceptions import CustomDataError
from executor.executors.variable_replacement import (
VariableReplacementHelper,
)
with pytest.raises(CustomDataError):
VariableReplacementHelper.replace_custom_data_variable(
prompt="{{custom_data.missing}}",
variable="custom_data.missing",
custom_data={"other": "value"},
)