LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
"""Executor log streaming to frontend via Socket.IO.
Tests cover:
- ExecutionContext round-trips log_events_id through to_dict/from_dict
- LogPublisher.log_progress() returns type: "PROGRESS" (not "LOG")
- LogPublisher.log_prompt() still returns type: "LOG" (unchanged)
- ExecutorToolShim with log_events_id: stream_log() publishes progress
- ExecutorToolShim without log_events_id: no publishing, no exceptions
- ExecutorToolShim with failing LogPublisher: no exception raised
- execute_extraction builds component dict when log_events_id present
- execute_extraction skips component dict when log_events_id absent
"""
from unittest.mock import MagicMock, patch
from unstract.sdk1.constants import LogLevel
from unstract.sdk1.execution.context import ExecutionContext
# ---------------------------------------------------------------------------
# 1A — ExecutionContext.log_events_id round-trip
# ---------------------------------------------------------------------------
class TestExecutionContextLogEventsId:
"""Verify log_events_id serialization in ExecutionContext."""
def test_log_events_id_default_is_none(self):
ctx = ExecutionContext(
executor_name="legacy",
operation="extract",
run_id="r1",
execution_source="ide",
)
assert ctx.log_events_id is None
def test_log_events_id_round_trips(self):
ctx = ExecutionContext(
executor_name="legacy",
operation="extract",
run_id="r1",
execution_source="ide",
log_events_id="session-abc",
)
d = ctx.to_dict()
assert d["log_events_id"] == "session-abc"
restored = ExecutionContext.from_dict(d)
assert restored.log_events_id == "session-abc"
def test_log_events_id_none_round_trips(self):
ctx = ExecutionContext(
executor_name="legacy",
operation="extract",
run_id="r1",
execution_source="ide",
)
d = ctx.to_dict()
assert d["log_events_id"] is None
restored = ExecutionContext.from_dict(d)
assert restored.log_events_id is None
def test_backward_compat_missing_key(self):
"""from_dict with old payload lacking log_events_id."""
old_payload = {
"executor_name": "legacy",
"operation": "extract",
"run_id": "r1",
"execution_source": "ide",
}
ctx = ExecutionContext.from_dict(old_payload)
assert ctx.log_events_id is None
# ---------------------------------------------------------------------------
# 1B-i — LogPublisher.log_progress() vs log_prompt()
# ---------------------------------------------------------------------------
class TestLogPublisherLogProgress:
"""Verify log_progress returns type PROGRESS, log_prompt returns LOG."""
def test_log_progress_type(self):
from unstract.core.pubsub_helper import LogPublisher
result = LogPublisher.log_progress(
component={"tool_id": "t1"},
level="INFO",
state="TOOL_RUN",
message="Extracting text...",
)
assert result["type"] == "PROGRESS"
assert result["service"] == "prompt"
assert result["message"] == "Extracting text..."
assert result["component"] == {"tool_id": "t1"}
assert "timestamp" in result
def test_log_prompt_type_unchanged(self):
from unstract.core.pubsub_helper import LogPublisher
result = LogPublisher.log_prompt(
component={"tool_id": "t1"},
level="INFO",
state="RUNNING",
message="test",
)
assert result["type"] == "LOG"
assert result["service"] == "prompt"
def test_log_progress_has_all_fields(self):
from unstract.core.pubsub_helper import LogPublisher
result = LogPublisher.log_progress(
component={"tool_id": "t1", "prompt_key": "pk"},
level="ERROR",
state="FAILED",
message="boom",
)
assert result["level"] == "ERROR"
assert result["state"] == "FAILED"
assert result["component"]["prompt_key"] == "pk"
# ---------------------------------------------------------------------------
# 1B-ii — ExecutorToolShim progress publishing
# ---------------------------------------------------------------------------
class TestExecutorToolShimProgress:
"""Verify ExecutorToolShim publishes progress via LogPublisher."""
@patch("executor.executor_tool_shim.LogPublisher")
def test_stream_log_publishes_when_log_events_id_set(self, mock_lp):
from executor.executor_tool_shim import ExecutorToolShim
component = {"tool_id": "t1", "run_id": "r1"}
shim = ExecutorToolShim(
platform_api_key="sk-test",
log_events_id="session-xyz",
component=component,
)
shim.stream_log("Extracting...", level=LogLevel.INFO)
mock_lp.log_progress.assert_called_once_with(
component=component,
level="INFO",
state="TOOL_RUN",
message="Extracting...",
)
mock_lp.publish.assert_called_once_with(
channel_id="session-xyz",
payload=mock_lp.log_progress.return_value,
)
@patch("executor.executor_tool_shim.LogPublisher")
def test_stream_log_no_publish_without_log_events_id(self, mock_lp):
from executor.executor_tool_shim import ExecutorToolShim
shim = ExecutorToolShim(platform_api_key="sk-test")
shim.stream_log("Hello", level=LogLevel.INFO)
mock_lp.log_progress.assert_not_called()
mock_lp.publish.assert_not_called()
@patch("executor.executor_tool_shim.LogPublisher")
def test_stream_log_empty_log_events_id_no_publish(self, mock_lp):
from executor.executor_tool_shim import ExecutorToolShim
shim = ExecutorToolShim(platform_api_key="sk-test", log_events_id="")
shim.stream_log("Hello", level=LogLevel.INFO)
mock_lp.log_progress.assert_not_called()
@patch("executor.executor_tool_shim.LogPublisher")
def test_stream_log_swallows_publish_error(self, mock_lp):
from executor.executor_tool_shim import ExecutorToolShim
mock_lp.publish.side_effect = ConnectionError("AMQP down")
shim = ExecutorToolShim(
platform_api_key="sk-test",
log_events_id="session-xyz",
component={"tool_id": "t1"},
)
# Should NOT raise
shim.stream_log("test", level=LogLevel.INFO)
@patch("executor.executor_tool_shim.LogPublisher")
def test_level_mapping(self, mock_lp):
from executor.executor_tool_shim import ExecutorToolShim
shim = ExecutorToolShim(
platform_api_key="sk-test",
log_events_id="s1",
component={},
)
# DEBUG is below the shim's log_level (INFO) so it should NOT
# be published to the frontend.
shim.stream_log("msg", level=LogLevel.DEBUG)
assert not mock_lp.log_progress.called, (
"DEBUG should be filtered out (below INFO threshold)"
)
# INFO and above should be published with the correct mapped level.
published_cases = [
(LogLevel.INFO, "INFO"),
(LogLevel.WARN, "WARN"),
(LogLevel.ERROR, "ERROR"),
(LogLevel.FATAL, "ERROR"),
]
for sdk_level, expected_wf_level in published_cases:
mock_lp.reset_mock()
shim.stream_log("msg", level=sdk_level)
call_kwargs = mock_lp.log_progress.call_args
assert call_kwargs.kwargs["level"] == expected_wf_level, (
f"SDK {sdk_level} should map to {expected_wf_level}"
)
@patch("executor.executor_tool_shim.LogPublisher")
def test_custom_stage_passed_through(self, mock_lp):
from executor.executor_tool_shim import ExecutorToolShim
shim = ExecutorToolShim(
platform_api_key="sk-test",
log_events_id="s1",
component={},
)
shim.stream_log("msg", level=LogLevel.INFO, stage="INDEXING")
call_kwargs = mock_lp.log_progress.call_args
assert call_kwargs.kwargs["state"] == "INDEXING"
# ---------------------------------------------------------------------------
# 1C — Component dict building in execute_extraction
# ---------------------------------------------------------------------------
class TestExecuteExtractionComponentDict:
"""Verify component dict is built from executor_params."""
@patch("executor.tasks.ExecutionOrchestrator")
def test_component_dict_built_when_log_events_id_present(self, mock_orch_cls):
mock_orch = MagicMock()
mock_orch.execute.return_value = MagicMock(
success=True, to_dict=lambda: {"success": True}
)
mock_orch_cls.return_value = mock_orch
from executor.tasks import execute_extraction
payload = {
"executor_name": "legacy",
"operation": "extract",
"run_id": "r1",
"execution_source": "ide",
"log_events_id": "session-abc",
"executor_params": {
"tool_id": "tool-123",
"file_name": "invoice.pdf",
},
}
execute_extraction(payload)
# Verify the context passed to orchestrator has _log_component
ctx = mock_orch.execute.call_args[0][0]
assert ctx._log_component == {
"tool_id": "tool-123",
"run_id": "r1",
"doc_name": "invoice.pdf",
"operation": "extract",
}
@patch("executor.tasks.ExecutionOrchestrator")
def test_component_dict_empty_when_no_log_events_id(self, mock_orch_cls):
mock_orch = MagicMock()
mock_orch.execute.return_value = MagicMock(
success=True, to_dict=lambda: {"success": True}
)
mock_orch_cls.return_value = mock_orch
from executor.tasks import execute_extraction
payload = {
"executor_name": "legacy",
"operation": "extract",
"run_id": "r1",
"execution_source": "ide",
"executor_params": {},
}
execute_extraction(payload)
ctx = mock_orch.execute.call_args[0][0]
assert ctx._log_component == {}
# ---------------------------------------------------------------------------
# 1D — LegacyExecutor passes log info to shim
# ---------------------------------------------------------------------------
class TestLegacyExecutorLogPassthrough:
"""Verify LegacyExecutor passes log_events_id and component to shim."""
@patch("executor.executors.legacy_executor.FileUtils.get_fs_instance")
@patch("executor.executors.legacy_executor.X2Text")
@patch("executor.executors.legacy_executor.ExecutorToolShim")
def test_extract_passes_log_info_to_shim(
self, mock_shim_cls, mock_x2text, mock_fs, tmp_path
):
from executor.executors.legacy_executor import LegacyExecutor
from unstract.sdk1.execution.registry import ExecutorRegistry
if "legacy" not in ExecutorRegistry.list_executors():
ExecutorRegistry._registry["legacy"] = LegacyExecutor
mock_shim = MagicMock()
mock_shim_cls.return_value = mock_shim
mock_x2t = MagicMock()
mock_x2t.process.return_value = MagicMock(extracted_text="hello")
mock_x2text.return_value = mock_x2t
ctx = ExecutionContext(
executor_name="legacy",
operation="extract",
run_id="r1",
execution_source="ide",
log_events_id="session-abc",
executor_params={
"x2text_instance_id": "x2t-1",
"file_path": str(tmp_path / "test.pdf"),
"platform_api_key": "sk-test",
},
)
ctx._log_component = {"tool_id": "t1", "run_id": "r1", "doc_name": "test.pdf"}
executor = LegacyExecutor()
result = executor.execute(ctx)
assert result.success
mock_shim_cls.assert_called_once_with(
platform_api_key="sk-test",
log_events_id="session-abc",
component={"tool_id": "t1", "run_id": "r1", "doc_name": "test.pdf"},
execution_id=None,
organization_id=None,
file_execution_id=None,
)
@patch("executor.executors.legacy_executor.FileUtils.get_fs_instance")
@patch("executor.executors.legacy_executor.X2Text")
@patch("executor.executors.legacy_executor.ExecutorToolShim")
def test_extract_no_log_info_when_absent(
self, mock_shim_cls, mock_x2text, mock_fs, tmp_path
):
from executor.executors.legacy_executor import LegacyExecutor
from unstract.sdk1.execution.registry import ExecutorRegistry
if "legacy" not in ExecutorRegistry.list_executors():
ExecutorRegistry._registry["legacy"] = LegacyExecutor
mock_shim = MagicMock()
mock_shim_cls.return_value = mock_shim
mock_x2t = MagicMock()
mock_x2t.process.return_value = MagicMock(extracted_text="hello")
mock_x2text.return_value = mock_x2t
ctx = ExecutionContext(
executor_name="legacy",
operation="extract",
run_id="r1",
execution_source="tool",
executor_params={
"x2text_instance_id": "x2t-1",
"file_path": str(tmp_path / "test.pdf"),
"platform_api_key": "sk-test",
},
)
executor = LegacyExecutor()
result = executor.execute(ctx)
assert result.success
mock_shim_cls.assert_called_once_with(
platform_api_key="sk-test",
log_events_id="",
component={},
execution_id=None,
organization_id=None,
file_execution_id=None,
)
@patch("executor.executors.legacy_executor.LegacyExecutor._get_prompt_deps")
@patch("executor.executors.legacy_executor.ExecutorToolShim")
def test_answer_prompt_enriches_component_with_prompt_key(
self, mock_shim_cls, mock_prompt_deps
):
"""Verify per-prompt shim includes prompt_key in component."""
from executor.executors.legacy_executor import LegacyExecutor
from unstract.sdk1.execution.registry import ExecutorRegistry
if "legacy" not in ExecutorRegistry.list_executors():
ExecutorRegistry._registry["legacy"] = LegacyExecutor
mock_shim = MagicMock()
mock_shim_cls.return_value = mock_shim
# Mock prompt deps
mock_answer_prompt_service = MagicMock()
mock_answer_prompt_service.extract_variable.return_value = "prompt text"
mock_retrieval_service = MagicMock()
mock_variable_replacement_service = MagicMock()
mock_variable_replacement_service.is_variables_present.return_value = False
mock_index = MagicMock()
mock_llm = MagicMock()
mock_embedding_compat = MagicMock()
mock_vector_db = MagicMock()
mock_prompt_deps.return_value = (
mock_answer_prompt_service,
mock_retrieval_service,
mock_variable_replacement_service,
mock_index,
mock_llm,
mock_embedding_compat,
mock_vector_db,
)
ctx = ExecutionContext(
executor_name="legacy",
operation="answer_prompt",
run_id="r1",
execution_source="ide",
log_events_id="session-abc",
executor_params={
"tool_id": "t1",
"outputs": [
{
"name": "invoice_number",
"prompt": "What is the invoice number?",
"chunk-size": 0,
"type": "text",
"retrieval-strategy": "simple",
"vector-db": "vdb1",
"embedding": "emb1",
"x2text_adapter": "x2t1",
"chunk-overlap": 0,
"llm": "llm1",
},
],
"tool_settings": {},
"PLATFORM_SERVICE_API_KEY": "sk-test",
},
)
ctx._log_component = {
"tool_id": "t1",
"run_id": "r1",
"doc_name": "test.pdf",
}
# Mock IndexingUtils
with patch(
"unstract.sdk1.utils.indexing.IndexingUtils.generate_index_key",
return_value="doc-id-1",
):
executor = LegacyExecutor()
# The handler will try retrieval which we need to mock
mock_retrieval_service.retrieve_complete_context.return_value = ["context"]
mock_answer_prompt_service.construct_and_run_prompt.return_value = "INV-001"
executor.execute(ctx)
# Check that shim was created with prompt_key in component
shim_call = mock_shim_cls.call_args
assert shim_call.kwargs["component"]["prompt_key"] == "invoice_number"
assert shim_call.kwargs["log_events_id"] == "session-abc"