Skip to content

LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows

23.2 KB
Raw Blame
"""Consolidated regression tests for the executor plugin migration: full
Operation-to-executor coverage, multi-executor coexistence, per-executor Celery
chains, cross-cutting highlight, dispatch/result flow, queue routing, graceful
degradation when cloud plugins are absent, and log_component for all operations.
"""

from unittest.mock import MagicMock, patch

import pytest
from unstract.sdk1.execution.context import ExecutionContext, Operation
from unstract.sdk1.execution.executor import BaseExecutor
from unstract.sdk1.execution.orchestrator import ExecutionOrchestrator
from unstract.sdk1.execution.registry import ExecutorRegistry
from unstract.sdk1.execution.result import ExecutionResult
from unstract.workflow_execution.executor_rpc import PgExecutionDispatcher

from .executor_dispatch_fakes import FakeExecutorTransport, queue_for

# ---------------------------------------------------------------------------
# Fixtures
# ---------------------------------------------------------------------------


@pytest.fixture(autouse=True)
def _clean_registry():
    ExecutorRegistry.clear()
    yield
    ExecutorRegistry.clear()


@pytest.fixture
def eager_app():
    """Configure executor Celery app for eager-mode testing."""
    from executor.worker import app

    original = {
        "task_always_eager": app.conf.task_always_eager,
        "task_eager_propagates": app.conf.task_eager_propagates,
        "result_backend": app.conf.result_backend,
    }
    app.conf.update(
        task_always_eager=True,
        task_eager_propagates=False,
        result_backend="cache+memory://",
    )
    yield app
    app.conf.update(original)


def _register_legacy():
    # Guarded because the import below may be the FIRST in the process, in which
    # case ``@ExecutorRegistry.register`` fires on it and registering again
    # raises a duplicate-name ValueError.
    from executor.executors.legacy_executor import LegacyExecutor

    if "legacy" not in ExecutorRegistry.list_executors():
        ExecutorRegistry.register(LegacyExecutor)


# Mock cloud executors for multi-executor tests
def _register_mock_cloud_executors():
    """Register mock cloud executors alongside LegacyExecutor."""

    @ExecutorRegistry.register
    class MockTableExecutor(BaseExecutor):
        @property
        def name(self) -> str:
            return "table"

        def execute(self, context):
            if context.operation != "table_extract":
                return ExecutionResult.failure(error=f"Unsupported: {context.operation}")
            return ExecutionResult(
                success=True,
                data={"output": "table_data", "metadata": {}},
            )

    @ExecutorRegistry.register
    class MockSmartTableExecutor(BaseExecutor):
        @property
        def name(self) -> str:
            return "smart_table"

        def execute(self, context):
            if context.operation != "smart_table_extract":
                return ExecutionResult.failure(error=f"Unsupported: {context.operation}")
            return ExecutionResult(
                success=True,
                data={"output": "smart_table_data", "metadata": {}},
            )

    @ExecutorRegistry.register
    class MockSPSExecutor(BaseExecutor):
        @property
        def name(self) -> str:
            return "simple_prompt_studio"

        def execute(self, context):
            if context.operation not in ("sps_answer_prompt", "sps_index"):
                return ExecutionResult.failure(error=f"Unsupported: {context.operation}")
            return ExecutionResult(
                success=True,
                data={"output": f"sps_{context.operation}", "metadata": {}},
            )

    @ExecutorRegistry.register
    class MockAgenticExecutor(BaseExecutor):
        _OPS = {
            "agentic_extract",
            "agentic_summarize",
            "agentic_uniformize",
            "agentic_finalize",
            "agentic_generate_prompt",
            "agentic_generate_prompt_pipeline",
            "agentic_compare",
            "agentic_tune_field",
        }

        @property
        def name(self) -> str:
            return "agentic"

        def execute(self, context):
            if context.operation not in self._OPS:
                return ExecutionResult.failure(error=f"Unsupported: {context.operation}")
            return ExecutionResult(
                success=True,
                data={"output": f"agentic_{context.operation}", "metadata": {}},
            )


# ---------------------------------------------------------------------------
# 1. Full Operation enum coverage — every operation has exactly one executor
# ---------------------------------------------------------------------------

# Map of every Operation value to the executor that handles it
OPERATION_TO_EXECUTOR = {
    # LegacyExecutor (OSS)
    "extract": "legacy",
    "index": "legacy",
    "answer_prompt": "legacy",
    "single_pass_extraction": "legacy",
    "summarize": "legacy",
    "ide_index": "legacy",
    "structure_pipeline": "legacy",
    # Cloud executors
    "table_extract": "table",
    "smart_table_extract": "smart_table",
    "sps_answer_prompt": "simple_prompt_studio",
    "sps_index": "simple_prompt_studio",
    "agentic_extract": "agentic",
    "agentic_summarize": "agentic",
    "agentic_uniformize": "agentic",
    "agentic_finalize": "agentic",
    "agentic_generate_prompt": "agentic",
    "agentic_generate_prompt_pipeline": "agentic",
    "agentic_compare": "agentic",
    "agentic_tune_field": "agentic",
}


class TestOperationEnumCoverage:
    def test_every_operation_is_mapped(self):
        """Every Operation enum value has an assigned executor."""
        for op in Operation:
            assert op.value in OPERATION_TO_EXECUTOR, (
                f"Operation {op.value} not mapped to any executor"
            )

    def test_no_extra_mappings(self):
        """No stale mappings for removed operations."""
        valid_ops = {op.value for op in Operation}
        for mapped_op in OPERATION_TO_EXECUTOR:
            assert mapped_op in valid_ops, (
                f"Mapped operation '{mapped_op}' not in Operation enum"
            )

    def test_operation_count(self):
        """Verify total operation count matches expectations."""
        assert len(Operation) == 19  # 7 legacy + 2 table + 2 sps + 8 agentic

    def test_legacy_operations_in_operation_map(self):
        """All legacy operations are in LegacyExecutor._OPERATION_MAP."""
        from executor.executors.legacy_executor import LegacyExecutor

        for op_value, executor_name in OPERATION_TO_EXECUTOR.items():
            if executor_name == "legacy":
                assert op_value in LegacyExecutor._OPERATION_MAP, (
                    f"Legacy operation {op_value} missing from _OPERATION_MAP"
                )

    def test_cloud_operations_not_in_legacy_map(self):
        """Cloud operations are NOT in LegacyExecutor._OPERATION_MAP."""
        from executor.executors.legacy_executor import LegacyExecutor

        for op_value, executor_name in OPERATION_TO_EXECUTOR.items():
            if executor_name != "legacy":
                assert op_value not in LegacyExecutor._OPERATION_MAP, (
                    f"Cloud operation {op_value} should NOT be in legacy map"
                )


# ---------------------------------------------------------------------------
# 2. Multi-executor coexistence in registry
# ---------------------------------------------------------------------------


class TestMultiExecutorCoexistence:
    def test_all_five_executors_registered(self):
        """Legacy + 4 cloud executors all coexist in registry."""
        _register_legacy()
        _register_mock_cloud_executors()

        executors = ExecutorRegistry.list_executors()
        assert "legacy" in executors
        assert "table" in executors
        assert "smart_table" in executors
        assert "simple_prompt_studio" in executors
        assert "agentic" in executors
        assert len(executors) == 5

    def test_each_executor_has_correct_name(self):
        _register_legacy()
        _register_mock_cloud_executors()

        for name in ["legacy", "table", "smart_table", "simple_prompt_studio", "agentic"]:
            executor = ExecutorRegistry.get(name)
            assert executor.name == name

    def test_wrong_executor_rejects_operation(self):
        """Dispatching a table operation to legacy returns failure."""
        _register_legacy()
        _register_mock_cloud_executors()

        legacy = ExecutorRegistry.get("legacy")
        ctx = ExecutionContext(
            executor_name="legacy",
            operation="table_extract",
            run_id="run-1",
            execution_source="tool",
        )
        result = legacy.execute(ctx)
        assert not result.success
        assert "does not support" in result.error

    def test_correct_executor_handles_operation(self):
        """Each operation routes to the right executor."""
        _register_legacy()
        _register_mock_cloud_executors()

        test_cases = [
            ("table", "table_extract"),
            ("smart_table", "smart_table_extract"),
            ("simple_prompt_studio", "sps_answer_prompt"),
            ("simple_prompt_studio", "sps_index"),
            ("agentic", "agentic_extract"),
            ("agentic", "agentic_compare"),
        ]
        for executor_name, operation in test_cases:
            executor = ExecutorRegistry.get(executor_name)
            ctx = ExecutionContext(
                executor_name=executor_name,
                operation=operation,
                run_id=f"run-{operation}",
                execution_source="tool",
            )
            result = executor.execute(ctx)
            assert result.success, f"{executor_name}/{operation} failed"


# ---------------------------------------------------------------------------
# 3. End-to-end Celery chain for cloud executors
# ---------------------------------------------------------------------------


class TestCeleryChainCloudExecutors:
    def test_table_extract_celery_chain(self, eager_app):
        """TABLE extraction through full Celery task chain."""
        _register_legacy()
        _register_mock_cloud_executors()

        ctx = ExecutionContext(
            executor_name="table",
            operation="table_extract",
            run_id="run-celery-table",
            execution_source="tool",
        )
        task = eager_app.tasks["execute_extraction"]
        result_dict = task.apply(args=[ctx.to_dict()]).get()
        result = ExecutionResult.from_dict(result_dict)

        assert result.success
        assert result.data["output"] == "table_data"

    def test_smart_table_extract_celery_chain(self, eager_app):
        """SMART TABLE extraction through full Celery task chain."""
        _register_legacy()
        _register_mock_cloud_executors()

        ctx = ExecutionContext(
            executor_name="smart_table",
            operation="smart_table_extract",
            run_id="run-celery-smart-table",
            execution_source="tool",
        )
        task = eager_app.tasks["execute_extraction"]
        result_dict = task.apply(args=[ctx.to_dict()]).get()
        result = ExecutionResult.from_dict(result_dict)

        assert result.success
        assert result.data["output"] == "smart_table_data"

    def test_sps_answer_prompt_celery_chain(self, eager_app):
        """SPS answer_prompt through full Celery task chain."""
        _register_legacy()
        _register_mock_cloud_executors()

        ctx = ExecutionContext(
            executor_name="simple_prompt_studio",
            operation="sps_answer_prompt",
            run_id="run-celery-sps",
            execution_source="tool",
        )
        task = eager_app.tasks["execute_extraction"]
        result_dict = task.apply(args=[ctx.to_dict()]).get()
        result = ExecutionResult.from_dict(result_dict)

        assert result.success

    def test_agentic_extract_celery_chain(self, eager_app):
        """Agentic extraction through full Celery task chain."""
        _register_legacy()
        _register_mock_cloud_executors()

        ctx = ExecutionContext(
            executor_name="agentic",
            operation="agentic_extract",
            run_id="run-celery-agentic",
            execution_source="tool",
        )
        task = eager_app.tasks["execute_extraction"]
        result_dict = task.apply(args=[ctx.to_dict()]).get()
        result = ExecutionResult.from_dict(result_dict)

        assert result.success

    def test_unregistered_executor_returns_failure(self, eager_app):
        """Dispatching to unregistered executor returns failure."""
        _register_legacy()
        # Don't register cloud executors

        ctx = ExecutionContext(
            executor_name="table",
            operation="table_extract",
            run_id="run-missing",
            execution_source="tool",
        )
        task = eager_app.tasks["execute_extraction"]
        result_dict = task.apply(args=[ctx.to_dict()]).get()
        result = ExecutionResult.from_dict(result_dict)

        assert not result.success
        assert "table" in result.error.lower()


# ---------------------------------------------------------------------------
# 4. Cross-cutting highlight plugin across executors
# ---------------------------------------------------------------------------


class TestCrossCuttingHighlight:
    @patch("importlib.metadata.entry_points", return_value=[])
    def test_highlight_plugin_not_installed_no_error(self, _mock_eps):
        """When highlight plugin not installed, extraction still works."""
        from executor.executors.plugins.loader import ExecutorPluginLoader

        ExecutorPluginLoader.clear()
        assert ExecutorPluginLoader.get("highlight-data") is None
        # No error — graceful degradation

    def test_mock_highlight_plugin_shared_across_executors(self, tmp_path):
        """Multiple executors can use the same highlight plugin instance."""
        from executor.executors.plugins.loader import ExecutorPluginLoader

        class FakeHighlight:
            def __init__(self, **kwargs):
                self.kwargs = kwargs

            def run(self, response, **kwargs):
                return {"highlighted": True}

            def get_highlight_data(self):
                return {"lines": [1, 2, 3]}

            def get_confidence_data(self):
                return {"confidence": 0.95}

        fake_ep = MagicMock()
        fake_ep.name = "highlight-data"
        fake_ep.load.return_value = FakeHighlight

        with patch(
            "importlib.metadata.entry_points",
            return_value=[fake_ep],
        ):
            ExecutorPluginLoader.clear()
            cls = ExecutorPluginLoader.get("highlight-data")
            assert cls is FakeHighlight

            # Both legacy and agentic contexts can create instances
            legacy_hl = cls(file_path=str(tmp_path / "doc.txt"), execution_source="ide")
            agentic_hl = cls(
                file_path=str(tmp_path / "other.txt"), execution_source="tool"
            )

            assert legacy_hl.get_highlight_data() == {"lines": [1, 2, 3]}
            assert agentic_hl.get_confidence_data() == {"confidence": 0.95}


# ---------------------------------------------------------------------------
# 5. Plugin loader → registration → dispatch → result flow
# ---------------------------------------------------------------------------


class TestPluginDiscoveryToDispatchFlow:
    def test_full_discovery_to_dispatch_flow(self):
        """Simulate: entry point discovery → register → dispatch → result."""

        # Step 1: "Discover" a cloud executor via entry point
        @ExecutorRegistry.register
        class DiscoveredExecutor(BaseExecutor):
            @property
            def name(self):
                return "discovered"

            def execute(self, context):
                return ExecutionResult(
                    success=True,
                    data={"output": "discovered_result"},
                )

        # Step 2: Verify registration
        assert "discovered" in ExecutorRegistry.list_executors()

        # Step 3: Dispatch via a fake executor transport
        transport = FakeExecutorTransport(
            result=ExecutionResult(
                success=True, data={"output": "discovered_result"}
            ).to_dict()
        )
        dispatcher = PgExecutionDispatcher(transport)
        ctx = ExecutionContext(
            executor_name="discovered",
            operation="custom_op",
            run_id="run-flow",
            execution_source="tool",
        )
        result = dispatcher.dispatch(ctx)

        # Step 4: Verify result
        assert result.success
        assert result.data["output"] == "discovered_result"

        # Step 5: Verify queue routing
        assert transport.queue == "celery_executor_discovered"


# ---------------------------------------------------------------------------
# 6. Queue routing for all executor names
# ---------------------------------------------------------------------------

EXECUTOR_QUEUE_MAP = {
    "legacy": "celery_executor_legacy",
    "table": "celery_executor_table",
    "smart_table": "celery_executor_smart_table",
    "simple_prompt_studio": "celery_executor_simple_prompt_studio",
    "agentic": "celery_executor_agentic",
}


class TestQueueRoutingAllExecutors:
    @pytest.mark.parametrize(
        "executor_name,expected_queue",
        list(EXECUTOR_QUEUE_MAP.items()),
    )
    def test_queue_name_for_executor(self, executor_name, expected_queue):
        assert queue_for(executor_name) == expected_queue


# ---------------------------------------------------------------------------
# 7. Graceful degradation when cloud plugins missing
# ---------------------------------------------------------------------------


class TestGracefulDegradation:
    def test_legacy_works_without_cloud_executors(self, eager_app):
        """Legacy operations work even when no cloud executors installed."""
        _register_legacy()

        # Only legacy should be in registry
        assert ExecutorRegistry.list_executors() == ["legacy"]

        # Legacy executor can be retrieved from the registry
        executor = ExecutorRegistry.get("legacy")
        assert executor is not None
        assert executor.name == "legacy"

    def test_cloud_op_on_legacy_returns_meaningful_error(self):
        """Attempting a cloud operation on legacy gives clear error."""
        _register_legacy()
        executor = ExecutorRegistry.get("legacy")

        for cloud_op in [
            "table_extract",
            "smart_table_extract",
            "sps_answer_prompt",
            "agentic_extract",
        ]:
            ctx = ExecutionContext(
                executor_name="legacy",
                operation=cloud_op,
                run_id=f"run-{cloud_op}",
                execution_source="tool",
            )
            result = executor.execute(ctx)
            assert not result.success
            assert "does not support" in result.error

    def test_missing_executor_via_orchestrator(self):
        """Orchestrator returns failure for unregistered executor."""
        _register_legacy()
        orchestrator = ExecutionOrchestrator()

        ctx = ExecutionContext(
            executor_name="table",
            operation="table_extract",
            run_id="run-no-table",
            execution_source="tool",
        )
        result = orchestrator.execute(ctx)
        assert not result.success
        assert "table" in result.error.lower()


# ---------------------------------------------------------------------------
# 8. tasks.py log_component for all operation types
# ---------------------------------------------------------------------------


class TestLogComponentAllOperations:
    """Verify tasks.py builds `_log_component` for the branches it special-cases.
    Drives the real `execute_extraction` (orchestrator patched out) so drift in
    the builder fails here instead of passing against a replica.
    """

    def _log_component(self, operation, params):
        with patch("executor.tasks.ExecutionOrchestrator") as mock_orch_cls:
            mock_orch = MagicMock()
            mock_orch.execute.return_value = MagicMock(
                success=True, to_dict=lambda: {"success": True}
            )
            mock_orch_cls.return_value = mock_orch

            from executor.tasks import execute_extraction

            execute_extraction(
                {
                    "executor_name": "legacy",
                    "operation": operation,
                    "run_id": "run-log",
                    "execution_source": "tool",
                    "log_events_id": "evt-1",
                    "executor_params": params,
                }
            )
            return mock_orch.execute.call_args[0][0]._log_component

    def test_ide_index_reads_nested_index_and_usage_params(self):
        comp = self._log_component(
            "ide_index",
            {
                "index_params": {"tool_id": "t-nested"},
                "extract_params": {"usage_kwargs": {"file_name": "nested.pdf"}},
            },
        )
        assert comp["tool_id"] == "t-nested"
        assert comp["doc_name"] == "nested.pdf"

    def test_structure_pipeline_reads_answer_and_pipeline_params(self):
        comp = self._log_component(
            "structure_pipeline",
            {
                "answer_params": {"tool_id": "t-pipe"},
                "pipeline_options": {"source_file_name": "pipe.pdf"},
            },
        )
        assert comp["tool_id"] == "t-pipe"
        assert comp["doc_name"] == "pipe.pdf"

    def test_default_branch_reads_direct_params(self):
        comp = self._log_component(
            "table_extract", {"tool_id": "t-1", "file_name": "doc.pdf"}
        )
        assert comp == {
            "tool_id": "t-1",
            "run_id": "run-log",
            "doc_name": "doc.pdf",
            "operation": "table_extract",
        }


# ---------------------------------------------------------------------------
# 9. ExecutionResult serialization round-trip
# ---------------------------------------------------------------------------


class TestResultRoundTrip:
    def test_success_result_round_trip(self):
        original = ExecutionResult(
            success=True,
            data={"output": {"field": "value"}, "metadata": {"tokens": 100}},
        )
        restored = ExecutionResult.from_dict(original.to_dict())
        assert restored.success == original.success
        assert restored.data == original.data

    def test_failure_result_round_trip(self):
        original = ExecutionResult.failure(error="Something went wrong")
        restored = ExecutionResult.from_dict(original.to_dict())
        assert not restored.success
        assert restored.error == "Something went wrong"

    def test_context_round_trip(self):
        original = ExecutionContext(
            executor_name="agentic",
            operation="agentic_extract",
            run_id="run-rt",
            execution_source="tool",
            organization_id="org-1",
            executor_params={"key": "value"},
            log_events_id="evt-1",
        )
        restored = ExecutionContext.from_dict(original.to_dict())
        assert restored.executor_name == "agentic"
        assert restored.operation == "agentic_extract"
        assert restored.organization_id == "org-1"
        assert restored.executor_params == {"key": "value"}
        assert restored.log_events_id == "evt-1"