Skip to content

LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows

10.5 KB
Raw Blame
"""Regression tests for synchronous staging failures in ``execute_workflow``.

When an API-deployment run fails synchronously at the "Staging files in API
storage" step (``SourceConnector.add_input_file_to_api_storage``, before async
dispatch), the PENDING ``WorkflowExecution`` row created earlier must be marked
ERROR — otherwise the UI shows the run as stuck/running forever. The
error-marking is isolated so the rate-limit slot release and storage cleanup
still run even if that DB write fails.

Unit tests: the real ``execute_workflow`` control flow runs with every
DB/storage-touching collaborator patched on the imported module, so no
database is needed.
"""

from unittest import mock
from unittest.mock import MagicMock

import pytest

import api_v2.deployment_helper as dh


@pytest.fixture
def collaborators():
    """Patch execute_workflow's collaborators; staging fails with 'boom'."""
    with mock.patch.multiple(
        dh,
        WorkflowExecutionServiceHelper=mock.DEFAULT,
        SourceConnector=mock.DEFAULT,
        DestinationConnector=mock.DEFAULT,
        APIDeploymentRateLimiter=mock.DEFAULT,
        WorkflowHelper=mock.DEFAULT,
        Tag=mock.DEFAULT,
        logger=mock.DEFAULT,
    ) as mocks:
        execution_row = MagicMock()
        execution_row.id = "exec-123"
        mocks[
            "WorkflowExecutionServiceHelper"
        ].create_workflow_execution.return_value = execution_row
        mocks["SourceConnector"].add_input_file_to_api_storage.side_effect = (
            RuntimeError("boom")
        )
        yield mocks


def _api() -> MagicMock:
    api = MagicMock()
    api.workflow.id = "wf-1"
    api.id = "pipe-1"
    api.organization.organization_id = "org-uuid-1"
    return api


def test_staging_failure_marks_execution_error(collaborators) -> None:
    """A staging failure marks the execution ERROR instead of leaving it PENDING."""
    # Must NOT raise — the failure should be handled, not propagated.
    dh.DeploymentHelper.execute_workflow(
        organization_name="org",
        api=_api(),
        file_objs=[],
        timeout=-1,
    )

    # The PENDING row is marked ERROR with the surfaced reason.
    collaborators[
        "WorkflowExecutionServiceHelper"
    ].update_execution_err.assert_called_once_with("exec-123", "boom")
    # And the slot/storage cleanup still runs.
    collaborators["APIDeploymentRateLimiter"].release_slot.assert_called_once()
    collaborators["DestinationConnector"].delete_api_storage_dir.assert_called_once()
    # Async dispatch is never reached when staging fails.
    collaborators["WorkflowHelper"].execute_workflow_async.assert_not_called()


def test_staging_failure_cleanup_survives_db_marking_error(collaborators) -> None:
    """If marking the row ERROR itself raises, cleanup must still run (not propagate)."""
    collaborators["WorkflowExecutionServiceHelper"].update_execution_err.side_effect = (
        RuntimeError("db down")
    )

    # Must NOT raise — a failed error-marking should not break cleanup.
    dh.DeploymentHelper.execute_workflow(
        organization_name="org",
        api=_api(),
        file_objs=[],
        timeout=-1,
    )

    # Cleanup still runs even though error-marking raised.
    collaborators["APIDeploymentRateLimiter"].release_slot.assert_called_once()
    collaborators["DestinationConnector"].delete_api_storage_dir.assert_called_once()


@pytest.fixture
def staging_rejects_everything():
    """Patch execute_workflow's collaborators; staging returns no dispatchable files."""
    with mock.patch.multiple(
        dh,
        WorkflowExecutionServiceHelper=mock.DEFAULT,
        SourceConnector=mock.DEFAULT,
        DestinationConnector=mock.DEFAULT,
        APIDeploymentRateLimiter=mock.DEFAULT,
        WorkflowHelper=mock.DEFAULT,
        ResultCacheUtils=mock.DEFAULT,
        PipelineUtils=mock.DEFAULT,
        Tag=mock.DEFAULT,
        logger=mock.DEFAULT,
    ) as mocks:
        execution_row = MagicMock()
        execution_row.id = "exec-123"
        mocks[
            "WorkflowExecutionServiceHelper"
        ].create_workflow_execution.return_value = execution_row
        mocks["SourceConnector"].add_input_file_to_api_storage.return_value = {}
        mocks["ResultCacheUtils"].get_api_results.return_value = [
            {"file": "evil.pdf", "status": "Failed", "error": "unsupported MIME type"}
        ]
        completed_row = MagicMock()
        completed_row.status = "COMPLETED"
        mocks[
            "WorkflowExecutionServiceHelper"
        ].update_execution_completed.return_value = completed_row
        yield mocks


def test_all_files_rejected_completes_without_dispatch(
    staging_rejects_everything,
) -> None:
    """A request whose every file is rejected must reach a terminal status.

    The worker short-circuits an empty file set without writing a status back, so
    dispatching one strands the execution in PENDING and the caller polls forever.
    """
    mocks = staging_rejects_everything
    # A non-empty upload whose staging result is empty. Passing [] instead would
    # leave the branch satisfied by `not file_objs` too, and the original bug -
    # dispatching a request whose files were all rejected - would pass this test.
    response = dh.DeploymentHelper.execute_workflow(
        organization_name="org",
        api=_api(),
        file_objs=[MagicMock()],
        timeout=-1,
    )

    # Nothing is dispatched...
    mocks["WorkflowHelper"].execute_workflow_async.assert_not_called()
    # ...the row is terminalised here instead of being left PENDING, and the
    # counters are written so the run does not read back as a clean success...
    mocks[
        "WorkflowExecutionServiceHelper"
    ].update_execution_completed.assert_called_once_with(
        "exec-123", total_files=1, failed_files=1
    )
    # ...the slot and staging dir are released. The slot must be released by org
    # id string: release_slot formats its argument into the Redis key, so passing
    # the model instance removes a non-member and silently holds the slot.
    mocks["APIDeploymentRateLimiter"].release_slot.assert_called_once_with(
        "org-uuid-1", "exec-123"
    )
    mocks["DestinationConnector"].delete_api_storage_dir.assert_called_once()
    # ...and the caller still sees why each file failed.
    assert response["execution_status"] == "COMPLETED"
    assert response["result"][0]["file"] == "evil.pdf"
    assert response["result"][0]["status"] == "Failed"


def test_all_files_rejected_acknowledges_and_notifies(
    staging_rejects_everything,
) -> None:
    """The early return owes the caller what the dispatch path would have done.

    It hands back the results in its own response and reaches a terminal status
    without going through WorkflowHelper, so both the acknowledgement and the
    subscriber notification have to happen here or they happen nowhere.
    """
    mocks = staging_rejects_everything
    completed_row = mocks[
        "WorkflowExecutionServiceHelper"
    ].update_execution_completed.return_value

    dh.DeploymentHelper.execute_workflow(
        organization_name="org",
        api=_api(),
        file_objs=[MagicMock()],
        timeout=-1,
    )

    # Results were served in this response, so a later GET /status must 406
    # rather than serve them again.
    mocks["WorkflowHelper"].set_result_acknowledge.assert_called_once_with(completed_row)
    # PipelineUtils is the only dispatcher of API deployment notifications.
    mocks["PipelineUtils"].update_pipeline_status.assert_called_once_with(
        pipeline_id="pipe-1", workflow_execution=completed_row
    )


def test_notification_survives_a_failed_acknowledgement(
    staging_rejects_everything,
) -> None:
    """Acknowledgement and notification are independent obligations.

    Sharing one try block would let a failed acknowledgement silence the
    notification that subscribers depend on.
    """
    mocks = staging_rejects_everything
    mocks["WorkflowHelper"].set_result_acknowledge.side_effect = Exception("db is down")

    response = dh.DeploymentHelper.execute_workflow(
        organization_name="org",
        api=_api(),
        file_objs=[MagicMock()],
        timeout=-1,
    )

    mocks["PipelineUtils"].update_pipeline_status.assert_called_once()
    assert response["execution_status"] == "COMPLETED"


def test_all_files_rejected_still_responds_if_notification_fails(
    staging_rejects_everything,
) -> None:
    """A failing webhook must not turn a handled rejection into a 500."""
    mocks = staging_rejects_everything
    mocks["PipelineUtils"].update_pipeline_status.side_effect = Exception("webhook down")

    response = dh.DeploymentHelper.execute_workflow(
        organization_name="org",
        api=_api(),
        file_objs=[MagicMock()],
        timeout=-1,
    )

    assert response["execution_status"] == "COMPLETED"
    assert response["result"][0]["file"] == "evil.pdf"


def test_files_staged_successfully_are_dispatched(staging_rejects_everything) -> None:
    """The short-circuit must not fire when staging did return files.

    Sibling to the test above: together they pin the branch to the staging result
    rather than to the upload list.
    """
    mocks = staging_rejects_everything
    mocks["SourceConnector"].add_input_file_to_api_storage.return_value = {
        "good.pdf": MagicMock()
    }

    dh.DeploymentHelper.execute_workflow(
        organization_name="org",
        api=_api(),
        file_objs=[MagicMock()],
        timeout=-1,
    )

    mocks["WorkflowHelper"].execute_workflow_async.assert_called_once()
    mocks["WorkflowExecutionServiceHelper"].update_execution_completed.assert_not_called()


def test_all_files_rejected_cleanup_survives_db_marking_error(
    staging_rejects_everything,
) -> None:
    """A failing status write must not strand the slot or the staging dir.

    update_execution_completed only catches DoesNotExist, so a lock timeout or a
    dropped connection propagates; without isolation the org's rate limit slot
    stays held for its full TTL and throttles every other call for that org.
    """
    mocks = staging_rejects_everything
    mocks[
        "WorkflowExecutionServiceHelper"
    ].update_execution_completed.side_effect = Exception("db is down")

    response = dh.DeploymentHelper.execute_workflow(
        organization_name="org",
        api=_api(),
        file_objs=[MagicMock()],
        timeout=-1,
    )

    mocks["APIDeploymentRateLimiter"].release_slot.assert_called_once()
    mocks["DestinationConnector"].delete_api_storage_dir.assert_called_once()
    # The row never reached COMPLETED, so the response must not claim it did.
    assert response["execution_status"] == "ERROR"