Skip to content

LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows

13.7 KB
Raw Blame
"""UN-3671: orchestration idempotency claim — the double-orchestration guard for
``async_execute_bin`` before replica scale-out.

Two layers:

1. **Gate + short-circuit** (no DB): the transport-gated
   ``_should_skip_duplicate_orchestration`` helper and the
   ``async_execute_bin_general`` no-op wiring, with the claim primitive mocked.
2. **Claim primitive** (real Postgres): ``try_claim_orchestration`` /
   ``release_orchestration_claim`` against the ``pg_orchestration_claim`` table —
   skips if unreachable/unmigrated.
"""

from __future__ import annotations

import logging
import os
import threading
from unittest.mock import MagicMock, patch

import psycopg2
import pytest
from general import tasks
from general.tasks import _should_skip_duplicate_orchestration
from queue_backend import pg_barrier
from queue_backend.pg_barrier import (
    release_orchestration_claim,
    try_claim_orchestration,
)
from queue_backend.pg_queue.connection import create_pg_connection

# --- Layer 1: the duplicate-orchestration gate (no DB) ---


class TestOrchestrationGate:
    """``_should_skip_duplicate_orchestration`` — the first delivery proceeds and
    a duplicate skips. The gate used to be transport-conditional (a no-op on
    Celery, which had no redelivery); with PG the only transport it always runs.
    """

    def test_pg_first_delivery_proceeds(self):
        with patch("general.tasks.try_claim_orchestration", return_value=True) as claim:
            assert _should_skip_duplicate_orchestration("e1", "org1") is False
            claim.assert_called_once_with("e1", "org1")  # org stamped for the reaper

    def test_pg_duplicate_delivery_skips(self):
        with patch("general.tasks.try_claim_orchestration", return_value=False):
            assert _should_skip_duplicate_orchestration("e1", "org1") is True

    def test_claim_error_propagates(self):
        # A DB error in the claim must PROPAGATE, not be swallowed — a swallow-and-
        # proceed would double-orchestrate, a swallow-and-skip would silently drop a
        # legitimate first delivery. Pin the no-try/except contract.
        with patch(
            "general.tasks.try_claim_orchestration",
            side_effect=psycopg2.OperationalError("db down"),
        ):
            with pytest.raises(psycopg2.OperationalError):
                _should_skip_duplicate_orchestration("e1", "org1")

    def test_skip_on_retry_logs_error_with_errorid(self, caplog):
        # A skip when retries > 0 is a SUPPRESSED retry (a prior attempt claimed
        # and crashed before releasing), not a benign duplicate — log it at ERROR
        # with an errorId so alerting can distinguish the two.
        with patch("general.tasks.try_claim_orchestration", return_value=False):
            with caplog.at_level(logging.ERROR, logger="general.tasks"):
                assert (
                    _should_skip_duplicate_orchestration("e1", "org1", 2)
                    is True
                )
        assert "ORCH_CLAIM_SUPPRESSED_RETRY" in caplog.text


class TestOrchestratorShortCircuit:
    """``async_execute_bin_general`` no-ops on a duplicate BEFORE any setup / arm /
    dispatch.
    """

    def test_duplicate_returns_skip_without_setup(self):
        # Exercise the REAL gate end-to-end: a PG duplicate (try_claim_orchestration
        # returns False) must short-circuit to the skip result before any setup /
        # barrier arm / dispatch. Reaching the skip dict is itself proof setup was
        # never called (it runs after the gate).
        #
        # Belt-and-braces: the worker *bootstrap* loads async_execute_bin_general
        # under a bare ``tasks`` module (a sys.path quirk) distinct from
        # ``general.tasks`` — that bootstrap doesn't run under pytest, so here
        # ``fn.__globals__`` IS ``general.tasks.__dict__`` and a plain
        # ``patch("general.tasks...")`` would also work. Patching the task's own
        # globals is simply robust to either loading.
        fn = tasks.async_execute_bin_general
        while hasattr(fn, "__wrapped__"):
            fn = fn.__wrapped__
        with patch.dict(
            fn.__globals__, {"try_claim_orchestration": lambda _eid, _org: False}
        ):
            out = tasks.async_execute_bin_general(
                schema_name="org",
                workflow_id="wf",
                execution_id="e-dup",
                hash_values_of_files={},
            )
        assert out == {
            "status": "skipped_duplicate_orchestration",
            "execution_id": "e-dup",
            "workflow_id": "wf",
        }

    @staticmethod
    def _task_globals():
        fn = tasks.async_execute_bin_general
        while hasattr(fn, "__wrapped__"):
            fn = fn.__wrapped__
        return fn.__globals__

    def test_no_release_when_claim_never_acquired(self):
        # G2: if the claim's own INSERT raises, the failure path must NOT call
        # release — nothing was acquired, and a spurious DELETE would reconnect just
        # to delete nothing and muddy the log.
        released: list[str] = []

        def _boom(_eid, _org):
            raise psycopg2.OperationalError("db down")

        with patch.dict(
            self._task_globals(),
            {
                "try_claim_orchestration": _boom,
                "release_orchestration_claim": lambda eid: released.append(eid),
            },
        ):
            with pytest.raises(psycopg2.OperationalError):
                tasks.async_execute_bin_general(
                    schema_name="org",
                    workflow_id="wf",
                    execution_id="e-x",
                    hash_values_of_files={},
                )
        assert released == []  # claim never acquired → nothing released

    def test_release_called_when_claimed_and_work_fails(self):
        # The claim was won (gate proceeds) but the work fails → release it so the
        # redelivery/retry can re-orchestrate.
        released: list[str] = []
        with patch.dict(
            self._task_globals(),
            {
                "try_claim_orchestration": lambda _eid, _org: True,  # won the claim
                "release_orchestration_claim": lambda eid: released.append(eid),
            },
        ):
            with patch.object(
                tasks.WorkerExecutionContext,
                "setup_execution_context",
                side_effect=RuntimeError("setup boom"),
            ):
                with pytest.raises(RuntimeError):
                    tasks.async_execute_bin_general(
                        schema_name="org",
                        workflow_id="wf",
                        execution_id="e-y",
                        hash_values_of_files={},
                    )
        assert released == ["e-y"]  # claimed then failed → released

    def test_terminal_execution_skips_and_keeps_claim(self):
        # L1: a PG redelivery that WON a fresh claim (the tombstone had been GC'd)
        # but finds the execution already terminal must skip re-orchestration
        # WITHOUT releasing the claim (re-establishing the tombstone), and never
        # reach the status-update / arm / dispatch below the guard.
        released: list[str] = []
        api_client = MagicMock()
        api_client.get_workflow_execution.return_value = MagicMock(
            success=True,
            data={
                "execution": {
                    "status": tasks.ExecutionStatus.COMPLETED.value,
                    "workflow_id": "wf",
                }
            },
        )
        with patch.dict(
            self._task_globals(),
            {
                "try_claim_orchestration": lambda _eid, _org: True,  # won the claim
                "release_orchestration_claim": lambda eid: released.append(eid),
            },
        ):
            with patch.object(
                tasks.WorkerExecutionContext,
                "setup_execution_context",
                return_value=(MagicMock(), api_client),
            ):
                out = tasks.async_execute_bin_general(
                    schema_name="org",
                    workflow_id="wf",
                    execution_id="e-term",
                    hash_values_of_files={},
                )
        assert out["status"] == "skipped_terminal_execution"
        assert released == []  # claim kept → tombstone re-established
        api_client.update_workflow_execution_status.assert_not_called()  # never re-armed

    # ``test_terminal_execution_not_skipped_on_celery`` is gone with the
    # transport: the terminal guard used to be is_pg-gated so the Celery path ran
    # past it. There is no un-guarded path left — the guard above
    # (``test_terminal_execution_skips_and_keeps_claim``) is now the only
    # behaviour.


# --- Layer 2: claim primitive against real Postgres ---


@pytest.fixture
def claim_db():
    """Inject a real autocommit connection into pg_barrier's thread-local so the
    claim primitives write to the real ``pg_orchestration_claim`` table.
    """
    os.environ.setdefault("TEST_DB_HOST", "127.0.0.1")
    try:
        conn = create_pg_connection(env_prefix="TEST_DB_")
    except psycopg2.OperationalError as exc:
        pytest.skip(f"Postgres not reachable: {exc}")
    conn.autocommit = True
    with conn.cursor() as cur:
        cur.execute("SELECT to_regclass('pg_orchestration_claim')")
        if cur.fetchone()[0] is None:
            conn.close()
            pytest.skip(
                "pg_orchestration_claim migration not applied (run backend migrate)"
            )
        cur.execute("DELETE FROM pg_orchestration_claim")
    pg_barrier._local.conn = conn
    yield conn
    with conn.cursor() as cur:
        cur.execute("DELETE FROM pg_orchestration_claim")
    conn.close()
    pg_barrier._local.conn = None


def _claim_count(conn, execution_id):
    with conn.cursor() as cur:
        cur.execute(
            "SELECT count(*) FROM pg_orchestration_claim WHERE execution_id = %s",
            (execution_id,),
        )
        return cur.fetchone()[0]


@pytest.mark.integration
class TestClaimOrchestration:
    def test_first_claim_wins(self, claim_db):
        assert try_claim_orchestration("exec-1", "org-1") is True

    def test_duplicate_claim_loses(self, claim_db):
        assert try_claim_orchestration("exec-1", "org-1") is True
        assert try_claim_orchestration("exec-1", "org-1") is False  # redelivery / sibling

    def test_distinct_executions_are_independent(self, claim_db):
        assert try_claim_orchestration("exec-1", "org-1") is True
        assert try_claim_orchestration("exec-2", "org-1") is True  # keyed on execution_id

    def test_claim_persists_as_tombstone(self, claim_db):
        # NOT cleared at finalise (unlike per-batch markers): a post-completion
        # redelivery still finds it, so a finished execution can't be re-orchestrated.
        assert try_claim_orchestration("exec-keep", "org-1") is True
        assert _claim_count(claim_db, "exec-keep") == 1
        assert try_claim_orchestration("exec-keep", "org-1") is False

    def test_release_allows_reclaim(self, claim_db):
        # The failure-path release frees the slot so a redelivery/retry can
        # re-orchestrate (this is what stops a transient failure permanently
        # no-oping every redelivery).
        assert try_claim_orchestration("exec-rel", "org-1") is True
        assert try_claim_orchestration("exec-rel", "org-1") is False  # held
        release_orchestration_claim("exec-rel")
        assert _claim_count(claim_db, "exec-rel") == 0
        assert (
            try_claim_orchestration("exec-rel", "org-1") is True
        )  # reclaimable after release

    def test_release_is_safe_when_absent(self, claim_db):
        release_orchestration_claim("never-claimed")  # no row → no error
        assert _claim_count(claim_db, "never-claimed") == 0

    def test_concurrent_claims_resolve_to_exactly_one_winner(self, claim_db):
        # The real reason this feature exists: N replicas racing the SAME
        # orchestration must resolve to exactly one True. Force the contended
        # ON CONFLICT path — pre-build N connections and release every claim at
        # once via a threading.Barrier — and repeat over distinct executions so a
        # non-atomic check-then-insert can't pass by luck. Each thread uses its own
        # connection (a libpq conn is not thread-safe across threads).
        n = 8
        conns = [create_pg_connection(env_prefix="TEST_DB_") for _ in range(n)]
        for c in conns:
            c.autocommit = True
        try:
            for trial in range(5):
                execution_id = f"exec-race-{trial}"
                gate = threading.Barrier(n)
                results: list[bool] = []
                lock = threading.Lock()

                # Bind loop vars as defaults (ruff B023): threads are created and
                # joined within the iteration, but defaults make the capture explicit.
                def run(conn, eid=execution_id, gate=gate, results=results, lock=lock):
                    pg_barrier._local.conn = conn
                    try:
                        gate.wait()  # align all claims at the same instant
                        won = try_claim_orchestration(eid, "org-1")
                        with lock:
                            results.append(won)
                    finally:
                        pg_barrier._local.conn = None

                threads = [threading.Thread(target=run, args=(c,)) for c in conns]
                for t in threads:
                    t.start()
                for t in threads:
                    t.join()

                assert sum(results) == 1, f"trial {trial}: {sum(results)} winners"
                assert _claim_count(claim_db, execution_id) == 1
        finally:
            for c in conns:
                c.close()
            pg_barrier._local.conn = None


if __name__ == "__main__":
    pytest.main([__file__, "-v"])