Skip to content

LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows

12.5 KB
Raw Blame
"""Unit tests for the PG finalization-strand fixes.

- L2: ``_update_execution_status_unified`` re-raises a failed write by default (so
  the failed finalization vt-redelivers / poison-drops). Since UN-4078 that is
  unconditional — the Celery swallow is gone — and ``raise_on_failure=False`` is
  reserved for the one caller already handling a failure.
- L3: ``_backoff_with_jitter`` spreads retries across [base/2, base] (equal jitter)
  so concurrent workers don't retry in synchronized waves. Unconditional since
  UN-4046 — it was gated on ``pg_queue_enabled`` to keep the Celery flow on the
  exact exponential cadence.
"""

from unittest.mock import MagicMock

import pytest


class TestBackoffJitter:
    """UN-4046: jitter is unconditional, so there is no flag to patch.

    The two flag-off cases (``flag off returns exact base``, ``Flipt error fails
    closed to no jitter``) are GONE — they asserted the exact exponential cadence
    the Celery flow kept, and there is no Celery flow and no Flipt call left. What
    they were really protecting is that the RESULT stays a sane backoff, which the
    band assertion below already pins from both sides. The remaining three drop
    their patch and assert the same properties directly.
    """

    def test_within_equal_jitter_band(self):
        from shared.clients.base_client import _backoff_with_jitter

        # base = backoff_factor(1.0) * 2**attempt(2) = 4.0 → equal jitter [2.0, 4.0]
        for _ in range(500):
            assert 2.0 <= _backoff_with_jitter(1.0, 2) <= 4.0

    def test_zero_base_returns_zero(self):
        from shared.clients.base_client import _backoff_with_jitter

        assert _backoff_with_jitter(0.0, 3) == pytest.approx(0.0)

    def test_is_not_constant(self):
        from shared.clients.base_client import _backoff_with_jitter

        # De-correlation: repeated calls must NOT all land on the same value
        # (that synchronized schedule is exactly the thundering herd we fix).
        assert len({_backoff_with_jitter(1.0, 3) for _ in range(64)}) > 1


class TestFinalizationReraise:
    AGG = {"total_files": 1, "successful_files": 1, "failed_files": 0}

    def _run(self, raise_on_failure: bool, fail: bool):
        from callback.tasks import _update_execution_status_unified

        api = MagicMock()
        if fail:
            api.update_workflow_execution_status.side_effect = RuntimeError("boom")
        return (
            api,
            lambda: _update_execution_status_unified(
                api_client=api,
                execution_id="exec-1",
                final_status="COMPLETED",
                aggregated_results=self.AGG,
                organization_id="org-1",
                raise_on_failure=raise_on_failure,
            ),
        )

    def test_reraises_on_write_failure(self):
        # The whole point: a failed finalization must propagate so the message
        # vt-redelivers instead of stranding the execution.
        _api, call = self._run(raise_on_failure=True, fail=True)
        with pytest.raises(RuntimeError):
            call()

    def test_opt_out_swallows_on_write_failure(self):
        # raise_on_failure=False: the ERROR-marking write inside an except block,
        # where a raise would mask the original failure.
        _api, call = self._run(raise_on_failure=False, fail=True)
        result = call()
        assert result["status"] == "failed"

    def test_success_path_returns_completed(self):
        api, call = self._run(raise_on_failure=True, fail=False)
        result = call()
        assert result["status"] == "completed"
        api.update_workflow_execution_status.assert_called_once()


class TestRecoverStuckClientResponseShape:
    """The recovery endpoint returns a FLAT body (no {"data": ...} envelope). The
    client must surface it as ``APIResponse.data`` so the reaper's observability
    block can read the counters — convert_dict_response() would zero them all."""

    def _client(self, post_return):
        from shared.clients.execution_client import ExecutionAPIClient

        client = ExecutionAPIClient.__new__(ExecutionAPIClient)
        client._build_url = MagicMock(return_value="/recover/")
        client.post = MagicMock(return_value=post_return)
        return client

    def test_flat_body_surfaced_as_data(self):
        body = {"recovered": 3, "skipped": 2, "scanned": 5, "failed": 0}
        resp = self._client(body).recover_stuck_pg_executions(stuck_seconds=9000)
        assert resp.data == body
        assert resp.data.get("recovered") == 3  # reaper reads a real count, not 0

    def test_non_dict_body_yields_empty_data(self):
        # A malformed (non-dict) body → empty data → the reaper's loud-guard fires.
        resp = self._client(None).recover_stuck_pg_executions()
        assert not resp.data


_RECOVERY_ENV = "WORKER_PG_STUCK_EXECUTION_RECOVERY_ENABLED"
_RECOVERY_SECONDS_ENV = "WORKER_PG_STUCK_EXECUTION_RECOVERY_SECONDS"


class TestStuckRecoveryWindowDefault:
    """The window must be a DEFAULT, not something an operator sets.

    It used to inherit the barrier stuck-timeout (~2.5 h), which answers a different
    question: that one bounds how long a batch may make no progress, where a single
    file legitimately can take hours. This one only needs to outlast a callback that
    is about to fire — seconds.

    The threshold is not what makes recovery safe; the endpoint skips unless EVERY
    file is terminal, so a running execution is never a candidate however short this
    is. Inheriting hours therefore bought nothing and left a stranded execution dead
    for ~2.5 h — the common shape being a Celery run whose callback worker was removed
    by a deploy (300 s grace, against file-processing's 7200 s).

    Pinned as a default because production and on-prem have **no operator and no
    Flipt**: a value that only exists as an env override is one those environments
    never receive, and they are precisely the ones that cannot diagnose a hung
    execution themselves.
    """

    def _seconds(self, monkeypatch, value=None):
        from queue_backend.pg_queue import reaper as R

        monkeypatch.delenv(_RECOVERY_SECONDS_ENV, raising=False)
        if value is not None:
            monkeypatch.setenv(_RECOVERY_SECONDS_ENV, value)
        return R._positive_duration_from_env(
            _RECOVERY_SECONDS_ENV, R._DEFAULT_STUCK_RECOVERY_SECONDS, int
        )

    def test_default_is_ten_minutes(self, monkeypatch):
        assert self._seconds(monkeypatch) == 600

    def test_default_is_NOT_the_barrier_stuck_timeout(self, monkeypatch):
        """The regression this replaces. Any multi-hour value here means a stranded
        execution stays dead for hours in an environment with nobody to notice."""
        assert self._seconds(monkeypatch) < 3600

    def test_env_still_overrides_for_tuning(self, monkeypatch):
        assert self._seconds(monkeypatch, "120") == 120

    def test_a_nonsense_value_raises_rather_than_silently_defaulting(self, monkeypatch):
        """Loud-on-misconfig, matching the other duration knobs: silently falling back
        would hide a typo'd ConfigMap behind behaviour that looks deliberate."""
        import pytest

        with pytest.raises(ValueError):
            self._seconds(monkeypatch, "not-a-number")


class TestStuckRecoveryDefaultOn:
    """Recovery is ON by default in the PG reaper, parsed like the codebase's other
    default-on flags (CACHE_REDIS_ENABLED / ENABLE_METRICS): unset → on; any non-
    'true' value disables — a reliable kill-switch an operator can't miss with a
    stray space or case. A default-off gate previously let strands accumulate."""

    def _flag(self, monkeypatch, value):
        from queue_backend.pg_queue.reaper import stuck_recovery_enabled_from_env

        monkeypatch.delenv(_RECOVERY_ENV, raising=False)
        if value is not None:
            monkeypatch.setenv(_RECOVERY_ENV, value)
        return stuck_recovery_enabled_from_env()

    def test_default_unset_is_enabled(self, monkeypatch):
        assert self._flag(monkeypatch, None) is True

    def test_true_stays_enabled(self, monkeypatch):
        assert self._flag(monkeypatch, "true") is True
        assert self._flag(monkeypatch, "TRUE") is True

    def test_false_disables(self, monkeypatch):
        assert self._flag(monkeypatch, "false") is False
        assert self._flag(monkeypatch, "FALSE") is False

    def test_disable_is_whitespace_and_case_tolerant(self, monkeypatch):
        # An incident rollback with a stray space/newline/case from a ConfigMap must
        # still disable — strip()+lower() before the compare.
        assert self._flag(monkeypatch, " false ") is False
        assert self._flag(monkeypatch, "  FALSE\n") is False

    def test_any_non_true_value_disables(self, monkeypatch):
        # Reliable kill-switch: 0/no/off/disabled/empty (and a typo) all turn it off,
        # so an operator trying to disable it never silently fails to.
        for value in ("0", "no", "off", "disabled", "", "nope"):
            assert self._flag(monkeypatch, value) is False


class TestStuckRecoverySweepGating:
    """The flag must actually GATE the sweep, not merely parse — a regression that
    drops/inverts the guard in _maybe_recover_stuck_executions would pass the parser
    tests above but break recovery."""

    def _reaper(self, enabled):
        from queue_backend.pg_queue.reaper import PgReaper

        reaper = PgReaper.__new__(PgReaper)
        reaper._stuck_recovery_enabled = enabled
        reaper._stuck_recovery_seconds = 9000
        api = MagicMock()
        resp = MagicMock()
        resp.data = {"recovered": 0, "skipped": 0, "scanned": 0, "failed": 0}
        api.recover_stuck_pg_executions.return_value = resp
        reaper._get_api_client = MagicMock(return_value=api)
        return reaper, api

    def test_disabled_skips_the_recovery_call(self):
        reaper, api = self._reaper(enabled=False)
        reaper._maybe_recover_stuck_executions()
        api.recover_stuck_pg_executions.assert_not_called()

    def test_enabled_makes_the_recovery_call(self):
        reaper, api = self._reaper(enabled=True)
        reaper._maybe_recover_stuck_executions()
        api.recover_stuck_pg_executions.assert_called_once_with(stuck_seconds=9000)


class TestFacadeExposesRecovery:
    """The reaper calls recover_stuck_pg_executions on the InternalAPIClient FACADE
    (what _get_api_client() returns), which must delegate to its ExecutionAPIClient.
    The mock-based gating test above CANNOT catch a missing delegator (a MagicMock
    has every attribute) — this pins the real facade wiring; the delegator was
    missing (added in UN-3718), a real AttributeError the MagicMock-based test can't
    surface."""

    def test_facade_delegates_to_execution_client(self):
        from shared.api.internal_client import InternalAPIClient

        # __new__ (not __init__) → no network/env setup; inject a mock sub-client.
        client = InternalAPIClient.__new__(InternalAPIClient)
        client.execution_client = MagicMock()
        sentinel = object()
        client.execution_client.recover_stuck_pg_executions.return_value = sentinel

        result = client.recover_stuck_pg_executions(stuck_seconds=600, limit=50)

        client.execution_client.recover_stuck_pg_executions.assert_called_once_with(
            stuck_seconds=600, limit=50
        )
        assert result is sentinel

    def test_facade_forwards_defaults(self):
        # The only production caller (reaper) passes just stuck_seconds and relies on
        # limit defaulting to None — exercise that forwarding path too.
        from shared.api.internal_client import InternalAPIClient

        client = InternalAPIClient.__new__(InternalAPIClient)
        client.execution_client = MagicMock()

        client.recover_stuck_pg_executions(stuck_seconds=600)

        client.execution_client.recover_stuck_pg_executions.assert_called_once_with(
            stuck_seconds=600, limit=None
        )


class TestReaperConstructorWiring:
    """__init__ sources _stuck_recovery_enabled from the helper (not a hardcoded
    literal) — pins the wiring so a stray `= True` can't slip through."""

    def test_init_sources_flag_from_helper(self, monkeypatch):
        import queue_backend.pg_queue.reaper as reaper_mod

        monkeypatch.setattr(reaper_mod, "stuck_recovery_enabled_from_env", lambda: False)
        # Mock lease: interval (default 5s) must be < lease_seconds; nothing else in
        # __init__ does I/O, so this constructs without env/DB/HTTP setup.
        reaper = reaper_mod.PgReaper(MagicMock(lease_seconds=30))
        assert reaper._stuck_recovery_enabled is False