Skip to content

LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows

68.6 KB
Raw Blame
"""Tests for the PG queue consumer (claim → run → ack).

Unit tests drive ``poll_once`` with a mocked client and *real* registered
Celery tasks (so ``apply()`` actually runs the body). The integration test
exercises the full enqueue → poll → execute → ack loop against real Postgres
(skips if unreachable / unmigrated).
"""

from __future__ import annotations

import logging
import os
import threading
import time
from unittest.mock import MagicMock, patch

import pytest
from celery import shared_task
from queue_backend.fairness import FAIRNESS_HEADER_NAME
from queue_backend.pg_queue import to_payload
from queue_backend.pg_queue.client import QueueMessage
from queue_backend.pg_queue.connection import CONN_DEAD_ERRORS
from queue_backend.pg_queue.consumer import PgQueueConsumer

# Registered test tasks (namespaced). apply() runs their bodies in-process.
_calls: list = []


@shared_task(name="test_pg_consumer.ok")
def _ok_task(x, y=0):
    _calls.append((x, y))
    return x + y


@shared_task(name="test_pg_consumer.boom")
def _boom_task():
    raise RuntimeError("boom")


@shared_task(name="test_pg_consumer.dt")
def _dt_task():
    # Returns a non-JSON-native value (datetime) — the self-chain must coerce it
    # before enqueue, else client.send's plain json.dumps would raise.
    from datetime import datetime

    return {"when": datetime(2020, 1, 1)}  # noqa: DTZ001 — fixed value for the test


@pytest.fixture(autouse=True)
def _clear_calls():
    _calls.clear()


def _msg(msg_id, payload, *, read_ct=1, queue_name="qa"):
    return QueueMessage(
        msg_id=msg_id, message=payload, read_ct=read_ct, queue_name=queue_name
    )


def _ok_payload(x, y=0):
    return {"task_name": "test_pg_consumer.ok", "args": [x], "kwargs": {"y": y}}


# --- poll_once (mocked client, real tasks) ---


class TestPollOnce:
    def test_runs_task_and_acks(self):
        client = MagicMock()
        client.read.return_value = [_msg(1, _ok_payload(3, 4))]
        assert PgQueueConsumer(["q"], client=client).poll_once() == 1
        assert _calls == [(3, 4)]  # task body ran
        client.delete.assert_called_once_with(1)  # acked

    def test_failed_task_is_not_acked_and_logs(self, caplog):
        client = MagicMock()
        client.read.return_value = [
            _msg(2, {"task_name": "test_pg_consumer.boom", "args": [], "kwargs": {}})
        ]
        with caplog.at_level(logging.ERROR, logger="queue_backend.pg_queue.consumer"):
            PgQueueConsumer(["q"], client=client).poll_once()
        client.delete.assert_not_called()  # left for vt-expiry redelivery
        assert "failed" in caplog.text  # the cycling signal is logged

    def test_unknown_task_is_dropped(self, caplog):
        client = MagicMock()
        client.read.return_value = [
            _msg(3, {"task_name": "nope.nope", "args": [], "kwargs": {}})
        ]
        with caplog.at_level(logging.ERROR, logger="queue_backend.pg_queue.consumer"):
            PgQueueConsumer(["q"], client=client).poll_once()
        client.delete.assert_called_once_with(3)  # dropped, not redelivered
        assert "unknown task" in caplog.text

    def test_missing_task_name_dropped_as_malformed(self, caplog):
        client = MagicMock()
        client.read.return_value = [_msg(4, {"args": [], "kwargs": {}})]  # no task_name
        with caplog.at_level(logging.ERROR, logger="queue_backend.pg_queue.consumer"):
            PgQueueConsumer(["q"], client=client).poll_once()
        client.delete.assert_called_once_with(4)
        assert "missing task_name" in caplog.text  # distinct from "unknown task"

    def test_poison_message_dropped_past_max_attempts(self, caplog):
        client = MagicMock()
        # boom task, claimed more than max_attempts times → drop instead of redeliver.
        client.read.return_value = [
            _msg(5, {"task_name": "test_pg_consumer.boom"}, read_ct=6)
        ]
        with caplog.at_level(logging.ERROR, logger="queue_backend.pg_queue.consumer"):
            PgQueueConsumer(["q"], client=client, max_attempts=5).poll_once()
        client.delete.assert_called_once_with(5)  # dropped as poison
        assert "poison" in caplog.text

    def test_fairness_header_rebuilt_for_run(self):
        # Mock app so we can inspect the apply() headers.
        fairness = {"org_id": "o", "workload_type": "api", "pipeline_priority": 5}
        task = MagicMock()
        app = MagicMock()
        app.tasks.get.return_value = task
        client = MagicMock()
        client.read.return_value = [
            _msg(
                6,
                {
                    "task_name": "t",
                    "args": [1],
                    "kwargs": {"k": "v"},
                    "fairness": fairness,
                },
            )
        ]
        PgQueueConsumer(["q"], client=client, app=app).poll_once()
        kwargs = task.apply.call_args.kwargs
        assert kwargs["args"] == [1]
        assert kwargs["kwargs"] == {"k": "v"}
        assert kwargs["headers"] == {FAIRNESS_HEADER_NAME: fairness}

    def test_payload_task_id_is_passed_to_apply(self):
        # Idempotency-critical: Celery's Task.apply does ``task_id = task_id or
        # uuid()``, so without an explicit task_id the task sees a FRESH request.id
        # on every delivery and any guard keyed on it (fan-out claims,
        # generation_task_id uniqueness, task-complete markers) dedupes nothing
        # across a redelivery. The producer's stable id must reach apply().
        task = MagicMock()
        app = MagicMock()
        app.tasks.get.return_value = task
        client = MagicMock()
        client.read.return_value = [
            _msg(6, {"task_name": "t", "args": [], "kwargs": {}, "task_id": "stable-1"})
        ]
        PgQueueConsumer(["q"], client=client, app=app).poll_once()
        assert task.apply.call_args.kwargs["task_id"] == "stable-1"

    def test_missing_task_id_falls_back_to_celery_uuid(self):
        # No task_id in the payload -> pass None so Celery mints one, rather than
        # passing "" (which apply() would take literally as the request id).
        task = MagicMock()
        app = MagicMock()
        app.tasks.get.return_value = task
        client = MagicMock()
        client.read.return_value = [_msg(6, {"task_name": "t", "args": [], "kwargs": {}})]
        PgQueueConsumer(["q"], client=client, app=app).poll_once()
        assert task.apply.call_args.kwargs["task_id"] is None

    def test_ack_finding_no_row_warns(self, caplog):
        client = MagicMock()
        client.delete.return_value = False  # row already gone (vt expired mid-run)
        client.read.return_value = [_msg(7, _ok_payload(1))]
        with caplog.at_level(logging.WARNING, logger="queue_backend.pg_queue.consumer"):
            PgQueueConsumer(["q"], client=client).poll_once()
        assert "possible double-run" in caplog.text

    def test_multi_message_batch(self):
        client = MagicMock()
        client.read.return_value = [
            _msg(10, _ok_payload(1)),
            _msg(11, {"task_name": "test_pg_consumer.boom"}),
            _msg(12, {"task_name": "nope.nope"}),
            _msg(13, _ok_payload(2)),
        ]
        assert PgQueueConsumer(["q"], client=client).poll_once() == 4
        assert _calls == [(1, 0), (2, 0)]  # ok tasks ran in order
        deleted = {c.args[0] for c in client.delete.call_args_list}
        assert deleted == {10, 12, 13}  # ok acked + unknown dropped; boom NOT acked

    def test_empty_batch_acks_nothing(self):
        client = MagicMock()
        client.read.return_value = []
        assert PgQueueConsumer(["q"], client=client).poll_once() == 0
        client.delete.assert_not_called()


def _callback_payload(execution_id="exec-1", organization_id="org-1"):
    """A barrier aggregating-callback payload: execution identity in kwargs, no
    reply_key / on_error (the sharpest silent-strand case).
    """
    return {
        "task_name": "process_batch_callback_api",
        "args": [[]],
        "kwargs": {
            "execution_id": execution_id,
            "organization_id": organization_id,
            "pipeline_id": "pipe-1",
        },
    }


class TestPipelineIdentity:
    """``_pipeline_identity`` extraction across the payload shapes."""

    def test_from_callback_kwargs(self):
        from queue_backend.pg_queue.consumer import _pipeline_identity

        assert _pipeline_identity(_callback_payload("e", "o")) == ("e", "o")

    def test_from_barrier_context_and_fairness(self):
        from queue_backend.pg_queue.consumer import _pipeline_identity

        payload = {
            "task_name": "process_file_batch",
            "kwargs": {"_barrier_context": {"execution_id": "e2"}},
            "fairness": {"org_id": "o2", "workload_type": "api"},
        }
        assert _pipeline_identity(payload) == ("e2", "o2")

    def test_org_from_barrier_callback_descriptor(self):
        from queue_backend.pg_queue.consumer import _pipeline_identity

        payload = {
            "kwargs": {
                "_barrier_context": {
                    "execution_id": "e3",
                    "callback_descriptor": {"kwargs": {"organization_id": "o3"}},
                }
            }
        }
        assert _pipeline_identity(payload) == ("e3", "o3")

    def test_non_pipeline_returns_none(self):
        from queue_backend.pg_queue.consumer import _pipeline_identity

        assert _pipeline_identity({"task_name": "t", "kwargs": {}}) == (None, "")

    def test_execution_without_org_returns_empty_org(self):
        from queue_backend.pg_queue.consumer import _pipeline_identity

        payload = {"kwargs": {"execution_id": "e4"}}
        assert _pipeline_identity(payload) == ("e4", "")

    def test_from_positional_orchestration_args(self):
        # async_execute_bin passes identity POSITIONALLY: args=[schema, workflow,
        # execution_id, hash]. The fallback keys off the payload's own task_name.
        from queue_backend.pg_queue.consumer import _pipeline_identity

        payload = {
            "task_name": "async_execute_bin",
            "args": ["org-schema", "wf-1", "exec-9", {}],
            "kwargs": {"pipeline_id": "p"},
        }
        assert _pipeline_identity(payload) == ("exec-9", "org-schema")
        # An unknown task_name leaves the positional map inert.
        assert _pipeline_identity({**payload, "task_name": "some.other"}) == (None, "")

    def test_positional_short_args_no_indexerror(self):
        # A short args list (missing the execution_id index) must not IndexError;
        # it yields no identity — harmless, since with no execution_id the poison
        # drop bare-deletes anyway (the org is only used alongside an execution_id).
        from queue_backend.pg_queue.consumer import _pipeline_identity

        payload = {"task_name": "async_execute_bin", "args": ["org-schema"]}
        assert _pipeline_identity(payload) == (None, "")

    def test_positional_non_sequence_args_no_crash(self):
        # A non-list args (malformed payload) must not TypeError/IndexError.
        from queue_backend.pg_queue.consumer import _pipeline_identity

        payload = {"task_name": "async_execute_bin", "args": {"not": "a list"}}
        assert _pipeline_identity(payload) == (None, "")


class TestPoisonDropMarksExecution:
    """UN-3670: a poison drop with no reply channel marks the execution ERROR
    (or re-parks if the mark can't be confirmed) instead of silently discarding.
    """

    def _poison(self, msg_id=5, payload=None, read_ct=6):
        return _msg(msg_id, payload or _callback_payload(), read_ct=read_ct)

    def test_marks_error_then_deletes(self, monkeypatch):
        marks = []
        monkeypatch.setattr(
            "queue_backend.pg_queue.recovery.mark_execution_error",
            lambda client, eid, org, *, error_message: marks.append((eid, org)) or True,
        )
        client = MagicMock()
        client.read.return_value = [self._poison()]
        PgQueueConsumer(
            ["q"], client=client, api_client=MagicMock(), max_attempts=5
        ).poll_once()
        assert marks == [("exec-1", "org-1")]  # marked ERROR
        client.delete.assert_called_once_with(5)  # then dropped
        client.set_vt.assert_not_called()

    def test_poison_log_enriched_with_org_and_read_ct(self, monkeypatch, caplog):
        # Q2 debuggability: the poison drop logs a clear "poison-dropped" message
        # carrying read_ct + org_id + queue so a dropped task is greppable/
        # Sentry-visible and its source queue is identifiable at a glance.
        monkeypatch.setattr(
            "queue_backend.pg_queue.recovery.mark_execution_error",
            lambda *a, **k: True,
        )
        client = MagicMock()
        payload = {**_callback_payload(), "queue": "agentic_callback"}
        client.read.return_value = [self._poison(payload=payload, read_ct=6)]
        with caplog.at_level(logging.ERROR, logger="queue_backend.pg_queue.consumer"):
            PgQueueConsumer(
                ["q"], client=client, api_client=MagicMock(), max_attempts=5
            ).poll_once()
        assert "poison-dropped" in caplog.text
        assert "read_ct=6" in caplog.text
        assert "org_id=org-1" in caplog.text  # org surfaced for the dropped task
        assert "queue=agentic_callback" in caplog.text  # source queue surfaced
        # read_ct is rendered ONCE (an earlier "after N attempts" phrasing printed
        # the same value twice under a name that misread as an execution count).
        assert caplog.text.count("6") >= 1
        assert "attempts (msg_id" not in caplog.text

    def test_poison_log_redacts_credentials_and_body(self, monkeypatch, caplog):
        # Security: notification payloads carry the subscriber's Authorization /
        # API-key headers and the clubbed webhook body (customer extracted data).
        # A poison drop must not dump either to stdout/Sentry.
        monkeypatch.setattr(
            "queue_backend.pg_queue.recovery.mark_execution_error",
            lambda *a, **k: True,
        )
        payload = {
            "task_name": "send_webhook_notification",
            "queue": "notifications",
            "args": [
                "https://hook.test",
                {"secret_field": "customer extracted data"},
                {"Authorization": "Bearer super-secret", "X-Api-Key": "ak_live_123"},
                30,
            ],
            "kwargs": {"organization_id": "org-1", "auth_token": "tok_abc"},
        }
        client = MagicMock()
        client.read.return_value = [self._poison(payload=payload, read_ct=6)]
        with caplog.at_level(logging.ERROR, logger="queue_backend.pg_queue.consumer"):
            PgQueueConsumer(
                ["q"], client=client, api_client=MagicMock(), max_attempts=5
            ).poll_once()
        # No secret value survives, by any route.
        assert "super-secret" not in caplog.text
        assert "ak_live_123" not in caplog.text
        assert "tok_abc" not in caplog.text
        # Customer body replaced by a type+size summary, not dumped.
        assert "customer extracted data" not in caplog.text
        assert "<dict len=1>" in caplog.text
        # Routing metadata IS preserved — that's what the drop is debugged from.
        assert "send_webhook_notification" in caplog.text
        assert "https://hook.test" in caplog.text
        assert "REDACTED" in caplog.text

    def test_positional_orchestration_poison_marks_error(self, monkeypatch):
        # H2 regression: a poisoned async_execute_bin carries execution_id
        # POSITIONALLY (args[2]) with no _barrier_context and — since its poison
        # precedes barrier arm + claim — no reaper handle at all. It must still
        # recover its identity and mark ERROR, not bare-delete into a silent strand.
        marks = []
        monkeypatch.setattr(
            "queue_backend.pg_queue.recovery.mark_execution_error",
            lambda client, eid, org, *, error_message: marks.append((eid, org)) or True,
        )
        payload = {
            "task_name": "async_execute_bin",
            "args": ["org-schema", "wf-1", "exec-9", {}],
            "kwargs": {"pipeline_id": "p"},
        }
        client = MagicMock()
        client.read.return_value = [_msg(7, payload, read_ct=6)]
        PgQueueConsumer(
            ["q"], client=client, api_client=MagicMock(), max_attempts=5
        ).poll_once()
        assert marks == [("exec-9", "org-schema")]  # recovered + marked
        client.delete.assert_called_once_with(7)  # then dropped

    def test_reparks_when_mark_unconfirmed(self, monkeypatch):
        monkeypatch.setattr(
            "queue_backend.pg_queue.recovery.mark_execution_error",
            lambda *a, **k: False,  # backend down
        )
        client = MagicMock()
        client.read.return_value = [self._poison(read_ct=6)]
        PgQueueConsumer(
            ["q"], client=client, api_client=MagicMock(), max_attempts=5
        ).poll_once()
        client.delete.assert_not_called()  # NOT dropped into a void
        client.set_vt.assert_called_once_with(5, 300)  # re-parked long

    def test_drops_after_repark_budget_exhausted(self, monkeypatch, caplog):
        monkeypatch.setattr(
            "queue_backend.pg_queue.recovery.mark_execution_error",
            lambda *a, **k: False,
        )
        client = MagicMock()
        # read_ct beyond max_attempts + budget → give up and drop.
        client.read.return_value = [self._poison(read_ct=11)]
        with caplog.at_level(logging.ERROR, logger="queue_backend.pg_queue.consumer"):
            PgQueueConsumer(
                ["q"],
                client=client,
                api_client=MagicMock(),
                max_attempts=5,
                poison_repark_budget=5,
            ).poll_once()
        client.set_vt.assert_not_called()
        client.delete.assert_called_once_with(5)
        assert "re-park budget" in caplog.text

    def test_reply_channel_keeps_existing_behavior(self, monkeypatch):
        # A message with on_error surfaces on its channel and is dropped — NOT
        # marked ERROR (it already has a failure channel).
        marked = MagicMock()
        monkeypatch.setattr(
            "queue_backend.pg_queue.recovery.mark_execution_error", marked
        )
        payload = {
            "task_name": "some.executor",
            "on_error": {"task_name": "cb", "queue": "q"},
            "kwargs": {"execution_id": "exec-1", "organization_id": "org-1"},
        }
        client = MagicMock()
        client.read.return_value = [self._poison(payload=payload)]
        consumer = PgQueueConsumer(["q"], client=client, max_attempts=5)
        monkeypatch.setattr(consumer, "_chain_continuation", MagicMock())
        consumer.poll_once()
        marked.assert_not_called()  # reply channel path, no execution mark
        client.delete.assert_called_once_with(5)
        client.set_vt.assert_not_called()

    def test_no_execution_id_drops_without_marking(self, monkeypatch):
        marked = MagicMock()
        monkeypatch.setattr(
            "queue_backend.pg_queue.recovery.mark_execution_error", marked
        )
        client = MagicMock()
        client.read.return_value = [
            _msg(5, {"task_name": "test_pg_consumer.boom"}, read_ct=6)
        ]
        PgQueueConsumer(["q"], client=client, max_attempts=5).poll_once()
        marked.assert_not_called()
        client.delete.assert_called_once_with(5)

    def test_repark_budget_boundary_still_reparks(self, monkeypatch):
        # Gap A: at exactly read_ct == max_attempts + budget (10) the message must
        # still re-park; only strictly beyond it drops. Guards the >/>= boundary.
        monkeypatch.setattr(
            "queue_backend.pg_queue.recovery.mark_execution_error",
            lambda *a, **k: False,
        )
        client = MagicMock()
        client.read.return_value = [self._poison(read_ct=10)]  # 5 + 5
        PgQueueConsumer(
            ["q"],
            client=client,
            api_client=MagicMock(),
            max_attempts=5,
            poison_repark_budget=5,
        ).poll_once()
        client.set_vt.assert_called_once_with(5, 300)  # re-parked, not dropped
        client.delete.assert_not_called()

    def test_no_org_poison_drops_immediately_without_marking(self, monkeypatch):
        # Gap B: an execution_id but no org can never be marked (org-scoped API),
        # so it drops immediately — no wasted re-park budget, no mark attempt.
        marked = MagicMock()
        monkeypatch.setattr(
            "queue_backend.pg_queue.recovery.mark_execution_error", marked
        )
        client = MagicMock()
        client.read.return_value = [
            self._poison(
                payload={
                    "task_name": "process_file_batch",
                    "kwargs": {"execution_id": "e4"},
                }
            )
        ]
        PgQueueConsumer(
            ["q"], client=client, api_client=MagicMock(), max_attempts=5
        ).poll_once()
        marked.assert_not_called()  # never attempted (no org)
        client.set_vt.assert_not_called()  # not re-parked (permanent)
        client.delete.assert_called_once_with(5)  # dropped now

    def test_api_client_build_failure_reparks(self, monkeypatch):
        # Gap C: with no injected api_client, a build failure is transient — the
        # message re-parks (not drops) so a recovered backend can still mark it.
        monkeypatch.setattr(
            "shared.api.InternalAPIClient",
            MagicMock(side_effect=RuntimeError("no config")),
        )
        marked = MagicMock()
        monkeypatch.setattr(
            "queue_backend.pg_queue.recovery.mark_execution_error", marked
        )
        client = MagicMock()
        client.read.return_value = [self._poison(read_ct=6)]  # org present
        PgQueueConsumer(["q"], client=client, max_attempts=5).poll_once()
        marked.assert_not_called()  # never reached — client build raised first
        client.set_vt.assert_called_once_with(5, 300)  # re-parked
        client.delete.assert_not_called()


class TestConstruction:
    def test_rejects_non_positive_params(self):
        client = MagicMock()
        for kw in (
            {"batch_size": 0},
            {"vt_seconds": -1},
            {"poll_interval": 0},
            {"backoff_max": 0},
            {"max_attempts": 0},
            {"poison_repark_vt_seconds": 0},
            {"poison_repark_budget": -1},
        ):
            with pytest.raises(ValueError):
                PgQueueConsumer(["q"], client=client, **kw)

    def test_rejects_backoff_max_below_poll_interval(self):
        # Otherwise backoff would shrink below poll_interval instead of growing.
        client = MagicMock()
        with pytest.raises(ValueError, match="backoff_max"):
            PgQueueConsumer(["q"], client=client, poll_interval=0.5, backoff_max=0.1)

    def test_default_backoff_max_stays_under_default_health_stale(self):
        # An idle loop sleeps up to backoff_max between heartbeats, so the
        # default cap must stay below the default staleness threshold or idle
        # consumers fail their liveness probe.
        from queue_backend.pg_queue import consumer as consumer_mod

        assert (
            consumer_mod._DEFAULT_POLL_INTERVAL
            <= consumer_mod._DEFAULT_BACKOFF_MAX
            < consumer_mod._DEFAULT_HEALTH_STALE_SECONDS
        )


class TestRunLoop:
    def test_run_stops_gracefully(self, monkeypatch):
        client = MagicMock()
        client.read.return_value = []  # always empty → backoff path
        consumer = PgQueueConsumer(["q"], client=client)
        # First empty poll → sleep (patched to request stop) → loop exits.
        monkeypatch.setattr(
            "queue_backend.pg_queue.consumer.time.sleep", lambda _s: consumer.stop()
        )
        consumer.run(install_signals=False)
        assert consumer._running is False

    def test_backoff_grows_then_resets(self, monkeypatch):
        client = MagicMock()
        # empty, empty, one message, empty → backoff doubles, resets, doubles.
        client.read.side_effect = [[], [], [_msg(1, _ok_payload(1))], []]
        consumer = PgQueueConsumer(
            ["q"], client=client, poll_interval=0.1, backoff_max=0.25
        )
        sleeps: list[float] = []

        def _sleep(secs):
            sleeps.append(secs)
            if len(sleeps) == 3:  # stop after the third sleep
                consumer.stop()

        monkeypatch.setattr("queue_backend.pg_queue.consumer.time.sleep", _sleep)
        consumer.run(install_signals=False)
        # empty→0.1, empty→0.2 (doubled), [msg] resets, empty→0.1 again.
        assert sleeps == [0.1, 0.2, 0.1]

    def test_poll_error_does_not_kill_loop(self, monkeypatch):
        client = MagicMock()
        # first poll raises (transient), then empty → loop must survive the raise.
        client.read.side_effect = [RuntimeError("blip"), []]
        consumer = PgQueueConsumer(["q"], client=client)
        sleeps: list[float] = []

        def _sleep(secs):
            sleeps.append(secs)
            if len(sleeps) == 2:  # stop after the post-error poll
                consumer.stop()

        monkeypatch.setattr("queue_backend.pg_queue.consumer.time.sleep", _sleep)
        consumer.run(install_signals=False)  # must not raise
        assert client.read.call_count == 2  # kept polling after the error

    def test_run_refuses_to_start_with_empty_registry(self, isolated_celery_registry):
        # A non-bootstrapped consumer would drop every message as "unknown" —
        # fail loudly instead. isolated_celery_registry makes "empty" genuinely
        # empty (Celery's global shared_task backlog would otherwise leak the
        # worker's tasks in and let the guard pass → infinite poll loop / hang).
        from celery import Celery

        # No tasks at all: isolated_celery_registry cleared the finalizer backlog,
        # which holds the worker's shared tasks *and* Celery's celery.* built-ins.
        empty_app = Celery("empty-no-tasks", set_as_current=False)
        consumer = PgQueueConsumer(["q"], client=MagicMock(), app=empty_app)
        with pytest.raises(RuntimeError, match="no application tasks"):
            consumer.run(install_signals=False)

    def test_run_starts_with_registered_tasks(self, monkeypatch):
        # Positive arm: a non-empty registry starts and polls. current_app has
        # the module's @shared_task tasks registered, so the guard passes.
        client = MagicMock()
        client.read.return_value = []  # empty → loop sleeps → we stop it
        consumer = PgQueueConsumer(["q"], client=client)
        monkeypatch.setattr(
            "queue_backend.pg_queue.consumer.time.sleep", lambda _s: consumer.stop()
        )
        consumer.run(install_signals=False)  # must not raise
        assert client.read.called

    def test_run_bypasses_guard_when_require_tasks_false(
        self, monkeypatch, isolated_celery_registry
    ):
        # require_tasks=False skips the guard even on an empty registry (lets a
        # caller opt out, e.g. a deliberately task-less smoke run).
        from celery import Celery

        empty_app = Celery("empty-no-tasks", set_as_current=False)
        client = MagicMock()
        client.read.return_value = []
        consumer = PgQueueConsumer(["q"], client=client, app=empty_app)
        monkeypatch.setattr(
            "queue_backend.pg_queue.consumer.time.sleep", lambda _s: consumer.stop()
        )
        consumer.run(install_signals=False, require_tasks=False)  # must not raise
        assert client.read.called

    def test_registered_task_count_excludes_celery_builtins(
        self, isolated_celery_registry
    ):
        # The guard counts *application* tasks only — a celery.*-named task must
        # not count toward the total. isolated_celery_registry clears the
        # finalizer backlog so the app starts genuinely empty; we then register
        # one celery.*-named task and one application task and assert only the
        # latter is counted (exercising the exclusion filter under test).
        from celery import Celery

        empty_app = Celery("empty-no-tasks", set_as_current=False)
        consumer = PgQueueConsumer(["q"], client=MagicMock(), app=empty_app)
        assert consumer._registered_task_count() == 0  # genuinely empty

        @empty_app.task(name="celery.builtin_like")
        def _builtin_like():
            return 0

        # A celery.*-named task is excluded → still zero application tasks.
        assert consumer._registered_task_count() == 0

        @empty_app.task(name="test_pg_consumer.demo")
        def _demo():
            return 1

        assert consumer._registered_task_count() == 1


# --- Liveness heartbeat (drives the health endpoint) ---


class TestPollHeartbeat:
    def test_poll_once_refreshes_heartbeat(self):
        client = MagicMock()
        client.read.return_value = []
        consumer = PgQueueConsumer(["q"], client=client)
        # Simulate a long-idle consumer, then poll → heartbeat resets to ~now.
        consumer._last_poll_monotonic -= 120
        assert consumer.seconds_since_last_poll() > 100
        consumer.poll_once()
        assert consumer.seconds_since_last_poll() < 1.0

    def test_heartbeat_stamped_before_read(self):
        # Pins the headline design: the stamp lands before each claim (not
        # after it), so a task running longer than the threshold still trips
        # the probe. A bottom-of-poll stamp would pass test_poll_once_refreshes
        # but fail here.
        client = MagicMock()
        consumer = PgQueueConsumer(["q"], client=client)
        before = consumer._last_poll_monotonic
        seen: dict[str, float] = {}
        client.read.side_effect = lambda *a, **k: (
            seen.setdefault("during", consumer._last_poll_monotonic),
            [],
        )[1]
        consumer.poll_once()
        assert seen["during"] > before  # refreshed BEFORE read ran, not after

    def test_heartbeat_stamped_before_multi_queue_claim(self):
        # The multi-queue path claims once per cycle (read_any); the stamp must
        # still land before that claim, not after the task it returns.
        client = MagicMock()
        consumer = PgQueueConsumer(["q1", "q2"], client=client)
        before = consumer._last_poll_monotonic
        seen: dict[str, float] = {}
        client.read_any.side_effect = lambda *a, **k: (
            seen.setdefault("during", consumer._last_poll_monotonic),
            [],
        )[1]
        consumer.poll_once()
        assert seen["during"] > before

    def test_health_server_disabled_without_port(self):
        # No port configured → no server bound (opt-in).
        from queue_backend.pg_queue.consumer import _maybe_start_health_server

        consumer = PgQueueConsumer(["q"], client=MagicMock())
        assert _maybe_start_health_server(consumer, port=None, stale_after=60) is None

    def test_liveness_server_reports_200_then_503(self):
        # Real endpoint: 200 while the poll loop is fresh, 503 once it goes
        # stale. Bind port 0 so the OS picks a free port (no fixed-port clash).
        import json
        import urllib.error
        import urllib.request

        from queue_backend.pg_queue.consumer import LivenessServer

        consumer = PgQueueConsumer(["q"], client=MagicMock())
        server = LivenessServer(consumer, port=0, stale_after=60)
        server.start()
        try:
            url = f"http://127.0.0.1:{server.bound_port}/health"
            with urllib.request.urlopen(url, timeout=5) as resp:
                assert resp.status == 200
                assert json.loads(resp.read())["status"] == "healthy"

            consumer._last_poll_monotonic -= 120  # force the loop stale
            with pytest.raises(urllib.error.HTTPError) as ei:
                urllib.request.urlopen(url, timeout=5)
            assert ei.value.code == 503
            assert json.loads(ei.value.read())["status"] == "unhealthy"
        finally:
            server.stop()

    def test_liveness_aliases_and_unknown_path(self):
        # All three probe aliases answer 200 (different orchestrators probe
        # different paths); an unknown path is 404 (guards against a regression
        # that makes every path pass).
        import urllib.error
        import urllib.request

        from queue_backend.pg_queue.consumer import LivenessServer

        consumer = PgQueueConsumer(["q"], client=MagicMock())
        server = LivenessServer(consumer, port=0, stale_after=60)
        server.start()
        try:
            base = f"http://127.0.0.1:{server.bound_port}"
            # Aliases, plus a query string (self.path includes it) must match.
            for path in ("/health", "/healthz", "/livez", "/health?probe=k8s"):
                with urllib.request.urlopen(f"{base}{path}", timeout=5) as resp:
                    assert resp.status == 200, path
            with pytest.raises(urllib.error.HTTPError) as ei:
                urllib.request.urlopen(f"{base}/nope", timeout=5)
            assert ei.value.code == 404
        finally:
            server.stop()

    def test_single_process_ready_is_200_once_serving(self):
        # UN-4136: the single-process server only starts after `import worker` and
        # the consumer build, so reaching /ready means loaded. The chart's
        # startupProbe hits /ready on every PG consumer, CONCURRENCY=1 included.
        import json
        import urllib.request

        from queue_backend.pg_queue.consumer import LivenessServer

        consumer = PgQueueConsumer(["q"], client=MagicMock())
        server = LivenessServer(consumer, port=0, stale_after=60)
        server.start()
        try:
            base = f"http://127.0.0.1:{server.bound_port}"
            for path in ("/ready", "/readyz", "/ready?probe=startup"):
                with urllib.request.urlopen(f"{base}{path}", timeout=5) as resp:
                    assert resp.status == 200, path
                    assert json.loads(resp.read())["status"] == "ready", path
        finally:
            server.stop()

    def test_ready_is_404_without_a_ready_fn(self):
        # A process with no readiness notion (e.g. the reaper) must NOT answer
        # /ready with a pass — a startupProbe mistakenly pointed at it has to fail.
        import urllib.error
        import urllib.request

        from queue_backend.pg_queue.liveness import LivenessServer as Base

        server = Base(
            freshness_fn=lambda: 0.0,
            stale_after=60,
            port=0,
            check_name="t",
            age_key="age",
        )
        server.start()
        try:
            with pytest.raises(urllib.error.HTTPError) as ei:
                urllib.request.urlopen(
                    f"http://127.0.0.1:{server.bound_port}/ready", timeout=5
                )
            assert ei.value.code == 404
        finally:
            server.stop()

    def test_ready_fn_that_raises_answers_503_and_leaves_health_alone(self):
        import json
        import urllib.error
        import urllib.request

        from queue_backend.pg_queue.liveness import LivenessServer as Base

        def _broken() -> bool:
            raise RuntimeError("boom")

        server = Base(
            freshness_fn=lambda: 0.0,
            stale_after=60,
            port=0,
            check_name="t",
            age_key="age",
            ready_fn=_broken,
        )
        server.start()
        try:
            base = f"http://127.0.0.1:{server.bound_port}"
            with pytest.raises(urllib.error.HTTPError) as ei:
                urllib.request.urlopen(f"{base}/ready", timeout=5)
            assert ei.value.code == 503
            assert json.loads(ei.value.read())["status"] == "starting"
            with urllib.request.urlopen(f"{base}/health", timeout=5) as resp:
                assert resp.status == 200  # readiness never flips liveness
        finally:
            server.stop()

    def test_double_start_is_rejected(self):
        from queue_backend.pg_queue.consumer import LivenessServer

        server = LivenessServer(
            PgQueueConsumer(["q"], client=MagicMock()), port=0, stale_after=60
        )
        server.start()
        try:
            with pytest.raises(RuntimeError, match="already started"):
                server.start()
        finally:
            server.stop()
        # stop() returns it to the inert state → can start again.
        server.start()
        server.stop()


# --- Integration: full enqueue → poll → execute → ack against real PG ---
# Uses the shared ``pg_client`` fixture from conftest.py.


class TestConsumerIntegration:
    def test_enqueue_poll_execute_ack(self, pg_client):
        queue_name = f"test_consumer_{os.getpid()}"
        try:
            pg_client.send(
                queue_name,
                to_payload("test_pg_consumer.ok", args=[5], kwargs={"y": 6}),
            )
            claimed = PgQueueConsumer([queue_name], client=pg_client).poll_once()
            assert claimed == 1
            assert (5, 6) in _calls  # task actually executed off Postgres
            # Row was acked (deleted) — nothing left to claim.
            assert pg_client.read(queue_name, vt_seconds=30, qty=10) == []
        finally:
            with pg_client.conn.cursor() as cur:
                cur.execute(
                    "DELETE FROM pg_queue_message WHERE queue_name = %s", (queue_name,)
                )
            pg_client.conn.commit()

    def test_multi_queue_poll_executes_and_acks_on_postgres(self, pg_client):
        # The multi-queue claim end to end: real read_any rows feed the real
        # dispatch and the delete() ack, which the stubbed tests can't see.
        base = f"test_consumer_mq_{os.getpid()}"
        queues = [f"{base}_a", f"{base}_empty", f"{base}_b"]
        try:
            pg_client.send(queues[0], to_payload("test_pg_consumer.ok", args=[11]))
            pg_client.send(queues[2], to_payload("test_pg_consumer.ok", args=[22]))
            consumer = PgQueueConsumer(queues, client=pg_client)
            claimed = [consumer.poll_once() for _ in range(3)]
            assert claimed == [1, 1, 0]
            assert (11, 0) in _calls and (22, 0) in _calls
            assert pg_client.read_any(queues, vt_seconds=30, qty=10) == []  # acked
        finally:
            with pg_client.conn.cursor() as cur:
                cur.execute(
                    "DELETE FROM pg_queue_message WHERE queue_name = ANY(%s)", (queues,)
                )
            pg_client.conn.commit()

    def test_busy_queues_take_equal_turns_around_an_empty_one(self, pg_client):
        # An empty queue between two busy ones must not give the queue after it
        # an extra turn: claims alternate.
        base = f"test_consumer_rr_{os.getpid()}"
        queues = [f"{base}_a", f"{base}_empty", f"{base}_b"]
        try:
            for _ in range(3):
                pg_client.send(queues[0], to_payload("test_pg_consumer.ok", args=[1]))
                pg_client.send(queues[2], to_payload("test_pg_consumer.ok", args=[1]))
            consumer = PgQueueConsumer(queues, client=pg_client)
            served: list[str] = []
            consumer._handle = lambda m: served.append(m.queue_name)
            for _ in range(6):
                consumer.poll_once()
            assert served == [queues[0], queues[2]] * 3
        finally:
            with pg_client.conn.cursor() as cur:
                cur.execute(
                    "DELETE FROM pg_queue_message WHERE queue_name = ANY(%s)", (queues,)
                )
            pg_client.conn.commit()


class TestMultiQueue:
    """9f: one consumer drains several queues — one claim per cycle (UN-4176)."""

    def test_multi_queue_claims_once_per_cycle(self):
        client = MagicMock()
        client.read_any.return_value = [_msg(1, _ok_payload(1))]
        claimed = PgQueueConsumer(["qa", "qb"], client=client).poll_once()
        assert claimed == 1
        client.read_any.assert_called_once()
        client.read.assert_not_called()  # never one statement per queue
        assert client.read_any.call_args.args[0] == ["qa", "qb"]

    def test_single_queue_keeps_the_single_queue_claim(self):
        client = MagicMock()
        client.read.return_value = []
        PgQueueConsumer(["q"], client=client).poll_once()
        client.read.assert_called_once()
        client.read_any.assert_not_called()

    def test_next_claim_starts_past_the_queue_served(self):
        # Rotating by one regardless of what was served would hand a busy queue
        # an extra turn for every empty queue before it; starting one past the
        # queue actually served keeps busy queues on equal turns.
        client = MagicMock()
        client.read_any.side_effect = [
            [_msg(1, _ok_payload(1), queue_name="qc")],  # start 0 fell through to qc
            [_msg(2, _ok_payload(2), queue_name="qa")],
            [],
        ]
        consumer = PgQueueConsumer(["qa", "qb", "qc"], client=client)
        for _ in range(3):
            consumer.poll_once()
        assert [c.kwargs["start"] for c in client.read_any.call_args_list] == [0, 0, 1]
        assert consumer._next_queue == 2  # empty cycle moves on by one

    @pytest.mark.parametrize(
        ("failing", "expected_done", "unstarted"),
        [(1, 0, [2, 3]), (2, 1, [3]), (3, 2, [])],
    )
    def test_handler_failure_stops_the_batch_and_counts_the_work_done(
        self, caplog, failing, expected_done, unstarted
    ):
        # Running the rest after a failure could, in a DB outage, run work whose
        # ack cannot land; so the batch stops, but work done still counts (0 on
        # a first-message failure, so run() backs off).
        client = MagicMock()
        client.read_any.return_value = [
            _msg(1, _ok_payload(1), queue_name="qa"),
            _msg(2, _ok_payload(2), queue_name="qa"),
            _msg(3, _ok_payload(3), queue_name="qb"),
        ]
        consumer = PgQueueConsumer(
            ["qa", "qb"], client=client, batch_size=3, lease_seconds=60, vt_seconds=60
        )
        handled: list[int] = []

        def _handle(message):
            if message.msg_id == failing:
                raise RuntimeError("boom")
            handled.append(message.msg_id)

        consumer._handle = _handle
        with caplog.at_level(logging.ERROR):
            assert consumer.poll_once() == expected_done
        assert handled == list(range(1, failing))
        assert f"msg_id={failing} " in caplog.text
        if unstarted:
            assert f"unstarted msg_ids {unstarted}" in caplog.text
        else:
            assert "unstarted" not in caplog.text
        # Rotation follows the claim, not what ran: the batch ended on qb.
        assert consumer._next_queue == 0

    def test_non_object_payload_is_dropped_not_raised(self):
        # A non-dict payload must not raise out of _handle (it would stop its
        # batch on every delivery); it is dropped and acked like any malformed one.
        client = MagicMock()
        client.read_any.return_value = [
            _msg(1, ["not", "an", "object"]),
            _msg(2, _ok_payload(7)),
        ]
        consumer = PgQueueConsumer(
            ["qa", "qb"], client=client, batch_size=2, lease_seconds=60, vt_seconds=60
        )
        assert consumer.poll_once() == 2
        assert (7, 0) in _calls  # the batch-mate still ran
        assert [c.args[0] for c in client.delete.call_args_list] == [1, 2]

    def test_rotation_follows_the_last_message_of_a_batch(self):
        client = MagicMock()
        client.read_any.side_effect = [
            [
                _msg(1, _ok_payload(1), queue_name="qa"),
                _msg(2, _ok_payload(2), queue_name="qc"),
            ],
            [],
        ]
        consumer = PgQueueConsumer(
            ["qa", "qb", "qc"],
            client=client,
            batch_size=3,
            lease_seconds=60,
            vt_seconds=60,
        )
        consumer._handle = lambda m: None
        consumer.poll_once()
        consumer.poll_once()
        # Batch ended on qc (index 2), so the next claim starts at qa (0).
        assert [c.kwargs["start"] for c in client.read_any.call_args_list] == [0, 0]

    def test_start_queue_rotates_every_cycle(self):
        # With nothing claimed the start still moves on, so every queue gets to
        # be probed first in turn.
        client = MagicMock()
        client.read_any.return_value = []
        consumer = PgQueueConsumer(["qa", "qb", "qc"], client=client)
        for _ in range(4):
            consumer.poll_once()
        assert [c.kwargs["start"] for c in client.read_any.call_args_list] == [
            0,
            1,
            2,
            0,
        ]

    def test_rejects_empty_queue_list(self):
        client = MagicMock()
        with pytest.raises(ValueError, match="queue_names"):
            PgQueueConsumer([], client=client)

    def test_claim_failure_counts_as_an_empty_cycle(self):
        """A failed claim is logged, not raised, and returns 0 so run() backs off."""
        client = MagicMock()
        client.read_any.side_effect = RuntimeError("boom")
        assert PgQueueConsumer(["qa", "qb"], client=client).poll_once() == 0

    def test_claim_uses_batch_size_qty(self):
        client = MagicMock()
        client.read_any.return_value = []
        PgQueueConsumer(["qa", "qb"], client=client, batch_size=2).poll_once()
        assert client.read_any.call_args.kwargs["qty"] == 2

    def test_duplicate_queues_are_deduped(self):
        """A duplicate (e.g. env "q,q") must not double-read the same queue."""
        client = MagicMock()
        client.read.return_value = []
        consumer = PgQueueConsumer(["q", "q"], client=client)
        assert consumer.queue_names == ["q"]
        consumer.poll_once()
        assert client.read.call_count == 1


def test_parse_queue_list():
    from queue_backend.pg_queue.consumer import _parse_queue_list

    assert _parse_queue_list("a") == ["a"]  # single value → one-element (back-compat)
    assert _parse_queue_list("a,b,c") == ["a", "b", "c"]
    assert _parse_queue_list(" a , b ,c ") == ["a", "b", "c"]  # whitespace stripped
    assert _parse_queue_list("a,,b") == ["a", "b"]  # empties dropped


class TestRequestReply:
    """Executor-RPC reply_key behaviour: store outcome + ack after one attempt;
    drop branches store a definitive failure; a store failure still acks (no
    expensive re-run). PgResultBackend is mocked so no DB is needed.
    """

    _RB = "queue_backend.pg_queue.consumer.PgResultBackend"

    def test_success_stores_result_and_acks(self):
        client = MagicMock()
        client.read.return_value = [_msg(1, {**_ok_payload(3, 4), "reply_key": "rk1"})]
        with patch(self._RB) as rb_cls:
            PgQueueConsumer(["q"], client=client).poll_once()
        rb = rb_cls.return_value
        rb.store_result.assert_called_once()
        assert rb.store_result.call_args.args[0] == "rk1"
        assert rb.store_result.call_args.kwargs["result"] == 7  # x + y
        assert rb.store_result.call_args.kwargs["error"] is None
        client.delete.assert_called_once_with(1)  # acked

    def test_task_raise_stores_error_and_acks(self):
        client = MagicMock()
        client.read.return_value = [
            _msg(2, {"task_name": "test_pg_consumer.boom", "reply_key": "rk2"})
        ]
        with patch(self._RB) as rb_cls:
            PgQueueConsumer(["q"], client=client).poll_once()
        rb = rb_cls.return_value
        assert rb.store_result.call_args.kwargs["error"] is not None
        assert rb.store_result.call_args.kwargs["result"] is None
        client.delete.assert_called_once_with(2)  # acked, NOT left for redelivery

    def test_unknown_task_stores_error_and_acks(self):
        client = MagicMock()
        client.read.return_value = [
            _msg(3, {"task_name": "nope.nope", "reply_key": "rk3"})
        ]
        with patch(self._RB) as rb_cls:
            PgQueueConsumer(["q"], client=client).poll_once()
        rb = rb_cls.return_value
        assert rb.store_result.call_args.kwargs["error"] is not None
        client.delete.assert_called_once_with(3)

    def test_store_failure_on_success_path_still_acks(self, caplog):
        client = MagicMock()
        client.read.return_value = [_msg(4, {**_ok_payload(1, 1), "reply_key": "rk4"})]
        with patch(self._RB) as rb_cls:
            rb_cls.return_value.store_result.side_effect = RuntimeError("db down")
            with caplog.at_level(logging.ERROR, logger="queue_backend.pg_queue.consumer"):
                PgQueueConsumer(["q"], client=client).poll_once()
        # A store failure must NOT block the ack (avoids an expensive re-run).
        client.delete.assert_called_once_with(4)
        assert "FAILED to store request-reply result" in caplog.text


class TestSelfChain:
    """Async/callback self-chaining (③c): with no reply_key, the consumer enqueues
    the on_success / on_error continuation onto its queue after the task runs, then
    acks regardless (a vt-redelivery would re-run the executor = LLM double-spend).
    Best-effort: a chain-enqueue failure still acks. ``client.send`` is the
    self-chain enqueue (the mock client is also read/delete).
    """

    @staticmethod
    def _spec(task="cb.done", queue="ide_callback"):
        return {
            "task_name": task,
            "kwargs": {"callback_kwargs": {"room": "r1"}},
            "queue": queue,
        }

    def test_success_chains_on_success_with_result_and_acks(self):
        client = MagicMock()
        payload = {**_ok_payload(3, 4), "on_success": self._spec(), "task_id": "tid"}
        client.read.return_value = [_msg(1, payload)]
        PgQueueConsumer(["q"], client=client).poll_once()
        client.delete.assert_called_once_with(1)  # executor msg acked
        client.send.assert_called_once()  # continuation enqueued
        queue_arg, payload_arg = client.send.call_args.args[:2]
        assert queue_arg == "ide_callback"
        assert payload_arg["task_name"] == "cb.done"
        assert payload_arg["args"] == [7]  # x + y prepended (Celery link parity)
        assert payload_arg["kwargs"] == {"callback_kwargs": {"room": "r1"}}

    def test_failure_chains_on_error_with_task_id_and_acks(self):
        client = MagicMock()
        payload = {
            "task_name": "test_pg_consumer.boom",
            "on_error": self._spec("cb.err"),
            "task_id": "tid-9",
        }
        client.read.return_value = [_msg(2, payload)]
        PgQueueConsumer(["q"], client=client).poll_once()
        client.delete.assert_called_once_with(2)  # acked, NOT left for redelivery
        client.send.assert_called_once()
        payload_arg = client.send.call_args.args[1]
        assert payload_arg["task_name"] == "cb.err"
        assert payload_arg["args"] == ["tid-9"]  # dispatch task_id as the failed id
        # The real error text rides callback_kwargs['error'] (no Celery AsyncResult
        # to recover it from on PG) so on_error callbacks surface it, not a default.
        assert "RuntimeError: boom" in payload_arg["kwargs"]["callback_kwargs"]["error"]

    def test_early_drop_branches_still_chain_on_error(self):
        # Critical: a dispatch_with_callback that hits malformed/unknown/poison must
        # still fire on_error — else the HTTP-202 caller hangs with no terminal event.
        err = self._spec("cb.err")
        cases = [
            # (msg_id, payload, read_ct)  — malformed / unknown / poison
            (10, {"on_error": err, "task_id": "t"}, 1),
            (11, {"task_name": "nope.nope", "on_error": err, "task_id": "t"}, 1),
            (
                12,
                {"task_name": "test_pg_consumer.boom", "on_error": err, "task_id": "t"},
                99,
            ),
        ]
        for msg_id, payload, read_ct in cases:
            client = MagicMock()
            client.read.return_value = [_msg(msg_id, payload, read_ct=read_ct)]
            PgQueueConsumer(["q"], client=client).poll_once()
            client.delete.assert_called_once_with(msg_id)  # dropped/acked
            client.send.assert_called_once()  # on_error STILL chained
            payload_arg = client.send.call_args.args[1]
            assert payload_arg["task_name"] == "cb.err"
            assert payload_arg["args"] == ["t"]
            assert payload_arg["kwargs"]["callback_kwargs"]["error"]  # the drop reason

    def test_non_json_safe_result_is_coerced_before_chaining(self):
        client = MagicMock()
        payload = {
            "task_name": "test_pg_consumer.dt",
            "on_success": self._spec(),
            "task_id": "tid",
        }
        client.read.return_value = [_msg(6, payload)]
        PgQueueConsumer(["q"], client=client).poll_once()
        client.send.assert_called_once()  # not swallowed by a json.dumps TypeError
        chained = client.send.call_args.args[1]["args"][0]
        assert isinstance(chained["when"], str)  # datetime → str (default=str)
        import json as _json

        _json.dumps(chained)  # the coerced payload is plain-json serialisable

    def test_on_success_only_failure_acks_not_redelivered(self):
        # An on_success-only callback dispatch (on_error omitted) whose executor
        # RAISES must ACK — not fall through to vt-redelivery and re-run the
        # executor (LLM double-spend). No on_error → nothing chained, but acked.
        client = MagicMock()
        payload = {
            "task_name": "test_pg_consumer.boom",
            "on_success": self._spec(),
            "task_id": "tid",
        }
        client.read.return_value = [_msg(7, payload)]
        PgQueueConsumer(["q"], client=client).poll_once()
        client.delete.assert_called_once_with(7)  # acked, NOT left for redelivery
        client.send.assert_not_called()  # success callback not fired on a raise

    def test_on_success_enqueue_failure_falls_back_to_on_error(self):
        # If the success hand-off can't be enqueued, surface on_error so the caller
        # still gets a terminal event instead of hanging after the LLM spend.
        client = MagicMock()
        client.send.side_effect = [RuntimeError("queue down"), 999]  # 1st fails, 2nd ok
        payload = {
            **_ok_payload(2, 2),
            "on_success": self._spec("cb.ok"),
            "on_error": self._spec("cb.err"),
            "task_id": "tid",
        }
        client.read.return_value = [_msg(8, payload)]
        PgQueueConsumer(["q"], client=client).poll_once()
        assert client.send.call_count == 2  # on_success attempt, then on_error fallback
        fallback = client.send.call_args_list[1].args[1]
        assert fallback["task_name"] == "cb.err"
        assert "delivery failed" in fallback["kwargs"]["callback_kwargs"]["error"]
        client.delete.assert_called_once_with(8)  # acked

    def test_no_continuation_is_plain_fire_and_forget(self):
        client = MagicMock()
        client.read.return_value = [_msg(3, _ok_payload(1, 1))]
        PgQueueConsumer(["q"], client=client).poll_once()
        client.send.assert_not_called()  # nothing self-chained
        client.delete.assert_called_once_with(3)  # acked

    def test_chain_failure_on_success_still_acks(self, caplog):
        client = MagicMock()
        client.send.side_effect = RuntimeError("queue down")
        payload = {**_ok_payload(2, 2), "on_success": self._spec()}
        client.read.return_value = [_msg(4, payload)]
        with caplog.at_level(logging.ERROR, logger="queue_backend.pg_queue.consumer"):
            PgQueueConsumer(["q"], client=client).poll_once()
        client.delete.assert_called_once_with(4)  # acked despite the chain failure
        assert "FAILED to self-chain" in caplog.text

    def test_continuation_org_extracted_from_context(self):
        # The chained callback inherits the executor request's org (context dict);
        # non-dict / absent args degrade to "" (callbacks aren't fairness-critical).
        org_of = PgQueueConsumer._continuation_org
        assert org_of({"args": [{"organization_id": "orgZ"}]}) == "orgZ"
        assert org_of({"args": [42]}) == ""
        assert org_of({}) == ""

    def test_reply_key_and_callback_are_mutually_exclusive(self):
        # The consumer checks reply_key first and would silently drop a callback —
        # so to_payload rejects the ambiguous combination at the build boundary.
        spec = self._spec()
        with pytest.raises(ValueError, match="mutually exclusive"):
            to_payload("execute_extraction", reply_key="rk", on_success=spec)
        with pytest.raises(ValueError, match="mutually exclusive"):
            to_payload("execute_extraction", reply_key="rk", on_error=spec)
        # Either alone is fine.
        assert to_payload("execute_extraction", reply_key="rk")["reply_key"] == "rk"
        assert to_payload("execute_extraction", on_success=spec)["on_success"] == spec


if __name__ == "__main__":
    pytest.main([__file__, "-v"])


class TestRecordTaskStatus:
    """UN-3693: a dispatch_with_callback task's terminal status is recorded in
    pg_task_result so the REST PromptStudio.task_status poll resolves under PG.
    completed unless the run raised (error) or the executor reported success=False
    (completed rows are status-only; failed rows carry the executor error text). TTL'd
    + best-effort — never wedges the ack.
    """

    _RB = "queue_backend.pg_queue.consumer.PgResultBackend"
    _RET = 86400

    @staticmethod
    def _consumer(client=None):
        return PgQueueConsumer(["q"], client=client or MagicMock())

    @staticmethod
    def _entered_rb(patched):  # object bound by `with PgResultBackend() as rb:`
        return patched.return_value.__enter__.return_value

    def test_success_records_completed(self):
        with patch(self._RB) as RB:
            self._consumer()._record_task_status(
                {"task_id": "tid"}, error=None, executor_result={"success": True, "data": {}}
            )
        self._entered_rb(RB).store_result.assert_called_once_with(
            "tid", result={}, retention_seconds=self._RET
        )

    def test_executor_reported_failure_records_failed(self):
        with patch(self._RB) as RB:
            self._consumer()._record_task_status(
                {"task_id": "tid"}, error=None,
                executor_result={"success": False, "error": "boom"},
            )
        self._entered_rb(RB).store_result.assert_called_once_with(
            "tid", error="boom", retention_seconds=self._RET
        )

    def test_run_raised_records_failed(self):
        with patch(self._RB) as RB:
            self._consumer()._record_task_status(
                {"task_id": "tid"}, error="RuntimeError: kaboom", executor_result="tid"
            )
        self._entered_rb(RB).store_result.assert_called_once_with(
            "tid", error="RuntimeError: kaboom", retention_seconds=self._RET
        )

    def test_no_task_id_is_noop(self):
        with patch(self._RB) as RB:
            self._consumer()._record_task_status(
                {}, error=None, executor_result={"success": True}
            )
        RB.assert_not_called()

    def test_backend_error_is_swallowed(self):
        with patch(self._RB) as RB:
            RB.side_effect = RuntimeError("pg down")
            self._consumer()._record_task_status(
                {"task_id": "tid"}, error=None, executor_result={"success": True}
            )  # must not raise

    def test_chain_continuation_records_status_even_if_enqueue_fails(self):
        # Single-site wiring: status is recorded before the enqueue, so a REST poll
        # resolves even when the callback continuation is lost.
        client = MagicMock()
        client.send.side_effect = RuntimeError("enqueue down")
        with patch(self._RB) as RB:
            ok = self._consumer(client)._chain_continuation(
                {"task_name": "cb.done", "queue": "ide_callback", "kwargs": {}},
                prepend={"success": True},
                payload={"task_id": "tid", "args": [{}]},
            )
        assert ok is False
        self._entered_rb(RB).store_result.assert_called_once_with(
            "tid", result={}, retention_seconds=self._RET
        )


class TestLeaseRenewal:
    """UN-3695: the claim is taken for the short lease and renewed while the task
    runs, so a dead worker's claim expires in ~lease (fast redelivery) instead of the
    full VT. VT_SECONDS stays the drain/max-runtime bound.
    """

    # --- construction / config ---

    def test_lease_clamped_to_vt_loudly(self, caplog):
        with caplog.at_level(logging.WARNING, logger="queue_backend.pg_queue.consumer"):
            c = PgQueueConsumer(
                ["q"], client=MagicMock(), vt_seconds=100, lease_seconds=200
            )
        assert c.lease_seconds == 100  # clamped — the lease can't outlast VT
        assert "clamped the claim window" in caplog.text  # loud, not silent

    def test_lease_and_renew_interval(self):
        c = PgQueueConsumer(["q"], client=MagicMock(), vt_seconds=9060, lease_seconds=120)
        assert c.lease_seconds == 120
        assert c._lease_renew_interval == 40  # lease // 3 → 2 attempts of slack

    def test_tiny_lease_renew_interval_floored_at_1(self):
        c = PgQueueConsumer(["q"], client=MagicMock(), vt_seconds=2, lease_seconds=2)
        assert c._lease_renew_interval == 1  # max(1, 2 // 3)

    def test_non_positive_lease_rejected(self):
        client = MagicMock()
        with pytest.raises(ValueError, match="lease_seconds must be positive"):
            PgQueueConsumer(["q"], client=client, lease_seconds=0)

    def test_batch_forced_to_1_when_lease_shorter_than_vt(self, caplog):
        with caplog.at_level(logging.WARNING, logger="queue_backend.pg_queue.consumer"):
            c = PgQueueConsumer(
                ["q"], client=MagicMock(), vt_seconds=9060, lease_seconds=120, batch_size=8
            )
        assert c.batch_size == 1  # renewal only covers the in-flight message
        assert "forced to 1" in caplog.text

    def test_batch_kept_when_lease_equals_vt(self):
        # lease == vt → no short renewal window → batch>1 is safe, left untouched.
        c = PgQueueConsumer(
            ["q"], client=MagicMock(), vt_seconds=50, lease_seconds=50, batch_size=4
        )
        assert c.lease_seconds == 50 and c.batch_size == 4

    def test_env_wires_lease_seconds(self, monkeypatch):
        from queue_backend.pg_queue import consumer as mod

        monkeypatch.setenv("WORKER_PG_QUEUE_CONSUMER_LEASE_SECONDS", "77")
        monkeypatch.setenv("WORKER_PG_QUEUE_CONSUMER_VT_SECONDS", "9060")
        with patch.object(mod, "PgQueueClient"):  # no real DB connection
            c = mod.build_consumer_from_env()
        assert c.lease_seconds == 77

    def test_env_wires_max_attempts(self, monkeypatch):
        # Per-worker override: an execution worker keeps the default (at-least-once),
        # the interactive orchestrator sets 1 (at-most-once, no LLM re-compute).
        from queue_backend.pg_queue import consumer as mod

        monkeypatch.setenv("WORKER_PG_QUEUE_CONSUMER_MAX_ATTEMPTS", "1")
        with patch.object(mod, "PgQueueClient"):  # no real DB connection
            c = mod.build_consumer_from_env()
        assert c.max_attempts == 1

    def test_max_attempts_defaults_to_5_when_unset(self, monkeypatch):
        from queue_backend.pg_queue import consumer as mod
        from queue_backend.pg_queue.consumer import _DEFAULT_MAX_ATTEMPTS

        monkeypatch.delenv("WORKER_PG_QUEUE_CONSUMER_MAX_ATTEMPTS", raising=False)
        with patch.object(mod, "PgQueueClient"):  # no real DB connection
            c = mod.build_consumer_from_env()
        # The code default stays 5 so execution workers remain at-least-once.
        assert c.max_attempts == _DEFAULT_MAX_ATTEMPTS == 5

    def test_env_invalid_max_attempts_raises_named_error(self, monkeypatch):
        from queue_backend.pg_queue import consumer as mod

        monkeypatch.setenv("WORKER_PG_QUEUE_CONSUMER_MAX_ATTEMPTS", "abc")
        with patch.object(mod, "PgQueueClient"):
            with pytest.raises(ValueError, match="WORKER_PG_QUEUE_CONSUMER_MAX_ATTEMPTS"):
                mod.build_consumer_from_env()

    def test_poll_claims_with_lease_not_vt(self):
        client = MagicMock()
        client.read.return_value = []
        c = PgQueueConsumer(["q"], client=client, vt_seconds=9060, lease_seconds=120)
        c.poll_once()
        client.read.assert_called_with("q", vt_seconds=120, qty=c.batch_size)

    # --- the renewal loop (own connection via the _make_renew_client factory) ---

    def _renewing(self, renew_client, lease_seconds=3):
        c = PgQueueConsumer(["q"], client=MagicMock(), lease_seconds=lease_seconds)
        c._make_renew_client = MagicMock(return_value=renew_client)
        return c

    def test_renews_until_stopped_then_closes_own_client(self):
        rc = MagicMock()
        rc.set_vt.return_value = True
        c = self._renewing(rc)
        stop = MagicMock()
        stop.wait.side_effect = [False, False, True]  # renew twice, then stop
        c._renew_lease_loop(99, stop)
        assert rc.set_vt.call_count == 2
        rc.set_vt.assert_called_with(99, 3)  # renews with the lease
        stop.wait.assert_called_with(c._lease_renew_interval)
        rc.close.assert_called_once()  # own connection released on exit

    def test_no_client_built_for_sub_interval_task(self):
        c = self._renewing(MagicMock())
        stop = MagicMock()
        stop.wait.side_effect = [True]  # stopped before the first renewal tick
        c._renew_lease_loop(99, stop)
        c._make_renew_client.assert_not_called()  # short task opens no 2nd connection

    def test_row_gone_logs_double_run_and_stops(self, caplog):
        rc = MagicMock()
        rc.set_vt.return_value = False  # reclaimed + acked elsewhere (we ack later)
        c = self._renewing(rc)
        stop = MagicMock()
        stop.wait.side_effect = [False, False, True]
        with caplog.at_level(logging.WARNING, logger="queue_backend.pg_queue.consumer"):
            c._renew_lease_loop(99, stop)
        assert rc.set_vt.call_count == 1  # stopped after the first gone
        assert "may double-run" in caplog.text  # the lost-lease signal is logged
        rc.close.assert_called_once()

    def test_conn_death_retried_then_escalates_past_lease(self, caplog):
        rc = MagicMock()
        rc.set_vt.side_effect = CONN_DEAD_ERRORS[0]("db down")
        c = self._renewing(rc, lease_seconds=3)
        stop = MagicMock()
        stop.wait.side_effect = [False, False, True]
        # last_ok=0; 1st failure at t=0 (< lease → WARNING); 2nd at t=10 (≥ lease → ERROR)
        with patch(
            "queue_backend.pg_queue.consumer.time.monotonic",
            side_effect=[0.0, 0.0, 10.0],
        ):
            with caplog.at_level(
                logging.WARNING, logger="queue_backend.pg_queue.consumer"
            ):
                c._renew_lease_loop(99, stop)  # must NOT raise
        assert rc.set_vt.call_count == 2
        assert "self-heals" in caplog.text  # within-slack retry
        assert "may double-run" in caplog.text  # escalated once past the lease

    def test_non_connection_error_propagates(self):
        # A programming bug is NOT swallowed as a "self-healing blip" forever.
        rc = MagicMock()
        rc.set_vt.side_effect = AttributeError("bug")
        c = self._renewing(rc)
        stop = MagicMock()
        stop.wait.side_effect = [False, False, True]
        with pytest.raises(AttributeError):
            c._renew_lease_loop(99, stop)
        rc.close.assert_called_once()  # still released via finally

    # --- the context manager + _handle wiring ---

    def test_ctx_runs_loop_and_sets_stop_on_exit(self):
        c = PgQueueConsumer(["q"], client=MagicMock(), lease_seconds=3)
        entered = False
        with patch.object(c, "_renew_lease_loop") as loop:
            with c._lease_renewal(42):
                entered = True  # CM body runs while the renewal thread is live
        assert entered
        loop.assert_called_once()
        msg_id, stop = loop.call_args.args
        assert msg_id == 42
        assert isinstance(stop, threading.Event) and stop.is_set()  # stopped on exit

    def test_ctx_logs_when_thread_wedged_past_join_timeout(self, caplog):
        c = PgQueueConsumer(["q"], client=MagicMock(), lease_seconds=3)
        entered = False
        # A loop that ignores stop → join times out → the thread is still alive.
        with patch.object(c, "_renew_lease_loop", lambda mid, s: time.sleep(0.3)):
            with patch(
                "queue_backend.pg_queue.consumer._LEASE_JOIN_TIMEOUT_SECONDS", 0.01
            ):
                with caplog.at_level(
                    logging.ERROR, logger="queue_backend.pg_queue.consumer"
                ):
                    with c._lease_renewal(7):
                        entered = True  # CM body runs even though the thread wedges
        assert entered
        assert "did not stop within" in caplog.text  # wedged thread is not silent

    def test_handle_wraps_task_run_in_lease_renewal(self):
        # Integration: a real run goes through _lease_renewal (dropping the `with`
        # in _handle would keep the unit tests green — this guards that regression).
        client = MagicMock()
        client.read.return_value = [_msg(1, _ok_payload(3, 4))]
        c = PgQueueConsumer(["q"], client=client)
        with patch.object(c, "_lease_renewal", wraps=c._lease_renewal) as lease:
            c.poll_once()
        lease.assert_called_once_with(1)  # the in-flight msg_id was leased
        assert _calls == [(3, 4)]  # task still ran
        client.delete.assert_called_once_with(1)  # and acked