Skip to content

LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows

11.8 KB
Raw Blame
"""The group-notification worker leg is entirely failure-path logic.

``_post_group_notification`` is the only thing standing between a transient
backend blip and a permanently unsent email, and every branch in it encodes a
different judgement about whether re-posting is safe. Two of those judgements
are easy to get backwards:

* A response the backend never sent (connect refused, DNS gone) is safe to
  re-post -- nothing happened.
* A response lost *after* the request arrived is not. The backend does not stop
  when the client disconnects, and the send path mails group by group with no
  checkpoint, so a re-post re-mails everyone who already received it.

These pin which exception lands on which side, plus the attempt cap, the
sub-500 break, and the fact that the timeout is per-phase rather than scalar --
httpx applies a scalar timeout to connect, write and read separately, so a
scalar one silently triples the task's worst-case wall time and can push it
past the queue's visibility timeout.
"""

from __future__ import annotations

import logging
from unittest.mock import MagicMock, patch

import httpx
import pytest
from notification.tasks import (
    _GROUP_NOTIFICATION_ATTEMPTS,
    _GROUP_NOTIFICATION_RETRY_DELAY,
    _GROUP_NOTIFICATION_TIMEOUT,
    _post_group_notification,
    notify_group_membership_changed,
    notify_resource_shared_with_group,
)

_ENV = {
    "INTERNAL_API_BASE_URL": "http://backend/internal",
    "INTERNAL_SERVICE_API_KEY": "k",
}


def _response(status: int) -> MagicMock:
    return MagicMock(status_code=status, text="body")


class _Client:
    """Stand-in for ``httpx.Client`` that records posts and replays outcomes."""

    def __init__(self, outcomes):
        self.outcomes = list(outcomes)
        self.calls = []

    def __call__(self, *a, **kw):
        return self

    def __enter__(self):
        return self

    def __exit__(self, *exc):
        return False

    def post(self, url, **kwargs):
        self.calls.append((url, kwargs))
        outcome = self.outcomes.pop(0) if self.outcomes else _response(200)
        if isinstance(outcome, Exception):
            raise outcome
        return outcome


def _run(outcomes, endpoint="resource-shared", payload=None):
    """Drive one ``_post_group_notification`` with ``outcomes`` per attempt."""
    client = _Client(outcomes)
    with (
        patch.dict("os.environ", _ENV, clear=False),
        patch("notification.tasks.httpx.Client", client),
        patch("notification.tasks.time.sleep") as sleep,
    ):
        raised = None
        try:
            _post_group_notification(endpoint, "org-a", payload or {"x": 1})
        except Exception as e:  # noqa: BLE001
            raised = e
    return client, raised, sleep


_DROP_PAYLOAD = {"group_ids": [7], "resource_id": "res-1", "share_action": "revoked"}


def _run_and_capture_drop(
    caplog: pytest.LogCaptureFixture, outcomes: list[object]
) -> tuple[_Client, Exception | None, MagicMock]:
    """``_run`` with ERROR logs captured, for the paths that drop the job."""
    with caplog.at_level(logging.ERROR, logger="notification.tasks"):
        client, raised, sleep = _run(outcomes, payload=_DROP_PAYLOAD)
    return client, raised, sleep


def _assert_drop_logged(caplog: pytest.LogCaptureFixture) -> None:
    """A dropped job is acked and deleted, so this log line is its only record."""
    dropped = [
        r.getMessage()
        for r in caplog.records
        if r.levelno == logging.ERROR
        and "metric=group_notification_dropped_total" in r.getMessage()
    ]
    assert len(dropped) == 1, dropped
    assert str(_DROP_PAYLOAD) in dropped[0], "payload missing from the drop log line"


class TestRetryClassification:
    def test_success_posts_once_and_returns(self):
        client, raised, _ = _run([_response(200)])
        assert raised is None
        assert len(client.calls) == 1

    def test_server_error_exhausts_the_attempt_cap_then_raises(self):
        client, raised, sleep = _run([_response(503)] * _GROUP_NOTIFICATION_ATTEMPTS)
        assert isinstance(raised, RuntimeError)
        assert len(client.calls) == _GROUP_NOTIFICATION_ATTEMPTS
        assert sleep.call_count == _GROUP_NOTIFICATION_ATTEMPTS - 1

    def test_client_error_is_dropped_without_raising(self, caplog):
        # A rejected payload will be rejected again; retrying only burns
        # budget, and raising would trigger queue redelivery of a message
        # that can never succeed.
        client, raised, sleep = _run_and_capture_drop(caplog, [_response(400)])
        assert raised is None
        assert len(client.calls) == 1
        assert sleep.call_count == 0
        _assert_drop_logged(caplog)

    @pytest.mark.parametrize(
        "exc",
        [
            httpx.ReadTimeout("read"),
            httpx.WriteTimeout("write"),
            httpx.ReadError("read err"),
            httpx.WriteError("write err"),
            httpx.RemoteProtocolError("server disconnected"),
        ],
    )
    def test_request_sent_outcome_unknown_is_never_re_posted(self, exc, caplog):
        """Each of these means the backend already has the request.

        Re-posting would re-mail every group that already succeeded, so the
        in-process attempts must end after one post -- and the task must not
        raise either, or queue redelivery does the exact re-post this guards
        against.
        """
        client, raised, sleep = _run_and_capture_drop(caplog, [exc])
        assert raised is None
        assert len(client.calls) == 1, f"{type(exc).__name__} was re-posted"
        assert sleep.call_count == 0
        _assert_drop_logged(caplog)

    @pytest.mark.parametrize(
        "exc",
        [
            httpx.ConnectTimeout("connect"),
            httpx.ConnectError("refused"),
            httpx.PoolTimeout("pool"),
        ],
    )
    def test_request_never_left_is_retried(self, exc):
        """Nothing reached the backend, so the full attempt cap is correct."""
        client, raised, _ = _run([exc] * _GROUP_NOTIFICATION_ATTEMPTS)
        assert isinstance(raised, RuntimeError)
        assert len(client.calls) == _GROUP_NOTIFICATION_ATTEMPTS

    def test_recovers_when_a_later_attempt_succeeds(self):
        client, raised, _ = _run([_response(502), _response(200)])
        assert raised is None
        assert len(client.calls) == 2

    def test_retryable_then_terminal_stops_and_drops_not_raises(self, caplog):
        # A transient 503 retries once, then a 400 on that retry is terminal --
        # the loop must stop there (not spend the 3rd attempt) and the final
        # classification (400 -> not retryable) decides drop-without-raising,
        # not the classification of the earlier, already-superseded attempt.
        client, raised, sleep = _run_and_capture_drop(
            caplog, [_response(503), _response(400)]
        )
        assert raised is None
        assert len(client.calls) == 2
        assert sleep.call_count == 1
        _assert_drop_logged(caplog)


class TestRequestShape:
    def test_missing_credentials_raise_before_any_post(self):
        client = _Client([])
        with (
            patch.dict(
                "os.environ",
                {"INTERNAL_API_BASE_URL": "", "INTERNAL_SERVICE_API_KEY": ""},
            ),
            patch("notification.tasks.httpx.Client", client),
        ):
            with pytest.raises(RuntimeError):
                _post_group_notification("resource-shared", "org-a", {})
        assert client.calls == []

    def test_url_auth_and_tenant_header(self):
        client, _, _ = _run([_response(200)], endpoint="membership-changed")
        url, kwargs = client.calls[0]
        assert url == "http://backend/internal/v1/group-notification/membership-changed/"
        assert kwargs["headers"]["Authorization"] == "Bearer k"
        # Without this the backend resolves no tenant and every org-scoped
        # query comes back empty -- a silent no-op rather than an error.
        assert kwargs["headers"]["X-Organization-ID"] == "org-a"

    def test_timeout_is_per_phase_not_scalar(self):
        """A scalar timeout is applied to connect, write AND read separately.

        Passing one would let a single post spend it three times over, which is
        what pushed this task past the consumer's visibility timeout.
        """
        client, _, _ = _run([_response(200)])
        timeout = client.calls[0][1]["timeout"]
        assert isinstance(timeout, httpx.Timeout)
        assert timeout is _GROUP_NOTIFICATION_TIMEOUT
        assert timeout.connect < timeout.read

    def test_timeout_constants_sum_under_the_visibility_and_health_stale_ceilings(self):
        """Consistency check on the constants, not a measurement of a real POST.

        VT is 300s for this worker in both deployments; the heartbeat is frozen
        for the task's duration, so health-stale (360s) is the other ceiling.
        Includes the sleep between retries -- a per-post-only sum undercounts
        the real wall time by (attempts - 1) * retry delay. Nothing here times
        an actual request -- this only proves the constants are still
        consistent with each other, not that a real POST stays under budget.
        """
        t = _GROUP_NOTIFICATION_TIMEOUT
        per_post = t.connect + t.write + t.read + t.pool
        sleeps = (_GROUP_NOTIFICATION_ATTEMPTS - 1) * _GROUP_NOTIFICATION_RETRY_DELAY
        worst = per_post * _GROUP_NOTIFICATION_ATTEMPTS + sleeps
        assert worst < 300, f"worst case {worst}s exceeds the 300s visibility timeout"
        assert worst < 360, f"worst case {worst}s exceeds the 360s health-stale ceiling"


class TestTaskPayloads:
    def test_resource_shared_task_sends_every_field_the_endpoint_requires(self):
        client = _Client([_response(200)])
        with (
            patch.dict("os.environ", _ENV, clear=False),
            patch("notification.tasks.httpx.Client", client),
        ):
            notify_resource_shared_with_group(
                group_ids=[2, 5],
                actor_id=7,
                resource_kind="workflow",
                resource_id="wf-1",
                organization_id="org-a",
                share_action="revoked",
                revoked_at="2026-01-01T00:00:00+00:00",
            )
        url, kwargs = client.calls[0]
        assert url.endswith("/resource-shared/")
        assert kwargs["json"] == {
            "group_ids": [2, 5],
            "actor_id": 7,
            "resource_kind": "workflow",
            "resource_id": "wf-1",
            "share_action": "revoked",
            "revoked_at": "2026-01-01T00:00:00+00:00",
        }

    def test_share_direction_sends_a_null_cutoff_rather_than_omitting_it(self):
        # The endpoint requires the key on both directions; omitting it is a 400.
        client = _Client([_response(200)])
        with (
            patch.dict("os.environ", _ENV, clear=False),
            patch("notification.tasks.httpx.Client", client),
        ):
            notify_resource_shared_with_group(
                group_ids=[2],
                actor_id=7,
                resource_kind="workflow",
                resource_id="wf-1",
                organization_id="org-a",
                share_action="shared",
                revoked_at=None,
            )
        assert client.calls[0][1]["json"]["revoked_at"] is None

    def test_membership_task_payload(self):
        client = _Client([_response(200)])
        with (
            patch.dict("os.environ", _ENV, clear=False),
            patch("notification.tasks.httpx.Client", client),
        ):
            notify_group_membership_changed(
                group_id=3,
                actor_id=7,
                membership_action="removed",
                user_ids=[11, 12],
                organization_id="org-a",
            )
        url, kwargs = client.calls[0]
        assert url.endswith("/membership-changed/")
        assert kwargs["json"] == {
            "group_id": 3,
            "actor_id": 7,
            "membership_action": "removed",
            "user_ids": [11, 12],
        }