Skip to content

LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows

UN-3445 [GATED-FEAT] PG-queue auxiliary-worker migration (OSS): loader fix + notification seam (#2217)

alie2f8a44e3218
diff --git a/backend/notification_v2/internal_api_views.py b/backend/notification_v2/internal_api_views.py
index be2bbe57e..42d0eeac9 100644
--- a/backend/notification_v2/internal_api_views.py
+++ b/backend/notification_v2/internal_api_views.py
@@ -14,6 +14,7 @@ import logging
from datetime import timedelta
from typing import Any, cast
+from account_v2.models import Organization
from api_v2.models import APIDeployment
from django.conf import settings
from django.db import transaction
@@ -37,12 +38,25 @@ from notification_v2.helper import (
webhook_url_hash,
)
from notification_v2.models import Notification, NotificationBuffer
+from notification_v2.notification_dispatch import (
+ PermanentDispatchError,
+ dispatch_webhook_notification,
+)
logger = logging.getLogger(__name__)
# Constants for error messages
INTERNAL_SERVER_ERROR_MSG = "Internal server error"
+# How many early transient-failure cycles get their SENDING claim refunded before the
+# claim starts sticking. Refunding forever makes net attempt progress zero, so
+# ``NOTIFICATION_MAX_DISPATCH_ATTEMPTS`` becomes unreachable and a permanently
+# recurring failure (not just a brief broker blip) re-dispatches every flush tick with
+# no termination condition. Small by design: a genuine blip clears well inside this,
+# and anything still failing after it is treated as repetitive and allowed to age
+# toward the cap.
+NOTIFICATION_TRANSIENT_REFUND_LIMIT = 3
+
def _load_execution(execution_id: str | None) -> WorkflowExecution | None:
"""Best-effort lookup; returns None on missing id or unknown row."""
@@ -513,6 +527,38 @@ def _reclaim_stale_sending() -> int:
return int(reclaimed)
+def _org_identifier(org_pk: int) -> str | None:
+ """Resolve the string ``Organization.organization_id`` from the buffer's org pk.
+
+ ``resolve_transport`` keys its Flipt decision on the org's string identifier,
+ but the buffer stores/uses the Organization pk. One indexed pk lookup per
+ dispatch group (post-commit) — negligible relative to the downstream webhook
+ dispatch.
+
+ Data-anomaly guard: ``NotificationBuffer.organization`` is
+ ``on_delete=CASCADE``, so a live buffer row with a missing org is unreachable
+ in normal operation. A ``None`` here therefore signals a dangling FK — we log
+ it (the only org-traceable breadcrumb; resolve_transport's own warning is keyed
+ on the random dispatch uuid) and fail closed to Celery in resolve_transport.
+ """
+ org_string_id = (
+ Organization.objects.filter(pk=org_pk)
+ .values_list("organization_id", flat=True)
+ .first()
+ )
+ if org_string_id is None:
+ # Sentry-routed (logger.error): a live buffer row with no org is a data
+ # anomaly (dangling FK / corruption) that shouldn't happen under the
+ # CASCADE constraint, not routine noise. Routing still fails closed to
+ # Celery in resolve_transport.
+ logger.error(
+ "metric=notification_org_identifier_missing_total org_pk=%s "
+ "(dangling FK; notification routing falls back to Celery)",
+ org_pk,
+ )
+ return org_string_id
+
+
def _send_clubbed(
*,
url: str,
@@ -521,7 +567,7 @@ def _send_clubbed(
platform: str,
max_retries: int,
buffer_ids: list[str],
- org_id: Any,
+ org_id: int,
) -> None:
"""Send the clubbed Celery task after the DB transition has committed.
@@ -540,8 +586,12 @@ def _send_clubbed(
``buffer_row_ids`` + ``organization_id`` to the worker so it can mark them.
"""
try:
- celery_app.send_task(
- "send_webhook_notification",
+ # Flag-gated transport (UN-3753): PG queue when pg_queue_enabled for this
+ # org, else Celery (byte-identical to the prior send_task). resolve_transport
+ # keys on the org STRING id, but the buffer/worker contract below uses the
+ # org pk — hence _org_identifier(org_id) for routing, org_id in kwargs.
+ dispatched = dispatch_webhook_notification(
+ celery_app=celery_app,
args=[url, body, headers, settings.NOTIFICATION_TIMEOUT],
kwargs={
"max_retries": max_retries,
@@ -557,16 +607,61 @@ def _send_clubbed(
"organization_id": org_id,
},
queue="notifications",
+ org_string_id=_org_identifier(org_id),
)
+ # transport= makes the rollout answerable from the logs: during a percentage
+ # ramp the question is "are PG-routed notifications succeeding at the same
+ # rate as Celery-routed ones?", which result=success alone cannot answer.
logger.info(
"metric=notification_batch_dispatched_total platform=%s result=success "
- "org_id=%s webhook_url_hash=%s rows=%d",
+ "transport=%s org_id=%s webhook_url_hash=%s rows=%d task_id=%s",
platform,
+ dispatched.transport,
org_id,
webhook_url_hash(url),
len(buffer_ids),
+ dispatched.task_id,
)
+ except PermanentDispatchError:
+ # PG-ONLY permanent failure. From this path the only reachable cause is
+ # payload JSON-serialization (e.g. a NaN/Infinity float that jsonb rejects
+ # at insert): this call passes no reply_key/callback and a default in-range
+ # priority, so enqueue_task's other permanent checks can't fire here (the
+ # full set lives on PermanentDispatchError's docstring). It fails
+ # identically every flush tick — dead-letter now (distinct metric) instead
+ # of reverting to PENDING and re-rendering + re-dispatching + emitting a
+ # broker_failure traceback until the attempt cap. The Celery path never
+ # raises this, so the flag-off flow is unchanged: a Celery send_task failure
+ # is an ordinary Exception handled by the transient branch below, exactly as
+ # before UN-3753. Guard on SENDING so a row the worker already resolved
+ # isn't clobbered.
+ logger.exception(
+ "metric=notification_batch_dispatched_total platform=%s "
+ "result=dispatch_error org_id=%s webhook_url_hash=%s rows=%d",
+ platform,
+ org_id,
+ webhook_url_hash(url),
+ len(buffer_ids),
+ )
+ NotificationBuffer.objects.filter(
+ id__in=buffer_ids,
+ status=BufferStatus.SENDING.value,
+ ).update(status=BufferStatus.DEAD_LETTER.value)
except Exception:
+ # TRANSIENT transport/broker failure — revert to PENDING (outside the
+ # committed txn) so the next flush tick retries. Guard on SENDING so a row
+ # the worker already marked terminal (broker raised post-delivery) isn't
+ # resurrected into a duplicate.
+ #
+ # The refund is BOUNDED, not unconditional: refunding the SENDING claim on
+ # every cycle makes net progress zero, so `NOTIFICATION_MAX_DISPATCH_ATTEMPTS`
+ # can never be reached and a permanently-recurring failure re-renders and
+ # re-dispatches forever (emitting a traceback each tick). Refunding is still
+ # correct for a genuinely transient blip — nothing was queued or sent — so we
+ # keep it while attempts are low and let the cap take over once a failure has
+ # proven itself repetitive. This matters more now that this branch also
+ # catches non-permanent PG enqueue failures (e.g. a DataError that will recur
+ # identically), not just "RabbitMQ is briefly down".
logger.exception(
"metric=notification_batch_dispatched_total platform=%s "
"result=broker_failure org_id=%s webhook_url_hash=%s rows=%d",
@@ -575,21 +670,24 @@ def _send_clubbed(
webhook_url_hash(url),
len(buffer_ids),
)
- # Revert to PENDING (outside the committed txn) so a transient broker
- # outage retries next tick; refund the SENDING-claim attempt since nothing
- # was queued or sent. Guard on SENDING so a row the worker already marked
- # terminal (broker raised post-delivery) isn't resurrected into a duplicate.
- NotificationBuffer.objects.filter(
+ reverted = NotificationBuffer.objects.filter(
id__in=buffer_ids,
status=BufferStatus.SENDING.value,
- ).update(
+ )
+ # Refund only the early attempts; past the limit the claim stands so the
+ # counter grows and the cap can eventually dead-letter the group.
+ reverted.filter(
+ dispatch_attempts__lte=NOTIFICATION_TRANSIENT_REFUND_LIMIT
+ ).update(dispatch_attempts=F("dispatch_attempts") - 1)
+ # Every claimed row goes back to PENDING for the next flush tick, refunded
+ # or not.
+ reverted.update(
status=BufferStatus.PENDING.value,
dispatched_at=None,
- dispatch_attempts=F("dispatch_attempts") - 1,
)
-def _penalize_render_failure(buffer_ids: list[str], org_id: Any, platform: str) -> None:
+def _penalize_render_failure(buffer_ids: list[str], org_id: int, platform: str) -> None:
"""Charge a dispatch attempt to a group whose payloads failed to render.
The SENDING-claim increment never runs on a render failure, so count it here
@@ -609,7 +707,7 @@ def _penalize_render_failure(buffer_ids: list[str], org_id: Any, platform: str)
def _dispatch_group(
- org_id: Any,
+ org_id: int,
webhook_url: str,
auth_sig: str,
platform: str,
diff --git a/backend/notification_v2/notification_dispatch.py b/backend/notification_v2/notification_dispatch.py
new file mode 100644
index 000000000..4b027182e
--- /dev/null
+++ b/backend/notification_v2/notification_dispatch.py
@@ -0,0 +1,180 @@
+"""Transport-routed dispatch for buffered webhook notifications (UN-3753).
+
+Routes the ``send_webhook_notification`` task through the same
+:func:`resolve_transport` flag as the execution path: the PG queue when
+``pg_queue_enabled`` for this org, else Celery. **Fail-closed** — with the gate
+off (the production default) it resolves to Celery, behaving exactly like the
+prior unconditional ``celery_app.send_task`` (zero regression).
+
+DEPLOYMENT PREREQUISITE (PG path): on PG the task lands on the ``notifications``
+queue and requires a pg-queue consumer configured with
+``WORKER_PG_QUEUE_CONSUMER_WORKER_TYPE=notification`` /
+``WORKER_PG_QUEUE_CONSUMER_QUEUE=notifications``. **No such service exists in this
+repo's compose yet** — none of the ``pg-queue-consumer`` services in
+``docker/docker-compose.yaml`` polls ``notifications``, and ``run-worker.sh``'s
+``PG_CONSUMER_ROLES`` has no notification role. Until one is deployed the flag must
+stay off for the org: enqueued rows would sit undrained (no TTL sweep covers
+``pg_queue_message``) and every buffered webhook for that org would be lost.
+
+``args`` and ``queue`` are forwarded verbatim on both paths. ``kwargs`` differ in
+exactly two keys, forced on the PG branch only (see the inline note at the branch):
+``raise_on_final_failure`` → ``False`` and ``max_retries`` → ``0``. On PG the
+payload is additionally JSON-normalized by ``enqueue_task`` (UUIDs/datetimes → str).
+"""
+
+from __future__ import annotations
+
+import logging
+import uuid
+from typing import Any, NamedTuple
+
+from pg_queue.producer import enqueue_task
+from workflow_manager.workflow_v2.transport import resolve_transport
+
+from unstract.core.data_models import is_pg_transport
+
+logger = logging.getLogger(__name__)
+
+# The fired task name — mirrors the Celery task registered by the notification
+# worker; kept as a local constant so the backend doesn't import the workers pkg.
+WEBHOOK_NOTIFICATION_TASK = "send_webhook_notification"
+
+
+# Transport labels for the dispatch metric. During a percentage ramp the one thing
+# you need from the logs is "are PG-routed notifications succeeding at the same rate
+# as Celery-routed ones?" — a bare task id can't answer that, since a minted PG id
+# and a Celery AsyncResult id are indistinguishable to the caller.
+PG_TRANSPORT = "pg_queue"
+CELERY_TRANSPORT = "celery"
+
+
+class DispatchResult(NamedTuple):
+ """Which transport actually took the dispatch, plus the resulting task id."""
+
+ transport: str
+ task_id: str
+
+
+class PermanentDispatchError(Exception):
+ """A dispatch failure that would fail identically on every retry.
+
+ Raised ONLY on the PG path, when ``enqueue_task`` rejects the message for a
+ permanent reason (priority range / reply_key+callback exclusivity validation,
+ or a payload that can't be JSON-serialized). The Celery path never raises it,
+ so a caller can dead-letter on this exception without altering the flag-off
+ (Celery) error flow — a Celery ``send_task`` failure stays an ordinary
+ ``Exception`` the caller's transient handler owns, exactly as before.
+ """
+
+
+def dispatch_webhook_notification(
+ *,
+ celery_app: Any,
+ args: list[Any],
+ kwargs: dict[str, Any],
+ queue: str,
+ org_string_id: str | None,
+) -> DispatchResult:
+ """Dispatch ``send_webhook_notification`` on the resolved transport.
+
+ ``args``/``queue`` are forwarded unchanged on both paths, and on the Celery
+ branch ``kwargs`` too — so the flag-off path is byte-identical to the legacy
+ ``send_task`` call. The PG branch overrides two retry-semantics kwargs (see
+ below); nothing else differs.
+
+ Args:
+ celery_app: Injected Celery app (the backend's ``celery_service.app``);
+ passed in rather than imported so this seam stays trivially testable.
+ args: Positional task args, forwarded verbatim.
+ kwargs: Keyword task args. Forwarded as-is on Celery; on PG,
+ ``raise_on_final_failure`` is overridden to ``False`` and
+ ``max_retries`` to ``0`` (the in-task retry loop is a no-op under the
+ consumer's eager ``apply()`` — see the inline note). For the buffered
+ path this
+ carries ``organization_id`` = the buffer's org **pk** (the worker's
+ buffer-mark contract) — deliberately a DIFFERENT identifier from the
+ ``org_string_id`` param below (the two must not be conflated).
+ queue: Target queue name, forwarded verbatim.
+ org_string_id: The org's **string** identifier
+ (``Organization.organization_id``), used solely for the Flipt
+ transport decision — NOT the org pk carried in ``kwargs``. ``None`` (or
+ empty) fails closed to Celery.
+
+ Returns:
+ A :class:`DispatchResult` carrying which transport took the dispatch and
+ the resulting task id (the Celery ``AsyncResult`` id, or the minted PG task
+ id). The transport is what makes the caller's dispatch metric answerable
+ during a ramp — the two id flavours are otherwise indistinguishable.
+ """
+ # A buffered notification is a single fire-and-forget task with no natural
+ # sticky entity, so mint a fresh id to drive Flipt's percentage bucketing and
+ # to serve as the PG task id.
+ dispatch_id = str(uuid.uuid4())
+ # resolve_transport already normalizes falsy input to Celery, so pass the id
+ # straight through (no `or None` needed).
+ transport = resolve_transport(
+ execution_id=dispatch_id,
+ organization_id=org_string_id,
+ )
+ # Use the shared is_pg_transport() — the single source for "what counts as PG
+ # transport" — rather than opening a second comparison site.
+ if is_pg_transport(transport):
+ # Two PG-only kwarg overrides, both about the SAME thing: the consumer runs
+ # the task eagerly via ``task.apply(..., throw=True)``, where Celery's
+ # in-task retry loop does not work and its terminal branch is what we need.
+ #
+ # 1. ``max_retries`` → 0. Under ``apply()`` ``self.request.retries`` is ALWAYS
+ # 0, so the worker's ``if self.request.retries < max_retries`` guard
+ # (workers/notification/tasks.py) is true on the FIRST failure for any
+ # max_retries >= 1 and calls ``self.retry(...)``, which raises ``Retry``;
+ # with ``throw=True`` that propagates straight out of ``apply()`` and the
+ # terminal branch is never reached — so the buffers are never marked
+ # DEAD_LETTER and ``raise_on_final_failure`` is never even read. The
+ # consumer's ``except Exception`` then leaves the row for vt-expiry
+ # redelivery, re-POSTing the subscriber every time. Forcing 0 sends the
+ # task down the terminal branch on the first failure instead.
+ # 2. ``raise_on_final_failure`` → False. On BOTH transports the worker marks
+ # the buffers DEAD_LETTER over the internal API *before* it re-raises, so
+ # the re-raise is only a FAILURE-state signal for Celery monitoring (no
+ # redelivery there). On the PG consumer that same raise is treated as a
+ # failure and leaves the row for redelivery, so it must not raise.
+ #
+ # Together: one POST, buffers dead-lettered, task returns None → the consumer
+ # acks (deletes) the row. Retry spacing belongs to the PG layer, not the task.
+ # (The Celery branch below keeps kwargs verbatim — byte-identical.)
+ pg_kwargs = {**kwargs, "raise_on_final_failure": False, "max_retries": 0}
+ try:
+ msg_id = enqueue_task(
+ task_name=WEBHOOK_NOTIFICATION_TASK,
+ queue=queue,
+ args=args,
+ kwargs=pg_kwargs,
+ # enqueue_task types org_id as str; None→"" here satisfies that
+ # type only, not runtime — enqueue_task itself re-coerces
+ # ``org_id or ""`` at insert (producer.py), so this has no runtime
+ # effect.
+ org_id=org_string_id or "",
+ task_id=dispatch_id,
+ )
+ except (ValueError, TypeError) as exc:
+ # PG-only permanent failure (enqueue_task validation / JSON encode):
+ # re-raise as PermanentDispatchError so the caller dead-letters it.
+ # A transient PG error (DB down) is NOT wrapped — it propagates as an
+ # ordinary Exception into the caller's retry (revert-to-PENDING) path.
+ raise PermanentDispatchError(str(exc)) from exc
+ # msg_id correlates this line with the producer's own ``msg_id=`` log, so a
+ # dropped notification can be traced across the seam/producer boundary.
+ logger.info(
+ "Webhook notification enqueued on PG '%s' queue (task_id=%s msg_id=%s)",
+ queue,
+ dispatch_id,
+ msg_id,
+ )
+ return DispatchResult(transport=PG_TRANSPORT, task_id=dispatch_id)
+ result = celery_app.send_task(
+ WEBHOOK_NOTIFICATION_TASK,
+ args=args,
+ kwargs=kwargs,
+ queue=queue,
+ )
+ return DispatchResult(transport=CELERY_TRANSPORT, task_id=result.id)
diff --git a/backend/notification_v2/tests/__init__.py b/backend/notification_v2/tests/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/backend/notification_v2/tests/test_notification_dispatch.py b/backend/notification_v2/tests/test_notification_dispatch.py
new file mode 100644
index 000000000..e08bf55ff
--- /dev/null
+++ b/backend/notification_v2/tests/test_notification_dispatch.py
@@ -0,0 +1,192 @@
+"""Unit tests for the buffered-webhook transport routing (UN-3753).
+
+``resolve_transport`` + ``enqueue_task`` are patched on the module, so no Flipt /
+DB is needed — these pin the routing contract: PG when the flag resolves PG,
+Celery otherwise (fail-closed), with identical args/queue on both paths. The
+kwargs are identical EXCEPT two retry-semantics keys forced on the PG branch
+(``raise_on_final_failure`` -> False, ``max_retries`` -> 0), because the consumer
+runs tasks eagerly via ``apply()`` where the in-task retry loop cannot work — see
+``test_pg_forces_terminal_branch_kwargs``.
+"""
+
+from __future__ import annotations
+
+from unittest.mock import MagicMock, patch
+
+import pytest
+
+import notification_v2.notification_dispatch as nd
+
+_ARGS = ["https://hook.test", {"text": "hi"}, {"Content-Type": "application/json"}, 30]
+_KWARGS = {
+ "max_retries": 3,
+ "retry_delay": 10,
+ "platform": "SLACK",
+ "raise_on_final_failure": True,
+ "buffer_row_ids": ["b1", "b2"],
+ "organization_id": 7, # the org pk (worker's buffer-mark contract)
+}
+_QUEUE = "notifications"
+
+
+def _dispatch(celery_app, org_string_id="org_x"):
+ return nd.dispatch_webhook_notification(
+ celery_app=celery_app,
+ args=_ARGS,
+ kwargs=_KWARGS,
+ queue=_QUEUE,
+ org_string_id=org_string_id,
+ )
+
+
+class TestDispatchWebhookNotification:
+ def test_routes_to_pg_when_flag_resolves_pg(self):
+ celery = MagicMock()
+ with (
+ patch.object(nd, "resolve_transport", return_value="pg_queue"),
+ patch.object(nd, "enqueue_task", return_value=42) as enqueue,
+ ):
+ result = _dispatch(celery)
+ task_id = result.task_id
+ assert result.transport == nd.PG_TRANSPORT # rollout metric can tell them apart
+ enqueue.assert_called_once()
+ kwargs = enqueue.call_args.kwargs
+ assert kwargs["task_name"] == "send_webhook_notification"
+ assert kwargs["queue"] == _QUEUE
+ assert kwargs["args"] == _ARGS
+ # Every kwarg is forwarded verbatim EXCEPT the two retry-semantics keys the
+ # PG branch forces (see test_pg_forces_terminal_branch_kwargs).
+ assert kwargs["kwargs"] == {
+ **_KWARGS,
+ "raise_on_final_failure": False,
+ "max_retries": 0,
+ }
+ assert kwargs["org_id"] == "org_x"
+ # The minted PG task id is returned and threaded into the enqueue row.
+ assert kwargs["task_id"] == task_id
+ celery.send_task.assert_not_called()
+
+ def test_routes_to_celery_when_flag_resolves_celery(self):
+ celery = MagicMock()
+ with (
+ patch.object(nd, "resolve_transport", return_value="celery"),
+ patch.object(nd, "enqueue_task") as enqueue,
+ ):
+ result = _dispatch(celery)
+ celery.send_task.assert_called_once_with(
+ "send_webhook_notification", args=_ARGS, kwargs=_KWARGS, queue=_QUEUE
+ )
+ enqueue.assert_not_called()
+ assert result.task_id is celery.send_task.return_value.id
+ assert result.transport == nd.CELERY_TRANSPORT
+
+ def test_pg_enqueue_failure_propagates_with_no_celery_fallback(self):
+ # No silent Celery fallback on a TRANSIENT PG failure — it propagates raw
+ # (not wrapped), so the caller (_send_clubbed) reverts rows to PENDING.
+ celery = MagicMock()
+ with (
+ patch.object(nd, "resolve_transport", return_value="pg_queue"),
+ patch.object(nd, "enqueue_task", side_effect=RuntimeError("pg down")),
+ ):
+ with pytest.raises(RuntimeError, match="pg down"):
+ _dispatch(celery)
+ celery.send_task.assert_not_called()
+
+ def test_pg_permanent_enqueue_error_is_wrapped(self):
+ # A PERMANENT enqueue error (ValueError/TypeError: validation / JSON encode)
+ # is re-raised as PermanentDispatchError so the caller dead-letters it —
+ # raised ONLY on the PG path, so the Celery flow is never affected.
+ celery = MagicMock()
+ for exc in (ValueError("priority out of range"), TypeError("not serializable")):
+ with (
+ patch.object(nd, "resolve_transport", return_value="pg_queue"),
+ patch.object(nd, "enqueue_task", side_effect=exc),
+ ):
+ with pytest.raises(nd.PermanentDispatchError):
+ _dispatch(celery)
+ celery.send_task.assert_not_called()
+
+ def test_none_org_fails_closed_to_celery(self):
+ # A missing org string (org deleted) must not route to PG — resolve_transport
+ # fails closed, and we pass organization_id=None straight through to it.
+ celery = MagicMock()
+ with (
+ patch.object(nd, "resolve_transport", return_value="celery") as resolve,
+ patch.object(nd, "enqueue_task") as enqueue,
+ ):
+ _dispatch(celery, org_string_id=None)
+ assert resolve.call_args.kwargs["organization_id"] is None
+ celery.send_task.assert_called_once()
+ enqueue.assert_not_called()
+
+ def test_resolve_transport_buckets_by_minted_dispatch_id(self):
+ # Fire-and-forget: entity_id is a freshly minted uuid (str), and it equals
+ # the PG task_id so the row and the Flipt bucket agree.
+ celery = MagicMock()
+ with (
+ patch.object(nd, "resolve_transport", return_value="pg_queue") as resolve,
+ patch.object(nd, "enqueue_task", return_value=1) as enqueue,
+ ):
+ task_id = _dispatch(celery).task_id
+ assert resolve.call_args.kwargs["execution_id"] == task_id
+ assert enqueue.call_args.kwargs["task_id"] == task_id
+
+ def test_args_and_queue_identical_on_both_paths(self):
+ # The consumer must behave the same regardless of transport — args/queue are
+ # byte-identical, and kwargs match apart from the transport-specific
+ # raise_on_final_failure override (asserted separately below).
+ celery = MagicMock()
+ with patch.object(nd, "resolve_transport", return_value="celery"):
+ _dispatch(celery)
+ celery_call = celery.send_task.call_args.kwargs
+ celery2 = MagicMock()
+ with (
+ patch.object(nd, "resolve_transport", return_value="pg_queue"),
+ patch.object(nd, "enqueue_task", return_value=1) as enqueue,
+ ):
+ _dispatch(celery2)
+ assert enqueue.call_args.kwargs["args"] == celery_call["args"]
+ assert enqueue.call_args.kwargs["queue"] == celery_call["queue"]
+ # kwargs differ ONLY by the two PG-forced retry-semantics keys.
+ pg_kwargs = enqueue.call_args.kwargs["kwargs"]
+ assert pg_kwargs == {
+ **celery_call["kwargs"],
+ "raise_on_final_failure": False,
+ "max_retries": 0,
+ }
+
+ def test_pg_forces_terminal_branch_kwargs(self):
+ # Regression (UN-3753): the PG consumer runs the task eagerly via
+ # ``task.apply(..., throw=True)``, where the in-task retry loop cannot work:
+ # * max_retries >= 1 -> the worker's ``request.retries < max_retries`` guard
+ # is true on the FIRST failure (retries is always 0 under apply()), so it
+ # raises Retry, which propagates out of apply() -- the terminal branch
+ # (mark DEAD_LETTER + honour raise_on_final_failure) never runs and the row
+ # is left for vt-expiry redelivery, re-POSTing the subscriber each time.
+ # * a terminal re-raise is likewise treated as failure -> redelivery.
+ # So the seam must force BOTH max_retries=0 and raise_on_final_failure=False on
+ # the PG branch. The Celery branch keeps the caller's values verbatim.
+ # (The end-to-end behaviour these kwargs buy — one POST + one dead-letter mark
+ # through a real ``task.apply()`` — is asserted in
+ # workers/tests/test_notification_pg_terminal.py.)
+ assert _KWARGS["raise_on_final_failure"] is True # guard the fixture premise
+ assert _KWARGS["max_retries"] == 3 # ... and that retries are configured
+ celery = MagicMock()
+ with (
+ patch.object(nd, "resolve_transport", return_value="pg_queue"),
+ patch.object(nd, "enqueue_task", return_value=1) as enqueue,
+ ):
+ _dispatch(celery)
+ pg_kwargs = enqueue.call_args.kwargs["kwargs"]
+ assert pg_kwargs["raise_on_final_failure"] is False
+ assert pg_kwargs["max_retries"] == 0
+ # The caller's dict is not mutated in place (a fresh dict is enqueued).
+ assert _KWARGS["raise_on_final_failure"] is True
+ assert _KWARGS["max_retries"] == 3
+
+ celery2 = MagicMock()
+ with patch.object(nd, "resolve_transport", return_value="celery"):
+ _dispatch(celery2)
+ sent_kwargs = celery2.send_task.call_args.kwargs["kwargs"]
+ assert sent_kwargs["raise_on_final_failure"] is True
+ assert sent_kwargs["max_retries"] == 3
diff --git a/backend/notification_v2/tests/test_send_clubbed.py b/backend/notification_v2/tests/test_send_clubbed.py
new file mode 100644
index 000000000..b296edfdf
--- /dev/null
+++ b/backend/notification_v2/tests/test_send_clubbed.py
@@ -0,0 +1,147 @@
+"""Call-site tests for ``_send_clubbed`` / ``_org_identifier`` (UN-3753).
+
+The seam's own routing is covered by ``test_notification_dispatch``; these lock
+the two highest-risk regressions at the buffer-flush call site:
+
+1. the two-org-identifier contract — the org **string** id routes the flag, while
+ the org **pk** stays in the worker kwargs (swapping them passes every seam test
+ yet mis-routes every org to Celery and strands the mark endpoint);
+2. failure recovery — a transient/broker error refunds and reverts SENDING rows to
+ PENDING, while a permanent PG enqueue error (surfaced as ``PermanentDispatchError``,
+ raised only on the PG path) dead-letters instead of retrying forever. A Celery
+ send_task failure is an ordinary ``Exception`` → the PENDING path, so flag-off is
+ byte-identical.
+
+Mock-based (no broker/DB): patch the module's collaborators.
+"""
+
+from __future__ import annotations
+
+from unittest.mock import patch
+
+from django.conf import settings
+from notification_v2 import internal_api_views as views
+from notification_v2.enums import BufferStatus
+
+_URL = "https://hook.test"
+_BODY = {"text": "x"}
+_HEADERS = {"Content-Type": "application/json"}
+_BUFFER_IDS = ["b1"]
+
+
+def _send(**overrides):
+ kw = {
+ "url": _URL,
+ "body": _BODY,
+ "headers": _HEADERS,
+ "platform": "SLACK",
+ "max_retries": 3,
+ "buffer_ids": _BUFFER_IDS,
+ "org_id": 7,
+ }
+ kw.update(overrides)
+ views._send_clubbed(**kw)
+
+
+class TestSendClubbedOrgContract:
+ def test_routes_string_id_and_keeps_pk_in_kwargs(self):
+ with (
+ patch.object(views, "_org_identifier", return_value="org-str-id") as ident,
+ patch.object(views, "dispatch_webhook_notification") as seam,
+ ):
+ _send(org_id=7)
+ ident.assert_called_once_with(7)
+ call = seam.call_args.kwargs
+ # Routing uses the STRING id; the worker buffer-mark contract keeps the PK.
+ assert call["org_string_id"] == "org-str-id"
+ assert call["kwargs"]["organization_id"] == 7
+ assert call["queue"] == "notifications"
+ # args carry the full webhook invocation, in order.
+ assert call["args"] == [_URL, _BODY, _HEADERS, settings.NOTIFICATION_TIMEOUT]
+ # buffer_row_ids is what REPLACED the Celery link/link_error callbacks: it is
+ # how the worker knows which rows to mark DISPATCHED / DEAD_LETTER. Drop it
+ # and every row strands in SENDING until the reclaim lease expires — silent,
+ # visible only as a growing SENDING backlog.
+ assert call["kwargs"]["buffer_row_ids"] == _BUFFER_IDS
+
+
+class TestSendClubbedFailureRecovery:
+ def test_transient_failure_reverts_sending_rows_to_pending(self):
+ with (
+ patch.object(views, "_org_identifier", return_value="o"),
+ patch.object(
+ views,
+ "dispatch_webhook_notification",
+ side_effect=RuntimeError("broker down"),
+ ),
+ patch.object(views, "NotificationBuffer") as buf,
+ ):
+ _send(buffer_ids=["b1", "b2"])
+ # Guarded on SENDING, reverted to PENDING.
+ fkw = buf.objects.filter.call_args.kwargs
+ assert fkw["status"] == BufferStatus.SENDING.value
+ assert set(fkw["id__in"]) == {"b1", "b2"}
+ # Every claimed row goes back to PENDING for the next tick.
+ ukw = buf.objects.filter.return_value.update.call_args.kwargs
+ assert ukw == {
+ "status": BufferStatus.PENDING.value,
+ "dispatched_at": None,
+ }
+ # The refund must be ASSERTED, not merely described in a comment: without it
+ # a transient outage burns no attempt (claim +1, refund -1 = net zero), so
+ # NOTIFICATION_MAX_DISPATCH_ATTEMPTS is unreachable and a permanently
+ # recurring failure re-dispatches every flush tick forever. It is also
+ # BOUNDED — only rows still under the limit are refunded, so a failure that
+ # keeps recurring eventually ages into the cap.
+ refund_qs = buf.objects.filter.return_value.filter
+ assert (
+ refund_qs.call_args.kwargs["dispatch_attempts__lte"]
+ == views.NOTIFICATION_TRANSIENT_REFUND_LIMIT
+ )
+ refund_kw = refund_qs.return_value.update.call_args.kwargs
+ assert set(refund_kw) == {"dispatch_attempts"}
+ assert "dispatch_attempts" in str(refund_kw["dispatch_attempts"]) # F(...) - 1
+
+ def test_permanent_pg_error_dead_letters(self):
+ # The PG path surfaces a permanent enqueue failure as PermanentDispatchError;
+ # only that dead-letters. (A raw ValueError from the Celery path can't occur,
+ # and would fall through to the transient PENDING branch, unchanged.)
+ with (
+ patch.object(views, "_org_identifier", return_value="o"),
+ patch.object(
+ views,
+ "dispatch_webhook_notification",
+ side_effect=views.PermanentDispatchError("priority out of range"),
+ ),
+ patch.object(views, "NotificationBuffer") as buf,
+ ):
+ _send()
+ # Permanent error → terminal DEAD_LETTER, no PENDING revert / refund.
+ # The dead-letter update is guarded to only touch rows still SENDING
+ # (same clobber-guard the transient-revert path asserts).
+ fkw = buf.objects.filter.call_args.kwargs
+ assert fkw["status"] == BufferStatus.SENDING.value
+ ukw = buf.objects.filter.return_value.update.call_args.kwargs
+ assert ukw == {"status": BufferStatus.DEAD_LETTER.value}
+
+
+class TestOrgIdentifier:
+ def test_returns_string_id(self):
+ with patch.object(views, "Organization") as org:
+ chain = org.objects.filter.return_value.values_list.return_value
+ chain.first.return_value = "org-uuid"
+ assert views._org_identifier(7) == "org-uuid"
+ org.objects.filter.assert_called_once_with(pk=7)
+
+ def test_missing_org_returns_none_and_logs_error(self):
+ with (
+ patch.object(views, "Organization") as org,
+ patch.object(views.logger, "error") as err,
+ ):
+ chain = org.objects.filter.return_value.values_list.return_value
+ chain.first.return_value = None
+ assert views._org_identifier(7) is None
+ # A dangling FK is a data anomaly: logged at error (Sentry-routed),
+ # org-traceable, rather than swallowed.
+ err.assert_called_once()
+ assert "org_pk" in err.call_args.args[0]
diff --git a/backend/pg_queue/producer.py b/backend/pg_queue/producer.py
index 3c4585fc4..028fddfb6 100644
--- a/backend/pg_queue/producer.py
+++ b/backend/pg_queue/producer.py
@@ -50,8 +50,15 @@ def _json_safe(value: Any) -> Any:
``PgQueueMessage.message`` is a plain ``JSONField`` (no Django encoder), and
the worker consumer already receives string ids on the existing PG dispatch
path, so coercing here keeps both transports consistent.
+
+ ``allow_nan=False`` rejects ``NaN``/``Infinity`` here with a ``ValueError``
+ rather than letting Python's default lenient encoder emit the non-standard
+ ``NaN``/``Infinity`` tokens: Postgres ``jsonb`` rejects those at insert with a
+ ``django.db.DataError`` (a permanent failure the notification dispatcher's
+ ``(ValueError, TypeError)`` seam would otherwise miss, looping the row). Fail
+ at the intended seam instead.
"""
- return json.loads(json.dumps(value, default=str))
+ return json.loads(json.dumps(value, default=str, allow_nan=False))
def enqueue_task(
@@ -97,33 +104,37 @@ def enqueue_task(
f"[{FAIRNESS_MIN_PRIORITY}, {FAIRNESS_MAX_PRIORITY}]: {priority!r}"
)
pg_queue = queue or DEFAULT_GENERAL_QUEUE
- message: TaskPayload = {
- "task_name": task_name,
- "args": _json_safe(list(args) if args is not None else []),
- "kwargs": _json_safe(dict(kwargs) if kwargs is not None else {}),
- "queue": pg_queue,
- # Coerce like args/kwargs/on_success/on_error: FairnessPayload may carry a
- # UUID/enum/datetime that a plain JSONField insert can't serialise, which
- # would raise at enqueue and drop the task on the PG path only.
- "fairness": _json_safe(fairness) if fairness is not None else fairness,
- }
- # Each optional key is set only when present — keeps fire-and-forget rows
- # byte-identical to before these fields existed.
- if reply_key is not None:
- message["reply_key"] = reply_key
- # Continuation specs carry a nested callback ``kwargs`` dict that may hold a
- # UUID/datetime — coerce like ``args``/``kwargs`` above, else the JSONField
- # insert raises at dispatch time (caller-visible).
- if on_success is not None:
- message["on_success"] = _json_safe(on_success)
- if on_error is not None:
- message["on_error"] = _json_safe(on_error)
- if task_id is not None:
- message["task_id"] = task_id
# Mirror the worker _enqueue_pg path: log the failure with breadcrumbs before
# it propagates, so a DB/constraint/serialization error isn't mislabeled by
- # the caller's broad handler.
+ # the caller's broad handler. Message construction (the _json_safe coercions)
+ # lives inside the try because serialization failures surface there — e.g. a
+ # NaN/Infinity float raises ValueError under allow_nan=False — and on the
+ # orchestrator path (no silent fallback) would otherwise drop the task with no
+ # task/queue/org breadcrumb for on-call.
try:
+ message: TaskPayload = {
+ "task_name": task_name,
+ "args": _json_safe(list(args) if args is not None else []),
+ "kwargs": _json_safe(dict(kwargs) if kwargs is not None else {}),
+ "queue": pg_queue,
+ # Coerce like args/kwargs/on_success/on_error: FairnessPayload may carry
+ # a UUID/enum/datetime that a plain JSONField insert can't serialise,
+ # which would raise at enqueue and drop the task on the PG path only.
+ "fairness": _json_safe(fairness) if fairness is not None else fairness,
+ }
+ # Each optional key is set only when present — keeps fire-and-forget rows
+ # byte-identical to before these fields existed.
+ if reply_key is not None:
+ message["reply_key"] = reply_key
+ # Continuation specs carry a nested callback ``kwargs`` dict that may hold a
+ # UUID/datetime — coerce like ``args``/``kwargs`` above, else the JSONField
+ # insert raises at dispatch time (caller-visible).
+ if on_success is not None:
+ message["on_success"] = _json_safe(on_success)
+ if on_error is not None:
+ message["on_error"] = _json_safe(on_error)
+ if task_id is not None:
+ message["task_id"] = task_id
row = PgQueueMessage.objects.create(
queue_name=pg_queue,
message=message,
diff --git a/backend/pg_queue/tests/test_producer.py b/backend/pg_queue/tests/test_producer.py
index 40df8a79f..c94d219f0 100644
--- a/backend/pg_queue/tests/test_producer.py
+++ b/backend/pg_queue/tests/test_producer.py
@@ -5,6 +5,7 @@ the JSON-coercion logic without needing a test database.
"""
import datetime
+import logging
import uuid
from unittest.mock import MagicMock, patch
@@ -106,6 +107,40 @@ class TestEnqueueTask:
when = model.objects.create.call_args.kwargs["message"]["kwargs"]["when"]
assert isinstance(when, str) and "2026-06-18" in when
+ @pytest.mark.parametrize(
+ "bad", [float("nan"), float("inf"), float("-inf")], ids=["nan", "inf", "-inf"]
+ )
+ @pytest.mark.parametrize("slot", ["args", "kwargs", "fairness"])
+ def test_json_safe_rejects_non_finite_floats(self, bad, slot, caplog):
+ # A non-finite float slips past the default lenient encoder and only fails at
+ # the jsonb insert (DataError); allow_nan=False surfaces it as a ValueError at
+ # the enqueue seam so the notification dispatcher can dead-letter it.
+ #
+ # Every _json_safe-coerced slot is covered, not just kwargs: the scenario this
+ # guard exists for is a webhook BODY carrying a non-finite float, and
+ # notification_dispatch puts the body in args[1]. inf/-inf are rejected by
+ # allow_nan=False exactly like nan.
+ payloads = {
+ "args": {"args": ["url", {"score": bad}]},
+ "kwargs": {"kwargs": {"score": bad}},
+ "fairness": {"fairness": {"org_id": "o", "weight": bad}},
+ }
+ with patch(_MODEL) as model:
+ with caplog.at_level(logging.ERROR, logger=producer.logger.name):
+ with pytest.raises(ValueError):
+ producer.enqueue_task(
+ task_name="send_webhook_notification",
+ queue="notifications",
+ org_id="org-1",
+ **payloads[slot],
+ )
+ model.objects.create.assert_not_called() # never reaches the DB insert
+ # The breadcrumb is the point of moving coercion inside the try — assert the
+ # rendered record actually carries it, not merely that .exception() was hit.
+ assert "send_webhook_notification" in caplog.text
+ assert "notifications" in caplog.text
+ assert "org-1" in caplog.text
+
def test_enqueue_failure_logs_and_propagates(self):
with patch(_MODEL) as model:
model.objects.create.side_effect = RuntimeError("db down")
diff --git a/workers/notification/tasks.py b/workers/notification/tasks.py
index 41d76f5cf..f1c780ae0 100644
--- a/workers/notification/tasks.py
+++ b/workers/notification/tasks.py
@@ -252,9 +252,15 @@ def send_webhook_notification(
retry_delay: The delay between retries in seconds
platform: Platform type from notification config (SLACK, API, etc.)
raise_on_final_failure: When True, re-raise on retry exhaustion so the
- task ends in FAILURE and any Celery ``link_error`` callback runs
- (used by the clubbed/buffered dispatch to dead-letter the rows).
- When False (default), preserve the legacy "return None" behavior.
+ task ends in FAILURE (a Celery monitoring signal). NOTE: the buffer
+ rows are dead-lettered by ``_mark_buffer_outcome(dispatched=False)``
+ over the internal API *before* this re-raise, on BOTH transports —
+ ``link``/``link_error`` callbacks are no longer wired (they routed to
+ the ``celery`` queue and were dropped as "unregistered task"), so the
+ raise itself dead-letters nothing. When False (default), preserve the
+ legacy "return None" behavior. The PG dispatch seam forces this False
+ (and ``max_retries`` 0) because under the consumer's eager
+ ``task.apply()`` a raise means redelivery, not a FAILURE state.
Returns:
None (matches original behavior)
diff --git a/workers/queue_backend/pg_queue/consumer.py b/workers/queue_backend/pg_queue/consumer.py
index 3e016d228..241a3d291 100644
--- a/workers/queue_backend/pg_queue/consumer.py
+++ b/workers/queue_backend/pg_queue/consumer.py
@@ -204,6 +204,62 @@ def _pipeline_identity(payload: TaskPayload) -> tuple[str | None, str]:
)
+# Header/kwarg names whose VALUES are secrets. Substring match on a lowercased key,
+# so ``Authorization``/``X-Api-Key``/``auth_token``/``client_secret`` all match.
+_SECRET_KEY_PARTS = ("authorization", "api-key", "api_key", "apikey", "token", "secret")
+
+# Payload slots that carry customer data rather than routing metadata. Logged as a
+# type+size summary so a drop stays debuggable without dumping document content.
+_BULKY_ARG_INDEX = 1 # args[1] is the webhook body on the notification payload
+
+
+def _redact_secrets(value: Any) -> Any:
+ """Recursively mask secret-valued keys in dicts/lists; other values pass through."""
+ if isinstance(value, dict):
+ return {
+ k: (
+ "***REDACTED***"
+ if isinstance(k, str) and any(p in k.lower() for p in _SECRET_KEY_PARTS)
+ else _redact_secrets(v)
+ )
+ for k, v in value.items()
+ }
+ if isinstance(value, list):
+ return [_redact_secrets(v) for v in value]
+ return value
+
+
+def _redact_payload(payload: dict) -> dict:
+ """A log-safe view of a queue payload.
+
+ Two exposures this closes, both introduced in practice once credential-bearing
+ notification payloads started flowing through the PG consumer:
+
+ * **Secrets** — ``build_webhook_headers`` writes the subscriber's bearer token /
+ API key into the header dict carried in ``args``/``kwargs``. Masked by key name.
+ * **Customer data** — the clubbed webhook body is the pipeline's extracted output.
+ Replaced with a ``<type len=N>`` summary rather than dumped.
+
+ Routing metadata (task_name, queue, task_id, read counts) is preserved verbatim,
+ since that is what a poison drop is actually debugged from.
+ """
+ safe = {k: v for k, v in payload.items() if k not in ("args", "kwargs")}
+ args = payload.get("args")
+ if isinstance(args, list):
+ redacted_args: list[Any] = []
+ for i, a in enumerate(args):
+ if i == _BULKY_ARG_INDEX and isinstance(a, (dict, list, str)):
+ redacted_args.append(f"<{type(a).__name__} len={len(a)}>")
+ else:
+ redacted_args.append(_redact_secrets(a))
+ safe["args"] = redacted_args
+ elif args is not None:
+ safe["args"] = _redact_secrets(args)
+ if payload.get("kwargs") is not None:
+ safe["kwargs"] = _redact_secrets(payload.get("kwargs"))
+ return safe
+
+
class PgQueueConsumer:
"""Polls one PG queue, runs each claimed task in-process, acks on success."""
@@ -505,11 +561,22 @@ class PgQueueConsumer:
headers = {FAIRNESS_HEADER_NAME: fairness} if fairness else None
# Renew the short lease while the (possibly long) task runs, so a dead
# worker's claim expires fast but a live one is never redelivered.
+ #
+ # ``task_id=`` is load-bearing for idempotency: Celery's ``Task.apply``
+ # does ``task_id = task_id or uuid()``, so WITHOUT this the task sees a
+ # FRESH ``self.request.id`` on every delivery. Any guard keyed on it
+ # (fan-out claim rows, ``generation_task_id`` uniqueness, task-complete
+ # markers) would then never collide across a redelivery and dedupe
+ # nothing. The producer already persisted a stable id in the payload —
+ # pass it through so redelivery re-runs with the SAME request id, matching
+ # Celery's own redelivery semantics. Falls back to Celery's uuid() when
+ # absent (never None, which apply() would reject).
with self._lease_renewal(message.msg_id):
eager = task.apply(
args=payload.get("args") or [],
kwargs=payload.get("kwargs") or {},
headers=headers,
+ task_id=payload.get("task_id") or None,
throw=True,
)
except Exception as exc:
@@ -789,14 +856,23 @@ class PgQueueConsumer:
"""
execution_id, organization_id = _pipeline_identity(payload)
logger.error(
- "PG-queue consumer: task %r (msg_id=%s) exceeded max_attempts=%s "
- "(read_ct=%s, execution_id=%s) — poison; full payload: %r",
+ "PG-queue consumer: poison-dropped task %r "
+ "(msg_id=%s, read_ct=%s, queue=%s, execution_id=%s, org_id=%s) — "
+ "exceeded max_attempts=%s; payload: %r",
task_name,
message.msg_id,
- self.max_attempts,
+ # read_ct counts READS, not executions — a lease expiry with no run still
+ # increments it. Logged once, under its own name (an "after N attempts"
+ # phrasing duplicated this same value and misread as execution count).
message.read_ct,
+ payload.get("queue"),
execution_id,
- payload,
+ # NB: identifier flavour is payload-dependent — see _pipeline_identity.
+ organization_id or None,
+ self.max_attempts,
+ # Redacted: payloads carry subscriber credentials (webhook Authorization
+ # / API-key headers) and customer data. Never log them raw.
+ _redact_payload(payload),
)
# Failure channel (request-reply / on_error) → surface there, then drop.
if payload.get("reply_key") or payload.get("on_error"):
diff --git a/workers/shared/enums/worker_enums_base.py b/workers/shared/enums/worker_enums_base.py
index 9c4dfbdb4..65465659c 100644
--- a/workers/shared/enums/worker_enums_base.py
+++ b/workers/shared/enums/worker_enums_base.py
@@ -58,13 +58,38 @@ class WorkerType(str, Enum):
if self.is_pluggable():
return f"pluggable_worker.{self.value}.tasks"
- # Map to actual directory structure
+ return f"{self.to_directory()}.tasks"
+
+ def to_directory(self) -> str:
+ """Return the on-disk directory name for this **non-pluggable** worker.
+
+ Single source of truth for the naming-convention mapping: enum values use
+ underscores (Python module names), while ``API_DEPLOYMENT``'s on-disk dir
+ uses a hyphen (``api-deployment``). ``to_import_path`` builds on this, and
+ the file-path task loader in ``worker.py`` reads it directly rather than
+ slicing the import path.
+
+ Raises:
+ ValueError: if called on a pluggable worker type. Pluggable code lives
+ under ``pluggable_worker/<value>``, so the mapping's passthrough
+ would return a bare, *plausible-looking* but wrong path. Today only
+ call ordering prevents that (``to_import_path`` guards with
+ ``is_pluggable()``, and ``worker.py`` early-returns for pluggable
+ workers before reaching here) — nothing enforced it. Since
+ ``worker.py`` now RAISES on a missing directory instead of logging,
+ a future caller that skips the guard would turn a silently-wrong
+ path into a boot crash-loop; fail loudly and locally instead.
+ """
+ if self.is_pluggable():
+ raise ValueError(
+ f"{self!r} is a pluggable worker; its code lives under "
+ f"pluggable_worker/{self.value} — use to_import_path() instead."
+ )
directory_mapping = {
- "api_deployment": "api-deployment",
+ WorkerType.API_DEPLOYMENT: "api-deployment",
# All others use same name for directory and module
}
- directory = directory_mapping.get(self.value, self.value)
- return f"{directory}.tasks"
+ return directory_mapping.get(self, self.value)
def is_pluggable(self) -> bool:
"""Check if this worker type is a pluggable worker.
diff --git a/workers/tests/conftest.py b/workers/tests/conftest.py
index bd4b090b4..faa90d759 100644
--- a/workers/tests/conftest.py
+++ b/workers/tests/conftest.py
@@ -410,6 +410,16 @@ def _restore_current_celery_app():
"""
from celery._state import default_app
+ # Guard against there being no default app at all: a module that imports no
+ # worker app (e.g. a pure-enum test running alone) would otherwise ERROR here on
+ # ``None.finalize()``. That previously forced such modules to call
+ # ``Celery(...).set_default()`` at import time as a workaround, which mutated
+ # process-global state for every later test in the session depending on
+ # collection order. Fix it here instead so leaf tests need no Celery app.
+ if default_app is None:
+ yield
+ return
+
default_app.finalize()
default_app.set_current()
try:
diff --git a/workers/tests/test_notification_pg_terminal.py b/workers/tests/test_notification_pg_terminal.py
new file mode 100644
index 000000000..8c3774029
--- /dev/null
+++ b/workers/tests/test_notification_pg_terminal.py
@@ -0,0 +1,110 @@
+"""``send_webhook_notification`` under the PG consumer's eager ``task.apply()``.
+
+The PG consumer runs tasks with ``task.apply(..., throw=True)`` rather than through
+a Celery worker. Under ``apply()`` Celery's in-task retry loop cannot work —
+``self.request.retries`` is ALWAYS 0, so the task's
+``if self.request.retries < max_retries`` guard is true on the FIRST failure for any
+``max_retries >= 1`` and raises ``Retry``, which ``throw=True`` propagates straight
+out of ``apply()``. The terminal branch (mark buffers DEAD_LETTER, honour
+``raise_on_final_failure``) is then never reached, so the consumer sees an exception,
+leaves the row for vt-expiry redelivery, and the subscriber is re-POSTed every time.
+
+That is why the dispatch seam (``backend/notification_v2/notification_dispatch.py``)
+forces BOTH ``max_retries=0`` and ``raise_on_final_failure=False`` on the PG branch.
+These tests drive the REAL task through ``apply()`` and assert the behaviour those
+kwargs buy — one POST + one DEAD_LETTER mark, no raise — rather than the kwarg values
+(which the seam's own unit tests already pin).
+"""
+
+from __future__ import annotations
+
+from unittest.mock import patch
+
+import pytest
+from celery.exceptions import Retry
+from notification.tasks import send_webhook_notification
+
+_URL = "https://hook.test/x"
+_BUFFER_IDS = ["b1", "b2"]
+_ORG = 7
+
+
+class _AlwaysFailingProvider:
+ """Webhook provider whose send() always reports failure, counting attempts."""
+
+ def __init__(self) -> None:
+ self.posts = 0
+
+ def send(self, notification_data: dict) -> dict:
+ self.posts += 1
+ return {"success": False, "message": "boom"}
+
+
+def _run(*, max_retries: int, raise_on_final_failure: bool):
+ """Drive the real task through the consumer's exact call shape.
+
+ Returns ``(provider, marks, raised)`` — POST count, the
+ ``_mark_buffer_outcome`` calls, and any exception propagated out of apply().
+ """
+ provider = _AlwaysFailingProvider()
+ marks: list[bool] = []
+ with (
+ patch(
+ "notification.tasks._get_webhook_provider_for_url", return_value=provider
+ ),
+ patch(
+ "notification.tasks._mark_buffer_outcome",
+ side_effect=lambda ids, org, *, dispatched: marks.append(dispatched),
+ ),
+ ):
+ raised: BaseException | None = None
+ try:
+ send_webhook_notification.apply(
+ args=[_URL, {"text": "hi"}, {"Content-Type": "application/json"}, 30],
+ kwargs={
+ "max_retries": max_retries,
+ "retry_delay": 10,
+ "platform": None,
+ "raise_on_final_failure": raise_on_final_failure,
+ "buffer_row_ids": _BUFFER_IDS,
+ "organization_id": _ORG,
+ },
+ throw=True,
+ )
+ except BaseException as exc: # noqa: BLE001 - we assert on the type below
+ raised = exc
+ return provider, marks, raised
+
+
+def test_pg_kwargs_give_one_post_and_one_dead_letter():
+ # The seam's PG kwargs (max_retries=0, raise_on_final_failure=False): the task
+ # takes the terminal branch on the first failure, marks the buffers DEAD_LETTER,
+ # and returns None -> the consumer acks the row. Subscriber hit exactly once.
+ provider, marks, raised = _run(max_retries=0, raise_on_final_failure=False)
+ assert raised is None # nothing propagates -> consumer acks (no redelivery)
+ assert provider.posts == 1 # subscriber POSTed once, not once per redelivery
+ assert marks == [False] # exactly one DEAD_LETTER mark
+
+
+def test_unforced_max_retries_would_escape_apply_without_dead_lettering():
+ # Regression guard for the bug the override fixes: with the caller's original
+ # max_retries >= 1, apply() raises Retry out of the task, the terminal branch
+ # never runs, and NO buffer mark happens -> the consumer would redeliver and
+ # re-POST. This is what the seam prevents by forcing max_retries=0.
+ provider, marks, raised = _run(max_retries=3, raise_on_final_failure=False)
+ assert isinstance(raised, Retry) # escapes apply() as an exception
+ assert provider.posts == 1 # ... after a single POST on this delivery
+ assert marks == [] # buffers NEVER dead-lettered -> rows strand/redeliver
+
+
[email protected]("raise_on_final", [True, False])
+def test_max_retries_zero_always_reaches_terminal_branch(raise_on_final):
+ # With max_retries=0 the terminal branch always runs, so the buffers are marked
+ # DEAD_LETTER regardless of raise_on_final_failure; the flag only decides whether
+ # the task additionally re-raises (a Celery FAILURE-state signal).
+ provider, marks, raised = _run(
+ max_retries=0, raise_on_final_failure=raise_on_final
+ )
+ assert provider.posts == 1
+ assert marks == [False] # dead-lettered on BOTH transports, before any re-raise
+ assert (raised is not None) is raise_on_final
diff --git a/workers/tests/test_pg_queue_consumer.py b/workers/tests/test_pg_queue_consumer.py
index 919e94e10..3e5d836dc 100644
--- a/workers/tests/test_pg_queue_consumer.py
+++ b/workers/tests/test_pg_queue_consumer.py
@@ -133,6 +133,33 @@ class TestPollOnce:
assert kwargs["kwargs"] == {"k": "v"}
assert kwargs["headers"] == {FAIRNESS_HEADER_NAME: fairness}
+ def test_payload_task_id_is_passed_to_apply(self):
+ # Idempotency-critical: Celery's Task.apply does ``task_id = task_id or
+ # uuid()``, so without an explicit task_id the task sees a FRESH request.id
+ # on every delivery and any guard keyed on it (fan-out claims,
+ # generation_task_id uniqueness, task-complete markers) dedupes nothing
+ # across a redelivery. The producer's stable id must reach apply().
+ task = MagicMock()
+ app = MagicMock()
+ app.tasks.get.return_value = task
+ client = MagicMock()
+ client.read.return_value = [
+ _msg(6, {"task_name": "t", "args": [], "kwargs": {}, "task_id": "stable-1"})
+ ]
+ PgQueueConsumer(["q"], client=client, app=app).poll_once()
+ assert task.apply.call_args.kwargs["task_id"] == "stable-1"
+
+ def test_missing_task_id_falls_back_to_celery_uuid(self):
+ # No task_id in the payload -> pass None so Celery mints one, rather than
+ # passing "" (which apply() would take literally as the request id).
+ task = MagicMock()
+ app = MagicMock()
+ app.tasks.get.return_value = task
+ client = MagicMock()
+ client.read.return_value = [_msg(6, {"task_name": "t", "args": [], "kwargs": {}})]
+ PgQueueConsumer(["q"], client=client, app=app).poll_once()
+ assert task.apply.call_args.kwargs["task_id"] is None
+
def test_ack_finding_no_row_warns(self, caplog):
client = MagicMock()
client.delete.return_value = False # row already gone (vt expired mid-run)
@@ -272,6 +299,67 @@ class TestPoisonDropMarksExecution:
client.delete.assert_called_once_with(5) # then dropped
client.set_vt.assert_not_called()
+ def test_poison_log_enriched_with_org_and_read_ct(self, monkeypatch, caplog):
+ # Q2 debuggability: the poison drop logs a clear "poison-dropped" message
+ # carrying read_ct + org_id + queue so a dropped task is greppable/
+ # Sentry-visible and its source queue is identifiable at a glance.
+ monkeypatch.setattr(
+ "queue_backend.pg_queue.recovery.mark_execution_error",
+ lambda *a, **k: True,
+ )
+ client = MagicMock()
+ payload = {**_callback_payload(), "queue": "agentic_callback"}
+ client.read.return_value = [self._poison(payload=payload, read_ct=6)]
+ with caplog.at_level(logging.ERROR, logger="queue_backend.pg_queue.consumer"):
+ PgQueueConsumer(
+ ["q"], client=client, api_client=MagicMock(), max_attempts=5
+ ).poll_once()
+ assert "poison-dropped" in caplog.text
+ assert "read_ct=6" in caplog.text
+ assert "org_id=org-1" in caplog.text # org surfaced for the dropped task
+ assert "queue=agentic_callback" in caplog.text # source queue surfaced
+ # read_ct is rendered ONCE (an earlier "after N attempts" phrasing printed
+ # the same value twice under a name that misread as an execution count).
+ assert caplog.text.count("6") >= 1
+ assert "attempts (msg_id" not in caplog.text
+
+ def test_poison_log_redacts_credentials_and_body(self, monkeypatch, caplog):
+ # Security: notification payloads carry the subscriber's Authorization /
+ # API-key headers and the clubbed webhook body (customer extracted data).
+ # A poison drop must not dump either to stdout/Sentry.
+ monkeypatch.setattr(
+ "queue_backend.pg_queue.recovery.mark_execution_error",
+ lambda *a, **k: True,
+ )
+ payload = {
+ "task_name": "send_webhook_notification",
+ "queue": "notifications",
+ "args": [
+ "https://hook.test",
+ {"secret_field": "customer extracted data"},
+ {"Authorization": "Bearer super-secret", "X-Api-Key": "ak_live_123"},
+ 30,
+ ],
+ "kwargs": {"organization_id": "org-1", "auth_token": "tok_abc"},
+ }
+ client = MagicMock()
+ client.read.return_value = [self._poison(payload=payload, read_ct=6)]
+ with caplog.at_level(logging.ERROR, logger="queue_backend.pg_queue.consumer"):
+ PgQueueConsumer(
+ ["q"], client=client, api_client=MagicMock(), max_attempts=5
+ ).poll_once()
+ # No secret value survives, by any route.
+ assert "super-secret" not in caplog.text
+ assert "ak_live_123" not in caplog.text
+ assert "tok_abc" not in caplog.text
+ # Customer body replaced by a type+size summary, not dumped.
+ assert "customer extracted data" not in caplog.text
+ assert "<dict len=1>" in caplog.text
+ # Routing metadata IS preserved — that's what the drop is debugged from.
+ assert "send_webhook_notification" in caplog.text
+ assert "https://hook.test" in caplog.text
+ assert "REDACTED" in caplog.text
+
def test_positional_orchestration_poison_marks_error(self, monkeypatch):
# H2 regression: a poisoned async_execute_bin carries execution_id
# POSITIONALLY (args[2]) with no _barrier_context and — since its poison
@@ -978,7 +1066,8 @@ class TestRecordTaskStatus:
pg_task_result so the REST PromptStudio.task_status poll resolves under PG.
completed unless the run raised (error) or the executor reported success=False
(completed rows are status-only; failed rows carry the executor error text). TTL'd
- + best-effort — never wedges the ack."""
+ + best-effort — never wedges the ack.
+ """
_RB = "queue_backend.pg_queue.consumer.PgResultBackend"
_RET = 86400
@@ -1104,6 +1193,34 @@ class TestLeaseRenewal:
c = mod.build_consumer_from_env()
assert c.lease_seconds == 77
+ def test_env_wires_max_attempts(self, monkeypatch):
+ # Per-worker override: an execution worker keeps the default (at-least-once),
+ # the interactive orchestrator sets 1 (at-most-once, no LLM re-compute).
+ from queue_backend.pg_queue import consumer as mod
+
+ monkeypatch.setenv("WORKER_PG_QUEUE_CONSUMER_MAX_ATTEMPTS", "1")
+ with patch.object(mod, "PgQueueClient"): # no real DB connection
+ c = mod.build_consumer_from_env()
+ assert c.max_attempts == 1
+
+ def test_max_attempts_defaults_to_5_when_unset(self, monkeypatch):
+ from queue_backend.pg_queue import consumer as mod
+ from queue_backend.pg_queue.consumer import _DEFAULT_MAX_ATTEMPTS
+
+ monkeypatch.delenv("WORKER_PG_QUEUE_CONSUMER_MAX_ATTEMPTS", raising=False)
+ with patch.object(mod, "PgQueueClient"): # no real DB connection
+ c = mod.build_consumer_from_env()
+ # The code default stays 5 so execution workers remain at-least-once.
+ assert c.max_attempts == _DEFAULT_MAX_ATTEMPTS == 5
+
+ def test_env_invalid_max_attempts_raises_named_error(self, monkeypatch):
+ from queue_backend.pg_queue import consumer as mod
+
+ monkeypatch.setenv("WORKER_PG_QUEUE_CONSUMER_MAX_ATTEMPTS", "abc")
+ with patch.object(mod, "PgQueueClient"):
+ with pytest.raises(ValueError, match="WORKER_PG_QUEUE_CONSUMER_MAX_ATTEMPTS"):
+ mod.build_consumer_from_env()
+
def test_poll_claims_with_lease_not_vt(self):
client = MagicMock()
client.read.return_value = []
diff --git a/workers/tests/test_worker_enums_directory.py b/workers/tests/test_worker_enums_directory.py
new file mode 100644
index 000000000..280a3ad59
--- /dev/null
+++ b/workers/tests/test_worker_enums_directory.py
@@ -0,0 +1,58 @@
+"""WorkerType.to_directory() — the single source for the on-disk dir mapping (UN-3798).
+
+worker.py's file-path task loader reads to_directory() directly instead of slicing
+to_import_path(); these pin the hyphen/underscore mapping and that to_import_path is
+built on top of it, so the two can't drift.
+
+No Celery app is set up here on purpose: this is a pure-enum module, and the autouse
+``_restore_current_celery_app`` fixture now tolerates a missing default app. An
+earlier version called ``Celery(...).set_default()`` at import time to satisfy that
+fixture, which mutated process-global Celery state for every later test in the
+session depending on collection order.
+"""
+
+from pathlib import Path
+
+import pytest
+from shared.enums.worker_enums_base import WorkerType
+
+_WORKERS_ROOT = Path(__file__).resolve().parents[1]
+
+
+def test_to_directory_maps_underscored_value_to_hyphenated_dir():
+ # The one worker whose on-disk dir differs from its enum value.
+ assert WorkerType.API_DEPLOYMENT.value == "api_deployment"
+ assert WorkerType.API_DEPLOYMENT.to_directory() == "api-deployment"
+
+
+def test_to_directory_passthrough_when_dir_equals_value():
+ assert WorkerType.GENERAL.to_directory() == "general"
+
+
+def test_to_import_path_is_built_on_to_directory():
+ # to_import_path must derive from to_directory (not a separate mapping), so the
+ # directory naming lives in exactly one place.
+ wt = WorkerType.API_DEPLOYMENT
+ assert wt.to_import_path() == f"{wt.to_directory()}.tasks"
+
+
[email protected]("wt", [w for w in WorkerType if not w.is_pluggable()])
+def test_every_worker_type_maps_to_an_existing_directory_with_tasks(wt):
+ # The failure mode UN-3798 exists for is enum-vs-disk DRIFT, and worker.py now
+ # RAISES on a missing directory (it previously logged and continued) — so drift
+ # is a container crash-loop across every replica, not a degraded worker. Pinning
+ # two members can't catch that; pin the whole mapping so adding a WorkerType
+ # without its directory is a red build instead of a production RuntimeError.
+ directory = _WORKERS_ROOT / wt.to_directory()
+ assert directory.is_dir(), f"{wt.value} -> {wt.to_directory()} does not exist"
+ assert (directory / "tasks.py").is_file(), f"{wt.to_directory()}/tasks.py missing"
+
+
[email protected]("wt", [w for w in WorkerType if w.is_pluggable()])
+def test_to_directory_rejects_pluggable_types(wt):
+ # Pluggable code lives under pluggable_worker/<value>, so the mapping's
+ # passthrough would return a wrong-but-plausible path. Only call ordering kept
+ # that unreachable; the precondition is now enforced.
+ with pytest.raises(ValueError, match="pluggable"):
+ wt.to_directory()
+ assert wt.to_import_path() == f"pluggable_worker.{wt.value}.tasks"
diff --git a/workers/worker.py b/workers/worker.py
index 3822e81a1..077fbd974 100755
--- a/workers/worker.py
+++ b/workers/worker.py
@@ -5,6 +5,7 @@ This module serves as the main entry point for all Celery workers.
It uses WorkerBuilder to ensure proper configuration including chord retry settings.
"""
+import importlib.util
import logging
import os
import sys
@@ -440,40 +441,97 @@ logger.info("🏗️ Initializing worker infrastructure (singleton pattern)...")
initialize_worker_infrastructure()
logger.info("✅ Worker infrastructure initialized successfully")
-# Import tasks from the worker-specific directory
-# Determine worker path dynamically based on worker type
-base_dir = os.path.dirname(os.path.abspath(__file__))
-if worker_type.is_pluggable():
- # Pluggable workers live inside workers/pluggable_worker/{worker_name}
- worker_directory = os.path.join("pluggable_worker", worker_type.value)
- worker_path = os.path.join(base_dir, worker_directory)
-else:
- # Enum values use underscores (Python module names); a few on-disk dirs
- # still use hyphens (e.g. api-deployment). Derive the directory from the
- # authoritative import-path map on WorkerType instead of a blind replace.
- worker_directory = worker_type.to_import_path().rsplit(".", 1)[0]
+
+def load_worker_tasks(worker_type: WorkerType) -> None:
+ """Register the worker type's Celery tasks.
+
+ For pluggable workers, ``build_celery_app()`` above already imported the
+ plugin package via ``WorkerBuilder._verify_pluggable_worker_exists``
+ (``importlib.import_module("pluggable_worker.<type>.worker")`` — a proper
+ PACKAGE import that RUNS the plugin's own registration code, e.g.
+ ``from . import tasks`` with relative imports such as
+ ``from .clients import ...``). That registration's ``@shared_task`` bindings
+ are applied when the app FINALIZES — which any read of ``app.tasks`` triggers
+ automatically (``finalize(auto=True)``), so they are in place by the time
+ anything inspects the registry. The generic file-path load below is SKIPPED for
+ pluggable workers because loading ``tasks.py`` under a bare ``"tasks"`` spec (no
+ parent package) would break the plugin's relative imports ("attempted relative
+ import with no known parent package") — not because of any binding-order
+ subtlety.
+
+ Non-pluggable (top-level) workers use absolute imports; their ``tasks.py`` is
+ loaded by file path with the worker directory on ``sys.path``. Their tasks bind
+ eagerly here, so a missing directory / ``tasks.py`` is a broken deploy and
+ hard-fails rather than booting a no-op worker.
+ """
+ if worker_type.is_pluggable():
+ logger.info(
+ f"✅ Pluggable worker {worker_type.value} tasks registered via "
+ "WorkerBuilder (package import); skipping file-path task load"
+ )
+ return
+
+ base_dir = os.path.dirname(os.path.abspath(__file__))
+ worker_directory = worker_type.to_directory()
worker_path = os.path.join(base_dir, worker_directory)
+ if not os.path.exists(worker_path):
+ # Non-pluggable only (pluggable returned above): tasks bind eagerly here
+ # with no later finalize step, so a missing worker dir is terminal.
+ raise RuntimeError(f"Worker directory not found: {worker_path}")
-# Add worker directory to path for task imports
-if os.path.exists(worker_path):
sys.path.append(worker_path)
logger.info(f"✅ Added {worker_directory} to Python path for task imports")
- # Import tasks module to register tasks
tasks_file = os.path.join(worker_path, "tasks.py")
- if os.path.exists(tasks_file):
- logger.info(f"📋 Loading tasks from: {tasks_file}")
- # Import the tasks module to register tasks with the app
- import importlib.util
-
- spec = importlib.util.spec_from_file_location("tasks", tasks_file)
- tasks_module = importlib.util.module_from_spec(spec)
- spec.loader.exec_module(tasks_module)
- logger.info(f"✅ Tasks loaded successfully from {worker_directory}")
+ if not os.path.exists(tasks_file):
+ raise RuntimeError(f"No tasks.py found for worker at: {tasks_file}")
+
+ logger.info(f"📋 Loading tasks from: {tasks_file}")
+ spec = importlib.util.spec_from_file_location("tasks", tasks_file)
+ tasks_module = importlib.util.module_from_spec(spec)
+ spec.loader.exec_module(tasks_module)
+ logger.info(f"✅ Tasks loaded successfully from {worker_directory}")
+
+
+load_worker_tasks(worker_type)
+
+# A worker that boots with no registered tasks starts but silently processes
+# nothing (Celery does not error on an empty registry). Surface that misconfig.
+#
+# NB: reading ``app.tasks`` below AUTO-FINALIZES the app (Celery's ``tasks``
+# property calls ``self.finalize(auto=True)``), so every pending ``@shared_task`` /
+# ``connect_on_app_finalize`` registration has already been applied by the time the
+# emptiness test is evaluated. The check is therefore exactly as accurate for
+# pluggable workers as for non-pluggable ones — an earlier version of this comment
+# claimed the opposite ("bind on finalize, which can be after this point") and used
+# it to justify the split; that premise was wrong. Two real consequences: the app is
+# finalized at import rather than at worker start, and an empty registry here is a
+# genuine misconfiguration for BOTH kinds.
+#
+# The split is kept deliberately, on deployment-risk grounds rather than accuracy:
+# a non-pluggable worker's tasks are bound eagerly by ``load_worker_tasks`` in this
+# same module, so an empty registry is unambiguously terminal — RAISE. Pluggable
+# workers register via plugin package import, whose availability is deployment
+# dependent; hard-failing there would turn one absent/misconfigured plugin into a
+# container crash-loop across the fleet, so it stays a loud WARN. Tightening this to
+# a raise is worthwhile but needs a pass confirming every deployed pluggable worker
+# registers at import time — tracked under UN-3445 hardening, not changed blind here.
+if not any(not name.startswith("celery.") for name in app.tasks):
+ _empty_registry_msg = (
+ f"No non-celery tasks registered for worker '{worker_type.value}' "
+ f"(pluggable={worker_type.is_pluggable()})."
+ )
+ if worker_type.is_pluggable():
+ logger.warning(
+ f"⚠️ {_empty_registry_msg} If this persists past app finalize the "
+ "worker will start but process nothing — check the plugin task "
+ "registration."
+ )
else:
- logger.warning(f"⚠️ No tasks.py found at: {tasks_file}")
-else:
- logger.error(f"❌ Worker directory not found: {worker_path}")
+ raise RuntimeError(
+ f"{_empty_registry_msg} The worker would start but process nothing — "
+ "check the worker task registration."
+ )
# Log successful configuration
logger.info(f"✅ Successfully loaded {worker_type} worker using WorkerBuilder")