LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
UN-3445 [GATED-FEAT] PG-queue auxiliary-worker migration (OSS): loader fix + notification seam (#2217)
alie2f8a44e3218
diff --git a/backend/notification_v2/internal_api_views.py b/backend/notification_v2/internal_api_views.pyindex be2bbe57e..42d0eeac9 100644--- a/backend/notification_v2/internal_api_views.py+++ b/backend/notification_v2/internal_api_views.py@@ -14,6 +14,7 @@ import loggingfrom datetime import timedeltafrom typing import Any, cast+from account_v2.models import Organizationfrom api_v2.models import APIDeploymentfrom django.conf import settingsfrom django.db import transaction@@ -37,12 +38,25 @@ from notification_v2.helper import (webhook_url_hash,)from notification_v2.models import Notification, NotificationBuffer+from notification_v2.notification_dispatch import (+ PermanentDispatchError,+ dispatch_webhook_notification,+)logger = logging.getLogger(__name__)# Constants for error messagesINTERNAL_SERVER_ERROR_MSG = "Internal server error"+# How many early transient-failure cycles get their SENDING claim refunded before the+# claim starts sticking. Refunding forever makes net attempt progress zero, so+# ``NOTIFICATION_MAX_DISPATCH_ATTEMPTS`` becomes unreachable and a permanently+# recurring failure (not just a brief broker blip) re-dispatches every flush tick with+# no termination condition. Small by design: a genuine blip clears well inside this,+# and anything still failing after it is treated as repetitive and allowed to age+# toward the cap.+NOTIFICATION_TRANSIENT_REFUND_LIMIT = 3+def _load_execution(execution_id: str | None) -> WorkflowExecution | None:"""Best-effort lookup; returns None on missing id or unknown row."""@@ -513,6 +527,38 @@ def _reclaim_stale_sending() -> int:return int(reclaimed)+def _org_identifier(org_pk: int) -> str | None:+ """Resolve the string ``Organization.organization_id`` from the buffer's org pk.++ ``resolve_transport`` keys its Flipt decision on the org's string identifier,+ but the buffer stores/uses the Organization pk. One indexed pk lookup per+ dispatch group (post-commit) — negligible relative to the downstream webhook+ dispatch.++ Data-anomaly guard: ``NotificationBuffer.organization`` is+ ``on_delete=CASCADE``, so a live buffer row with a missing org is unreachable+ in normal operation. A ``None`` here therefore signals a dangling FK — we log+ it (the only org-traceable breadcrumb; resolve_transport's own warning is keyed+ on the random dispatch uuid) and fail closed to Celery in resolve_transport.+ """+ org_string_id = (+ Organization.objects.filter(pk=org_pk)+ .values_list("organization_id", flat=True)+ .first()+ )+ if org_string_id is None:+ # Sentry-routed (logger.error): a live buffer row with no org is a data+ # anomaly (dangling FK / corruption) that shouldn't happen under the+ # CASCADE constraint, not routine noise. Routing still fails closed to+ # Celery in resolve_transport.+ logger.error(+ "metric=notification_org_identifier_missing_total org_pk=%s "+ "(dangling FK; notification routing falls back to Celery)",+ org_pk,+ )+ return org_string_id++def _send_clubbed(*,url: str,@@ -521,7 +567,7 @@ def _send_clubbed(platform: str,max_retries: int,buffer_ids: list[str],- org_id: Any,+ org_id: int,) -> None:"""Send the clubbed Celery task after the DB transition has committed.@@ -540,8 +586,12 @@ def _send_clubbed(``buffer_row_ids`` + ``organization_id`` to the worker so it can mark them."""try:- celery_app.send_task(- "send_webhook_notification",+ # Flag-gated transport (UN-3753): PG queue when pg_queue_enabled for this+ # org, else Celery (byte-identical to the prior send_task). resolve_transport+ # keys on the org STRING id, but the buffer/worker contract below uses the+ # org pk — hence _org_identifier(org_id) for routing, org_id in kwargs.+ dispatched = dispatch_webhook_notification(+ celery_app=celery_app,args=[url, body, headers, settings.NOTIFICATION_TIMEOUT],kwargs={"max_retries": max_retries,@@ -557,16 +607,61 @@ def _send_clubbed("organization_id": org_id,},queue="notifications",+ org_string_id=_org_identifier(org_id),)+ # transport= makes the rollout answerable from the logs: during a percentage+ # ramp the question is "are PG-routed notifications succeeding at the same+ # rate as Celery-routed ones?", which result=success alone cannot answer.logger.info("metric=notification_batch_dispatched_total platform=%s result=success "- "org_id=%s webhook_url_hash=%s rows=%d",+ "transport=%s org_id=%s webhook_url_hash=%s rows=%d task_id=%s",platform,+ dispatched.transport,org_id,webhook_url_hash(url),len(buffer_ids),+ dispatched.task_id,)+ except PermanentDispatchError:+ # PG-ONLY permanent failure. From this path the only reachable cause is+ # payload JSON-serialization (e.g. a NaN/Infinity float that jsonb rejects+ # at insert): this call passes no reply_key/callback and a default in-range+ # priority, so enqueue_task's other permanent checks can't fire here (the+ # full set lives on PermanentDispatchError's docstring). It fails+ # identically every flush tick — dead-letter now (distinct metric) instead+ # of reverting to PENDING and re-rendering + re-dispatching + emitting a+ # broker_failure traceback until the attempt cap. The Celery path never+ # raises this, so the flag-off flow is unchanged: a Celery send_task failure+ # is an ordinary Exception handled by the transient branch below, exactly as+ # before UN-3753. Guard on SENDING so a row the worker already resolved+ # isn't clobbered.+ logger.exception(+ "metric=notification_batch_dispatched_total platform=%s "+ "result=dispatch_error org_id=%s webhook_url_hash=%s rows=%d",+ platform,+ org_id,+ webhook_url_hash(url),+ len(buffer_ids),+ )+ NotificationBuffer.objects.filter(+ id__in=buffer_ids,+ status=BufferStatus.SENDING.value,+ ).update(status=BufferStatus.DEAD_LETTER.value)except Exception:+ # TRANSIENT transport/broker failure — revert to PENDING (outside the+ # committed txn) so the next flush tick retries. Guard on SENDING so a row+ # the worker already marked terminal (broker raised post-delivery) isn't+ # resurrected into a duplicate.+ #+ # The refund is BOUNDED, not unconditional: refunding the SENDING claim on+ # every cycle makes net progress zero, so `NOTIFICATION_MAX_DISPATCH_ATTEMPTS`+ # can never be reached and a permanently-recurring failure re-renders and+ # re-dispatches forever (emitting a traceback each tick). Refunding is still+ # correct for a genuinely transient blip — nothing was queued or sent — so we+ # keep it while attempts are low and let the cap take over once a failure has+ # proven itself repetitive. This matters more now that this branch also+ # catches non-permanent PG enqueue failures (e.g. a DataError that will recur+ # identically), not just "RabbitMQ is briefly down".logger.exception("metric=notification_batch_dispatched_total platform=%s ""result=broker_failure org_id=%s webhook_url_hash=%s rows=%d",@@ -575,21 +670,24 @@ def _send_clubbed(webhook_url_hash(url),len(buffer_ids),)- # Revert to PENDING (outside the committed txn) so a transient broker- # outage retries next tick; refund the SENDING-claim attempt since nothing- # was queued or sent. Guard on SENDING so a row the worker already marked- # terminal (broker raised post-delivery) isn't resurrected into a duplicate.- NotificationBuffer.objects.filter(+ reverted = NotificationBuffer.objects.filter(id__in=buffer_ids,status=BufferStatus.SENDING.value,- ).update(+ )+ # Refund only the early attempts; past the limit the claim stands so the+ # counter grows and the cap can eventually dead-letter the group.+ reverted.filter(+ dispatch_attempts__lte=NOTIFICATION_TRANSIENT_REFUND_LIMIT+ ).update(dispatch_attempts=F("dispatch_attempts") - 1)+ # Every claimed row goes back to PENDING for the next flush tick, refunded+ # or not.+ reverted.update(status=BufferStatus.PENDING.value,dispatched_at=None,- dispatch_attempts=F("dispatch_attempts") - 1,)-def _penalize_render_failure(buffer_ids: list[str], org_id: Any, platform: str) -> None:+def _penalize_render_failure(buffer_ids: list[str], org_id: int, platform: str) -> None:"""Charge a dispatch attempt to a group whose payloads failed to render.The SENDING-claim increment never runs on a render failure, so count it here@@ -609,7 +707,7 @@ def _penalize_render_failure(buffer_ids: list[str], org_id: Any, platform: str)def _dispatch_group(- org_id: Any,+ org_id: int,webhook_url: str,auth_sig: str,platform: str,diff --git a/backend/notification_v2/notification_dispatch.py b/backend/notification_v2/notification_dispatch.pynew file mode 100644index 000000000..4b027182e--- /dev/null+++ b/backend/notification_v2/notification_dispatch.py@@ -0,0 +1,180 @@+"""Transport-routed dispatch for buffered webhook notifications (UN-3753).++Routes the ``send_webhook_notification`` task through the same+:func:`resolve_transport` flag as the execution path: the PG queue when+``pg_queue_enabled`` for this org, else Celery. **Fail-closed** — with the gate+off (the production default) it resolves to Celery, behaving exactly like the+prior unconditional ``celery_app.send_task`` (zero regression).++DEPLOYMENT PREREQUISITE (PG path): on PG the task lands on the ``notifications``+queue and requires a pg-queue consumer configured with+``WORKER_PG_QUEUE_CONSUMER_WORKER_TYPE=notification`` /+``WORKER_PG_QUEUE_CONSUMER_QUEUE=notifications``. **No such service exists in this+repo's compose yet** — none of the ``pg-queue-consumer`` services in+``docker/docker-compose.yaml`` polls ``notifications``, and ``run-worker.sh``'s+``PG_CONSUMER_ROLES`` has no notification role. Until one is deployed the flag must+stay off for the org: enqueued rows would sit undrained (no TTL sweep covers+``pg_queue_message``) and every buffered webhook for that org would be lost.++``args`` and ``queue`` are forwarded verbatim on both paths. ``kwargs`` differ in+exactly two keys, forced on the PG branch only (see the inline note at the branch):+``raise_on_final_failure`` → ``False`` and ``max_retries`` → ``0``. On PG the+payload is additionally JSON-normalized by ``enqueue_task`` (UUIDs/datetimes → str).+"""++from __future__ import annotations++import logging+import uuid+from typing import Any, NamedTuple++from pg_queue.producer import enqueue_task+from workflow_manager.workflow_v2.transport import resolve_transport++from unstract.core.data_models import is_pg_transport++logger = logging.getLogger(__name__)++# The fired task name — mirrors the Celery task registered by the notification+# worker; kept as a local constant so the backend doesn't import the workers pkg.+WEBHOOK_NOTIFICATION_TASK = "send_webhook_notification"+++# Transport labels for the dispatch metric. During a percentage ramp the one thing+# you need from the logs is "are PG-routed notifications succeeding at the same rate+# as Celery-routed ones?" — a bare task id can't answer that, since a minted PG id+# and a Celery AsyncResult id are indistinguishable to the caller.+PG_TRANSPORT = "pg_queue"+CELERY_TRANSPORT = "celery"+++class DispatchResult(NamedTuple):+ """Which transport actually took the dispatch, plus the resulting task id."""++ transport: str+ task_id: str+++class PermanentDispatchError(Exception):+ """A dispatch failure that would fail identically on every retry.++ Raised ONLY on the PG path, when ``enqueue_task`` rejects the message for a+ permanent reason (priority range / reply_key+callback exclusivity validation,+ or a payload that can't be JSON-serialized). The Celery path never raises it,+ so a caller can dead-letter on this exception without altering the flag-off+ (Celery) error flow — a Celery ``send_task`` failure stays an ordinary+ ``Exception`` the caller's transient handler owns, exactly as before.+ """+++def dispatch_webhook_notification(+ *,+ celery_app: Any,+ args: list[Any],+ kwargs: dict[str, Any],+ queue: str,+ org_string_id: str | None,+) -> DispatchResult:+ """Dispatch ``send_webhook_notification`` on the resolved transport.++ ``args``/``queue`` are forwarded unchanged on both paths, and on the Celery+ branch ``kwargs`` too — so the flag-off path is byte-identical to the legacy+ ``send_task`` call. The PG branch overrides two retry-semantics kwargs (see+ below); nothing else differs.++ Args:+ celery_app: Injected Celery app (the backend's ``celery_service.app``);+ passed in rather than imported so this seam stays trivially testable.+ args: Positional task args, forwarded verbatim.+ kwargs: Keyword task args. Forwarded as-is on Celery; on PG,+ ``raise_on_final_failure`` is overridden to ``False`` and+ ``max_retries`` to ``0`` (the in-task retry loop is a no-op under the+ consumer's eager ``apply()`` — see the inline note). For the buffered+ path this+ carries ``organization_id`` = the buffer's org **pk** (the worker's+ buffer-mark contract) — deliberately a DIFFERENT identifier from the+ ``org_string_id`` param below (the two must not be conflated).+ queue: Target queue name, forwarded verbatim.+ org_string_id: The org's **string** identifier+ (``Organization.organization_id``), used solely for the Flipt+ transport decision — NOT the org pk carried in ``kwargs``. ``None`` (or+ empty) fails closed to Celery.++ Returns:+ A :class:`DispatchResult` carrying which transport took the dispatch and+ the resulting task id (the Celery ``AsyncResult`` id, or the minted PG task+ id). The transport is what makes the caller's dispatch metric answerable+ during a ramp — the two id flavours are otherwise indistinguishable.+ """+ # A buffered notification is a single fire-and-forget task with no natural+ # sticky entity, so mint a fresh id to drive Flipt's percentage bucketing and+ # to serve as the PG task id.+ dispatch_id = str(uuid.uuid4())+ # resolve_transport already normalizes falsy input to Celery, so pass the id+ # straight through (no `or None` needed).+ transport = resolve_transport(+ execution_id=dispatch_id,+ organization_id=org_string_id,+ )+ # Use the shared is_pg_transport() — the single source for "what counts as PG+ # transport" — rather than opening a second comparison site.+ if is_pg_transport(transport):+ # Two PG-only kwarg overrides, both about the SAME thing: the consumer runs+ # the task eagerly via ``task.apply(..., throw=True)``, where Celery's+ # in-task retry loop does not work and its terminal branch is what we need.+ #+ # 1. ``max_retries`` → 0. Under ``apply()`` ``self.request.retries`` is ALWAYS+ # 0, so the worker's ``if self.request.retries < max_retries`` guard+ # (workers/notification/tasks.py) is true on the FIRST failure for any+ # max_retries >= 1 and calls ``self.retry(...)``, which raises ``Retry``;+ # with ``throw=True`` that propagates straight out of ``apply()`` and the+ # terminal branch is never reached — so the buffers are never marked+ # DEAD_LETTER and ``raise_on_final_failure`` is never even read. The+ # consumer's ``except Exception`` then leaves the row for vt-expiry+ # redelivery, re-POSTing the subscriber every time. Forcing 0 sends the+ # task down the terminal branch on the first failure instead.+ # 2. ``raise_on_final_failure`` → False. On BOTH transports the worker marks+ # the buffers DEAD_LETTER over the internal API *before* it re-raises, so+ # the re-raise is only a FAILURE-state signal for Celery monitoring (no+ # redelivery there). On the PG consumer that same raise is treated as a+ # failure and leaves the row for redelivery, so it must not raise.+ #+ # Together: one POST, buffers dead-lettered, task returns None → the consumer+ # acks (deletes) the row. Retry spacing belongs to the PG layer, not the task.+ # (The Celery branch below keeps kwargs verbatim — byte-identical.)+ pg_kwargs = {**kwargs, "raise_on_final_failure": False, "max_retries": 0}+ try:+ msg_id = enqueue_task(+ task_name=WEBHOOK_NOTIFICATION_TASK,+ queue=queue,+ args=args,+ kwargs=pg_kwargs,+ # enqueue_task types org_id as str; None→"" here satisfies that+ # type only, not runtime — enqueue_task itself re-coerces+ # ``org_id or ""`` at insert (producer.py), so this has no runtime+ # effect.+ org_id=org_string_id or "",+ task_id=dispatch_id,+ )+ except (ValueError, TypeError) as exc:+ # PG-only permanent failure (enqueue_task validation / JSON encode):+ # re-raise as PermanentDispatchError so the caller dead-letters it.+ # A transient PG error (DB down) is NOT wrapped — it propagates as an+ # ordinary Exception into the caller's retry (revert-to-PENDING) path.+ raise PermanentDispatchError(str(exc)) from exc+ # msg_id correlates this line with the producer's own ``msg_id=`` log, so a+ # dropped notification can be traced across the seam/producer boundary.+ logger.info(+ "Webhook notification enqueued on PG '%s' queue (task_id=%s msg_id=%s)",+ queue,+ dispatch_id,+ msg_id,+ )+ return DispatchResult(transport=PG_TRANSPORT, task_id=dispatch_id)+ result = celery_app.send_task(+ WEBHOOK_NOTIFICATION_TASK,+ args=args,+ kwargs=kwargs,+ queue=queue,+ )+ return DispatchResult(transport=CELERY_TRANSPORT, task_id=result.id)diff --git a/backend/notification_v2/tests/__init__.py b/backend/notification_v2/tests/__init__.pynew file mode 100644index 000000000..e69de29bbdiff --git a/backend/notification_v2/tests/test_notification_dispatch.py b/backend/notification_v2/tests/test_notification_dispatch.pynew file mode 100644index 000000000..e08bf55ff--- /dev/null+++ b/backend/notification_v2/tests/test_notification_dispatch.py@@ -0,0 +1,192 @@+"""Unit tests for the buffered-webhook transport routing (UN-3753).++``resolve_transport`` + ``enqueue_task`` are patched on the module, so no Flipt /+DB is needed — these pin the routing contract: PG when the flag resolves PG,+Celery otherwise (fail-closed), with identical args/queue on both paths. The+kwargs are identical EXCEPT two retry-semantics keys forced on the PG branch+(``raise_on_final_failure`` -> False, ``max_retries`` -> 0), because the consumer+runs tasks eagerly via ``apply()`` where the in-task retry loop cannot work — see+``test_pg_forces_terminal_branch_kwargs``.+"""++from __future__ import annotations++from unittest.mock import MagicMock, patch++import pytest++import notification_v2.notification_dispatch as nd++_ARGS = ["https://hook.test", {"text": "hi"}, {"Content-Type": "application/json"}, 30]+_KWARGS = {+ "max_retries": 3,+ "retry_delay": 10,+ "platform": "SLACK",+ "raise_on_final_failure": True,+ "buffer_row_ids": ["b1", "b2"],+ "organization_id": 7, # the org pk (worker's buffer-mark contract)+}+_QUEUE = "notifications"+++def _dispatch(celery_app, org_string_id="org_x"):+ return nd.dispatch_webhook_notification(+ celery_app=celery_app,+ args=_ARGS,+ kwargs=_KWARGS,+ queue=_QUEUE,+ org_string_id=org_string_id,+ )+++class TestDispatchWebhookNotification:+ def test_routes_to_pg_when_flag_resolves_pg(self):+ celery = MagicMock()+ with (+ patch.object(nd, "resolve_transport", return_value="pg_queue"),+ patch.object(nd, "enqueue_task", return_value=42) as enqueue,+ ):+ result = _dispatch(celery)+ task_id = result.task_id+ assert result.transport == nd.PG_TRANSPORT # rollout metric can tell them apart+ enqueue.assert_called_once()+ kwargs = enqueue.call_args.kwargs+ assert kwargs["task_name"] == "send_webhook_notification"+ assert kwargs["queue"] == _QUEUE+ assert kwargs["args"] == _ARGS+ # Every kwarg is forwarded verbatim EXCEPT the two retry-semantics keys the+ # PG branch forces (see test_pg_forces_terminal_branch_kwargs).+ assert kwargs["kwargs"] == {+ **_KWARGS,+ "raise_on_final_failure": False,+ "max_retries": 0,+ }+ assert kwargs["org_id"] == "org_x"+ # The minted PG task id is returned and threaded into the enqueue row.+ assert kwargs["task_id"] == task_id+ celery.send_task.assert_not_called()++ def test_routes_to_celery_when_flag_resolves_celery(self):+ celery = MagicMock()+ with (+ patch.object(nd, "resolve_transport", return_value="celery"),+ patch.object(nd, "enqueue_task") as enqueue,+ ):+ result = _dispatch(celery)+ celery.send_task.assert_called_once_with(+ "send_webhook_notification", args=_ARGS, kwargs=_KWARGS, queue=_QUEUE+ )+ enqueue.assert_not_called()+ assert result.task_id is celery.send_task.return_value.id+ assert result.transport == nd.CELERY_TRANSPORT++ def test_pg_enqueue_failure_propagates_with_no_celery_fallback(self):+ # No silent Celery fallback on a TRANSIENT PG failure — it propagates raw+ # (not wrapped), so the caller (_send_clubbed) reverts rows to PENDING.+ celery = MagicMock()+ with (+ patch.object(nd, "resolve_transport", return_value="pg_queue"),+ patch.object(nd, "enqueue_task", side_effect=RuntimeError("pg down")),+ ):+ with pytest.raises(RuntimeError, match="pg down"):+ _dispatch(celery)+ celery.send_task.assert_not_called()++ def test_pg_permanent_enqueue_error_is_wrapped(self):+ # A PERMANENT enqueue error (ValueError/TypeError: validation / JSON encode)+ # is re-raised as PermanentDispatchError so the caller dead-letters it —+ # raised ONLY on the PG path, so the Celery flow is never affected.+ celery = MagicMock()+ for exc in (ValueError("priority out of range"), TypeError("not serializable")):+ with (+ patch.object(nd, "resolve_transport", return_value="pg_queue"),+ patch.object(nd, "enqueue_task", side_effect=exc),+ ):+ with pytest.raises(nd.PermanentDispatchError):+ _dispatch(celery)+ celery.send_task.assert_not_called()++ def test_none_org_fails_closed_to_celery(self):+ # A missing org string (org deleted) must not route to PG — resolve_transport+ # fails closed, and we pass organization_id=None straight through to it.+ celery = MagicMock()+ with (+ patch.object(nd, "resolve_transport", return_value="celery") as resolve,+ patch.object(nd, "enqueue_task") as enqueue,+ ):+ _dispatch(celery, org_string_id=None)+ assert resolve.call_args.kwargs["organization_id"] is None+ celery.send_task.assert_called_once()+ enqueue.assert_not_called()++ def test_resolve_transport_buckets_by_minted_dispatch_id(self):+ # Fire-and-forget: entity_id is a freshly minted uuid (str), and it equals+ # the PG task_id so the row and the Flipt bucket agree.+ celery = MagicMock()+ with (+ patch.object(nd, "resolve_transport", return_value="pg_queue") as resolve,+ patch.object(nd, "enqueue_task", return_value=1) as enqueue,+ ):+ task_id = _dispatch(celery).task_id+ assert resolve.call_args.kwargs["execution_id"] == task_id+ assert enqueue.call_args.kwargs["task_id"] == task_id++ def test_args_and_queue_identical_on_both_paths(self):+ # The consumer must behave the same regardless of transport — args/queue are+ # byte-identical, and kwargs match apart from the transport-specific+ # raise_on_final_failure override (asserted separately below).+ celery = MagicMock()+ with patch.object(nd, "resolve_transport", return_value="celery"):+ _dispatch(celery)+ celery_call = celery.send_task.call_args.kwargs+ celery2 = MagicMock()+ with (+ patch.object(nd, "resolve_transport", return_value="pg_queue"),+ patch.object(nd, "enqueue_task", return_value=1) as enqueue,+ ):+ _dispatch(celery2)+ assert enqueue.call_args.kwargs["args"] == celery_call["args"]+ assert enqueue.call_args.kwargs["queue"] == celery_call["queue"]+ # kwargs differ ONLY by the two PG-forced retry-semantics keys.+ pg_kwargs = enqueue.call_args.kwargs["kwargs"]+ assert pg_kwargs == {+ **celery_call["kwargs"],+ "raise_on_final_failure": False,+ "max_retries": 0,+ }++ def test_pg_forces_terminal_branch_kwargs(self):+ # Regression (UN-3753): the PG consumer runs the task eagerly via+ # ``task.apply(..., throw=True)``, where the in-task retry loop cannot work:+ # * max_retries >= 1 -> the worker's ``request.retries < max_retries`` guard+ # is true on the FIRST failure (retries is always 0 under apply()), so it+ # raises Retry, which propagates out of apply() -- the terminal branch+ # (mark DEAD_LETTER + honour raise_on_final_failure) never runs and the row+ # is left for vt-expiry redelivery, re-POSTing the subscriber each time.+ # * a terminal re-raise is likewise treated as failure -> redelivery.+ # So the seam must force BOTH max_retries=0 and raise_on_final_failure=False on+ # the PG branch. The Celery branch keeps the caller's values verbatim.+ # (The end-to-end behaviour these kwargs buy — one POST + one dead-letter mark+ # through a real ``task.apply()`` — is asserted in+ # workers/tests/test_notification_pg_terminal.py.)+ assert _KWARGS["raise_on_final_failure"] is True # guard the fixture premise+ assert _KWARGS["max_retries"] == 3 # ... and that retries are configured+ celery = MagicMock()+ with (+ patch.object(nd, "resolve_transport", return_value="pg_queue"),+ patch.object(nd, "enqueue_task", return_value=1) as enqueue,+ ):+ _dispatch(celery)+ pg_kwargs = enqueue.call_args.kwargs["kwargs"]+ assert pg_kwargs["raise_on_final_failure"] is False+ assert pg_kwargs["max_retries"] == 0+ # The caller's dict is not mutated in place (a fresh dict is enqueued).+ assert _KWARGS["raise_on_final_failure"] is True+ assert _KWARGS["max_retries"] == 3++ celery2 = MagicMock()+ with patch.object(nd, "resolve_transport", return_value="celery"):+ _dispatch(celery2)+ sent_kwargs = celery2.send_task.call_args.kwargs["kwargs"]+ assert sent_kwargs["raise_on_final_failure"] is True+ assert sent_kwargs["max_retries"] == 3diff --git a/backend/notification_v2/tests/test_send_clubbed.py b/backend/notification_v2/tests/test_send_clubbed.pynew file mode 100644index 000000000..b296edfdf--- /dev/null+++ b/backend/notification_v2/tests/test_send_clubbed.py@@ -0,0 +1,147 @@+"""Call-site tests for ``_send_clubbed`` / ``_org_identifier`` (UN-3753).++The seam's own routing is covered by ``test_notification_dispatch``; these lock+the two highest-risk regressions at the buffer-flush call site:++1. the two-org-identifier contract — the org **string** id routes the flag, while+ the org **pk** stays in the worker kwargs (swapping them passes every seam test+ yet mis-routes every org to Celery and strands the mark endpoint);+2. failure recovery — a transient/broker error refunds and reverts SENDING rows to+ PENDING, while a permanent PG enqueue error (surfaced as ``PermanentDispatchError``,+ raised only on the PG path) dead-letters instead of retrying forever. A Celery+ send_task failure is an ordinary ``Exception`` → the PENDING path, so flag-off is+ byte-identical.++Mock-based (no broker/DB): patch the module's collaborators.+"""++from __future__ import annotations++from unittest.mock import patch++from django.conf import settings+from notification_v2 import internal_api_views as views+from notification_v2.enums import BufferStatus++_URL = "https://hook.test"+_BODY = {"text": "x"}+_HEADERS = {"Content-Type": "application/json"}+_BUFFER_IDS = ["b1"]+++def _send(**overrides):+ kw = {+ "url": _URL,+ "body": _BODY,+ "headers": _HEADERS,+ "platform": "SLACK",+ "max_retries": 3,+ "buffer_ids": _BUFFER_IDS,+ "org_id": 7,+ }+ kw.update(overrides)+ views._send_clubbed(**kw)+++class TestSendClubbedOrgContract:+ def test_routes_string_id_and_keeps_pk_in_kwargs(self):+ with (+ patch.object(views, "_org_identifier", return_value="org-str-id") as ident,+ patch.object(views, "dispatch_webhook_notification") as seam,+ ):+ _send(org_id=7)+ ident.assert_called_once_with(7)+ call = seam.call_args.kwargs+ # Routing uses the STRING id; the worker buffer-mark contract keeps the PK.+ assert call["org_string_id"] == "org-str-id"+ assert call["kwargs"]["organization_id"] == 7+ assert call["queue"] == "notifications"+ # args carry the full webhook invocation, in order.+ assert call["args"] == [_URL, _BODY, _HEADERS, settings.NOTIFICATION_TIMEOUT]+ # buffer_row_ids is what REPLACED the Celery link/link_error callbacks: it is+ # how the worker knows which rows to mark DISPATCHED / DEAD_LETTER. Drop it+ # and every row strands in SENDING until the reclaim lease expires — silent,+ # visible only as a growing SENDING backlog.+ assert call["kwargs"]["buffer_row_ids"] == _BUFFER_IDS+++class TestSendClubbedFailureRecovery:+ def test_transient_failure_reverts_sending_rows_to_pending(self):+ with (+ patch.object(views, "_org_identifier", return_value="o"),+ patch.object(+ views,+ "dispatch_webhook_notification",+ side_effect=RuntimeError("broker down"),+ ),+ patch.object(views, "NotificationBuffer") as buf,+ ):+ _send(buffer_ids=["b1", "b2"])+ # Guarded on SENDING, reverted to PENDING.+ fkw = buf.objects.filter.call_args.kwargs+ assert fkw["status"] == BufferStatus.SENDING.value+ assert set(fkw["id__in"]) == {"b1", "b2"}+ # Every claimed row goes back to PENDING for the next tick.+ ukw = buf.objects.filter.return_value.update.call_args.kwargs+ assert ukw == {+ "status": BufferStatus.PENDING.value,+ "dispatched_at": None,+ }+ # The refund must be ASSERTED, not merely described in a comment: without it+ # a transient outage burns no attempt (claim +1, refund -1 = net zero), so+ # NOTIFICATION_MAX_DISPATCH_ATTEMPTS is unreachable and a permanently+ # recurring failure re-dispatches every flush tick forever. It is also+ # BOUNDED — only rows still under the limit are refunded, so a failure that+ # keeps recurring eventually ages into the cap.+ refund_qs = buf.objects.filter.return_value.filter+ assert (+ refund_qs.call_args.kwargs["dispatch_attempts__lte"]+ == views.NOTIFICATION_TRANSIENT_REFUND_LIMIT+ )+ refund_kw = refund_qs.return_value.update.call_args.kwargs+ assert set(refund_kw) == {"dispatch_attempts"}+ assert "dispatch_attempts" in str(refund_kw["dispatch_attempts"]) # F(...) - 1++ def test_permanent_pg_error_dead_letters(self):+ # The PG path surfaces a permanent enqueue failure as PermanentDispatchError;+ # only that dead-letters. (A raw ValueError from the Celery path can't occur,+ # and would fall through to the transient PENDING branch, unchanged.)+ with (+ patch.object(views, "_org_identifier", return_value="o"),+ patch.object(+ views,+ "dispatch_webhook_notification",+ side_effect=views.PermanentDispatchError("priority out of range"),+ ),+ patch.object(views, "NotificationBuffer") as buf,+ ):+ _send()+ # Permanent error → terminal DEAD_LETTER, no PENDING revert / refund.+ # The dead-letter update is guarded to only touch rows still SENDING+ # (same clobber-guard the transient-revert path asserts).+ fkw = buf.objects.filter.call_args.kwargs+ assert fkw["status"] == BufferStatus.SENDING.value+ ukw = buf.objects.filter.return_value.update.call_args.kwargs+ assert ukw == {"status": BufferStatus.DEAD_LETTER.value}+++class TestOrgIdentifier:+ def test_returns_string_id(self):+ with patch.object(views, "Organization") as org:+ chain = org.objects.filter.return_value.values_list.return_value+ chain.first.return_value = "org-uuid"+ assert views._org_identifier(7) == "org-uuid"+ org.objects.filter.assert_called_once_with(pk=7)++ def test_missing_org_returns_none_and_logs_error(self):+ with (+ patch.object(views, "Organization") as org,+ patch.object(views.logger, "error") as err,+ ):+ chain = org.objects.filter.return_value.values_list.return_value+ chain.first.return_value = None+ assert views._org_identifier(7) is None+ # A dangling FK is a data anomaly: logged at error (Sentry-routed),+ # org-traceable, rather than swallowed.+ err.assert_called_once()+ assert "org_pk" in err.call_args.args[0]diff --git a/backend/pg_queue/producer.py b/backend/pg_queue/producer.pyindex 3c4585fc4..028fddfb6 100644--- a/backend/pg_queue/producer.py+++ b/backend/pg_queue/producer.py@@ -50,8 +50,15 @@ def _json_safe(value: Any) -> Any:``PgQueueMessage.message`` is a plain ``JSONField`` (no Django encoder), andthe worker consumer already receives string ids on the existing PG dispatchpath, so coercing here keeps both transports consistent.++ ``allow_nan=False`` rejects ``NaN``/``Infinity`` here with a ``ValueError``+ rather than letting Python's default lenient encoder emit the non-standard+ ``NaN``/``Infinity`` tokens: Postgres ``jsonb`` rejects those at insert with a+ ``django.db.DataError`` (a permanent failure the notification dispatcher's+ ``(ValueError, TypeError)`` seam would otherwise miss, looping the row). Fail+ at the intended seam instead."""- return json.loads(json.dumps(value, default=str))+ return json.loads(json.dumps(value, default=str, allow_nan=False))def enqueue_task(@@ -97,33 +104,37 @@ def enqueue_task(f"[{FAIRNESS_MIN_PRIORITY}, {FAIRNESS_MAX_PRIORITY}]: {priority!r}")pg_queue = queue or DEFAULT_GENERAL_QUEUE- message: TaskPayload = {- "task_name": task_name,- "args": _json_safe(list(args) if args is not None else []),- "kwargs": _json_safe(dict(kwargs) if kwargs is not None else {}),- "queue": pg_queue,- # Coerce like args/kwargs/on_success/on_error: FairnessPayload may carry a- # UUID/enum/datetime that a plain JSONField insert can't serialise, which- # would raise at enqueue and drop the task on the PG path only.- "fairness": _json_safe(fairness) if fairness is not None else fairness,- }- # Each optional key is set only when present — keeps fire-and-forget rows- # byte-identical to before these fields existed.- if reply_key is not None:- message["reply_key"] = reply_key- # Continuation specs carry a nested callback ``kwargs`` dict that may hold a- # UUID/datetime — coerce like ``args``/``kwargs`` above, else the JSONField- # insert raises at dispatch time (caller-visible).- if on_success is not None:- message["on_success"] = _json_safe(on_success)- if on_error is not None:- message["on_error"] = _json_safe(on_error)- if task_id is not None:- message["task_id"] = task_id# Mirror the worker _enqueue_pg path: log the failure with breadcrumbs before# it propagates, so a DB/constraint/serialization error isn't mislabeled by- # the caller's broad handler.+ # the caller's broad handler. Message construction (the _json_safe coercions)+ # lives inside the try because serialization failures surface there — e.g. a+ # NaN/Infinity float raises ValueError under allow_nan=False — and on the+ # orchestrator path (no silent fallback) would otherwise drop the task with no+ # task/queue/org breadcrumb for on-call.try:+ message: TaskPayload = {+ "task_name": task_name,+ "args": _json_safe(list(args) if args is not None else []),+ "kwargs": _json_safe(dict(kwargs) if kwargs is not None else {}),+ "queue": pg_queue,+ # Coerce like args/kwargs/on_success/on_error: FairnessPayload may carry+ # a UUID/enum/datetime that a plain JSONField insert can't serialise,+ # which would raise at enqueue and drop the task on the PG path only.+ "fairness": _json_safe(fairness) if fairness is not None else fairness,+ }+ # Each optional key is set only when present — keeps fire-and-forget rows+ # byte-identical to before these fields existed.+ if reply_key is not None:+ message["reply_key"] = reply_key+ # Continuation specs carry a nested callback ``kwargs`` dict that may hold a+ # UUID/datetime — coerce like ``args``/``kwargs`` above, else the JSONField+ # insert raises at dispatch time (caller-visible).+ if on_success is not None:+ message["on_success"] = _json_safe(on_success)+ if on_error is not None:+ message["on_error"] = _json_safe(on_error)+ if task_id is not None:+ message["task_id"] = task_idrow = PgQueueMessage.objects.create(queue_name=pg_queue,message=message,diff --git a/backend/pg_queue/tests/test_producer.py b/backend/pg_queue/tests/test_producer.pyindex 40df8a79f..c94d219f0 100644--- a/backend/pg_queue/tests/test_producer.py+++ b/backend/pg_queue/tests/test_producer.py@@ -5,6 +5,7 @@ the JSON-coercion logic without needing a test database."""import datetime+import loggingimport uuidfrom unittest.mock import MagicMock, patch@@ -106,6 +107,40 @@ class TestEnqueueTask:when = model.objects.create.call_args.kwargs["message"]["kwargs"]["when"]assert isinstance(when, str) and "2026-06-18" in when+ @pytest.mark.parametrize(+ "bad", [float("nan"), float("inf"), float("-inf")], ids=["nan", "inf", "-inf"]+ )+ @pytest.mark.parametrize("slot", ["args", "kwargs", "fairness"])+ def test_json_safe_rejects_non_finite_floats(self, bad, slot, caplog):+ # A non-finite float slips past the default lenient encoder and only fails at+ # the jsonb insert (DataError); allow_nan=False surfaces it as a ValueError at+ # the enqueue seam so the notification dispatcher can dead-letter it.+ #+ # Every _json_safe-coerced slot is covered, not just kwargs: the scenario this+ # guard exists for is a webhook BODY carrying a non-finite float, and+ # notification_dispatch puts the body in args[1]. inf/-inf are rejected by+ # allow_nan=False exactly like nan.+ payloads = {+ "args": {"args": ["url", {"score": bad}]},+ "kwargs": {"kwargs": {"score": bad}},+ "fairness": {"fairness": {"org_id": "o", "weight": bad}},+ }+ with patch(_MODEL) as model:+ with caplog.at_level(logging.ERROR, logger=producer.logger.name):+ with pytest.raises(ValueError):+ producer.enqueue_task(+ task_name="send_webhook_notification",+ queue="notifications",+ org_id="org-1",+ **payloads[slot],+ )+ model.objects.create.assert_not_called() # never reaches the DB insert+ # The breadcrumb is the point of moving coercion inside the try — assert the+ # rendered record actually carries it, not merely that .exception() was hit.+ assert "send_webhook_notification" in caplog.text+ assert "notifications" in caplog.text+ assert "org-1" in caplog.text+def test_enqueue_failure_logs_and_propagates(self):with patch(_MODEL) as model:model.objects.create.side_effect = RuntimeError("db down")diff --git a/workers/notification/tasks.py b/workers/notification/tasks.pyindex 41d76f5cf..f1c780ae0 100644--- a/workers/notification/tasks.py+++ b/workers/notification/tasks.py@@ -252,9 +252,15 @@ def send_webhook_notification(retry_delay: The delay between retries in secondsplatform: Platform type from notification config (SLACK, API, etc.)raise_on_final_failure: When True, re-raise on retry exhaustion so the- task ends in FAILURE and any Celery ``link_error`` callback runs- (used by the clubbed/buffered dispatch to dead-letter the rows).- When False (default), preserve the legacy "return None" behavior.+ task ends in FAILURE (a Celery monitoring signal). NOTE: the buffer+ rows are dead-lettered by ``_mark_buffer_outcome(dispatched=False)``+ over the internal API *before* this re-raise, on BOTH transports —+ ``link``/``link_error`` callbacks are no longer wired (they routed to+ the ``celery`` queue and were dropped as "unregistered task"), so the+ raise itself dead-letters nothing. When False (default), preserve the+ legacy "return None" behavior. The PG dispatch seam forces this False+ (and ``max_retries`` 0) because under the consumer's eager+ ``task.apply()`` a raise means redelivery, not a FAILURE state.Returns:None (matches original behavior)diff --git a/workers/queue_backend/pg_queue/consumer.py b/workers/queue_backend/pg_queue/consumer.pyindex 3e016d228..241a3d291 100644--- a/workers/queue_backend/pg_queue/consumer.py+++ b/workers/queue_backend/pg_queue/consumer.py@@ -204,6 +204,62 @@ def _pipeline_identity(payload: TaskPayload) -> tuple[str | None, str]:)+# Header/kwarg names whose VALUES are secrets. Substring match on a lowercased key,+# so ``Authorization``/``X-Api-Key``/``auth_token``/``client_secret`` all match.+_SECRET_KEY_PARTS = ("authorization", "api-key", "api_key", "apikey", "token", "secret")++# Payload slots that carry customer data rather than routing metadata. Logged as a+# type+size summary so a drop stays debuggable without dumping document content.+_BULKY_ARG_INDEX = 1 # args[1] is the webhook body on the notification payload+++def _redact_secrets(value: Any) -> Any:+ """Recursively mask secret-valued keys in dicts/lists; other values pass through."""+ if isinstance(value, dict):+ return {+ k: (+ "***REDACTED***"+ if isinstance(k, str) and any(p in k.lower() for p in _SECRET_KEY_PARTS)+ else _redact_secrets(v)+ )+ for k, v in value.items()+ }+ if isinstance(value, list):+ return [_redact_secrets(v) for v in value]+ return value+++def _redact_payload(payload: dict) -> dict:+ """A log-safe view of a queue payload.++ Two exposures this closes, both introduced in practice once credential-bearing+ notification payloads started flowing through the PG consumer:++ * **Secrets** — ``build_webhook_headers`` writes the subscriber's bearer token /+ API key into the header dict carried in ``args``/``kwargs``. Masked by key name.+ * **Customer data** — the clubbed webhook body is the pipeline's extracted output.+ Replaced with a ``<type len=N>`` summary rather than dumped.++ Routing metadata (task_name, queue, task_id, read counts) is preserved verbatim,+ since that is what a poison drop is actually debugged from.+ """+ safe = {k: v for k, v in payload.items() if k not in ("args", "kwargs")}+ args = payload.get("args")+ if isinstance(args, list):+ redacted_args: list[Any] = []+ for i, a in enumerate(args):+ if i == _BULKY_ARG_INDEX and isinstance(a, (dict, list, str)):+ redacted_args.append(f"<{type(a).__name__} len={len(a)}>")+ else:+ redacted_args.append(_redact_secrets(a))+ safe["args"] = redacted_args+ elif args is not None:+ safe["args"] = _redact_secrets(args)+ if payload.get("kwargs") is not None:+ safe["kwargs"] = _redact_secrets(payload.get("kwargs"))+ return safe++class PgQueueConsumer:"""Polls one PG queue, runs each claimed task in-process, acks on success."""@@ -505,11 +561,22 @@ class PgQueueConsumer:headers = {FAIRNESS_HEADER_NAME: fairness} if fairness else None# Renew the short lease while the (possibly long) task runs, so a dead# worker's claim expires fast but a live one is never redelivered.+ #+ # ``task_id=`` is load-bearing for idempotency: Celery's ``Task.apply``+ # does ``task_id = task_id or uuid()``, so WITHOUT this the task sees a+ # FRESH ``self.request.id`` on every delivery. Any guard keyed on it+ # (fan-out claim rows, ``generation_task_id`` uniqueness, task-complete+ # markers) would then never collide across a redelivery and dedupe+ # nothing. The producer already persisted a stable id in the payload —+ # pass it through so redelivery re-runs with the SAME request id, matching+ # Celery's own redelivery semantics. Falls back to Celery's uuid() when+ # absent (never None, which apply() would reject).with self._lease_renewal(message.msg_id):eager = task.apply(args=payload.get("args") or [],kwargs=payload.get("kwargs") or {},headers=headers,+ task_id=payload.get("task_id") or None,throw=True,)except Exception as exc:@@ -789,14 +856,23 @@ class PgQueueConsumer:"""execution_id, organization_id = _pipeline_identity(payload)logger.error(- "PG-queue consumer: task %r (msg_id=%s) exceeded max_attempts=%s "- "(read_ct=%s, execution_id=%s) — poison; full payload: %r",+ "PG-queue consumer: poison-dropped task %r "+ "(msg_id=%s, read_ct=%s, queue=%s, execution_id=%s, org_id=%s) — "+ "exceeded max_attempts=%s; payload: %r",task_name,message.msg_id,- self.max_attempts,+ # read_ct counts READS, not executions — a lease expiry with no run still+ # increments it. Logged once, under its own name (an "after N attempts"+ # phrasing duplicated this same value and misread as execution count).message.read_ct,+ payload.get("queue"),execution_id,- payload,+ # NB: identifier flavour is payload-dependent — see _pipeline_identity.+ organization_id or None,+ self.max_attempts,+ # Redacted: payloads carry subscriber credentials (webhook Authorization+ # / API-key headers) and customer data. Never log them raw.+ _redact_payload(payload),)# Failure channel (request-reply / on_error) → surface there, then drop.if payload.get("reply_key") or payload.get("on_error"):diff --git a/workers/shared/enums/worker_enums_base.py b/workers/shared/enums/worker_enums_base.pyindex 9c4dfbdb4..65465659c 100644--- a/workers/shared/enums/worker_enums_base.py+++ b/workers/shared/enums/worker_enums_base.py@@ -58,13 +58,38 @@ class WorkerType(str, Enum):if self.is_pluggable():return f"pluggable_worker.{self.value}.tasks"- # Map to actual directory structure+ return f"{self.to_directory()}.tasks"++ def to_directory(self) -> str:+ """Return the on-disk directory name for this **non-pluggable** worker.++ Single source of truth for the naming-convention mapping: enum values use+ underscores (Python module names), while ``API_DEPLOYMENT``'s on-disk dir+ uses a hyphen (``api-deployment``). ``to_import_path`` builds on this, and+ the file-path task loader in ``worker.py`` reads it directly rather than+ slicing the import path.++ Raises:+ ValueError: if called on a pluggable worker type. Pluggable code lives+ under ``pluggable_worker/<value>``, so the mapping's passthrough+ would return a bare, *plausible-looking* but wrong path. Today only+ call ordering prevents that (``to_import_path`` guards with+ ``is_pluggable()``, and ``worker.py`` early-returns for pluggable+ workers before reaching here) — nothing enforced it. Since+ ``worker.py`` now RAISES on a missing directory instead of logging,+ a future caller that skips the guard would turn a silently-wrong+ path into a boot crash-loop; fail loudly and locally instead.+ """+ if self.is_pluggable():+ raise ValueError(+ f"{self!r} is a pluggable worker; its code lives under "+ f"pluggable_worker/{self.value} — use to_import_path() instead."+ )directory_mapping = {- "api_deployment": "api-deployment",+ WorkerType.API_DEPLOYMENT: "api-deployment",# All others use same name for directory and module}- directory = directory_mapping.get(self.value, self.value)- return f"{directory}.tasks"+ return directory_mapping.get(self, self.value)def is_pluggable(self) -> bool:"""Check if this worker type is a pluggable worker.diff --git a/workers/tests/conftest.py b/workers/tests/conftest.pyindex bd4b090b4..faa90d759 100644--- a/workers/tests/conftest.py+++ b/workers/tests/conftest.py@@ -410,6 +410,16 @@ def _restore_current_celery_app():"""from celery._state import default_app+ # Guard against there being no default app at all: a module that imports no+ # worker app (e.g. a pure-enum test running alone) would otherwise ERROR here on+ # ``None.finalize()``. That previously forced such modules to call+ # ``Celery(...).set_default()`` at import time as a workaround, which mutated+ # process-global state for every later test in the session depending on+ # collection order. Fix it here instead so leaf tests need no Celery app.+ if default_app is None:+ yield+ return+default_app.finalize()default_app.set_current()try:diff --git a/workers/tests/test_notification_pg_terminal.py b/workers/tests/test_notification_pg_terminal.pynew file mode 100644index 000000000..8c3774029--- /dev/null+++ b/workers/tests/test_notification_pg_terminal.py@@ -0,0 +1,110 @@+"""``send_webhook_notification`` under the PG consumer's eager ``task.apply()``.++The PG consumer runs tasks with ``task.apply(..., throw=True)`` rather than through+a Celery worker. Under ``apply()`` Celery's in-task retry loop cannot work —+``self.request.retries`` is ALWAYS 0, so the task's+``if self.request.retries < max_retries`` guard is true on the FIRST failure for any+``max_retries >= 1`` and raises ``Retry``, which ``throw=True`` propagates straight+out of ``apply()``. The terminal branch (mark buffers DEAD_LETTER, honour+``raise_on_final_failure``) is then never reached, so the consumer sees an exception,+leaves the row for vt-expiry redelivery, and the subscriber is re-POSTed every time.++That is why the dispatch seam (``backend/notification_v2/notification_dispatch.py``)+forces BOTH ``max_retries=0`` and ``raise_on_final_failure=False`` on the PG branch.+These tests drive the REAL task through ``apply()`` and assert the behaviour those+kwargs buy — one POST + one DEAD_LETTER mark, no raise — rather than the kwarg values+(which the seam's own unit tests already pin).+"""++from __future__ import annotations++from unittest.mock import patch++import pytest+from celery.exceptions import Retry+from notification.tasks import send_webhook_notification++_URL = "https://hook.test/x"+_BUFFER_IDS = ["b1", "b2"]+_ORG = 7+++class _AlwaysFailingProvider:+ """Webhook provider whose send() always reports failure, counting attempts."""++ def __init__(self) -> None:+ self.posts = 0++ def send(self, notification_data: dict) -> dict:+ self.posts += 1+ return {"success": False, "message": "boom"}+++def _run(*, max_retries: int, raise_on_final_failure: bool):+ """Drive the real task through the consumer's exact call shape.++ Returns ``(provider, marks, raised)`` — POST count, the+ ``_mark_buffer_outcome`` calls, and any exception propagated out of apply().+ """+ provider = _AlwaysFailingProvider()+ marks: list[bool] = []+ with (+ patch(+ "notification.tasks._get_webhook_provider_for_url", return_value=provider+ ),+ patch(+ "notification.tasks._mark_buffer_outcome",+ side_effect=lambda ids, org, *, dispatched: marks.append(dispatched),+ ),+ ):+ raised: BaseException | None = None+ try:+ send_webhook_notification.apply(+ args=[_URL, {"text": "hi"}, {"Content-Type": "application/json"}, 30],+ kwargs={+ "max_retries": max_retries,+ "retry_delay": 10,+ "platform": None,+ "raise_on_final_failure": raise_on_final_failure,+ "buffer_row_ids": _BUFFER_IDS,+ "organization_id": _ORG,+ },+ throw=True,+ )+ except BaseException as exc: # noqa: BLE001 - we assert on the type below+ raised = exc+ return provider, marks, raised+++def test_pg_kwargs_give_one_post_and_one_dead_letter():+ # The seam's PG kwargs (max_retries=0, raise_on_final_failure=False): the task+ # takes the terminal branch on the first failure, marks the buffers DEAD_LETTER,+ # and returns None -> the consumer acks the row. Subscriber hit exactly once.+ provider, marks, raised = _run(max_retries=0, raise_on_final_failure=False)+ assert raised is None # nothing propagates -> consumer acks (no redelivery)+ assert provider.posts == 1 # subscriber POSTed once, not once per redelivery+ assert marks == [False] # exactly one DEAD_LETTER mark+++def test_unforced_max_retries_would_escape_apply_without_dead_lettering():+ # Regression guard for the bug the override fixes: with the caller's original+ # max_retries >= 1, apply() raises Retry out of the task, the terminal branch+ # never runs, and NO buffer mark happens -> the consumer would redeliver and+ # re-POST. This is what the seam prevents by forcing max_retries=0.+ provider, marks, raised = _run(max_retries=3, raise_on_final_failure=False)+ assert isinstance(raised, Retry) # escapes apply() as an exception+ assert provider.posts == 1 # ... after a single POST on this delivery+ assert marks == [] # buffers NEVER dead-lettered -> rows strand/redeliver++[email protected]("raise_on_final", [True, False])+def test_max_retries_zero_always_reaches_terminal_branch(raise_on_final):+ # With max_retries=0 the terminal branch always runs, so the buffers are marked+ # DEAD_LETTER regardless of raise_on_final_failure; the flag only decides whether+ # the task additionally re-raises (a Celery FAILURE-state signal).+ provider, marks, raised = _run(+ max_retries=0, raise_on_final_failure=raise_on_final+ )+ assert provider.posts == 1+ assert marks == [False] # dead-lettered on BOTH transports, before any re-raise+ assert (raised is not None) is raise_on_finaldiff --git a/workers/tests/test_pg_queue_consumer.py b/workers/tests/test_pg_queue_consumer.pyindex 919e94e10..3e5d836dc 100644--- a/workers/tests/test_pg_queue_consumer.py+++ b/workers/tests/test_pg_queue_consumer.py@@ -133,6 +133,33 @@ class TestPollOnce:assert kwargs["kwargs"] == {"k": "v"}assert kwargs["headers"] == {FAIRNESS_HEADER_NAME: fairness}+ def test_payload_task_id_is_passed_to_apply(self):+ # Idempotency-critical: Celery's Task.apply does ``task_id = task_id or+ # uuid()``, so without an explicit task_id the task sees a FRESH request.id+ # on every delivery and any guard keyed on it (fan-out claims,+ # generation_task_id uniqueness, task-complete markers) dedupes nothing+ # across a redelivery. The producer's stable id must reach apply().+ task = MagicMock()+ app = MagicMock()+ app.tasks.get.return_value = task+ client = MagicMock()+ client.read.return_value = [+ _msg(6, {"task_name": "t", "args": [], "kwargs": {}, "task_id": "stable-1"})+ ]+ PgQueueConsumer(["q"], client=client, app=app).poll_once()+ assert task.apply.call_args.kwargs["task_id"] == "stable-1"++ def test_missing_task_id_falls_back_to_celery_uuid(self):+ # No task_id in the payload -> pass None so Celery mints one, rather than+ # passing "" (which apply() would take literally as the request id).+ task = MagicMock()+ app = MagicMock()+ app.tasks.get.return_value = task+ client = MagicMock()+ client.read.return_value = [_msg(6, {"task_name": "t", "args": [], "kwargs": {}})]+ PgQueueConsumer(["q"], client=client, app=app).poll_once()+ assert task.apply.call_args.kwargs["task_id"] is None+def test_ack_finding_no_row_warns(self, caplog):client = MagicMock()client.delete.return_value = False # row already gone (vt expired mid-run)@@ -272,6 +299,67 @@ class TestPoisonDropMarksExecution:client.delete.assert_called_once_with(5) # then droppedclient.set_vt.assert_not_called()+ def test_poison_log_enriched_with_org_and_read_ct(self, monkeypatch, caplog):+ # Q2 debuggability: the poison drop logs a clear "poison-dropped" message+ # carrying read_ct + org_id + queue so a dropped task is greppable/+ # Sentry-visible and its source queue is identifiable at a glance.+ monkeypatch.setattr(+ "queue_backend.pg_queue.recovery.mark_execution_error",+ lambda *a, **k: True,+ )+ client = MagicMock()+ payload = {**_callback_payload(), "queue": "agentic_callback"}+ client.read.return_value = [self._poison(payload=payload, read_ct=6)]+ with caplog.at_level(logging.ERROR, logger="queue_backend.pg_queue.consumer"):+ PgQueueConsumer(+ ["q"], client=client, api_client=MagicMock(), max_attempts=5+ ).poll_once()+ assert "poison-dropped" in caplog.text+ assert "read_ct=6" in caplog.text+ assert "org_id=org-1" in caplog.text # org surfaced for the dropped task+ assert "queue=agentic_callback" in caplog.text # source queue surfaced+ # read_ct is rendered ONCE (an earlier "after N attempts" phrasing printed+ # the same value twice under a name that misread as an execution count).+ assert caplog.text.count("6") >= 1+ assert "attempts (msg_id" not in caplog.text++ def test_poison_log_redacts_credentials_and_body(self, monkeypatch, caplog):+ # Security: notification payloads carry the subscriber's Authorization /+ # API-key headers and the clubbed webhook body (customer extracted data).+ # A poison drop must not dump either to stdout/Sentry.+ monkeypatch.setattr(+ "queue_backend.pg_queue.recovery.mark_execution_error",+ lambda *a, **k: True,+ )+ payload = {+ "task_name": "send_webhook_notification",+ "queue": "notifications",+ "args": [+ "https://hook.test",+ {"secret_field": "customer extracted data"},+ {"Authorization": "Bearer super-secret", "X-Api-Key": "ak_live_123"},+ 30,+ ],+ "kwargs": {"organization_id": "org-1", "auth_token": "tok_abc"},+ }+ client = MagicMock()+ client.read.return_value = [self._poison(payload=payload, read_ct=6)]+ with caplog.at_level(logging.ERROR, logger="queue_backend.pg_queue.consumer"):+ PgQueueConsumer(+ ["q"], client=client, api_client=MagicMock(), max_attempts=5+ ).poll_once()+ # No secret value survives, by any route.+ assert "super-secret" not in caplog.text+ assert "ak_live_123" not in caplog.text+ assert "tok_abc" not in caplog.text+ # Customer body replaced by a type+size summary, not dumped.+ assert "customer extracted data" not in caplog.text+ assert "<dict len=1>" in caplog.text+ # Routing metadata IS preserved — that's what the drop is debugged from.+ assert "send_webhook_notification" in caplog.text+ assert "https://hook.test" in caplog.text+ assert "REDACTED" in caplog.text+def test_positional_orchestration_poison_marks_error(self, monkeypatch):# H2 regression: a poisoned async_execute_bin carries execution_id# POSITIONALLY (args[2]) with no _barrier_context and — since its poison@@ -978,7 +1066,8 @@ class TestRecordTaskStatus:pg_task_result so the REST PromptStudio.task_status poll resolves under PG.completed unless the run raised (error) or the executor reported success=False(completed rows are status-only; failed rows carry the executor error text). TTL'd- + best-effort — never wedges the ack."""+ + best-effort — never wedges the ack.+ """_RB = "queue_backend.pg_queue.consumer.PgResultBackend"_RET = 86400@@ -1104,6 +1193,34 @@ class TestLeaseRenewal:c = mod.build_consumer_from_env()assert c.lease_seconds == 77+ def test_env_wires_max_attempts(self, monkeypatch):+ # Per-worker override: an execution worker keeps the default (at-least-once),+ # the interactive orchestrator sets 1 (at-most-once, no LLM re-compute).+ from queue_backend.pg_queue import consumer as mod++ monkeypatch.setenv("WORKER_PG_QUEUE_CONSUMER_MAX_ATTEMPTS", "1")+ with patch.object(mod, "PgQueueClient"): # no real DB connection+ c = mod.build_consumer_from_env()+ assert c.max_attempts == 1++ def test_max_attempts_defaults_to_5_when_unset(self, monkeypatch):+ from queue_backend.pg_queue import consumer as mod+ from queue_backend.pg_queue.consumer import _DEFAULT_MAX_ATTEMPTS++ monkeypatch.delenv("WORKER_PG_QUEUE_CONSUMER_MAX_ATTEMPTS", raising=False)+ with patch.object(mod, "PgQueueClient"): # no real DB connection+ c = mod.build_consumer_from_env()+ # The code default stays 5 so execution workers remain at-least-once.+ assert c.max_attempts == _DEFAULT_MAX_ATTEMPTS == 5++ def test_env_invalid_max_attempts_raises_named_error(self, monkeypatch):+ from queue_backend.pg_queue import consumer as mod++ monkeypatch.setenv("WORKER_PG_QUEUE_CONSUMER_MAX_ATTEMPTS", "abc")+ with patch.object(mod, "PgQueueClient"):+ with pytest.raises(ValueError, match="WORKER_PG_QUEUE_CONSUMER_MAX_ATTEMPTS"):+ mod.build_consumer_from_env()+def test_poll_claims_with_lease_not_vt(self):client = MagicMock()client.read.return_value = []diff --git a/workers/tests/test_worker_enums_directory.py b/workers/tests/test_worker_enums_directory.pynew file mode 100644index 000000000..280a3ad59--- /dev/null+++ b/workers/tests/test_worker_enums_directory.py@@ -0,0 +1,58 @@+"""WorkerType.to_directory() — the single source for the on-disk dir mapping (UN-3798).++worker.py's file-path task loader reads to_directory() directly instead of slicing+to_import_path(); these pin the hyphen/underscore mapping and that to_import_path is+built on top of it, so the two can't drift.++No Celery app is set up here on purpose: this is a pure-enum module, and the autouse+``_restore_current_celery_app`` fixture now tolerates a missing default app. An+earlier version called ``Celery(...).set_default()`` at import time to satisfy that+fixture, which mutated process-global Celery state for every later test in the+session depending on collection order.+"""++from pathlib import Path++import pytest+from shared.enums.worker_enums_base import WorkerType++_WORKERS_ROOT = Path(__file__).resolve().parents[1]+++def test_to_directory_maps_underscored_value_to_hyphenated_dir():+ # The one worker whose on-disk dir differs from its enum value.+ assert WorkerType.API_DEPLOYMENT.value == "api_deployment"+ assert WorkerType.API_DEPLOYMENT.to_directory() == "api-deployment"+++def test_to_directory_passthrough_when_dir_equals_value():+ assert WorkerType.GENERAL.to_directory() == "general"+++def test_to_import_path_is_built_on_to_directory():+ # to_import_path must derive from to_directory (not a separate mapping), so the+ # directory naming lives in exactly one place.+ wt = WorkerType.API_DEPLOYMENT+ assert wt.to_import_path() == f"{wt.to_directory()}.tasks"++[email protected]("wt", [w for w in WorkerType if not w.is_pluggable()])+def test_every_worker_type_maps_to_an_existing_directory_with_tasks(wt):+ # The failure mode UN-3798 exists for is enum-vs-disk DRIFT, and worker.py now+ # RAISES on a missing directory (it previously logged and continued) — so drift+ # is a container crash-loop across every replica, not a degraded worker. Pinning+ # two members can't catch that; pin the whole mapping so adding a WorkerType+ # without its directory is a red build instead of a production RuntimeError.+ directory = _WORKERS_ROOT / wt.to_directory()+ assert directory.is_dir(), f"{wt.value} -> {wt.to_directory()} does not exist"+ assert (directory / "tasks.py").is_file(), f"{wt.to_directory()}/tasks.py missing"++[email protected]("wt", [w for w in WorkerType if w.is_pluggable()])+def test_to_directory_rejects_pluggable_types(wt):+ # Pluggable code lives under pluggable_worker/<value>, so the mapping's+ # passthrough would return a wrong-but-plausible path. Only call ordering kept+ # that unreachable; the precondition is now enforced.+ with pytest.raises(ValueError, match="pluggable"):+ wt.to_directory()+ assert wt.to_import_path() == f"pluggable_worker.{wt.value}.tasks"diff --git a/workers/worker.py b/workers/worker.pyindex 3822e81a1..077fbd974 100755--- a/workers/worker.py+++ b/workers/worker.py@@ -5,6 +5,7 @@ This module serves as the main entry point for all Celery workers.It uses WorkerBuilder to ensure proper configuration including chord retry settings."""+import importlib.utilimport loggingimport osimport sys@@ -440,40 +441,97 @@ logger.info("🏗️ Initializing worker infrastructure (singleton pattern)...")initialize_worker_infrastructure()logger.info("✅ Worker infrastructure initialized successfully")-# Import tasks from the worker-specific directory-# Determine worker path dynamically based on worker type-base_dir = os.path.dirname(os.path.abspath(__file__))-if worker_type.is_pluggable():- # Pluggable workers live inside workers/pluggable_worker/{worker_name}- worker_directory = os.path.join("pluggable_worker", worker_type.value)- worker_path = os.path.join(base_dir, worker_directory)-else:- # Enum values use underscores (Python module names); a few on-disk dirs- # still use hyphens (e.g. api-deployment). Derive the directory from the- # authoritative import-path map on WorkerType instead of a blind replace.- worker_directory = worker_type.to_import_path().rsplit(".", 1)[0]++def load_worker_tasks(worker_type: WorkerType) -> None:+ """Register the worker type's Celery tasks.++ For pluggable workers, ``build_celery_app()`` above already imported the+ plugin package via ``WorkerBuilder._verify_pluggable_worker_exists``+ (``importlib.import_module("pluggable_worker.<type>.worker")`` — a proper+ PACKAGE import that RUNS the plugin's own registration code, e.g.+ ``from . import tasks`` with relative imports such as+ ``from .clients import ...``). That registration's ``@shared_task`` bindings+ are applied when the app FINALIZES — which any read of ``app.tasks`` triggers+ automatically (``finalize(auto=True)``), so they are in place by the time+ anything inspects the registry. The generic file-path load below is SKIPPED for+ pluggable workers because loading ``tasks.py`` under a bare ``"tasks"`` spec (no+ parent package) would break the plugin's relative imports ("attempted relative+ import with no known parent package") — not because of any binding-order+ subtlety.++ Non-pluggable (top-level) workers use absolute imports; their ``tasks.py`` is+ loaded by file path with the worker directory on ``sys.path``. Their tasks bind+ eagerly here, so a missing directory / ``tasks.py`` is a broken deploy and+ hard-fails rather than booting a no-op worker.+ """+ if worker_type.is_pluggable():+ logger.info(+ f"✅ Pluggable worker {worker_type.value} tasks registered via "+ "WorkerBuilder (package import); skipping file-path task load"+ )+ return++ base_dir = os.path.dirname(os.path.abspath(__file__))+ worker_directory = worker_type.to_directory()worker_path = os.path.join(base_dir, worker_directory)+ if not os.path.exists(worker_path):+ # Non-pluggable only (pluggable returned above): tasks bind eagerly here+ # with no later finalize step, so a missing worker dir is terminal.+ raise RuntimeError(f"Worker directory not found: {worker_path}")-# Add worker directory to path for task imports-if os.path.exists(worker_path):sys.path.append(worker_path)logger.info(f"✅ Added {worker_directory} to Python path for task imports")- # Import tasks module to register taskstasks_file = os.path.join(worker_path, "tasks.py")- if os.path.exists(tasks_file):- logger.info(f"📋 Loading tasks from: {tasks_file}")- # Import the tasks module to register tasks with the app- import importlib.util-- spec = importlib.util.spec_from_file_location("tasks", tasks_file)- tasks_module = importlib.util.module_from_spec(spec)- spec.loader.exec_module(tasks_module)- logger.info(f"✅ Tasks loaded successfully from {worker_directory}")+ if not os.path.exists(tasks_file):+ raise RuntimeError(f"No tasks.py found for worker at: {tasks_file}")++ logger.info(f"📋 Loading tasks from: {tasks_file}")+ spec = importlib.util.spec_from_file_location("tasks", tasks_file)+ tasks_module = importlib.util.module_from_spec(spec)+ spec.loader.exec_module(tasks_module)+ logger.info(f"✅ Tasks loaded successfully from {worker_directory}")+++load_worker_tasks(worker_type)++# A worker that boots with no registered tasks starts but silently processes+# nothing (Celery does not error on an empty registry). Surface that misconfig.+#+# NB: reading ``app.tasks`` below AUTO-FINALIZES the app (Celery's ``tasks``+# property calls ``self.finalize(auto=True)``), so every pending ``@shared_task`` /+# ``connect_on_app_finalize`` registration has already been applied by the time the+# emptiness test is evaluated. The check is therefore exactly as accurate for+# pluggable workers as for non-pluggable ones — an earlier version of this comment+# claimed the opposite ("bind on finalize, which can be after this point") and used+# it to justify the split; that premise was wrong. Two real consequences: the app is+# finalized at import rather than at worker start, and an empty registry here is a+# genuine misconfiguration for BOTH kinds.+#+# The split is kept deliberately, on deployment-risk grounds rather than accuracy:+# a non-pluggable worker's tasks are bound eagerly by ``load_worker_tasks`` in this+# same module, so an empty registry is unambiguously terminal — RAISE. Pluggable+# workers register via plugin package import, whose availability is deployment+# dependent; hard-failing there would turn one absent/misconfigured plugin into a+# container crash-loop across the fleet, so it stays a loud WARN. Tightening this to+# a raise is worthwhile but needs a pass confirming every deployed pluggable worker+# registers at import time — tracked under UN-3445 hardening, not changed blind here.+if not any(not name.startswith("celery.") for name in app.tasks):+ _empty_registry_msg = (+ f"No non-celery tasks registered for worker '{worker_type.value}' "+ f"(pluggable={worker_type.is_pluggable()})."+ )+ if worker_type.is_pluggable():+ logger.warning(+ f"⚠️ {_empty_registry_msg} If this persists past app finalize the "+ "worker will start but process nothing — check the plugin task "+ "registration."+ )else:- logger.warning(f"⚠️ No tasks.py found at: {tasks_file}")-else:- logger.error(f"❌ Worker directory not found: {worker_path}")+ raise RuntimeError(+ f"{_empty_registry_msg} The worker would start but process nothing — "+ "check the worker task registration."+ )# Log successful configurationlogger.info(f"✅ Successfully loaded {worker_type} worker using WorkerBuilder")