LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
UN-4046 [MISC] Roll the PG queue out of the pg_queue_enabled feature flag (#2262)
alia8cab8c17406
diff --git a/backend/backend/internal_api_constants.py b/backend/backend/internal_api_constants.pyindex 52c0838db..37b19f578 100644--- a/backend/backend/internal_api_constants.py+++ b/backend/backend/internal_api_constants.py@@ -64,11 +64,6 @@ class InternalEndpoints:"""File execution status endpoint."""return build_internal_endpoint(f"file-execution/{file_execution_id}/status")- @staticmethod- def webhook_send() -> str:- """Webhook send endpoint."""- return build_internal_endpoint("webhook/send")-@staticmethoddef organization(org_id: str = "{org_id}") -> str:"""Organization endpoint."""diff --git a/backend/entrypoint.sh b/backend/entrypoint.shindex 9766b01ad..db6ef7298 100755--- a/backend/entrypoint.sh+++ b/backend/entrypoint.sh@@ -35,11 +35,11 @@ if [ "$migrate" = true ]; then## READ THIS BEFORE ASSUMING A RESTART IS INERT. This is NOT the earlier# `--mirror-only` invocation, and it does NOT leave Beat alone: with- # PG_SCHEDULER_ENABLED=true it flips pg_owned and DISABLES the Beat PeriodicTask- # row for every mirrored pipeline THE pg_queue_enabled FLAG SELECTS — ownership is- # additionally per-schedule Flipt-gated and fails closed, so with Flipt blind or the- # flag at 0% this moves nothing and Beat keeps every row. A rolling restart therefore can move schedules- # off Beat. (A comment here used to describe --mirror-only and state that+ # PG_SCHEDULER_ENABLED it flips pg_owned and DISABLES the Beat PeriodicTask row+ # for EVERY mirrored pipeline. There is no second gate any more: UN-4046 removed+ # the pg_queue_enabled flag, and PG_SCHEDULER_ENABLED now DEFAULTS TO TRUE, so the+ # first start after an upgrade moves every schedule off Beat unless you set it to+ # false. Do not read this comment as "a restart is inert" — it no longer is. (A comment here used to describe --mirror-only and state that# "ownership hand-over stays an explicit operator action" — it survived the switch# to converge_pg_scheduler and told an SRE auditing exactly this question the# opposite of the truth.)diff --git a/backend/notification_v2/internal_api_views.py b/backend/notification_v2/internal_api_views.pyindex 42d0eeac9..4b24cf98d 100644--- a/backend/notification_v2/internal_api_views.py+++ b/backend/notification_v2/internal_api_views.py@@ -28,7 +28,6 @@ from pipeline_v2.models import Pipelinefrom utils.organization_utils import filter_queryset_by_organizationfrom workflow_manager.workflow_v2.models.execution import WorkflowExecution-from backend.celery_service import app as celery_appfrom notification_v2.clubbed_renderer import MAX_BATCH_SIZE, render_clubbed_messagefrom notification_v2.enums import BufferStatusfrom notification_v2.helper import (@@ -530,16 +529,16 @@ def _reclaim_stale_sending() -> int:def _org_identifier(org_pk: int) -> str | None:"""Resolve the string ``Organization.organization_id`` from the buffer's org pk.- ``resolve_transport`` keys its Flipt decision on the org's string identifier,- but the buffer stores/uses the Organization pk. One indexed pk lookup per- dispatch group (post-commit) — negligible relative to the downstream webhook- dispatch.+ The PG queue row records the org's string identifier (used for per-org+ fairness), but the buffer stores/uses the Organization pk. One indexed pk+ lookup per dispatch group (post-commit) — negligible relative to the+ downstream webhook dispatch.Data-anomaly guard: ``NotificationBuffer.organization`` is``on_delete=CASCADE``, so a live buffer row with a missing org is unreachablein normal operation. A ``None`` here therefore signals a dangling FK — we log- it (the only org-traceable breadcrumb; resolve_transport's own warning is keyed- on the random dispatch uuid) and fail closed to Celery in resolve_transport.+ it (the only org-traceable breadcrumb) and enqueue with an empty org id, which+ dispatches correctly but without fairness attribution."""org_string_id = (Organization.objects.filter(pk=org_pk)@@ -549,11 +548,11 @@ def _org_identifier(org_pk: int) -> str | None:if org_string_id is None:# Sentry-routed (logger.error): a live buffer row with no org is a data# anomaly (dangling FK / corruption) that shouldn't happen under the- # CASCADE constraint, not routine noise. Routing still fails closed to- # Celery in resolve_transport.+ # CASCADE constraint, not routine noise. The enqueue still succeeds — it+ # just carries an empty org id, so the row loses fairness attribution.logger.error("metric=notification_org_identifier_missing_total org_pk=%s "- "(dangling FK; notification routing falls back to Celery)",+ "(dangling FK; enqueued without fairness attribution)",org_pk,)return org_string_id@@ -586,12 +585,10 @@ def _send_clubbed(``buffer_row_ids`` + ``organization_id`` to the worker so it can mark them."""try:- # Flag-gated transport (UN-3753): PG queue when pg_queue_enabled for this- # org, else Celery (byte-identical to the prior send_task). resolve_transport- # keys on the org STRING id, but the buffer/worker contract below uses the- # org pk — hence _org_identifier(org_id) for routing, org_id in kwargs.- dispatched = dispatch_webhook_notification(- celery_app=celery_app,+ # The queue row records the org STRING id, but the buffer/worker contract+ # below uses the org pk — hence _org_identifier(org_id) on the row,+ # org_id in kwargs. The two must not be conflated.+ dispatch_task_id = dispatch_webhook_notification(args=[url, body, headers, settings.NOTIFICATION_TIMEOUT],kwargs={"max_retries": max_retries,@@ -609,18 +606,14 @@ def _send_clubbed(queue="notifications",org_string_id=_org_identifier(org_id),)- # transport= makes the rollout answerable from the logs: during a percentage- # ramp the question is "are PG-routed notifications succeeding at the same- # rate as Celery-routed ones?", which result=success alone cannot answer.logger.info("metric=notification_batch_dispatched_total platform=%s result=success "- "transport=%s org_id=%s webhook_url_hash=%s rows=%d task_id=%s",+ "org_id=%s webhook_url_hash=%s rows=%d task_id=%s",platform,- dispatched.transport,org_id,webhook_url_hash(url),len(buffer_ids),- dispatched.task_id,+ dispatch_task_id,)except PermanentDispatchError:# PG-ONLY permanent failure. From this path the only reachable cause isdiff --git a/backend/notification_v2/internal_serializers.py b/backend/notification_v2/internal_serializers.pyindex 94669d64a..ab2b55711 100644--- a/backend/notification_v2/internal_serializers.py+++ b/backend/notification_v2/internal_serializers.py@@ -30,66 +30,6 @@ class NotificationSerializer(serializers.ModelSerializer):]-class WebhookNotificationRequestSerializer(serializers.Serializer):- """Serializer for webhook notification requests."""-- notification_id = serializers.UUIDField(required=False)- url = serializers.URLField(required=True)- payload = serializers.JSONField(required=True)- authorization_type = serializers.ChoiceField(- choices=AuthorizationType.choices(), default=AuthorizationType.NONE.value- )- authorization_key = serializers.CharField(required=False, allow_blank=True)- authorization_header = serializers.CharField(required=False, allow_blank=True)- headers = serializers.DictField(required=False, default=dict)- timeout = serializers.IntegerField(default=30, min_value=1, max_value=300)- max_retries = serializers.IntegerField(default=3, min_value=0, max_value=10)- retry_delay = serializers.IntegerField(default=60, min_value=1, max_value=3600)---class WebhookNotificationResponseSerializer(serializers.Serializer):- """Serializer for webhook notification responses."""-- task_id = serializers.CharField()- notification_id = serializers.UUIDField(required=False)- url = serializers.URLField()- status = serializers.CharField()- queued_at = serializers.DateTimeField()---class WebhookStatusSerializer(serializers.Serializer):- """Serializer for webhook delivery status."""-- task_id = serializers.CharField()- status = serializers.CharField()- url = serializers.CharField()- attempts = serializers.IntegerField()- success = serializers.BooleanField()- error_message = serializers.CharField(required=False, allow_null=True)---class WebhookBatchRequestSerializer(serializers.Serializer):- """Serializer for batch webhook requests."""-- batch_name = serializers.CharField(required=False, max_length=255)- webhooks = serializers.ListField(- child=WebhookNotificationRequestSerializer(), min_length=1, max_length=100- )- delay_between_requests = serializers.IntegerField(- default=0, min_value=0, max_value=60- )---class WebhookBatchResponseSerializer(serializers.Serializer):- """Serializer for batch webhook responses."""-- batch_id = serializers.CharField()- batch_name = serializers.CharField()- total_webhooks = serializers.IntegerField()- queued_webhooks = serializers.ListField(child=WebhookNotificationResponseSerializer())- failed_webhooks = serializers.ListField(child=serializers.DictField())--class WebhookConfigurationSerializer(serializers.Serializer):"""Serializer for webhook configuration."""diff --git a/backend/notification_v2/internal_urls.py b/backend/notification_v2/internal_urls.pyindex 90da468fb..2a35dcaf3 100644--- a/backend/notification_v2/internal_urls.py+++ b/backend/notification_v2/internal_urls.py@@ -7,12 +7,8 @@ from rest_framework.routers import DefaultRouterfrom . import internal_api_viewsfrom .internal_views import (- WebhookBatchAPIView,- WebhookBatchStatusAPIView,WebhookInternalViewSet,WebhookMetricsAPIView,- WebhookSendAPIView,- WebhookStatusAPIView,WebhookTestAPIView,)@@ -66,13 +62,7 @@ urlpatterns = [name="get_api_data",),# Webhook operation endpoints- path("send/", WebhookSendAPIView.as_view(), name="webhook-send"),- path("batch/", WebhookBatchAPIView.as_view(), name="webhook-batch"),path("test/", WebhookTestAPIView.as_view(), name="webhook-test"),- path("status/<str:task_id>/", WebhookStatusAPIView.as_view(), name="webhook-status"),- path(- "batch-status/", WebhookBatchStatusAPIView.as_view(), name="webhook-batch-status"- ),path("metrics/", WebhookMetricsAPIView.as_view(), name="webhook-metrics"),# Webhook configuration CRUD (via router)path("", include(router.urls)),diff --git a/backend/notification_v2/internal_views.py b/backend/notification_v2/internal_views.pyindex e352f56db..596945d01 100644--- a/backend/notification_v2/internal_views.py+++ b/backend/notification_v2/internal_views.py@@ -3,12 +3,8 @@ Handles webhook notification related endpoints for internal services."""import logging-import uuidfrom typing import Any-from celery import current_app as celery_app-from celery.result import AsyncResult-from django.utils import timezonefrom rest_framework import status, viewsetsfrom rest_framework.decorators import actionfrom rest_framework.response import Response@@ -21,12 +17,7 @@ from notification_v2.enums import AuthorizationType, NotificationType, PlatformTfrom notification_v2.internal_serializers import (NotificationListSerializer,NotificationSerializer,- WebhookBatchRequestSerializer,- WebhookBatchResponseSerializer,WebhookConfigurationSerializer,- WebhookNotificationRequestSerializer,- WebhookNotificationResponseSerializer,- WebhookStatusSerializer,WebhookTestSerializer,)from notification_v2.models import Notification@@ -110,219 +101,6 @@ class WebhookInternalViewSet(viewsets.ReadOnlyModelViewSet):)-class WebhookSendAPIView(APIView):- """Internal API endpoint for sending webhook notifications."""-- def post(self, request):- """Send a webhook notification."""- try:- serializer = WebhookNotificationRequestSerializer(data=request.data)-- if not serializer.is_valid():- return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST)-- validated_data = serializer.validated_data-- # Build headers based on authorization type- headers = self._build_headers(validated_data)-- # Send webhook notification task- task = celery_app.send_task(- "send_webhook_notification",- args=[- validated_data["url"],- validated_data["payload"],- headers,- validated_data["timeout"],- ],- kwargs={- "max_retries": validated_data["max_retries"],- "retry_delay": validated_data["retry_delay"],- },- )-- # Prepare response- response_data = {- "task_id": task.id,- "notification_id": validated_data.get("notification_id"),- "url": validated_data["url"],- "status": "queued",- "queued_at": timezone.now(),- }-- response_serializer = WebhookNotificationResponseSerializer(response_data)-- logger.info(- f"Queued webhook notification task {task.id} for URL {validated_data['url']}"- )-- return Response(response_serializer.data, status=status.HTTP_202_ACCEPTED)-- except Exception as e:- logger.error(f"Failed to send webhook notification: {str(e)}")- return Response(- {"error": "Failed to send webhook notification", "detail": str(e)},- status=status.HTTP_500_INTERNAL_SERVER_ERROR,- )-- def _build_headers(self, validated_data: dict[str, Any]) -> dict[str, str]:- """Build headers based on authorization configuration."""- headers = {"Content-Type": APPLICATION_JSON}-- auth_type = validated_data.get("authorization_type", AuthorizationType.NONE.value)- auth_key = validated_data.get("authorization_key")- auth_header = validated_data.get("authorization_header")-- if validated_data.get("headers"):- headers.update(validated_data["headers"])-- if auth_type == AuthorizationType.BEARER.value and auth_key:- headers["Authorization"] = f"Bearer {auth_key}"- elif auth_type == AuthorizationType.API_KEY.value and auth_key:- headers["Authorization"] = auth_key- elif (- auth_type == AuthorizationType.CUSTOM_HEADER.value- and auth_header- and auth_key- ):- headers[auth_header] = auth_key-- return headers---class WebhookStatusAPIView(APIView):- """Internal API endpoint for checking webhook delivery status."""-- def get(self, request, task_id):- """Get webhook delivery status by task ID."""- try:- task_result = AsyncResult(task_id, app=celery_app)-- status_data = {- "task_id": task_id,- "status": task_result.status,- "url": "unknown",- "attempts": 0,- "success": task_result.successful(),- "error_message": None,- }-- if task_result.failed():- status_data["error_message"] = str(task_result.result)- elif task_result.successful():- status_data["attempts"] = getattr(task_result.result, "attempts", 1)-- serializer = WebhookStatusSerializer(status_data)- return Response(serializer.data)-- except Exception as e:- logger.error(f"Failed to get webhook status for task {task_id}: {str(e)}")- return Response(- {"error": "Failed to get webhook status", "detail": str(e)},- status=status.HTTP_500_INTERNAL_SERVER_ERROR,- )---class WebhookBatchAPIView(APIView):- """Internal API endpoint for sending batch webhook notifications."""-- def post(self, request):- """Send multiple webhook notifications in batch."""- try:- serializer = WebhookBatchRequestSerializer(data=request.data)-- if not serializer.is_valid():- return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST)-- validated_data = serializer.validated_data- webhooks = validated_data["webhooks"]- delay_between = validated_data.get("delay_between_requests", 0)-- batch_id = str(uuid.uuid4())- queued_webhooks = []- failed_webhooks = []-- for i, webhook_data in enumerate(webhooks):- try:- headers = self._build_headers(webhook_data)- countdown = i * delay_between if delay_between > 0 else 0-- task = celery_app.send_task(- "send_webhook_notification",- args=[- webhook_data["url"],- webhook_data["payload"],- headers,- webhook_data["timeout"],- ],- kwargs={- "max_retries": webhook_data["max_retries"],- "retry_delay": webhook_data["retry_delay"],- },- countdown=countdown,- )-- queued_webhooks.append(- {- "task_id": task.id,- "notification_id": webhook_data.get("notification_id"),- "url": webhook_data["url"],- "status": "queued",- "queued_at": timezone.now(),- }- )-- except Exception as e:- failed_webhooks.append({"url": webhook_data["url"], "error": str(e)})-- response_data = {- "batch_id": batch_id,- "batch_name": validated_data.get("batch_name", f"Batch-{batch_id[:8]}"),- "total_webhooks": len(webhooks),- "queued_webhooks": queued_webhooks,- "failed_webhooks": failed_webhooks,- }-- response_serializer = WebhookBatchResponseSerializer(response_data)-- logger.info(- f"Queued batch {batch_id} with {len(queued_webhooks)} webhooks, {len(failed_webhooks)} failed"- )-- return Response(response_serializer.data, status=status.HTTP_202_ACCEPTED)-- except Exception as e:- logger.error(f"Failed to send webhook batch: {str(e)}")- return Response(- {"error": "Failed to send webhook batch", "detail": str(e)},- status=status.HTTP_500_INTERNAL_SERVER_ERROR,- )-- def _build_headers(self, webhook_data: dict[str, Any]) -> dict[str, str]:- """Build headers for webhook request."""- headers = {"Content-Type": APPLICATION_JSON}-- auth_type = webhook_data.get("authorization_type", AuthorizationType.NONE.value)- auth_key = webhook_data.get("authorization_key")- auth_header = webhook_data.get("authorization_header")-- if webhook_data.get("headers"):- headers.update(webhook_data["headers"])-- if auth_type == AuthorizationType.BEARER.value and auth_key:- headers["Authorization"] = f"Bearer {auth_key}"- elif auth_type == AuthorizationType.API_KEY.value and auth_key:- headers["Authorization"] = auth_key- elif (- auth_type == AuthorizationType.CUSTOM_HEADER.value- and auth_header- and auth_key- ):- headers[auth_header] = auth_key-- return headers--class WebhookTestAPIView(APIView):"""Internal API endpoint for testing webhook configurations."""@@ -406,73 +184,6 @@ class WebhookTestAPIView(APIView):return headers-class WebhookBatchStatusAPIView(APIView):- """Internal API endpoint for checking batch webhook delivery status."""-- def get(self, request):- """Get batch webhook delivery status."""- try:- batch_id = request.query_params.get("batch_id")- task_ids = request.query_params.get("task_ids", "").split(",")-- if not batch_id and not task_ids:- return Response(- {"error": "Either batch_id or task_ids parameter is required"},- status=status.HTTP_400_BAD_REQUEST,- )-- batch_results = []-- if task_ids and task_ids[0]: # task_ids is not empty- for task_id in task_ids:- if task_id.strip():- try:- task_result = AsyncResult(task_id.strip(), app=celery_app)-- batch_results.append(- {- "task_id": task_id.strip(),- "status": task_result.status,- "success": task_result.successful(),- "error_message": str(task_result.result)- if task_result.failed()- else None,- }- )- except Exception as e:- batch_results.append(- {- "task_id": task_id.strip(),- "status": "ERROR",- "success": False,- "error_message": f"Failed to get task status: {str(e)}",- }- )-- response_data = {- "batch_id": batch_id,- "total_tasks": len(batch_results),- "results": batch_results,- "summary": {- "completed": sum(- 1 for r in batch_results if r["status"] == "SUCCESS"- ),- "failed": sum(1 for r in batch_results if r["status"] == "FAILURE"),- "pending": sum(1 for r in batch_results if r["status"] == "PENDING"),- "running": sum(1 for r in batch_results if r["status"] == "STARTED"),- },- }-- return Response(response_data)-- except Exception as e:- logger.error(f"Failed to get batch webhook status: {str(e)}")- return Response(- {"error": "Failed to get batch webhook status", "detail": str(e)},- status=status.HTTP_500_INTERNAL_SERVER_ERROR,- )--class WebhookMetricsAPIView(APIView):"""Internal API endpoint for webhook delivery metrics."""diff --git a/backend/notification_v2/notification_dispatch.py b/backend/notification_v2/notification_dispatch.pyindex 4b027182e..535450931 100644--- a/backend/notification_v2/notification_dispatch.py+++ b/backend/notification_v2/notification_dispatch.py@@ -1,37 +1,25 @@-"""Transport-routed dispatch for buffered webhook notifications (UN-3753).+"""PG-queue dispatch for buffered webhook notifications (UN-3753, UN-4046).-Routes the ``send_webhook_notification`` task through the same-:func:`resolve_transport` flag as the execution path: the PG queue when-``pg_queue_enabled`` for this org, else Celery. **Fail-closed** — with the gate-off (the production default) it resolves to Celery, behaving exactly like the-prior unconditional ``celery_app.send_task`` (zero regression).+Enqueues ``send_webhook_notification`` onto the PG queue the notification+consumer polls (``WORKER_PG_QUEUE_CONSUMER_WORKER_TYPE=notification``).-DEPLOYMENT PREREQUISITE (PG path): on PG the task lands on the ``notifications``-queue and requires a pg-queue consumer configured with-``WORKER_PG_QUEUE_CONSUMER_WORKER_TYPE=notification`` /-``WORKER_PG_QUEUE_CONSUMER_QUEUE=notifications``. **No such service exists in this-repo's compose yet** — none of the ``pg-queue-consumer`` services in-``docker/docker-compose.yaml`` polls ``notifications``, and ``run-worker.sh``'s-``PG_CONSUMER_ROLES`` has no notification role. Until one is deployed the flag must-stay off for the org: enqueued rows would sit undrained (no TTL sweep covers-``pg_queue_message``) and every buffered webhook for that org would be lost.+This used to route through ``resolve_transport``/``pg_queue_enabled`` and fall+back to ``celery_app.send_task``. The flag is gone (UN-4046) and PG is the only+transport, so the Celery branch went with it — along with the ``DispatchResult``+transport label, which existed only to tell the two apart during a ramp.-``args`` and ``queue`` are forwarded verbatim on both paths. ``kwargs`` differ in-exactly two keys, forced on the PG branch only (see the inline note at the branch):-``raise_on_final_failure`` → ``False`` and ``max_retries`` → ``0``. On PG the-payload is additionally JSON-normalized by ``enqueue_task`` (UUIDs/datetimes → str).+``args`` and ``queue`` are forwarded verbatim; the payload is JSON-normalized by+``enqueue_task`` (UUIDs/datetimes → str). Two ``kwargs`` are forced — see the+inline note at the enqueue."""from __future__ import annotationsimport loggingimport uuid-from typing import Any, NamedTuple+from typing import Anyfrom pg_queue.producer import enqueue_task-from workflow_manager.workflow_v2.transport import resolve_transport--from unstract.core.data_models import is_pg_transportlogger = logging.getLogger(__name__)@@ -40,141 +28,94 @@ logger = logging.getLogger(__name__)WEBHOOK_NOTIFICATION_TASK = "send_webhook_notification"-# Transport labels for the dispatch metric. During a percentage ramp the one thing-# you need from the logs is "are PG-routed notifications succeeding at the same rate-# as Celery-routed ones?" — a bare task id can't answer that, since a minted PG id-# and a Celery AsyncResult id are indistinguishable to the caller.-PG_TRANSPORT = "pg_queue"-CELERY_TRANSPORT = "celery"---class DispatchResult(NamedTuple):- """Which transport actually took the dispatch, plus the resulting task id."""-- transport: str- task_id: str--class PermanentDispatchError(Exception):"""A dispatch failure that would fail identically on every retry.- Raised ONLY on the PG path, when ``enqueue_task`` rejects the message for a- permanent reason (priority range / reply_key+callback exclusivity validation,- or a payload that can't be JSON-serialized). The Celery path never raises it,- so a caller can dead-letter on this exception without altering the flag-off- (Celery) error flow — a Celery ``send_task`` failure stays an ordinary- ``Exception`` the caller's transient handler owns, exactly as before.+ Raised when ``enqueue_task`` rejects the message for a permanent reason+ (priority range / reply_key+callback exclusivity validation, or a payload that+ can't be JSON-serialized), so a caller can dead-letter on it. A transient PG+ error (DB down) is deliberately NOT wrapped — it propagates as an ordinary+ ``Exception`` into the caller's retry path."""def dispatch_webhook_notification(*,- celery_app: Any,args: list[Any],kwargs: dict[str, Any],queue: str,org_string_id: str | None,-) -> DispatchResult:- """Dispatch ``send_webhook_notification`` on the resolved transport.-- ``args``/``queue`` are forwarded unchanged on both paths, and on the Celery- branch ``kwargs`` too — so the flag-off path is byte-identical to the legacy- ``send_task`` call. The PG branch overrides two retry-semantics kwargs (see- below); nothing else differs.+) -> str:+ """Enqueue ``send_webhook_notification`` on the PG queue.Args:- celery_app: Injected Celery app (the backend's ``celery_service.app``);- passed in rather than imported so this seam stays trivially testable.args: Positional task args, forwarded verbatim.- kwargs: Keyword task args. Forwarded as-is on Celery; on PG,- ``raise_on_final_failure`` is overridden to ``False`` and- ``max_retries`` to ``0`` (the in-task retry loop is a no-op under the- consumer's eager ``apply()`` — see the inline note). For the buffered- path this- carries ``organization_id`` = the buffer's org **pk** (the worker's- buffer-mark contract) — deliberately a DIFFERENT identifier from the- ``org_string_id`` param below (the two must not be conflated).+ kwargs: Keyword task args. ``raise_on_final_failure`` is overridden to+ ``False`` and ``max_retries`` to ``0`` (the in-task retry loop is a+ no-op under the consumer's eager ``apply()`` — see the inline note).+ For the buffered path this carries ``organization_id`` = the buffer's+ org **pk** (the worker's buffer-mark contract) — deliberately a+ DIFFERENT identifier from ``org_string_id`` (the two must not be+ conflated).queue: Target queue name, forwarded verbatim.org_string_id: The org's **string** identifier- (``Organization.organization_id``), used solely for the Flipt- transport decision — NOT the org pk carried in ``kwargs``. ``None`` (or- empty) fails closed to Celery.+ (``Organization.organization_id``), recorded on the queue row for+ fairness/routing.Returns:- A :class:`DispatchResult` carrying which transport took the dispatch and- the resulting task id (the Celery ``AsyncResult`` id, or the minted PG task- id). The transport is what makes the caller's dispatch metric answerable- during a ramp — the two id flavours are otherwise indistinguishable.+ The minted PG task id."""# A buffered notification is a single fire-and-forget task with no natural- # sticky entity, so mint a fresh id to drive Flipt's percentage bucketing and- # to serve as the PG task id.+ # sticky entity, so mint a fresh id to serve as the PG task id.dispatch_id = str(uuid.uuid4())- # resolve_transport already normalizes falsy input to Celery, so pass the id- # straight through (no `or None` needed).- transport = resolve_transport(- execution_id=dispatch_id,- organization_id=org_string_id,- )- # Use the shared is_pg_transport() — the single source for "what counts as PG- # transport" — rather than opening a second comparison site.- if is_pg_transport(transport):- # Two PG-only kwarg overrides, both about the SAME thing: the consumer runs- # the task eagerly via ``task.apply(..., throw=True)``, where Celery's- # in-task retry loop does not work and its terminal branch is what we need.- #- # 1. ``max_retries`` → 0. Under ``apply()`` ``self.request.retries`` is ALWAYS- # 0, so the worker's ``if self.request.retries < max_retries`` guard- # (workers/notification/tasks.py) is true on the FIRST failure for any- # max_retries >= 1 and calls ``self.retry(...)``, which raises ``Retry``;- # with ``throw=True`` that propagates straight out of ``apply()`` and the- # terminal branch is never reached — so the buffers are never marked- # DEAD_LETTER and ``raise_on_final_failure`` is never even read. The- # consumer's ``except Exception`` then leaves the row for vt-expiry- # redelivery, re-POSTing the subscriber every time. Forcing 0 sends the- # task down the terminal branch on the first failure instead.- # 2. ``raise_on_final_failure`` → False. On BOTH transports the worker marks- # the buffers DEAD_LETTER over the internal API *before* it re-raises, so- # the re-raise is only a FAILURE-state signal for Celery monitoring (no- # redelivery there). On the PG consumer that same raise is treated as a- # failure and leaves the row for redelivery, so it must not raise.- #- # Together: one POST, buffers dead-lettered, task returns None → the consumer- # acks (deletes) the row. Retry spacing belongs to the PG layer, not the task.- # (The Celery branch below keeps kwargs verbatim — byte-identical.)- pg_kwargs = {**kwargs, "raise_on_final_failure": False, "max_retries": 0}- try:- msg_id = enqueue_task(- task_name=WEBHOOK_NOTIFICATION_TASK,- queue=queue,- args=args,- kwargs=pg_kwargs,- # enqueue_task types org_id as str; None→"" here satisfies that- # type only, not runtime — enqueue_task itself re-coerces- # ``org_id or ""`` at insert (producer.py), so this has no runtime- # effect.- org_id=org_string_id or "",- task_id=dispatch_id,- )- except (ValueError, TypeError) as exc:- # PG-only permanent failure (enqueue_task validation / JSON encode):- # re-raise as PermanentDispatchError so the caller dead-letters it.- # A transient PG error (DB down) is NOT wrapped — it propagates as an- # ordinary Exception into the caller's retry (revert-to-PENDING) path.- raise PermanentDispatchError(str(exc)) from exc- # msg_id correlates this line with the producer's own ``msg_id=`` log, so a- # dropped notification can be traced across the seam/producer boundary.- logger.info(- "Webhook notification enqueued on PG '%s' queue (task_id=%s msg_id=%s)",- queue,- dispatch_id,- msg_id,+ # Two kwarg overrides, both about the SAME thing: the consumer runs the task+ # eagerly via ``task.apply(..., throw=True)``, where Celery's in-task retry+ # loop does not work and its terminal branch is what we need.+ #+ # 1. ``max_retries`` → 0. Under ``apply()`` ``self.request.retries`` is ALWAYS+ # 0, so the worker's ``if self.request.retries < max_retries`` guard+ # (workers/notification/tasks.py) is true on the FIRST failure for any+ # max_retries >= 1 and calls ``self.retry(...)``, which raises ``Retry``;+ # with ``throw=True`` that propagates straight out of ``apply()`` and the+ # terminal branch is never reached — so the buffers are never marked+ # DEAD_LETTER and ``raise_on_final_failure`` is never even read. The+ # consumer's ``except Exception`` then leaves the row for vt-expiry+ # redelivery, re-POSTing the subscriber every time. Forcing 0 sends the+ # task down the terminal branch on the first failure instead.+ # 2. ``raise_on_final_failure`` → False. The worker marks the buffers+ # DEAD_LETTER over the internal API *before* it re-raises, so the re-raise+ # carries no state the caller needs — and on the PG consumer that raise is+ # treated as a failure and leaves the row for redelivery, so it must not+ # raise.+ #+ # Together: one POST, buffers dead-lettered, task returns None → the consumer+ # acks (deletes) the row. Retry spacing belongs to the PG layer, not the task.+ pg_kwargs = {**kwargs, "raise_on_final_failure": False, "max_retries": 0}+ try:+ msg_id = enqueue_task(+ task_name=WEBHOOK_NOTIFICATION_TASK,+ queue=queue,+ args=args,+ kwargs=pg_kwargs,+ # enqueue_task types org_id as str; None→"" here satisfies that+ # type only, not runtime — enqueue_task itself re-coerces+ # ``org_id or ""`` at insert (producer.py), so this has no runtime+ # effect.+ org_id=org_string_id or "",+ task_id=dispatch_id,)- return DispatchResult(transport=PG_TRANSPORT, task_id=dispatch_id)- result = celery_app.send_task(- WEBHOOK_NOTIFICATION_TASK,- args=args,- kwargs=kwargs,- queue=queue,+ except (ValueError, TypeError) as exc:+ # PG-only permanent failure (enqueue_task validation / JSON encode):+ # re-raise as PermanentDispatchError so the caller dead-letters it.+ # A transient PG error (DB down) is NOT wrapped — it propagates as an+ # ordinary Exception into the caller's retry (revert-to-PENDING) path.+ raise PermanentDispatchError(str(exc)) from exc+ # msg_id correlates this line with the producer's own ``msg_id=`` log, so a+ # dropped notification can be traced across the seam/producer boundary.+ logger.info(+ "Webhook notification enqueued on PG '%s' queue (task_id=%s msg_id=%s)",+ queue,+ dispatch_id,+ msg_id,)- return DispatchResult(transport=CELERY_TRANSPORT, task_id=result.id)+ return dispatch_iddiff --git a/backend/notification_v2/tests/test_notification_dispatch.py b/backend/notification_v2/tests/test_notification_dispatch.pyindex e08bf55ff..450287a94 100644--- a/backend/notification_v2/tests/test_notification_dispatch.py+++ b/backend/notification_v2/tests/test_notification_dispatch.py@@ -1,17 +1,20 @@-"""Unit tests for the buffered-webhook transport routing (UN-3753).--``resolve_transport`` + ``enqueue_task`` are patched on the module, so no Flipt /-DB is needed — these pin the routing contract: PG when the flag resolves PG,-Celery otherwise (fail-closed), with identical args/queue on both paths. The-kwargs are identical EXCEPT two retry-semantics keys forced on the PG branch-(``raise_on_final_failure`` -> False, ``max_retries`` -> 0), because the consumer-runs tasks eagerly via ``apply()`` where the in-task retry loop cannot work — see-``test_pg_forces_terminal_branch_kwargs``.+"""Unit tests for the buffered-webhook PG dispatch (UN-3753, UN-4046).++``enqueue_task`` is patched on the module, so no DB is needed. These pin the+payload contract the notification consumer depends on, the permanent-vs-transient+error split the caller branches on, and the two retry-semantics kwargs the seam+forces because the consumer runs tasks eagerly via ``apply()``.++The transport-routing tests went with the ``pg_queue_enabled`` flag (UN-4046) —+"routes to celery", "fails closed to celery on a missing org", "buckets by minted+dispatch id" and "identical on both paths" no longer describe anything. What the+missing-org case still asserts (the row carries an empty org id rather than+``None``) is kept below, because ``enqueue_task`` types ``org_id`` as ``str``."""from __future__ import annotations-from unittest.mock import MagicMock, patch+from unittest.mock import patchimport pytest@@ -29,9 +32,8 @@ _KWARGS = {_QUEUE = "notifications"-def _dispatch(celery_app, org_string_id="org_x"):+def _dispatch(org_string_id="org_x"):return nd.dispatch_webhook_notification(- celery_app=celery_app,args=_ARGS,kwargs=_KWARGS,queue=_QUEUE,@@ -40,22 +42,16 @@ def _dispatch(celery_app, org_string_id="org_x"):class TestDispatchWebhookNotification:- def test_routes_to_pg_when_flag_resolves_pg(self):- celery = MagicMock()- with (- patch.object(nd, "resolve_transport", return_value="pg_queue"),- patch.object(nd, "enqueue_task", return_value=42) as enqueue,- ):- result = _dispatch(celery)- task_id = result.task_id- assert result.transport == nd.PG_TRANSPORT # rollout metric can tell them apart+ def test_enqueues_with_the_consumer_payload_contract(self):+ with patch.object(nd, "enqueue_task", return_value=42) as enqueue:+ task_id = _dispatch()enqueue.assert_called_once()kwargs = enqueue.call_args.kwargsassert kwargs["task_name"] == "send_webhook_notification"assert kwargs["queue"] == _QUEUEassert kwargs["args"] == _ARGS- # Every kwarg is forwarded verbatim EXCEPT the two retry-semantics keys the- # PG branch forces (see test_pg_forces_terminal_branch_kwargs).+ # Every kwarg forwarded verbatim EXCEPT the two retry-semantics keys the+ # seam forces (see test_forces_terminal_branch_kwargs).assert kwargs["kwargs"] == {**_KWARGS,"raise_on_final_failure": False,@@ -64,98 +60,31 @@ class TestDispatchWebhookNotification:assert kwargs["org_id"] == "org_x"# The minted PG task id is returned and threaded into the enqueue row.assert kwargs["task_id"] == task_id- celery.send_task.assert_not_called()-- def test_routes_to_celery_when_flag_resolves_celery(self):- celery = MagicMock()- with (- patch.object(nd, "resolve_transport", return_value="celery"),- patch.object(nd, "enqueue_task") as enqueue,- ):- result = _dispatch(celery)- celery.send_task.assert_called_once_with(- "send_webhook_notification", args=_ARGS, kwargs=_KWARGS, queue=_QUEUE- )- enqueue.assert_not_called()- assert result.task_id is celery.send_task.return_value.id- assert result.transport == nd.CELERY_TRANSPORT- def test_pg_enqueue_failure_propagates_with_no_celery_fallback(self):- # No silent Celery fallback on a TRANSIENT PG failure — it propagates raw- # (not wrapped), so the caller (_send_clubbed) reverts rows to PENDING.- celery = MagicMock()- with (- patch.object(nd, "resolve_transport", return_value="pg_queue"),- patch.object(nd, "enqueue_task", side_effect=RuntimeError("pg down")),- ):+ def test_transient_enqueue_failure_propagates_raw(self):+ # A TRANSIENT PG failure propagates unwrapped, so the caller (_send_clubbed)+ # reverts the rows to PENDING rather than dead-lettering them.+ with patch.object(nd, "enqueue_task", side_effect=RuntimeError("pg down")):with pytest.raises(RuntimeError, match="pg down"):- _dispatch(celery)- celery.send_task.assert_not_called()+ _dispatch()- def test_pg_permanent_enqueue_error_is_wrapped(self):+ def test_permanent_enqueue_error_is_wrapped(self):# A PERMANENT enqueue error (ValueError/TypeError: validation / JSON encode)- # is re-raised as PermanentDispatchError so the caller dead-letters it —- # raised ONLY on the PG path, so the Celery flow is never affected.- celery = MagicMock()+ # is re-raised as PermanentDispatchError so the caller dead-letters it+ # instead of retrying something that will fail identically every time.for exc in (ValueError("priority out of range"), TypeError("not serializable")):- with (- patch.object(nd, "resolve_transport", return_value="pg_queue"),- patch.object(nd, "enqueue_task", side_effect=exc),- ):+ with patch.object(nd, "enqueue_task", side_effect=exc):with pytest.raises(nd.PermanentDispatchError):- _dispatch(celery)- celery.send_task.assert_not_called()-- def test_none_org_fails_closed_to_celery(self):- # A missing org string (org deleted) must not route to PG — resolve_transport- # fails closed, and we pass organization_id=None straight through to it.- celery = MagicMock()- with (- patch.object(nd, "resolve_transport", return_value="celery") as resolve,- patch.object(nd, "enqueue_task") as enqueue,- ):- _dispatch(celery, org_string_id=None)- assert resolve.call_args.kwargs["organization_id"] is None- celery.send_task.assert_called_once()- enqueue.assert_not_called()+ _dispatch()- def test_resolve_transport_buckets_by_minted_dispatch_id(self):- # Fire-and-forget: entity_id is a freshly minted uuid (str), and it equals- # the PG task_id so the row and the Flipt bucket agree.- celery = MagicMock()- with (- patch.object(nd, "resolve_transport", return_value="pg_queue") as resolve,- patch.object(nd, "enqueue_task", return_value=1) as enqueue,- ):- task_id = _dispatch(celery).task_id- assert resolve.call_args.kwargs["execution_id"] == task_id- assert enqueue.call_args.kwargs["task_id"] == task_id+ def test_none_org_is_coerced_to_empty_string_on_the_row(self):+ # enqueue_task types org_id as str; a deleted org must not put None on the+ # row. (The producer re-coerces too, so this pins the seam's own contract.)+ with patch.object(nd, "enqueue_task", return_value=1) as enqueue:+ _dispatch(org_string_id=None)+ assert enqueue.call_args.kwargs["org_id"] == ""- def test_args_and_queue_identical_on_both_paths(self):- # The consumer must behave the same regardless of transport — args/queue are- # byte-identical, and kwargs match apart from the transport-specific- # raise_on_final_failure override (asserted separately below).- celery = MagicMock()- with patch.object(nd, "resolve_transport", return_value="celery"):- _dispatch(celery)- celery_call = celery.send_task.call_args.kwargs- celery2 = MagicMock()- with (- patch.object(nd, "resolve_transport", return_value="pg_queue"),- patch.object(nd, "enqueue_task", return_value=1) as enqueue,- ):- _dispatch(celery2)- assert enqueue.call_args.kwargs["args"] == celery_call["args"]- assert enqueue.call_args.kwargs["queue"] == celery_call["queue"]- # kwargs differ ONLY by the two PG-forced retry-semantics keys.- pg_kwargs = enqueue.call_args.kwargs["kwargs"]- assert pg_kwargs == {- **celery_call["kwargs"],- "raise_on_final_failure": False,- "max_retries": 0,- }-- def test_pg_forces_terminal_branch_kwargs(self):+ def test_forces_terminal_branch_kwargs(self):# Regression (UN-3753): the PG consumer runs the task eagerly via# ``task.apply(..., throw=True)``, where the in-task retry loop cannot work:# * max_retries >= 1 -> the worker's ``request.retries < max_retries`` guard@@ -164,29 +93,17 @@ class TestDispatchWebhookNotification:# (mark DEAD_LETTER + honour raise_on_final_failure) never runs and the row# is left for vt-expiry redelivery, re-POSTing the subscriber each time.# * a terminal re-raise is likewise treated as failure -> redelivery.- # So the seam must force BOTH max_retries=0 and raise_on_final_failure=False on- # the PG branch. The Celery branch keeps the caller's values verbatim.+ # So the seam must force BOTH max_retries=0 and raise_on_final_failure=False.# (The end-to-end behaviour these kwargs buy — one POST + one dead-letter mark# through a real ``task.apply()`` — is asserted in# workers/tests/test_notification_pg_terminal.py.)assert _KWARGS["raise_on_final_failure"] is True # guard the fixture premiseassert _KWARGS["max_retries"] == 3 # ... and that retries are configured- celery = MagicMock()- with (- patch.object(nd, "resolve_transport", return_value="pg_queue"),- patch.object(nd, "enqueue_task", return_value=1) as enqueue,- ):- _dispatch(celery)+ with patch.object(nd, "enqueue_task", return_value=1) as enqueue:+ _dispatch()pg_kwargs = enqueue.call_args.kwargs["kwargs"]assert pg_kwargs["raise_on_final_failure"] is Falseassert pg_kwargs["max_retries"] == 0# The caller's dict is not mutated in place (a fresh dict is enqueued).assert _KWARGS["raise_on_final_failure"] is Trueassert _KWARGS["max_retries"] == 3-- celery2 = MagicMock()- with patch.object(nd, "resolve_transport", return_value="celery"):- _dispatch(celery2)- sent_kwargs = celery2.send_task.call_args.kwargs["kwargs"]- assert sent_kwargs["raise_on_final_failure"] is True- assert sent_kwargs["max_retries"] == 3diff --git a/backend/pg_queue/executor_rpc.py b/backend/pg_queue/executor_rpc.pyindex 1bed4a03d..b76b05787 100644--- a/backend/pg_queue/executor_rpc.py+++ b/backend/pg_queue/executor_rpc.py@@ -1,14 +1,14 @@"""Executor-RPC for the PG path — backend (Django) transport adapter.-The gate + reply_key/timeout orchestration + routing live ONCE in+The reply_key/timeout orchestration lives ONCE in``unstract.workflow_execution.executor_rpc`` (shared with the workers). This moduleis the thin Django half: a :class:`DjangoQueueTransport` that enqueues via the ORM(``enqueue_task``) and polls ``PgTaskResult``, plus the :func:`get_executor_dispatcher`factory that wires them together.-Zero-regression: with the ``pg_queue_enabled`` Flipt flag off the routing dispatcher-delegates every mode to the unchanged Celery ``ExecutionDispatcher`` and no-``pg_task_result`` row is created.+Since UN-4046 the factory returns the PG dispatcher unconditionally: there is no+gate, no routing dispatcher and no Celery fall-through, so a ``pg_task_result``+row is written on every request-reply dispatch."""from __future__ import annotations@@ -21,14 +21,11 @@ from django.db import close_old_connectionsfrom pg_queue.models import PgTaskResultfrom pg_queue.producer import enqueue_taskfrom unstract.core.polling import poll_for_row-from unstract.sdk1.execution.dispatcher import ExecutionDispatcherfrom unstract.workflow_execution.executor_rpc import (EXECUTE_TASK,ExecResultRow,PgExecutionDispatcher,QueueTransport,- RoutingExecutionDispatcher,- resolve_pg_transport,)if TYPE_CHECKING:@@ -41,20 +38,10 @@ logger = logging.getLogger(__name__)__all__ = ["DjangoQueueTransport","PgExecutionDispatcher",- "RoutingExecutionDispatcher","get_executor_dispatcher",- "resolve_executor_transport",]-def resolve_executor_transport(context: ExecutionContext) -> bool:- """True → route this executor dispatch over PG; False → Celery (default).-- The single ``pg_queue_enabled`` Flipt flag (fail-closed).- """- return resolve_pg_transport(context)--class DjangoQueueTransport(QueueTransport):""":class:`QueueTransport` over the Django ORM (the backend half).@@ -122,12 +109,15 @@ class DjangoQueueTransport(QueueTransport):return row-def get_executor_dispatcher(- celery_app: object | None = None,-) -> RoutingExecutionDispatcher:- """Factory: the gate-routed executor dispatcher (PG when enabled, else Celery)."""- return RoutingExecutionDispatcher(- celery=ExecutionDispatcher(celery_app=celery_app),- pg=PgExecutionDispatcher(DjangoQueueTransport()),- resolve=resolve_executor_transport,- )+def get_executor_dispatcher() -> PgExecutionDispatcher:+ """Factory: the executor dispatcher.++ Takes no arguments. It used to accept a ``celery_app`` that fed the Celery+ branch of the routing dispatcher; that branch went with the ``pg_queue_enabled``+ flag (UN-4046), and the parameter was kept for a while so the call sites did not+ all need editing at once. Keeping an ignored parameter made the signature+ decoration rather than a contract — and it read as "this dispatcher may use+ Celery", which is exactly the belief that left a ``headers=`` argument at three+ call sites and broke every extraction. Removed.+ """+ return PgExecutionDispatcher(DjangoQueueTransport())diff --git a/backend/pg_queue/flags.py b/backend/pg_queue/flags.pydeleted file mode 100644index cf190134c..000000000--- a/backend/pg_queue/flags.py+++ /dev/null@@ -1,12 +0,0 @@-"""Single source of truth for the PG-queue rollout flag key.--One Flipt flag gates the whole PG-queue feature — execution-(``workflow_v2/transport.py``), scheduler (``scheduler/ownership.py``), and-executor (``pg_queue/executor_rpc.py``) all read this one key. Kept in a neutral-leaf module so the three resolvers import a single constant instead of-duplicating the literal (a grep on ``PG_QUEUE_FLAG_KEY`` finds every use), making-"one flag" a structural guarantee. This flag is the **sole** rollout control —-fail-closed to Celery on a blind/unreachable Flipt or any error.-"""--PG_QUEUE_FLAG_KEY = "pg_queue_enabled"diff --git a/backend/pg_queue/management/commands/reconcile_pg_schedules.py b/backend/pg_queue/management/commands/reconcile_pg_schedules.pyindex f5e9cd6f4..af7e163fb 100644--- a/backend/pg_queue/management/commands/reconcile_pg_schedules.py+++ b/backend/pg_queue/management/commands/reconcile_pg_schedules.py@@ -260,7 +260,7 @@ class Command(BaseCommand):# Preview only — read the would-be owner (no DB write) so an# operator can see how many a ramp change would hand to PG.reconciled += 1- if resolve_schedule_owner(str(row.pipeline_id), row.organization_id):+ if resolve_schedule_owner():pg_owned += 1continue# mirror.enabled tracks pipeline.active (dual-write); use it as thediff --git a/backend/pg_queue/tests/test_converge_pg_scheduler.py b/backend/pg_queue/tests/test_converge_pg_scheduler.pyindex f0edae1fe..b66ba222e 100644--- a/backend/pg_queue/tests/test_converge_pg_scheduler.py+++ b/backend/pg_queue/tests/test_converge_pg_scheduler.py@@ -48,15 +48,17 @@ class TestDirection:assert kwargs["mirror_only"] is Trueassert kwargs["release_stale"] is True- def test_an_unset_gate_converges_to_beat(self, monkeypatch):- """Absent must mean Beat, not "do nothing" — the default has to be the- safe direction, since that is what an environment that never opted in has.+ def test_an_unset_gate_converges_to_pg(self, monkeypatch):+ """Absent must mean PG, not "do nothing" — the default has to be the+ direction the deployment actually runs, and PG_SCHEDULER_ENABLED now+ defaults on (UN-4046)."""monkeypatch.delenv("PG_SCHEDULER_ENABLED", raising=False)with patch(f"{_CMD}.call_command") as sub:call_command("converge_pg_scheduler")kwargs = {c.args[0]: c.kwargs for c in sub.call_args_list}- assert kwargs["reconcile_pg_schedules"]["release_stale"] is True+ # Hand-over direction: adopt, not release.+ assert "release_stale" not in kwargs["reconcile_pg_schedules"]class TestPeriodicsAreOptIn:diff --git a/backend/pg_queue/tests/test_executor_rpc.py b/backend/pg_queue/tests/test_executor_rpc.pyindex 697d53d46..0a51ecb91 100644--- a/backend/pg_queue/tests/test_executor_rpc.py+++ b/backend/pg_queue/tests/test_executor_rpc.py@@ -10,14 +10,11 @@ single-Flipt-flag gate, and the factory wiring.from unittest.mock import MagicMock, patch-from unstract.workflow_execution.executor_rpc import ExecResultRow-from pg_queue.executor_rpc import (DjangoQueueTransport,- RoutingExecutionDispatcher,get_executor_dispatcher,- resolve_executor_transport,)+from unstract.workflow_execution.executor_rpc import ExecResultRow, PgExecutionDispatcher_MOD = "pg_queue.executor_rpc"@@ -34,7 +31,9 @@ class TestDjangoQueueTransportEnqueue:def test_enqueue_calls_enqueue_task_with_request_reply_fields(self):with patch(f"{_MOD}.enqueue_task") as enq:DjangoQueueTransport().enqueue(- queue="celery_executor_legacy", context=_ctx(), org_id="org9",+ queue="celery_executor_legacy",+ context=_ctx(),+ org_id="org9",reply_key="rk1",)kw = enq.call_args.kwargs@@ -49,8 +48,11 @@ class TestDjangoQueueTransportEnqueue:spec = {"task_name": "ide_prompt_complete", "kwargs": {}, "queue": "ide_callback"}with patch(f"{_MOD}.enqueue_task") as enq:DjangoQueueTransport().enqueue(- queue="celery_executor_legacy", context=_ctx(), org_id="o",- on_success=spec, task_id="tid-7",+ queue="celery_executor_legacy",+ context=_ctx(),+ org_id="o",+ on_success=spec,+ task_id="tid-7",)kw = enq.call_args.kwargsassert kw["on_success"] == spec and kw["task_id"] == "tid-7"@@ -116,23 +118,10 @@ class TestDjangoQueueTransportWait:qs.filter.return_value.update.assert_not_called()-class TestResolveExecutorTransport:- def test_delegates_to_shared_flipt_resolver_true(self):- with patch(f"{_MOD}.resolve_pg_transport", return_value=True) as r:- assert resolve_executor_transport(_ctx()) is True- r.assert_called_once()- # No master-gate is threaded any more — Flipt is the sole gate.- assert "master_gate_enabled" not in r.call_args.kwargs-- def test_delegates_to_shared_flipt_resolver_false(self):- with patch(f"{_MOD}.resolve_pg_transport", return_value=False):- assert resolve_executor_transport(_ctx()) is False--class TestFactoryWiring:- def test_factory_wires_routing_with_django_transport(self):- d = get_executor_dispatcher(celery_app="app")- assert isinstance(d, RoutingExecutionDispatcher)- # PG dispatcher uses the backend ORM transport; gate = the Flipt resolver.- assert isinstance(d._pg._transport, DjangoQueueTransport)- assert d._resolve is resolve_executor_transport+ def test_factory_wires_the_django_transport(self):+ # The factory takes no arguments (UN-4046) and returns the PG dispatcher+ # wired with the backend ORM transport.+ d = get_executor_dispatcher()+ assert isinstance(d, PgExecutionDispatcher)+ assert isinstance(d._transport, DjangoQueueTransport)diff --git a/backend/pipeline_v2/piepline_api_execution_views.py b/backend/pipeline_v2/piepline_api_execution_views.pyindex ca0ba8fa3..5b78e020e 100644--- a/backend/pipeline_v2/piepline_api_execution_views.py+++ b/backend/pipeline_v2/piepline_api_execution_views.py@@ -5,7 +5,6 @@ from rest_framework import status, viewsfrom rest_framework.request import Requestfrom rest_framework.response import Response-from backend.celery_service import app as celery_appfrom pipeline_v2.deployment_helper import DeploymentHelperfrom pipeline_v2.models import Pipelinefrom pipeline_v2.pipeline_dispatch import dispatch_pipeline_trigger@@ -30,10 +29,7 @@ class PipelineApiExecution(views.APIView):def post(self, request: Request, org_name: str, pipeline_id: str, pipeline: Pipeline) -> Response:- # Route through the transport flag instead of hardcoding Celery- # (see dispatch_pipeline_trigger).dispatch_pipeline_trigger(- celery_app=celery_app,org_id=org_name,pipeline_id=pipeline_id,pipeline_name=pipeline.pipeline_name,diff --git a/backend/pipeline_v2/pipeline_dispatch.py b/backend/pipeline_v2/pipeline_dispatch.pyindex c8d8002d0..7fc39f35e 100644--- a/backend/pipeline_v2/pipeline_dispatch.py+++ b/backend/pipeline_v2/pipeline_dispatch.py@@ -1,29 +1,24 @@-"""Transport-routed dispatch for the API-triggered pipeline execution trigger.+"""PG-queue dispatch for the API-triggered pipeline execution trigger.-Routes ``scheduler.tasks.execute_pipeline_task`` through the same-:func:`resolve_transport` flag as the rest of the execution path: the PG queue-when enabled for this pipeline/org, else Celery. **Fail-closed** — with the-master gate off (the production default) it resolves to Celery, behaving exactly-like the prior unconditional ``send_task`` (zero regression). The-scheduled-pipeline path already enqueues this task onto the PG ``scheduler``-queue (``pg_scheduler.dispatch_due_schedules``, run by ``worker-pg-scheduler``);-this brings the API-trigger view to parity.+Enqueues ``scheduler.tasks.execute_pipeline_task`` onto the PG ``scheduler``+queue, where ``worker-pg-scheduler`` polls it — the same queue and task the+scheduled-pipeline path uses (``pg_scheduler.dispatch_due_schedules``), so the+API trigger and the schedule trigger are at parity.-The same ``args`` are sent on both paths; on PG they're JSON-normalized by-``enqueue_task`` (UUIDs/datetimes → str), a no-op for these string/bool values,-so the consumer sees the same payload.+This used to route through ``resolve_transport``/``pg_queue_enabled`` and fall+back to ``celery_app.send_task``. The flag is gone (UN-4046) and PG is the only+transport, so the Celery branch went with it.++``args`` are JSON-normalized by ``enqueue_task`` (UUIDs/datetimes → str), a no-op+for these string/bool values."""from __future__ import annotationsimport logging-from typing import Anyfrom uuid import UUIDfrom pg_queue.producer import enqueue_task-from workflow_manager.workflow_v2.transport import resolve_transport--from unstract.core.data_models import is_pg_transportlogger = logging.getLogger(__name__)@@ -36,40 +31,26 @@ SCHEDULER_QUEUE = "scheduler"def dispatch_pipeline_trigger(*,- celery_app: Any,org_id: str | UUID,pipeline_id: str | UUID,pipeline_name: str,) -> None:- """Dispatch the pipeline-trigger task on the resolved transport.+ """Enqueue the pipeline-trigger task on the PG scheduler queue.- The positional args match ``execute_pipeline_task``'s signature on **both**- paths: ``(workflow_id, org_schema, execution_action, execution_id,- pipeline_id, with_logs, name)``. ``org_id`` / ``pipeline_id`` accept ``UUID``- (what ``resolve_transport`` takes) and are str-coerced into the task args.+ The positional args match ``execute_pipeline_task``'s signature:+ ``(workflow_id, org_schema, execution_action, execution_id, pipeline_id,+ with_logs, name)``. ``org_id`` / ``pipeline_id`` accept ``UUID`` and are+ str-coerced into the task args."""args = ["", str(org_id), "", "", str(pipeline_id), True, pipeline_name]- # No execution exists yet (it's created inside the task), so the trigger- # buckets the flag by pipeline_id as the sticky entity.- transport = resolve_transport(- execution_id=pipeline_id,- organization_id=org_id,- pipeline_id=pipeline_id,+ msg_id = enqueue_task(+ task_name=PIPELINE_TRIGGER_TASK,+ queue=SCHEDULER_QUEUE,+ args=args,+ org_id=str(org_id or ""),+ )+ logger.info(+ "Pipeline %s trigger enqueued on PG scheduler queue (msg_id=%s)",+ pipeline_id,+ msg_id,)- # Use the shared is_pg_transport() — the single source for "what counts as PG- # transport" — rather than opening a second comparison site.- if is_pg_transport(transport):- msg_id = enqueue_task(- task_name=PIPELINE_TRIGGER_TASK,- queue=SCHEDULER_QUEUE,- args=args,- org_id=str(org_id or ""),- )- logger.info(- "Pipeline %s trigger enqueued on PG scheduler queue (msg_id=%s)",- pipeline_id,- msg_id,- )- else:- celery_app.send_task(PIPELINE_TRIGGER_TASK, args=args)- logger.info("Pipeline %s trigger dispatched on Celery", pipeline_id)diff --git a/backend/pipeline_v2/tests/test_pipeline_dispatch.py b/backend/pipeline_v2/tests/test_pipeline_dispatch.pyindex ac8bd63e0..64ad6b7d2 100644--- a/backend/pipeline_v2/tests/test_pipeline_dispatch.py+++ b/backend/pipeline_v2/tests/test_pipeline_dispatch.py@@ -1,14 +1,19 @@-"""Unit tests for the API-triggered pipeline-trigger transport routing (UN-3616).+"""Unit tests for the API-triggered pipeline-trigger dispatch (UN-3616, UN-4046).-`resolve_transport` + `enqueue_task` are patched on the module, so no Flipt / DB-is needed — these pin the routing contract: PG when the flag resolves PG, Celery-otherwise (fail-closed), with byte-identical args on both paths.+``enqueue_task`` is patched on the module, so no DB is needed. These pin the+payload contract the ``scheduler`` consumer depends on, and the no-silent-fallback+property.++The transport-routing tests are gone with the ``pg_queue_enabled`` flag (UN-4046):+there is no Celery branch left to choose, so "routes to celery", "resolve_transport+called with the pipeline as entity" and "args identical on both paths" no longer+describe anything. What survives is what the consumer actually sees."""from __future__ import annotationsimport uuid-from unittest.mock import MagicMock, patch+from unittest.mock import patchimport pytest@@ -17,9 +22,8 @@ import pipeline_v2.pipeline_dispatch as pd_EXPECTED_ARGS = ["", "org_x", "", "", "pid-1", True, "My Pipeline"]-def _dispatch(celery_app):+def _dispatch():return pd.dispatch_pipeline_trigger(- celery_app=celery_app,org_id="org_x",pipeline_id="pid-1",pipeline_name="My Pipeline",@@ -27,83 +31,29 @@ def _dispatch(celery_app):class TestDispatchPipelineTrigger:- def test_routes_to_pg_when_flag_resolves_pg(self):- celery = MagicMock()- with (- patch.object(pd, "resolve_transport", return_value="pg_queue"),- patch.object(pd, "enqueue_task", return_value=42) as enqueue,- ):- _dispatch(celery)+ def test_enqueues_on_the_pg_scheduler_queue(self):+ with patch.object(pd, "enqueue_task", return_value=42) as enqueue:+ _dispatch()enqueue.assert_called_once()kwargs = enqueue.call_args.kwargsassert kwargs["task_name"] == "scheduler.tasks.execute_pipeline_task"assert kwargs["queue"] == "scheduler"assert kwargs["args"] == _EXPECTED_ARGSassert kwargs["org_id"] == "org_x"- celery.send_task.assert_not_called()- def test_pg_enqueue_failure_propagates_with_no_celery_fallback(self):- # The dispatcher has no try/except → a PG enqueue failure must surface- # (no silent Celery fallback, which would risk a double-dispatch).- celery = MagicMock()- with (- patch.object(pd, "resolve_transport", return_value="pg_queue"),- patch.object(pd, "enqueue_task", side_effect=RuntimeError("pg down")),- ):+ def test_pg_enqueue_failure_propagates(self):+ # The dispatcher has no try/except → a PG enqueue failure must surface to+ # the caller rather than being swallowed into a silent no-dispatch.+ with patch.object(pd, "enqueue_task", side_effect=RuntimeError("pg down")):with pytest.raises(RuntimeError, match="pg down"):- _dispatch(celery)- celery.send_task.assert_not_called()+ _dispatch()def test_uuid_pipeline_id_is_coerced_to_str_in_args(self):- # resolve_transport accepts a UUID, but the task args must carry strings.+ # The signature accepts a UUID, but the task args must carry strings —+ # enqueue_task JSON-normalizes, and the consumer unpacks positionally.pid = uuid.UUID("b1f16024-45f2-4e39-8756-d40e24148e30")- celery = MagicMock()- with (- patch.object(pd, "resolve_transport", return_value="pg_queue") as resolve,- patch.object(pd, "enqueue_task", return_value=1) as enqueue,- ):+ with patch.object(pd, "enqueue_task", return_value=1) as enqueue:pd.dispatch_pipeline_trigger(- celery_app=celery, org_id="org_x", pipeline_id=pid, pipeline_name="P"+ org_id="org_x", pipeline_id=pid, pipeline_name="P")assert enqueue.call_args.kwargs["args"][4] == str(pid)- # The raw UUID is passed to resolve_transport as the sticky entity.- assert resolve.call_args.kwargs["execution_id"] == pid-- def test_routes_to_celery_when_flag_resolves_celery(self):- celery = MagicMock()- with (- patch.object(pd, "resolve_transport", return_value="celery"),- patch.object(pd, "enqueue_task") as enqueue,- ):- _dispatch(celery)- celery.send_task.assert_called_once_with(- "scheduler.tasks.execute_pipeline_task", args=_EXPECTED_ARGS- )- enqueue.assert_not_called()-- def test_resolve_transport_called_with_pipeline_as_entity(self):- celery = MagicMock()- with (- patch.object(pd, "resolve_transport", return_value="celery") as resolve,- patch.object(pd, "enqueue_task"),- ):- _dispatch(celery)- kwargs = resolve.call_args.kwargs- assert kwargs["execution_id"] == "pid-1" # buckets by pipeline_id- assert kwargs["organization_id"] == "org_x"- assert kwargs["pipeline_id"] == "pid-1"-- def test_args_identical_on_both_paths(self):- # The consumer must behave the same regardless of transport — assert the- # PG-path args equal the Celery-path args.- celery = MagicMock()- with patch.object(pd, "resolve_transport", return_value="celery"):- _dispatch(celery)- celery_args = celery.send_task.call_args.kwargs["args"]- celery2 = MagicMock()- with (- patch.object(pd, "resolve_transport", return_value="pg_queue"),- patch.object(pd, "enqueue_task", return_value=1) as enqueue,- ):- _dispatch(celery2)- assert enqueue.call_args.kwargs["args"] == celery_argsdiff --git a/backend/prompt_studio/prompt_studio_core_v2/prompt_studio_helper.py b/backend/prompt_studio/prompt_studio_core_v2/prompt_studio_helper.pyindex ba157801c..94f3850c6 100644--- a/backend/prompt_studio/prompt_studio_core_v2/prompt_studio_helper.py+++ b/backend/prompt_studio/prompt_studio_core_v2/prompt_studio_helper.py@@ -26,7 +26,6 @@ from utils.file_storage.constants import FileStorageKeysfrom utils.file_storage.helpers.prompt_studio_file_helper import PromptStudioFileHelperfrom utils.local_context import StateStore-from backend.celery_service import app as celery_appfrom prompt_studio.lookup_utils import (get_lookup_config,get_lookup_configs_for_tool,@@ -327,15 +326,17 @@ class PromptStudioHelper:def _get_dispatcher():"""Executor dispatcher for the executor worker.- Gate-routed: when ``pg_queue_enabled`` is on the blocking- ``dispatch()`` rides the PG request-reply transport; otherwise — and for- all async/callback dispatches — it is the unchanged Celery- ``ExecutionDispatcher``. The decision is read per dispatch, so flipping- the flag is an instant, no-redeploy rollout/rollback.+ Always the PG request-reply dispatcher since UN-4046. This used to be+ gate-routed per dispatch on ``pg_queue_enabled``, falling back to the+ Celery ``ExecutionDispatcher``; with the flag gone the factory returns+ the PG dispatcher directly. It is kept as a factory call rather than a+ direct construction on purpose — UN-3779 was a hardcoded+ ``ExecutionDispatcher`` here that published to RabbitMQ and hung for its+ full 1200s timeout with no consumer."""from pg_queue.executor_rpc import get_executor_dispatcher- return get_executor_dispatcher(celery_app=celery_app)+ return get_executor_dispatcher()@staticmethoddef _get_platform_api_key(org_id: str) -> str:diff --git a/backend/prompt_studio/prompt_studio_core_v2/tests/test_task_status.py b/backend/prompt_studio/prompt_studio_core_v2/tests/test_task_status.pyindex ab0754ee1..64864bc2c 100644--- a/backend/prompt_studio/prompt_studio_core_v2/tests/test_task_status.py+++ b/backend/prompt_studio/prompt_studio_core_v2/tests/test_task_status.py@@ -1,11 +1,13 @@-"""UN-3693: PromptStudioCoreView.task_status — flag-gated.+"""UN-3693 / UN-4046: PromptStudioCoreView.task_status.-Gate on the per-org ``pg_queue_enabled`` flag (the same the executor dispatch uses):-flag ON → read ``pg_task_result`` (completed / failed / none→processing); flag OFF-(default) → Celery ``AsyncResult`` unchanged, and the PG table is never queried on-the hot poll path. ``check_feature_flag_status`` already fails closed to ``False`` on-any Flipt error, so "flag off / Flipt down" is one path. A transient PG DB error-degrades to "processing", not a bare 500.+The task rode PG, so status is read from ``pg_task_result``: completed / failed /+no row → processing. A transient PG DB error degrades to "processing", not a bare+500 — a status-code-keyed client would misread a 500 as a terminal task failure.++This used to be gated on the per-org ``pg_queue_enabled`` flag, with a Celery+``AsyncResult`` fall-through when it was off. The flag is gone (UN-4046), so the+flag-off suite went with it — there is no Celery result backend to poll (the eager+PG executor never wrote one, which is why the gate existed at all).NOTE: run via a Django-bootstrapped harness (no pytest-django in this repo yet — theCI-gating of these view tests is tracked in UN-3692). Verified locally green.@@ -15,7 +17,6 @@ from unittest.mock import MagicMock, patchfrom prompt_studio.prompt_studio_core_v2.views import PromptStudioCoreView-_VIEWS = "prompt_studio.prompt_studio_core_v2.views"_TID = "task-abc"@@ -25,14 +26,6 @@ def _view():return view-def _flag(enabled):- return patch(f"{_VIEWS}.check_feature_flag_status", return_value=enabled)---def _org():- return patch(f"{_VIEWS}.UserSessionUtils.get_organization_id", return_value="org1")--def _stub_pg(pg_row=None, *, error=None):"""Patch pg_task_result read: .filter().values().first() → pg_row, or raise."""m = MagicMock()@@ -43,72 +36,30 @@ def _stub_pg(pg_row=None, *, error=None):return patch("pg_queue.models.PgTaskResult", m), m-def _async(ready, successful=True, result=""):- ar = MagicMock()- ar.return_value.ready.return_value = ready- ar.return_value.successful.return_value = successful- ar.return_value.result = result- return ar---class TestTaskStatusFlagOn:- def test_pg_completed(self):+class TestTaskStatus:+ def test_completed(self):p_pg, _ = _stub_pg({"status": "completed", "error": ""})- with _org(), _flag(True), p_pg:+ with p_pg:resp = _view().task_status(MagicMock(), task_id=_TID)assert resp.data == {"task_id": _TID, "status": "completed"}- def test_pg_failed(self):+ def test_failed(self):p_pg, _ = _stub_pg({"status": "failed", "error": "extraction blew up"})- with _org(), _flag(True), p_pg:+ with p_pg:resp = _view().task_status(MagicMock(), task_id=_TID)assert resp.data["status"] == "failed"assert resp.data["error"] == "extraction blew up"assert resp.status_code == 500- def test_no_pg_row_is_processing(self):+ def test_no_row_is_processing(self):p_pg, _ = _stub_pg(None) # PG task not done yet — no "pending" row- with _org(), _flag(True), p_pg:+ with p_pg:resp = _view().task_status(MagicMock(), task_id=_TID)assert resp.data == {"task_id": _TID, "status": "processing"}- def test_pg_db_error_degrades_to_processing(self):+ def test_db_error_degrades_to_processing(self):# A transient PG read error must not 500; degrade to "processing".p_pg, _ = _stub_pg(error=RuntimeError("db blip"))- with _org(), _flag(True), p_pg:- resp = _view().task_status(MagicMock(), task_id=_TID)- assert resp.data == {"task_id": _TID, "status": "processing"}---class TestTaskStatusFlagOff:- def test_celery_completed_never_queries_pg(self):- # flag off (also the Flipt-error fail-closed path): Celery + PG untouched.- p_pg, pg = _stub_pg({"status": "completed"}) # would resolve, if queried- with (- _org(),- _flag(False),- p_pg,- patch(f"{_VIEWS}.AsyncResult", _async(ready=True)),- ):- resp = _view().task_status(MagicMock(), task_id=_TID)- assert resp.data == {"task_id": _TID, "status": "completed"}- pg.objects.filter.assert_not_called() # default path never touches PG-- def test_celery_failed(self):- with (- _org(),- _flag(False),- patch(- f"{_VIEWS}.AsyncResult",- _async(ready=True, successful=False, result="celery boom"),- ),- ):- resp = _view().task_status(MagicMock(), task_id=_TID)- assert resp.data["status"] == "failed"- assert "celery boom" in resp.data["error"]- assert resp.status_code == 500-- def test_pending(self):- with _org(), _flag(False), patch(f"{_VIEWS}.AsyncResult", _async(ready=False)):+ with p_pg:resp = _view().task_status(MagicMock(), task_id=_TID)assert resp.data == {"task_id": _TID, "status": "processing"}diff --git a/backend/prompt_studio/prompt_studio_core_v2/views.py b/backend/prompt_studio/prompt_studio_core_v2/views.pyindex b3d42cf5f..8990e870e 100644--- a/backend/prompt_studio/prompt_studio_core_v2/views.py+++ b/backend/prompt_studio/prompt_studio_core_v2/views.py@@ -9,7 +9,6 @@ from typing import Anyimport magicfrom account_v2.custom_exceptions import DuplicateDatafrom celery import signature-from celery.result import AsyncResultfrom django.db import IntegrityErrorfrom django.db.models import Count, OuterRef, QuerySet, Subqueryfrom django.http import HttpRequest, HttpResponse@@ -20,7 +19,6 @@ from permissions.membership_views import OwnerManagementMixinfrom permissions.permission import IsOwner, IsOwnerOrSharedUserOrSharedToOrgfrom permissions.resource_share_views import ResourceShareManagementMixinfrom permissions.roles import ResourceRole-from pg_queue.flags import PG_QUEUE_FLAG_KEYfrom plugins import get_pluginfrom rest_framework import status, viewsetsfrom rest_framework.decorators import action@@ -33,7 +31,6 @@ from utils.pagination import OptionalPaginationfrom utils.user_context import UserContextfrom utils.user_session import UserSessionUtils-from backend.celery_service import app as celery_appfrom prompt_studio.lookup_utils import (get_latest_lookup_mutation_for_tool,get_lookup_validation_for_tool,@@ -86,7 +83,6 @@ from prompt_studio.tool_usage import (join_deployment_types,)from unstract.core.data_models import PgTaskStatus-from unstract.flags.feature_flag import check_feature_flag_statusfrom unstract.sdk1.utils.common import Utils as CommonUtilsfrom .models import CustomTool@@ -865,57 +861,34 @@ class PromptStudioCoreView(# Verify the user has access to this tool (triggers permission check)self.get_object()- # Gate on the same per-org ``pg_queue_enabled`` flag the executor dispatch- # uses (``resolve_pg_transport`` — bucketed per org): flag ON → the task rode- # PG, which records its terminal state in ``pg_task_result`` (the eager PG- # executor never writes a Celery result backend, so ``AsyncResult`` alone- # would poll "processing" forever — UN-3693); flag OFF (default) → Celery,- # unchanged, and PG is not touched. ``entity_id``/``context`` match- # ``resolve_pg_transport`` so the per-org bucket agrees with the dispatch.- # No wrapper: ``check_feature_flag_status`` already fails closed to ``False``- # (its own try/except + warning) on any Flipt error → reads as "off" → Celery.- org_id = str(UserSessionUtils.get_organization_id(request) or "")- pg_enabled = check_feature_flag_status(- flag_key=PG_QUEUE_FLAG_KEY,- entity_id=org_id or "default",- context={"organization_id": org_id},- )- if pg_enabled:- from pg_queue.models import PgTaskResult+ # The task rode PG, which records its terminal state in ``pg_task_result``.+ # (The eager PG executor never writes a Celery result backend, so the old+ # ``AsyncResult`` poll would have said "processing" forever — UN-3693.)+ from pg_queue.models import PgTaskResult- try:- row = (- PgTaskResult.objects.filter(task_id=task_id)- .values("status", "error")- .first()- )- except Exception:- # A transient DB blip degrades to "processing" (a poll retries) rather- # than a bare 500 that a status-code-keyed client would misread as a- # terminal task failure.- logger.exception(- "task_status: pg_task_result read failed for %s; treating as pending",- task_id,- )- row = None- # No row → not finished yet (there is no "pending" row). Status compared- # against PgTaskStatus (the writer's source of truth), not a bare literal.- if row is None:- return Response({"task_id": task_id, "status": "processing"})- if row["status"] == PgTaskStatus.COMPLETED.value:- return Response({"task_id": task_id, "status": "completed"})- return Response(- {"task_id": task_id, "status": "failed", "error": row["error"] or ""},- status=status.HTTP_500_INTERNAL_SERVER_ERROR,+ try:+ row = (+ PgTaskResult.objects.filter(task_id=task_id)+ .values("status", "error")+ .first())-- result = AsyncResult(task_id, app=celery_app)- if not result.ready():+ except Exception:+ # A transient DB blip degrades to "processing" (a poll retries) rather+ # than a bare 500 that a status-code-keyed client would misread as a+ # terminal task failure.+ logger.exception(+ "task_status: pg_task_result read failed for %s; treating as pending",+ task_id,+ )+ row = None+ # No row → not finished yet (there is no "pending" row). Status compared+ # against PgTaskStatus (the writer's source of truth), not a bare literal.+ if row is None:return Response({"task_id": task_id, "status": "processing"})- if result.successful():+ if row["status"] == PgTaskStatus.COMPLETED.value:return Response({"task_id": task_id, "status": "completed"})return Response(- {"task_id": task_id, "status": "failed", "error": str(result.result)},+ {"task_id": task_id, "status": "failed", "error": row["error"] or ""},status=status.HTTP_500_INTERNAL_SERVER_ERROR,)diff --git a/backend/scheduler/helper.py b/backend/scheduler/helper.pyindex d4ea878c4..dc3421c2b 100644--- a/backend/scheduler/helper.py+++ b/backend/scheduler/helper.py@@ -75,11 +75,10 @@ class SchedulerHelper:cron_string=cron_string,enabled=pipeline.active,)- # Ramp control (Phase 9, ②c): align this schedule's firer (Beat vs PG)- # with the rollout. Inert until the gate + pg_scheduler_enabled flag are- # on (fails closed to Beat), so this is a no-op during normal operation;- # when a schedule is owned by PG it disables the Beat PeriodicTask in the- # same transaction so the two never both fire. Best-effort (never raises).+ # Align this schedule's firer (Beat vs PG). NOT a no-op: since UN-4046+ # PG_SCHEDULER_ENABLED defaults to true, so the normal path here hands the+ # schedule to PG and disables the Beat PeriodicTask in the same transaction+ # (which is what stops the two both firing). Best-effort (never raises).# organization_id is the org identifier string (what the mirror stores).reconcile_ownership_for(str(pipeline.pk), organization_id, active=pipeline.active)diff --git a/backend/scheduler/ownership.py b/backend/scheduler/ownership.pyindex c7c9c2817..a53c3d21e 100644--- a/backend/scheduler/ownership.py+++ b/backend/scheduler/ownership.py@@ -4,24 +4,23 @@ from Celery Beat to the Postgres scheduler, per-schedule and reversibly.A schedule is owned by exactly one firer. ``reconcile_ownership_for``applies that decision atomically:- pg_periodic_schedule.pg_owned = resolve_schedule_owner(...) # PG fires it+ pg_periodic_schedule.pg_owned = resolve_schedule_owner() # PG fires itPeriodicTask.enabled = active AND NOT pg_owned # Beat fires itDoing both in one transaction is what makes "never double-fires" real (it was*conditional* on this slice): a ``pg_owned`` row always has its Beat``PeriodicTask`` disabled, so the two can't both fire.-**Two gates, and both must be on.** ``PG_SCHEDULER_ENABLED`` (env, default off) comes-first; the ``pg_queue_enabled`` Flipt flag then decides per schedule, failing closed to-Beat on a blind Flipt or any error.+**One gate, and it defaults ON.** ``PG_SCHEDULER_ENABLED`` (env) is the whole decision.+UN-4046 removed the second one — the per-schedule ``pg_queue_enabled`` Flipt flag that+used to fail closed to Beat — and flipped this env's default from ``false`` to ``true``.-They are separate because Beat stays the **sole scheduler for the whole PG rollout**:-pipelines already reach PG without this module (``execute_pipeline_task_v2`` →-``complete_execution`` → ``execute_workflow_async`` → ``resolve_transport``), so the PG-scheduler exists only to retire the Beat *deployment* — a later, deliberate step. While-the env gate is off this module writes **nothing at all**, so Beat's tables are untouched-for the entire rollout and turning the flag off again is a pure Flipt flip with nothing-to restore.+The consequence is the opposite of what this docstring said before, so it is worth+stating plainly: on a default deployment this module DOES write, on every schedule save+and on every ``--migrate`` start, handing pipelines to PG and disabling Beat's+``PeriodicTask`` rows. Rolling back is not a flag flip with nothing to restore — it is+``PG_SCHEDULER_ENABLED=false`` **plus** a ``converge_pg_scheduler`` run to release the+rows already adopted. Beat coming back does not re-enable rows this module disabled."""from __future__ import annotations@@ -32,89 +31,46 @@ import osfrom django.db import transactionfrom django.utils import timezonefrom django_celery_beat.models import PeriodicTask, PeriodicTasks-from pg_queue.flags import PG_QUEUE_FLAG_KEYfrom pg_queue.models import PgPeriodicSchedule-from unstract.flags.feature_flag import check_feature_flag_status-logger = logging.getLogger(__name__)-# Gating uses the single shared PG-queue flag (pg_queue.flags.PG_QUEUE_FLAG_KEY,-# imported above) — one flip gates execution + scheduler + executor. The scheduler-# buckets the %-rollout on pipeline_id (each subsystem keys on its own entity).---# Second gate, ahead of the Flipt flag: hand-over to the PG scheduler happens only-# when this is explicitly on. Referenced as "pg_scheduler_enabled" by this module's-# callers' comments (scheduler/helper.py, reconcile_pg_schedules) since the ramp was-# designed, but never actually implemented — resolve_schedule_owner gated on-# pg_queue_enabled alone, so the two flips were welded together.-#-# They must be separable, because Beat stays the SOLE scheduler for the whole PG-# rollout: pipelines already reach PG without it (execute_pipeline_task_v2 →-# complete_execution → execute_workflow_async → resolve_transport), so the PG-# scheduler exists only to retire the Beat *deployment* — a later, deliberate step.-#-# Without this gate, turning pg_queue_enabled on would hand schedules to a PG-# scheduler that is not running: reconcile_ownership_for disables the Beat-# PeriodicTask, nothing polls the PG side, and the pipeline has no firer at all. It-# runs in the backend on every schedule save, so scaling Beat to zero does not avoid-# it and no ramp command is needed to trigger it — a user saving a schedule suffices.+# The sole gate on schedule hand-over. It used to sit ahead of the+# ``pg_queue_enabled`` Flipt flag; the flag is gone (UN-4046) and this remains,+# because it guards a failure mode of its own: ``reconcile_ownership_for``+# disables the Beat ``PeriodicTask`` whenever a schedule is PG-owned, so handing+# schedules over while no PG scheduler is running leaves the pipeline with no+# firer at all. It runs in the backend on every schedule save, so scaling Beat to+# zero does not avoid it — a user saving a schedule suffices._PG_SCHEDULER_ENABLED_ENV = "PG_SCHEDULER_ENABLED"def pg_scheduler_enabled() -> bool:- """Whether schedule hand-over to the PG scheduler is switched on at all.+ """Whether schedule hand-over to the PG scheduler is switched on.- Defaults **off**. Flip it only alongside deploying the PG scheduler worker, as- part of retiring Celery Beat.+ Defaults **on** (UN-4046): PG is the only transport and the PG scheduler ships+ with the fleet. Set ``PG_SCHEDULER_ENABLED=false`` only to keep Beat as the+ firer in a deployment that deliberately runs without ``worker-pg-scheduler``."""- return os.environ.get(_PG_SCHEDULER_ENABLED_ENV, "false").strip().lower() == "true"+ return os.environ.get(_PG_SCHEDULER_ENABLED_ENV, "true").strip().lower() == "true"-def resolve_schedule_owner(pipeline_id: str, organization_id: str | None) -> bool:+def resolve_schedule_owner() -> bool:"""True → the PG scheduler owns this schedule; False → Celery Beat does.- Mirrors ``resolve_transport``: gated by the single ``pg_queue_enabled`` Flipt- flag, keyed on ``pipeline_id`` for a stable percentage bucket. **Fails closed to- Beat** on a blind Flipt or any error — so a schedule never silently loses its- firer.+ Gated solely by ``PG_SCHEDULER_ENABLED``. This used to *also* require the+ ``pg_queue_enabled`` Flipt flag, because during the rollout the flag could be+ on while the PG scheduler was still dark and a schedule handed to a scheduler+ that is not running would simply stop firing. The flag is gone (UN-4046), so+ the env gate is the sole dial — and it still has to be, for the same reason:+ ``reconcile_ownership_for`` disables the Beat ``PeriodicTask`` when this+ returns True, so turning it on without a running PG scheduler leaves the+ pipeline with no firer at all.- Requires ``PG_SCHEDULER_ENABLED`` **as well as** the flag: during the PG rollout- the flag is on while the PG scheduler is still dark, and a schedule handed to a- scheduler that is not running would simply stop firing.+ ``organization_id`` is retained for call-site compatibility; it fed the Flipt+ context and is now unused."""- if not pg_scheduler_enabled():- return False- if os.environ.get("FLIPT_SERVICE_AVAILABLE", "false").lower() != "true":- logger.warning(- "resolve_schedule_owner: FLIPT_SERVICE_AVAILABLE != true "- "(Flipt blind) for pipeline %s; leaving on Beat",- pipeline_id,- )- return False-- # Flipt context is a gRPC map<string,string>; coerce values to str (a non-str- # makes the client swallow it as False). entity_id str-coerced too so the- # %-rollout bucket is stable across str/UUID call sites.- context = {"pipeline_id": str(pipeline_id)}- if organization_id:- context["organization_id"] = str(organization_id)- try:- owned = check_feature_flag_status(- flag_key=PG_QUEUE_FLAG_KEY, entity_id=str(pipeline_id), context=context- )- except Exception:- # Expected, recoverable (fail-closed to Beat) and runs on every schedule- # edit — warn with traceback rather than logger.exception so a persistently- # down Flipt doesn't bury real errors as a per-edit Sentry exception.- logger.warning(- "resolve_schedule_owner: Flipt check failed for pipeline %s; leaving on Beat",- pipeline_id,- exc_info=True,- )- return False- return bool(owned)+ return pg_scheduler_enabled()def _has_stale_pg_ownership(pipeline_id: str) -> bool:@@ -149,7 +105,7 @@ def reconcile_ownership_for(window where both fire. On rollback (``pg_owned`` → False) ``next_run_at`` isalso cleared so a later re-hand-over re-enters the PG tick's NULL baseline(no burst). ``organization_id`` is the org *identifier* string (what the- mirror stores), used for Flipt per-org segmenting. ``active`` is keyword-only+ mirror stores), carried for logging/traceability. ``active`` is keyword-only(a fire/don't-fire boolean trap otherwise).Best-effort: a DB failure is logged and swallowed so it can never break the@@ -195,7 +151,7 @@ def reconcile_ownership_for()pg_owned = Falseelse:- pg_owned = resolve_schedule_owner(pipeline_id, organization_id)+ pg_owned = resolve_schedule_owner()try:with transaction.atomic():was_pg_owned = (diff --git a/backend/scheduler/tests/test_pg_schedule_ownership.py b/backend/scheduler/tests/test_pg_schedule_ownership.pyindex 4f05e8d15..0b6e5ee18 100644--- a/backend/scheduler/tests/test_pg_schedule_ownership.py+++ b/backend/scheduler/tests/test_pg_schedule_ownership.py@@ -18,47 +18,25 @@ _ORG = "org_abc"class TestResolveScheduleOwner:- @pytest.fixture(autouse=True)- def _scheduler_gate_on(self, monkeypatch):- """These pin the FINAL-phase behaviour, when hand-over is switched on.+ """Ownership is gated solely by PG_SCHEDULER_ENABLED (UN-4046).- PG_SCHEDULER_ENABLED defaults off for the whole rollout (Beat stays the sole- scheduler), so without this every case below would short-circuit to Beat and- assert nothing about the logic it was written for.- """- monkeypatch.setenv("PG_SCHEDULER_ENABLED", "true")+ The ``pg_queue_enabled`` Flipt cases (unavailable / true / false / error) are+ gone with the flag. The env gate still has to hold in both directions:+ ``reconcile_ownership_for`` disables the Beat PeriodicTask whenever this says+ PG, so a deployment without a running PG scheduler must be able to keep Beat.+ """- def test_flipt_unavailable_is_beat(self, monkeypatch):- monkeypatch.setenv("FLIPT_SERVICE_AVAILABLE", "false")- with patch("scheduler.ownership.check_feature_flag_status") as flag:- assert ownership.resolve_schedule_owner(_PID, _ORG) is False- flag.assert_not_called()-- def test_flag_true_is_pg(self, monkeypatch):- monkeypatch.setenv("FLIPT_SERVICE_AVAILABLE", "true")- with patch(- "scheduler.ownership.check_feature_flag_status", return_value=True- ) as flag:- assert ownership.resolve_schedule_owner(_PID, _ORG) is True- # entity_id = pipeline_id (stable %-bucket); org in context.- assert flag.call_args.kwargs["entity_id"] == _PID- assert flag.call_args.kwargs["context"]["pipeline_id"] == _PID- assert flag.call_args.kwargs["context"]["organization_id"] == _ORG-- def test_flag_false_is_beat(self, monkeypatch):- monkeypatch.setenv("FLIPT_SERVICE_AVAILABLE", "true")- with patch(- "scheduler.ownership.check_feature_flag_status", return_value=False- ):- assert ownership.resolve_schedule_owner(_PID, _ORG) is False+ def test_defaults_to_pg(self, monkeypatch):+ monkeypatch.delenv("PG_SCHEDULER_ENABLED", raising=False)+ assert ownership.resolve_schedule_owner() is True- def test_flipt_error_fails_closed_to_beat(self, monkeypatch):- monkeypatch.setenv("FLIPT_SERVICE_AVAILABLE", "true")- with patch(- "scheduler.ownership.check_feature_flag_status",- side_effect=RuntimeError("flipt down"),- ):- assert ownership.resolve_schedule_owner(_PID, _ORG) is False+ def test_env_gate_off_keeps_beat(self, monkeypatch):+ monkeypatch.setenv("PG_SCHEDULER_ENABLED", "false")+ assert ownership.resolve_schedule_owner() is False++ def test_env_gate_on_is_pg(self, monkeypatch):+ monkeypatch.setenv("PG_SCHEDULER_ENABLED", "true")+ assert ownership.resolve_schedule_owner() is Trueclass TestReconcileOwnership:@@ -66,9 +44,9 @@ class TestReconcileOwnership:def _scheduler_gate_on(self, monkeypatch):"""These pin the FINAL-phase behaviour, when hand-over is switched on.- PG_SCHEDULER_ENABLED defaults off for the whole rollout (Beat stays the sole- scheduler), so without this every case below would short-circuit to Beat and- assert nothing about the logic it was written for.+ PG_SCHEDULER_ENABLED is the sole ownership gate. It now defaults ON, so+ this is belt-and-braces: it keeps these cases independent of the default+ should a future change flip it."""monkeypatch.setenv("PG_SCHEDULER_ENABLED", "true")@@ -82,9 +60,7 @@ class TestReconcileOwnership:def _patches(self, *, owner: bool, rows_matched: int = 1):sched = patch("scheduler.ownership.PgPeriodicSchedule")pt = patch("scheduler.ownership.PeriodicTask")- resolve = patch(- "scheduler.ownership.resolve_schedule_owner", return_value=owner- )+ resolve = patch("scheduler.ownership.resolve_schedule_owner", return_value=owner)# transaction.atomic() as a no-op context manager.txn = patch("scheduler.ownership.transaction.atomic",@@ -101,14 +77,11 @@ class TestReconcileOwnership:assert result is True# mirror pg_owned set Trueassert (- Sched.objects.filter.return_value.update.call_args.kwargs["pg_owned"]- is True+ Sched.objects.filter.return_value.update.call_args.kwargs["pg_owned"] is True)# Beat PeriodicTask disabled (active AND NOT pg_owned == False)PT.objects.filter.assert_called_once_with(name=_PID)- assert (- PT.objects.filter.return_value.update.call_args.kwargs["enabled"] is False- )+ assert PT.objects.filter.return_value.update.call_args.kwargs["enabled"] is Falsedef test_not_pg_owned_enables_beat_and_clears_next_run(self):sched, pt, resolve, txn = self._patches(owner=False)@@ -120,9 +93,7 @@ class TestReconcileOwnership:assert update_kwargs["pg_owned"] is False# Rollback to Beat clears next_run_at so a re-hand-over re-baselines.assert update_kwargs["next_run_at"] is None- assert (- PT.objects.filter.return_value.update.call_args.kwargs["enabled"] is True- )+ assert PT.objects.filter.return_value.update.call_args.kwargs["enabled"] is Truedef test_an_ALREADY_pg_owned_schedule_does_not_clear_next_run(self):"""Narrowed: this is the True→True case, not "adopt never clears".@@ -152,9 +123,7 @@ class TestReconcileOwnership:ownership.reconcile_ownership_for(_PID, _ORG, active=False)# active=False → Beat stays disabled regardless of ownership.- assert (- PT.objects.filter.return_value.update.call_args.kwargs["enabled"] is False- )+ assert PT.objects.filter.return_value.update.call_args.kwargs["enabled"] is Falsedef test_missing_mirror_row_skips_and_reports_beat(self):sched, pt, resolve, txn = self._patches(owner=True)@@ -178,7 +147,8 @@ class TestReconcileOwnership:clobbering a concurrent reconcile), which bypasses django-celery-beat'spost_save signal. Without an explicit PeriodicTasks.update_changed() bump,DatabaseScheduler never reloads and Beat keeps firing the handed-over- schedule from its stale in-memory copy (breaking no-double-fire)."""+ schedule from its stale in-memory copy (breaking no-double-fire).+ """sched, pt, resolve, txn = self._patches(owner=True)with sched as Sched, pt, resolve, txn:Sched.objects.filter.return_value.update.return_value = 1@@ -187,7 +157,8 @@ class TestReconcileOwnership:def test_beat_reload_not_signalled_when_no_mirror_row(self, _mock_periodic_tasks):"""No mirror row → the method returns before touching the PeriodicTask, so- no reload should be signalled either."""+ no reload should be signalled either.+ """sched, pt, resolve, txn = self._patches(owner=True)with sched as Sched, pt, resolve, txn:Sched.objects.filter.return_value.update.return_value = 0@@ -196,34 +167,30 @@ class TestReconcileOwnership:class TestPgSchedulerGate:- """The gate that keeps Beat the sole scheduler for the whole PG rollout.+ """The gate deciding whether the PG scheduler or Beat owns a schedule.- Pipelines already reach PG without the PG scheduler (execute_pipeline_task_v2 →- complete_execution → execute_workflow_async → resolve_transport), so the PG- scheduler exists only to retire the Beat *deployment* — a later, deliberate step.- Handing a schedule over before then leaves it with NO firer: Beat's PeriodicTask- disabled, nothing polling the PG side.+ It defaults ON (UN-4046) — PG is the only transport and the PG scheduler ships+ with the fleet. It remains a gate because handing a schedule over while no PG+ scheduler is running leaves it with NO firer: Beat's PeriodicTask disabled,+ nothing polling the PG side."""- def test_defaults_off(self, monkeypatch):+ def test_defaults_on(self, monkeypatch):monkeypatch.delenv("PG_SCHEDULER_ENABLED", raising=False)- assert ownership.pg_scheduler_enabled() is False+ assert ownership.pg_scheduler_enabled() is True@pytest.mark.parametrize("value", ["false", "False", "0", "", "yes", "TRUE "])- def test_only_an_exact_true_opts_in(self, monkeypatch, value):+ def test_only_an_exact_true_stays_on(self, monkeypatch, value):+ """Opting OUT is the deliberate act now — anything but an exact ``true``+ (case/whitespace-insensitive) turns the gate off.+ """monkeypatch.setenv("PG_SCHEDULER_ENABLED", value)assert ownership.pg_scheduler_enabled() is (value.strip().lower() == "true")- def test_owner_stays_beat_even_with_the_flag_on(self, monkeypatch):- """The rollout state: pg_queue_enabled on, PG scheduler still dark."""- monkeypatch.delenv("PG_SCHEDULER_ENABLED", raising=False)- monkeypatch.setenv("FLIPT_SERVICE_AVAILABLE", "true")- with patch(- "scheduler.ownership.check_feature_flag_status", return_value=True- ) as flag:- assert ownership.resolve_schedule_owner(_PID, _ORG) is False- # Short-circuits before Flipt — no evaluation on every schedule save.- flag.assert_not_called()+ def test_owner_stays_beat_when_the_gate_is_off(self, monkeypatch):+ """A deployment that deliberately runs without worker-pg-scheduler."""+ monkeypatch.setenv("PG_SCHEDULER_ENABLED", "false")+ assert ownership.resolve_schedule_owner() is Falsedef test_reconcile_writes_NEITHER_beat_table(self, monkeypatch):"""The regression this gate exists to prevent.@@ -233,13 +200,11 @@ class TestPgSchedulerGate:on every schedule save — writing back a value Beat already had and forcing areload, on a table we promised not to touch."""- monkeypatch.delenv("PG_SCHEDULER_ENABLED", raising=False)- monkeypatch.setenv("FLIPT_SERVICE_AVAILABLE", "true")+ monkeypatch.setenv("PG_SCHEDULER_ENABLED", "false")with (patch("scheduler.ownership.PeriodicTask") as PT,patch("scheduler.ownership.PeriodicTasks") as PTs,patch("scheduler.ownership.PgPeriodicSchedule") as Sched,- patch("scheduler.ownership.check_feature_flag_status", return_value=True),):# A CLEAN row: not pg_owned, so there is nothing to repair and the# write-free path must be taken. Must be set explicitly — a bare@@ -286,7 +251,7 @@ class TestStalePgOwnershipIsReleased:patch.stopall()def test_a_stale_row_is_handed_back_to_beat(self, monkeypatch):- monkeypatch.delenv("PG_SCHEDULER_ENABLED", raising=False)+ monkeypatch.setenv("PG_SCHEDULER_ENABLED", "false")sched = self._mocks(stale=True)with (patch("scheduler.ownership.PeriodicTask") as PT,@@ -315,7 +280,7 @@ class TestStalePgOwnershipIsReleased:def test_a_paused_pipeline_is_not_resurrected_by_the_repair(self, monkeypatch):"""Repair restores the FIRER, never the on/off state the user chose."""- monkeypatch.delenv("PG_SCHEDULER_ENABLED", raising=False)+ monkeypatch.setenv("PG_SCHEDULER_ENABLED", "false")self._mocks(stale=True)with (patch("scheduler.ownership.PeriodicTask") as PT,@@ -334,22 +299,17 @@ class TestStalePgOwnershipIsReleased:Asking Flipt could return True and re-hand the schedule to a PG schedulerthat is not running — turning a double-fire into no firer at all."""- monkeypatch.delenv("PG_SCHEDULER_ENABLED", raising=False)- monkeypatch.setenv("FLIPT_SERVICE_AVAILABLE", "true")+ monkeypatch.setenv("PG_SCHEDULER_ENABLED", "false")self._mocks(stale=True)with (patch("scheduler.ownership.PeriodicTask"),patch("scheduler.ownership.PeriodicTasks"),- patch(- "scheduler.ownership.check_feature_flag_status", return_value=True- ) as flag,):ownership.reconcile_ownership_for(_PID, _ORG, active=True)- flag.assert_not_called()def test_a_db_error_on_the_check_falls_back_to_writing_nothing(self, monkeypatch):"""A repair that cannot confirm it is needed must not run."""- monkeypatch.delenv("PG_SCHEDULER_ENABLED", raising=False)+ monkeypatch.setenv("PG_SCHEDULER_ENABLED", "false")sched = patch("scheduler.ownership.PgPeriodicSchedule").start()sched.objects.filter.side_effect = Exception("db down")with (@@ -412,9 +372,7 @@ class TestReconcileAtomicityRealDB:patch("scheduler.ownership.resolve_schedule_owner", return_value=True),patch("scheduler.ownership.PeriodicTask", failing_pt),):- result = ownership.reconcile_ownership_for(- pid, "org_atomic", active=True- )+ result = ownership.reconcile_ownership_for(pid, "org_atomic", active=True)assert result is None # failure signalled# The pg_owned=True write was rolled back with the failed PT update.assert PgPeriodicSchedule.objects.get(pipeline_id=pid).pg_owned is False@@ -445,7 +403,6 @@ class TestNextRunBaselineOnTransition:def _reconcile(self, monkeypatch, *, was_pg_owned, now_pg_owned):monkeypatch.setenv("PG_SCHEDULER_ENABLED", "true")- monkeypatch.setenv("FLIPT_SERVICE_AVAILABLE", "true")with (patch("scheduler.ownership.PgPeriodicSchedule") as Sched,patch("scheduler.ownership.PeriodicTask"),@@ -455,7 +412,7 @@ class TestNextRunBaselineOnTransition:return_value=contextlib.nullcontext(),),patch(- "scheduler.ownership.check_feature_flag_status",+ "scheduler.ownership.resolve_schedule_owner",return_value=now_pg_owned,),):@@ -503,7 +460,6 @@ class TestBeatClockBaselineOnRelease:def _reconcile(self, monkeypatch, *, was_pg_owned, now_pg_owned, active=True):monkeypatch.setenv("PG_SCHEDULER_ENABLED", "true")- monkeypatch.setenv("FLIPT_SERVICE_AVAILABLE", "true")with (patch("scheduler.ownership.PgPeriodicSchedule") as Sched,patch("scheduler.ownership.PeriodicTask") as Beat,@@ -513,7 +469,7 @@ class TestBeatClockBaselineOnRelease:return_value=contextlib.nullcontext(),),patch(- "scheduler.ownership.check_feature_flag_status",+ "scheduler.ownership.resolve_schedule_owner",return_value=now_pg_owned,),):@@ -534,7 +490,8 @@ class TestBeatClockBaselineOnRelease:def test_handing_over_to_pg_does_NOT_touch_beats_clock(self, monkeypatch):"""Adoption switches Beat OFF, so its clock is irrelevant — and overwriting it- would destroy the value the eventual release needs to restore from."""+ would destroy the value the eventual release needs to restore from.+ """beat = self._reconcile(monkeypatch, was_pg_owned=False, now_pg_owned=True)assert beat["enabled"] is Falseassert "last_run_at" not in beat@@ -542,7 +499,8 @@ class TestBeatClockBaselineOnRelease:def test_an_unchanged_owner_is_NOT_re_stamped(self, monkeypatch):"""Same transition-scoping reason as next_run_at: reconcile runs on everypipeline save, and stamping unconditionally would push Beat's clock forward- on an ordinary edit and silently skip a due fire."""+ on an ordinary edit and silently skip a due fire.+ """beat = self._reconcile(monkeypatch, was_pg_owned=False, now_pg_owned=False)assert "last_run_at" not in beat@@ -550,7 +508,8 @@ class TestBeatClockBaselineOnRelease:self, monkeypatch):"""A paused schedule comes back paused — but if it is later resumed, Beat must- not then replay the backlog it accrued while PG owned it."""+ not then replay the backlog it accrued while PG owned it.+ """beat = self._reconcile(monkeypatch, was_pg_owned=True, now_pg_owned=False, active=False)diff --git a/backend/workflow_manager/internal_api_views.py b/backend/workflow_manager/internal_api_views.pyindex bdc09a0a5..c54199f16 100644--- a/backend/workflow_manager/internal_api_views.py+++ b/backend/workflow_manager/internal_api_views.py@@ -18,9 +18,9 @@ from rest_framework.decorators import api_viewfrom rest_framework.response import Responsefrom tool_instance_v2.models import ToolInstance+from unstract.core.data_models import WorkflowTransportfrom workflow_manager.workflow_v2.enums import ExecutionStatusfrom workflow_manager.workflow_v2.models import Workflow, WorkflowExecution-from workflow_manager.workflow_v2.transport import resolve_transportlogger = logging.getLogger(__name__)@@ -321,23 +321,14 @@ def create_workflow_execution(request):# Handle tags logic if neededpass- # Resolve the transport this execution rides (9e). Decided once here, at- # the creation chokepoint, and returned so the caller carries it in the- # dispatched task's payload — not persisted on the row. Flipt (gated by- # the env master-switch) decides; fails closed to "celery".- transport = resolve_transport(- execution_id=str(execution.id),- organization_id=org_id,- workflow_id=str(workflow.id),- pipeline_id=data.get("pipeline_id"),- )-return Response({"execution_id": str(execution.id),"status": execution.status,"execution_log_id": execution.execution_log_id, # Return for workers to use- "transport": transport,+ # Wire field the workers still branch on. PG is the only+ # transport now; the field goes when those branches do.+ "transport": WorkflowTransport.PG_QUEUE.value,})@@ -389,72 +380,3 @@ def compile_workflow(request):{"error": "Internal server error"},status=status.HTTP_500_INTERNAL_SERVER_ERROR,)---@csrf_exempt # Safe: Internal API with Bearer token auth, no session/cookies-@api_view(["POST"])-def submit_file_batch_for_processing(request):- """Submit file batch for processing by workers.-- This endpoint receives batch data and returns immediately,- as actual processing is handled by Celery workers.-- Returns:- JSON response with batch submission status- """- try:- batch_data = request.data-- # Get organization from header- org_id = request.headers.get("X-Organization-ID")- if not org_id:- return Response(- {"error": "X-Organization-ID header is required"},- status=status.HTTP_400_BAD_REQUEST,- )-- # Add organization ID to file_data where WorkerFileData expects it- if "file_data" in batch_data:- batch_data["file_data"]["organization_id"] = org_id- else:- # Fallback: add at top level for backward compatibility- batch_data["organization_id"] = org_id-- # Submit to file processing worker queue using Celery- try:- from backend.celery_service import app as celery_app-- # Submit the batch data to the file processing worker using send_task- # This calls the task by name without needing to import it- task_result = celery_app.send_task(- "process_file_batch", # Task name as defined in workers/file_processing/tasks.py- args=[batch_data], # Pass batch_data as first argument- queue="file_processing", # Send to file processing queue- )-- logger.info(- f"Successfully submitted file batch {batch_data.get('batch_id')} to worker queue (task: {task_result.id})"- )-- return Response(- {- "success": True,- "batch_id": batch_data.get("batch_id"),- "celery_task_id": task_result.id,- "message": "Batch submitted for processing",- }- )-- except Exception as e:- logger.error(f"Failed to submit batch to worker queue: {e}")- return Response(- {"error": f"Failed to submit batch for processing: {str(e)}"},- status=status.HTTP_500_INTERNAL_SERVER_ERROR,- )-- except Exception as e:- logger.error(f"Error submitting file batch: {e}")- return Response(- {"error": "Internal server error"},- status=status.HTTP_500_INTERNAL_SERVER_ERROR,- )diff --git a/backend/workflow_manager/internal_urls.py b/backend/workflow_manager/internal_urls.pyindex 6fac0be62..35db8645f 100644--- a/backend/workflow_manager/internal_urls.py+++ b/backend/workflow_manager/internal_urls.py@@ -34,12 +34,6 @@ urlpatterns = [internal_api_views.compile_workflow,name="compile_workflow",),- # File batch processing- path(- "file-batch/submit/",- internal_api_views.submit_file_batch_for_processing,- name="submit_file_batch_for_processing",- ),# Workflow definition and type detection (using sophisticated class-based views)path("workflow/<str:workflow_id>/",diff --git a/backend/workflow_manager/workflow_v2/execution_log_internal_views.py b/backend/workflow_manager/workflow_v2/execution_log_internal_views.pyindex 0c07b0474..7a3d63fc6 100644--- a/backend/workflow_manager/workflow_v2/execution_log_internal_views.py+++ b/backend/workflow_manager/workflow_v2/execution_log_internal_views.py@@ -190,7 +190,8 @@ class SweepUndispatchedExecutionsAPIView(APIView):sweep also needs the rate limiter — both backend-side.Transport-agnostic on purpose: the create-then-dispatch window sits upstream of- ``resolve_transport``, so this recovers Celery-path strands too.+ transport selection, so this still recovers strands left by the pre-UN-4046+ Celery path."""def post(self, request: Request) -> Response:diff --git a/backend/workflow_manager/workflow_v2/tests/test_dispatch_orchestrator.py b/backend/workflow_manager/workflow_v2/tests/test_dispatch_orchestrator.pyindex 6b0a03553..3ee117480 100644--- a/backend/workflow_manager/workflow_v2/tests/test_dispatch_orchestrator.py+++ b/backend/workflow_manager/workflow_v2/tests/test_dispatch_orchestrator.py@@ -6,7 +6,7 @@ decision (PG enqueue vs Celery send), the bare-msg_id task-id contract, theWorkloadType value, and the two org sentinels."""-from unittest.mock import MagicMock, patch+from unittest.mock import patchfrom utils.constants import CeleryQueuefrom workflow_manager.workflow_v2.workflow_helper import WorkflowHelper@@ -16,10 +16,9 @@ _CELERY = "workflow_manager.workflow_v2.workflow_helper.celery_app"class TestDispatchOrchestratorTask:- def test_pg_transport_enqueues_to_pg_not_celery(self):+ def test_enqueues_to_the_pg_queue(self):with patch(_PG, return_value=4242) as pg, patch(_CELERY) as celery:task_id = WorkflowHelper._dispatch_orchestrator_task(- transport="pg_queue",queue=CeleryQueue.CELERY_API_DEPLOYMENTS,args=["a"],kwargs={"transport": "pg_queue"},@@ -37,7 +36,7 @@ class TestDispatchOrchestratorTask:def test_pg_general_queue_uses_non_api_workload(self):with patch(_PG, return_value=7) as pg, patch(_CELERY):WorkflowHelper._dispatch_orchestrator_task(- transport="pg_queue", queue=None, args=[], kwargs={}, org_schema="org1"+ queue=None, args=[], kwargs={}, org_schema="org1")assert pg.call_args.kwargs["fairness"]["workload_type"] == "non_api"@@ -45,18 +44,8 @@ class TestDispatchOrchestratorTask:"""Row org_id column is NOT NULL → ""; fairness org_id is str|None → None."""with patch(_PG, return_value=1) as pg, patch(_CELERY):WorkflowHelper._dispatch_orchestrator_task(- transport="pg_queue", queue=None, args=[], kwargs={}, org_schema=""+ queue=None, args=[], kwargs={}, org_schema="")kw = pg.call_args.kwargsassert kw["org_id"] == ""assert kw["fairness"]["org_id"] is None-- def test_celery_transport_uses_send_task_not_pg(self):- with patch(_PG) as pg, patch(_CELERY) as celery:- celery.send_task.return_value = MagicMock(id="celery-task-1")- task_id = WorkflowHelper._dispatch_orchestrator_task(- transport="celery", queue=None, args=[], kwargs={}, org_schema="org1"- )- assert task_id == "celery-task-1"- pg.assert_not_called()- celery.send_task.assert_called_once()diff --git a/backend/workflow_manager/workflow_v2/tests/test_execute_workflow_async_wait.py b/backend/workflow_manager/workflow_v2/tests/test_execute_workflow_async_wait.pyindex 74586ab53..91a5e1bcb 100644--- a/backend/workflow_manager/workflow_v2/tests/test_execute_workflow_async_wait.py+++ b/backend/workflow_manager/workflow_v2/tests/test_execute_workflow_async_wait.py@@ -27,14 +27,11 @@ class TestTimeoutWaitSurvivesBookkeepingFailure:exec_row = MagicMock(status=ExecutionStatus.EXECUTING.value)with (- patch(f"{_MOD}.resolve_transport", return_value="pg_queue"),patch(f"{_MOD}.UserContext") as user_ctx,patch(f"{_MOD}.StateStore") as state_store,patch(f"{_MOD}.time"), # no-op sleeppatch(f"{_MOD}.WorkflowExecution") as wf_exec,- patch.object(- WorkflowHelper, "_dispatch_orchestrator_task", return_value="1"- ),+ patch.object(WorkflowHelper, "_dispatch_orchestrator_task", return_value="1"),patch.object(WorkflowHelper,"_record_dispatch_handle",diff --git a/backend/workflow_manager/workflow_v2/tests/test_record_dispatch_handle.py b/backend/workflow_manager/workflow_v2/tests/test_record_dispatch_handle.pyindex ec40a18bb..dcba6ac65 100644--- a/backend/workflow_manager/workflow_v2/tests/test_record_dispatch_handle.py+++ b/backend/workflow_manager/workflow_v2/tests/test_record_dispatch_handle.py@@ -23,20 +23,17 @@ from workflow_manager.workflow_v2.workflow_helper import WorkflowHelper_MODEL = "workflow_manager.workflow_v2.execution.WorkflowExecution"-_HELPER = (- "workflow_manager.workflow_v2.workflow_helper.WorkflowExecutionServiceHelper"-)+_HELPER = "workflow_manager.workflow_v2.workflow_helper.WorkflowExecutionServiceHelper"_EXEC = "exec-123"class TestRecordDispatchHandle:- def test_pg_handle_goes_to_queue_message_id_as_int_not_task_id(self):+ def test_handle_goes_to_queue_message_id_as_int(self):"""The PG msg_id (bigint string) must be stored in queue_message_id as anint — never in the UUID task_id (the bug that crashed the wait loop)."""with patch(_HELPER) as helper:WorkflowHelper._record_dispatch_handle(execution_id=_EXEC,- transport="pg_queue",dispatch_handle="1527",org_schema="org1",file_count=3,@@ -45,32 +42,15 @@ class TestRecordDispatchHandle:execution_id=_EXEC, queue_message_id=1527)assert isinstance(- helper.update_execution_queue_message_id.call_args.kwargs[- "queue_message_id"- ],+ helper.update_execution_queue_message_id.call_args.kwargs["queue_message_id"],int,)helper.update_execution_task.assert_not_called() # task_id stays NULL- def test_celery_handle_goes_to_task_id_not_queue_message_id(self):- with patch(_HELPER) as helper:- WorkflowHelper._record_dispatch_handle(- execution_id=_EXEC,- transport="celery",- dispatch_handle="b1b2c3d4-0000-0000-0000-000000000000",- org_schema="org1",- file_count=1,- )- helper.update_execution_task.assert_called_once_with(- execution_id=_EXEC, task_id="b1b2c3d4-0000-0000-0000-000000000000"- )- helper.update_execution_queue_message_id.assert_not_called()-def test_empty_handle_records_nothing(self):with patch(_HELPER) as helper:WorkflowHelper._record_dispatch_handle(execution_id=_EXEC,- transport="celery",dispatch_handle=None,org_schema="org1",file_count=0,@@ -78,7 +58,7 @@ class TestRecordDispatchHandle:helper.update_execution_task.assert_not_called()helper.update_execution_queue_message_id.assert_not_called()- def test_non_numeric_pg_handle_records_nothing(self):+ def test_non_numeric_handle_records_nothing(self):"""A malformed PG handle (not a bigint) must be parsed defensively — noValueError out of the helper, and nothing recorded. Today this can'thappen (``_dispatch_orchestrator_task`` always returns ``str(msg_id)``),@@ -86,7 +66,6 @@ class TestRecordDispatchHandle:with patch(_HELPER) as helper:WorkflowHelper._record_dispatch_handle(execution_id=_EXEC,- transport="pg_queue",dispatch_handle="not-a-number",org_schema="org1",file_count=1,diff --git a/backend/workflow_manager/workflow_v2/tests/test_step_execution_transport.py b/backend/workflow_manager/workflow_v2/tests/test_step_execution_transport.pyindex 2e179ad0e..38603d648 100644--- a/backend/workflow_manager/workflow_v2/tests/test_step_execution_transport.py+++ b/backend/workflow_manager/workflow_v2/tests/test_step_execution_transport.py@@ -1,17 +1,17 @@-"""Tests for the PG guard on single-step execution (UN-3445 acceptance gate).+"""Tests for the single-step execution guard (UN-3445 acceptance gate, UN-4046).-Single-step is the one entry path whose fan-out was never transport-gated: it+Single-step is the one entry path whose fan-out was never moved onto PG: itreaches the Celery chord in ``process_input_files`` directly, while the normal-path's PG fan-out lives in the general worker. With the flag on and the Celery-file_processing workers scaled to zero, those batches would never be consumed-and the execution would hang in EXECUTING with no PG reaper aware of it.+path's PG fan-out lives in the general worker. With no Celery file_processing+workers those batches would never be consumed and the execution would hang in+EXECUTING with no PG reaper aware of it — so it refuses loudly instead.-So the guard has to hold in *both* directions, and that is what these pin:-Celery must still run step execution unchanged (it is what staging and-production run today), and PG must refuse it loudly rather than dispatch into a-queue with no consumer.+It used to refuse only when the ``pg_queue_enabled`` flag resolved to PG. The+flag is gone (UN-4046) and PG is the only transport, so the refusal is+unconditional and the "celery still runs it" / "transport is resolved on the+execution id" cases no longer describe anything.-DB-free: WorkflowExecution, resolve_transport and run_workflow are all mocked.+DB-free: WorkflowExecution and run_workflow are mocked."""from unittest.mock import MagicMock, patch@@ -21,7 +21,6 @@ from workflow_manager.workflow_v2.exceptions import WorkflowExecutionErrorfrom workflow_manager.workflow_v2.models import Workflowfrom workflow_manager.workflow_v2.workflow_helper import WorkflowHelper-_TRANSPORT = "workflow_manager.workflow_v2.workflow_helper.resolve_transport"_EXECUTION = "workflow_manager.workflow_v2.workflow_helper.WorkflowExecution"_RUN = "workflow_manager.workflow_v2.workflow_helper.WorkflowHelper.run_workflow"@@ -49,11 +48,9 @@ def _patched_execution():return model-class TestStepExecutionTransportGuard:- def test_pg_transport_refuses_instead_of_dispatching_to_celery(self):- with patch(_EXECUTION, _patched_execution()), patch(- _TRANSPORT, return_value="pg_queue"- ), patch(_RUN) as run:+class TestStepExecutionGuard:+ def test_refuses_instead_of_dispatching_into_an_unconsumed_queue(self):+ with patch(_EXECUTION, _patched_execution()), patch(_RUN) as run:with pytest.raises(WorkflowExecutionError, match="not supported"):WorkflowHelper.step_execution(workflow=_workflow(),@@ -63,58 +60,3 @@ class TestStepExecutionTransportGuard:# The point of the guard: nothing reaches the chord. Asserting the raise# alone would still pass if run_workflow had already fanned out.run.assert_not_called()-- def test_celery_transport_still_runs_step_execution(self):- """Flag-off is what staging and production run — it must be untouched."""- with patch(_EXECUTION, _patched_execution()), patch(- _TRANSPORT, return_value="celery"- ), patch(_RUN, return_value="ran") as run:- result = WorkflowHelper.step_execution(- workflow=_workflow(),- execution_action=_START,- execution_id=_EXECUTION_ID,- )- assert result == "ran"- assert run.call_args.kwargs["single_step"] is True-- def test_transport_is_resolved_on_the_execution_id(self):- """entity_id must be the execution id — it is what Flipt buckets on, and- what keeps one execution from re-bucketing across transports."""- with patch(_EXECUTION, _patched_execution()), patch(- _TRANSPORT, return_value="celery"- ) as resolve, patch(_RUN):- WorkflowHelper.step_execution(- workflow=_workflow(),- execution_action=_START,- execution_id=_EXECUTION_ID,- )- assert resolve.call_args.kwargs["execution_id"] == _EXECUTION_ID- assert resolve.call_args.kwargs["organization_id"] == "org_acme"-- def test_no_execution_id_creates_a_step_execution_without_resolving(self):- """The START-without-id branch only mints a row; it dispatches nothing, so- it must not consume a Flipt evaluation (and must not be blocked)."""- with patch(_TRANSPORT) as resolve, patch.object(- WorkflowHelper, "create_and_make_execution_response", return_value="created"- ):- result = WorkflowHelper.step_execution(- workflow=_workflow(), execution_action=_START, execution_id=None- )- assert result == "created"- resolve.assert_not_called()-- def test_missing_execution_falls_back_to_creating_one(self):- """Pre-existing behaviour: a stale execution_id re-mints rather than 404s.- Pinned so the guard's placement inside the try/except cannot change it."""- model = _patched_execution()- model.objects.get.side_effect = model.DoesNotExist- with patch(_EXECUTION, model), patch(_TRANSPORT) as resolve, patch.object(- WorkflowHelper, "create_and_make_execution_response", return_value="created"- ):- result = WorkflowHelper.step_execution(- workflow=_workflow(),- execution_action=_START,- execution_id=_EXECUTION_ID,- )- assert result == "created"- resolve.assert_not_called()diff --git a/backend/workflow_manager/workflow_v2/tests/test_transport.py b/backend/workflow_manager/workflow_v2/tests/test_transport.pydeleted file mode 100644index a93c926f4..000000000--- a/backend/workflow_manager/workflow_v2/tests/test_transport.py+++ /dev/null@@ -1,161 +0,0 @@-"""Tests for the 9e transport-resolution seam (PR 3 — Flipt canary wiring).--``resolve_transport`` is the single chokepoint that decides whether a new-execution rides the legacy Celery transport or the Postgres queue. It is gated by a-single ``pg_queue_enabled`` Flipt boolean flag; it fails closed to Celery on any-problem. These tests pin that contract.-"""--from unittest.mock import MagicMock, patch--import pytest-from unstract.core.data_models import (- DEFAULT_WORKFLOW_TRANSPORT,- WorkflowTransport,- normalize_transport,-)--from workflow_manager.workflow_v2.transport import (- PG_QUEUE_FLAG_KEY,- resolve_transport,-)--# Where ``check_feature_flag_status`` is looked up (imported into the module).-_FLIPT = "workflow_manager.workflow_v2.transport.check_feature_flag_status"---class TestWorkflowTransportEnum:- def test_values(self):- assert WorkflowTransport.CELERY.value == "celery"- assert WorkflowTransport.PG_QUEUE.value == "pg_queue"-- def test_default_is_celery(self):- assert DEFAULT_WORKFLOW_TRANSPORT == WorkflowTransport.CELERY.value---class TestResolveTransport:- @pytest.fixture(autouse=True)- def _flipt_available(self, monkeypatch):- """resolve_transport short-circuits to celery when Flipt is marked- unavailable; default it available so the flag-evaluation tests exercise- the real path. Individual tests override as needed.- """- monkeypatch.setenv("FLIPT_SERVICE_AVAILABLE", "true")-- def test_missing_organization_forces_celery(self):- """No org context → can't segment safely → fail closed; Flipt not called- (str(None)/"" must never reach the Flipt org segment).- """- with patch(_FLIPT) as flipt:- result = resolve_transport(execution_id="e1", organization_id="")- assert result == WorkflowTransport.CELERY.value- flipt.assert_not_called()-- def test_flipt_unavailable_forces_celery(self, monkeypatch):- """Flipt service unavailable → celery, loudly — not a silent masquerade as a- healthy 100%-celery rollout.- """- monkeypatch.setenv("FLIPT_SERVICE_AVAILABLE", "false")- with patch(_FLIPT) as flipt:- result = resolve_transport(execution_id="e1", organization_id="org1")- assert result == WorkflowTransport.CELERY.value- flipt.assert_not_called()-- def test_flipt_true_resolves_pg_queue(self):- with patch(_FLIPT, return_value=True):- result = resolve_transport(execution_id="e1", organization_id="org1")- assert result == WorkflowTransport.PG_QUEUE.value-- def test_flipt_false_resolves_celery(self):- with patch(_FLIPT, return_value=False):- result = resolve_transport(execution_id="e1", organization_id="org1")- assert result == WorkflowTransport.CELERY.value-- def test_flipt_exception_fails_closed_to_celery(self):- """A Flipt outage must never break execution creation."""- with patch(_FLIPT, side_effect=RuntimeError("flipt down")):- result = resolve_transport(execution_id="e1", organization_id="org1")- assert result == WorkflowTransport.CELERY.value-- def test_passes_execution_id_as_entity_and_builds_context(self):- """entity_id = execution_id (sticky bucketing); context carries the- org/workflow/pipeline for segment rules.- """- with patch(_FLIPT, return_value=True) as flipt:- resolve_transport(- execution_id="exec-42",- organization_id="org1",- workflow_id="wf1",- pipeline_id="pl1",- )- flipt.assert_called_once_with(- flag_key=PG_QUEUE_FLAG_KEY,- entity_id="exec-42",- context={- "organization_id": "org1",- "workflow_id": "wf1",- "pipeline_id": "pl1",- },- )-- def test_non_string_ids_are_coerced_for_flipt(self):- """Callers pass UUID objects for the ids. Flipt's context is a gRPC- map<string,string> and entity_id must hash stably, so every value must- reach check_feature_flag_status as a plain str — otherwise the client- swallows the serialization error as False and silently forces celery.- Regression test for that exact dev-test finding.- """- import uuid-- ex = uuid.UUID("8c091789-9dde-45e7-bb85-06f23fe120eb")- wf = uuid.UUID("ebed2834-c9fb-4b6c-8df3-9dd841f616bb")- pl = uuid.UUID("eaca3b0e-083a-4c75-8b25-85349d54145b")- with patch(_FLIPT, return_value=True) as flipt:- result = resolve_transport(- execution_id=ex, organization_id="org1", workflow_id=wf, pipeline_id=pl- )- assert result == WorkflowTransport.PG_QUEUE.value- _, kwargs = flipt.call_args- assert kwargs["entity_id"] == str(ex)- assert all(isinstance(v, str) for v in kwargs["context"].values())- assert kwargs["context"]["workflow_id"] == str(wf)- assert kwargs["context"]["pipeline_id"] == str(pl)-- def test_context_omits_unset_optional_ids(self):- with patch(_FLIPT, return_value=True) as flipt:- resolve_transport(execution_id="exec-42", organization_id="org1")- _, kwargs = flipt.call_args- assert kwargs["context"] == {"organization_id": "org1"}-- def test_result_is_a_valid_transport_value(self):- valid = {t.value for t in WorkflowTransport}- with patch(_FLIPT, return_value=True):- assert (- resolve_transport(execution_id="e1", organization_id="org1") in valid- )---class TestNormalizeTransport:- """The fail-closed coercion used at every untrusted read boundary."""-- def test_recognized_values_pass_through(self):- assert normalize_transport("celery") == "celery"- assert normalize_transport("pg_queue") == "pg_queue"-- def test_unrecognized_value_falls_back_to_celery(self):- assert normalize_transport("celary") == DEFAULT_WORKFLOW_TRANSPORT- assert normalize_transport("pg-queue") == DEFAULT_WORKFLOW_TRANSPORT-- def test_none_and_empty_fall_back_to_celery(self):- assert normalize_transport(None) == DEFAULT_WORKFLOW_TRANSPORT- assert normalize_transport("") == DEFAULT_WORKFLOW_TRANSPORT-- def test_invalid_value_logs_a_warning_when_logger_given(self):- log = MagicMock()- assert normalize_transport("bogus", logger=log, context=" [exec:x]") == "celery"- log.warning.assert_called_once()-- def test_valid_value_does_not_warn(self):- log = MagicMock()- normalize_transport("pg_queue", logger=log)- log.warning.assert_not_called()diff --git a/backend/workflow_manager/workflow_v2/transport.py b/backend/workflow_manager/workflow_v2/transport.pydeleted file mode 100644index 08b2f31c2..000000000--- a/backend/workflow_manager/workflow_v2/transport.py+++ /dev/null@@ -1,145 +0,0 @@-"""Transport resolution for a workflow execution (9e).--A workflow execution rides one transport end-to-end — legacy Celery/RabbitMQ-or the bespoke Postgres queue. The choice is resolved **once**, at the-execution-creation chokepoint, and returned to the caller so it can be carried-in the dispatched task's payload (see ``workers/queue_backend/pg_queue/-9e-design.md``). It is deliberately **not** persisted on the ``WorkflowExecution``-row: the payload is the single carrier, durable for PG via the queue row's-JSONB, and the giant shared table is never migrated for this work.--The transport is resolved from a single Flipt evaluation:-- Flipt boolean (``pg_queue_enabled``) → transport--Routing onto PG needs **both**: Flipt reachable (``FLIPT_SERVICE_AVAILABLE=true``)-and the single ``pg_queue_enabled`` flag enabled for this execution. The flag is the-sole rollout control — flip it to ramp PG, flip it off (or a Flipt outage) to fall-back to Celery.--- **Flipt** decides per-execution: ``entity_id = execution_id`` drives the- percentage-rollout hashing (an execution resolves exactly once, so it can- never re-bucket mid-flight); ``context`` carries org/workflow/pipeline for- segment rules. The flag contract is fixed in 9e-design §2.-- **Fail-closed to Celery**: a blind/unreachable Flipt, a missing org, or any error- must never break execution creation — it resolves to Celery, mirroring- ``normalize_transport`` on the read side. The decision is logged so a "flag on but- still all Celery" situation (e.g. a blind Flipt) is visible rather than silent.-"""--from __future__ import annotations--import logging-import os-from typing import TYPE_CHECKING--from pg_queue.flags import PG_QUEUE_FLAG_KEY--from unstract.core.data_models import WorkflowTransport-from unstract.flags.feature_flag import check_feature_flag_status--if TYPE_CHECKING:- from uuid import UUID--logger = logging.getLogger(__name__)--# The single PG-queue rollout flag (Boolean, default false) is defined once in-# pg_queue.flags and imported above; execution, scheduler, and executor all read-# that one key. Re-exported here for callers/tests that import it from this module.---def resolve_transport(- *,- execution_id: str | UUID,- organization_id: str | UUID | None,- workflow_id: str | UUID | None = None,- pipeline_id: str | UUID | None = None,-) -> str:- """Resolve the transport for a new workflow execution.-- The id parameters are typed ``str | UUID`` because callers pass either- (e.g. the view path pre-coerces with ``str(...)``, the helper path passes- raw UUIDs). Coercion to the wire types Flipt needs is kept internal here so- callers never have to know about it.-- Args:- execution_id: The execution's id. Used as the Flipt ``entity_id`` so the- percentage rollout buckets per execution and stays sticky (one- execution is resolved exactly once).- organization_id: The org's string identifier- (``Organization.organization_id`` — the ``X-Organization-ID`` header- value, *not* the DB pk). May be ``None`` on the helper path- (``StateStore`` not populated) → resolves to celery. Carried in the- Flipt ``context`` for per-org segment rollouts.- workflow_id: Optional, carried in ``context`` for future segment rules.- pipeline_id: Optional, carried in ``context`` for future segment rules.-- Returns:- A :class:`WorkflowTransport` value string — ``"pg_queue"`` only when Flipt- is reachable and says yes for this execution; ``"celery"`` otherwise- (including any error — fail-closed).- """- celery = WorkflowTransport.CELERY.value-- # No org context → per-org segment matching can't be trusted (str(None) would- # ship a bogus "None" org into the Flipt context and mis-segment). The view- # path validates the header non-None, but the helper path reads it from- # StateStore, which can be empty. Fail closed rather than mis-bucket.- if not organization_id:- logger.warning(- "resolve_transport: no organization_id for execution %s; forcing celery",- execution_id,- )- return celery-- # FliptClient returns False for ALL flags when the service is marked- # unavailable — indistinguishable from "rollout says no". Surface it loudly so- # a blind Flipt doesn't masquerade as a healthy 100%-Celery rollout. Parse- # exactly as FliptClient does (``.lower()``, no ``.strip()``) so the two can- # never disagree on a value like ``" true"``.- if os.environ.get("FLIPT_SERVICE_AVAILABLE", "false").lower() != "true":- logger.warning(- "resolve_transport: FLIPT_SERVICE_AVAILABLE != true "- "(Flipt is blind) for execution %s; forcing celery",- execution_id,- )- return celery-- # Flipt's context is a gRPC map<string,string>; callers pass UUID objects for- # the ids, so coerce every value to str. A non-str value makes the client's- # serialization fail and check_feature_flag_status swallow it as False —- # silently forcing celery. entity_id is str-coerced for the same reason (and- # so the %-rollout hash is stable across str/UUID call sites).- context = {"organization_id": str(organization_id)}- if workflow_id:- context["workflow_id"] = str(workflow_id)- if pipeline_id:- context["pipeline_id"] = str(pipeline_id)-- # Defense in depth: check_feature_flag_status already wraps its body in- # try/except → False, but an explicit fail-closed wrap here keeps a future- # change to that helper from ever letting a Flipt problem break execution- # creation.- try:- enabled = check_feature_flag_status(- flag_key=PG_QUEUE_FLAG_KEY,- entity_id=str(execution_id),- context=context,- )- except Exception:- logger.warning(- "resolve_transport: Flipt evaluation failed for execution %s; "- "falling back to Celery",- execution_id,- exc_info=True,- )- return celery-- result = WorkflowTransport.PG_QUEUE.value if enabled else celery- logger.info(- "resolve_transport: execution %s resolved to %s (flipt enabled=%s)",- execution_id,- result,- enabled,- )- return resultdiff --git a/backend/workflow_manager/workflow_v2/undispatched_sweep.py b/backend/workflow_manager/workflow_v2/undispatched_sweep.pyindex cc3d77f35..290f63522 100644--- a/backend/workflow_manager/workflow_v2/undispatched_sweep.py+++ b/backend/workflow_manager/workflow_v2/undispatched_sweep.py@@ -21,9 +21,10 @@ executions were left PENDING** — 2914 execution rows from 2444 requests, becaurow outlived the request. ``pg_queue_message`` was completely empty, confirming thework was never enqueued rather than enqueued-and-lost.-**Not PG-specific.** The create-then-dispatch ordering sits *upstream* of-``resolve_transport``, so the Celery path has the identical window. The predicate here-is transport-aware for exactly that reason.+**Not PG-specific.** The create-then-dispatch ordering sits *upstream* of transport+selection, so the Celery path had the identical window (UN-4046 removed that path;+the predicate stays transport-aware because rows dispatched before the upgrade carry+``task_id`` rather than ``queue_message_id``).**The predicate.** ``workflow_helper.py:566/570`` stamps ``queue_message_id`` (PG) or``task_id`` (Celery) immediately after a successful dispatch, and the model documentsdiff --git a/backend/workflow_manager/workflow_v2/workflow_helper.py b/backend/workflow_manager/workflow_v2/workflow_helper.pyindex 971fb7a8c..da5ad380c 100644--- a/backend/workflow_manager/workflow_v2/workflow_helper.py+++ b/backend/workflow_manager/workflow_v2/workflow_helper.py@@ -27,7 +27,7 @@ from utils.local_context import StateStorefrom utils.user_context import UserContextfrom backend.celery_service import app as celery_app-from unstract.core.data_models import WorkloadType, is_pg_transport+from unstract.core.data_models import WorkflowTransport, WorkloadTypefrom unstract.workflow_execution.enums import LogStagefrom workflow_manager.endpoint_v2.destination import DestinationConnectorfrom workflow_manager.endpoint_v2.dto import FileHash@@ -59,7 +59,6 @@ from workflow_manager.workflow_v2.execution import WorkflowExecutionServiceHelpefrom workflow_manager.workflow_v2.file_history_helper import FileHistoryHelperfrom workflow_manager.workflow_v2.models.execution import WorkflowExecutionfrom workflow_manager.workflow_v2.models.workflow import Workflow-from workflow_manager.workflow_v2.transport import resolve_transportlogger = logging.getLogger(__name__)@@ -483,96 +482,80 @@ class WorkflowHelper:@staticmethoddef _dispatch_orchestrator_task(*,- transport: str,queue: str | None,args: list[Any],kwargs: dict[str, Any],org_schema: str,) -> str | None:- """Dispatch ``async_execute_bin`` on the resolved transport; return its id.+ """Enqueue ``async_execute_bin`` on the PG queue; return its id.- ``pg_queue`` → enqueue to ``pg_queue_message`` (a PG consumer runs it);- ``celery`` → ``celery_app.send_task``. The returned id is the **bare**- ``str(msg_id)`` (PG) or the Celery task id — one format across entry+ The returned id is the **bare** ``str(msg_id)`` — one format across entrypaths, matching the worker PG dispatch (``PgDispatchHandle.id``)."""- if is_pg_transport(transport):- is_api_execution = queue == CeleryQueue.CELERY_API_DEPLOYMENTS- msg_id = pg_enqueue_task(- task_name="async_execute_bin",- # None → "celery" (general); else celery_api_deployments.- queue=queue,- args=args,- kwargs=kwargs,- # Two sentinels, deliberately: the row's org_id column is NOT NULL- # (wants ""), while the fairness payload's org_id is str | None (a- # missing org means "no segment", not the literal "None").- org_id=org_schema or "",- priority=PG_DEFAULT_PRIORITY,- fairness={- "org_id": org_schema or None,- "workload_type": (- WorkloadType.API.value- if is_api_execution- else WorkloadType.NON_API.value- ),- "pipeline_priority": PG_DEFAULT_PRIORITY,- },- )- return str(msg_id)- async_execution = celery_app.send_task(- "async_execute_bin", args=args, kwargs=kwargs, queue=queue+ is_api_execution = queue == CeleryQueue.CELERY_API_DEPLOYMENTS+ msg_id = pg_enqueue_task(+ task_name="async_execute_bin",+ # None → "celery" (general); else celery_api_deployments.+ queue=queue,+ args=args,+ kwargs=kwargs,+ # Two sentinels, deliberately: the row's org_id column is NOT NULL+ # (wants ""), while the fairness payload's org_id is str | None (a+ # missing org means "no segment", not the literal "None").+ org_id=org_schema or "",+ priority=PG_DEFAULT_PRIORITY,+ fairness={+ "org_id": org_schema or None,+ "workload_type": (+ WorkloadType.API.value+ if is_api_execution+ else WorkloadType.NON_API.value+ ),+ "pipeline_priority": PG_DEFAULT_PRIORITY,+ },)- return async_execution.id+ return str(msg_id)@staticmethoddef _record_dispatch_handle(*,execution_id: str,- transport: str,dispatch_handle: str | None,org_schema: str,file_count: int,) -> None:- """Persist the transport's dispatch handle on the execution row.+ """Persist the dispatch handle on the execution row.- Celery → the Celery task UUID into ``task_id`` (UUIDField). PG → the- ``pg_queue_message.msg_id`` into ``queue_message_id`` (BigIntegerField);- ``task_id`` stays NULL because there is no Celery task on the PG path.- Each id lives in its own correctly-typed column so a bigint msg_id is- never forced into the UUID ``task_id``.+ The ``pg_queue_message.msg_id`` goes into ``queue_message_id``+ (BigIntegerField). ``task_id`` stays NULL — it held the Celery task UUID+ and there is no Celery task on the PG path."""if not dispatch_handle:- # PG always yields a truthy msg_id, so an empty handle is Celery-only.+ # PG always yields a truthy msg_id, so an empty handle means the+ # enqueue did not return one — record it rather than silently+ # leaving the execution without a handle.logger.warning(- f"[{org_schema}] Empty dispatch handle (transport={transport}) "- f"for execution_id '{execution_id}'."+ f"[{org_schema}] Empty dispatch handle for execution_id '{execution_id}'.")return- if is_pg_transport(transport):- # The PG handle is the bigint msg_id as a string. Parse defensively- # so a malformed/future handle format surfaces its specific cause- # here instead of being absorbed by the caller's generic guard.- try:- msg_id = int(dispatch_handle)- except (TypeError, ValueError):- logger.error(- f"[{org_schema}] PG dispatch handle {dispatch_handle!r} is not "- f"a valid bigint msg_id for execution_id '{execution_id}'; "- "queue_message_id not recorded"- )- return- WorkflowExecutionServiceHelper.update_execution_queue_message_id(- execution_id=execution_id, queue_message_id=msg_id- )- else:- WorkflowExecutionServiceHelper.update_execution_task(- execution_id=execution_id, task_id=dispatch_handle+ # The handle is the bigint msg_id as a string. Parse defensively so a+ # malformed/future handle format surfaces its specific cause here+ # instead of being absorbed by the caller's generic guard.+ try:+ msg_id = int(dispatch_handle)+ except (TypeError, ValueError):+ logger.error(+ f"[{org_schema}] PG dispatch handle {dispatch_handle!r} is not "+ f"a valid bigint msg_id for execution_id '{execution_id}'; "+ "queue_message_id not recorded")+ return+ WorkflowExecutionServiceHelper.update_execution_queue_message_id(+ execution_id=execution_id, queue_message_id=msg_id+ )logger.info(- f"[{org_schema}] Job '{dispatch_handle}' enqueued "- f"(transport={transport}) for execution_id '{execution_id}', "- f"'{file_count}' files"+ f"[{org_schema}] Job '{dispatch_handle}' enqueued for "+ f"execution_id '{execution_id}', '{file_count}' files")@classmethod@@ -621,19 +604,6 @@ class WorkflowHelper:# end-to-end. Flipt (gated by the env master-switch) decides; fails# closed to "celery".#- # NOTE (deliberate, two resolution sites): transport is resolved here- # for the API/manual/async paths and separately in- # internal_api_views.create_workflow_execution for the scheduler- # path. These are DISTINCT entry paths — never a double-resolution of- # the same execution. entity_id is execution_id, so each execution- # buckets exactly once and can't split across transports even if the- # rollout % is re-rolled between the two sites.- transport = resolve_transport(- execution_id=execution_id,- organization_id=org_schema,- workflow_id=workflow_id,- pipeline_id=pipeline_id,- )dispatch_args = [org_schema, # schema_nameworkflow_id, # workflow_id@@ -650,13 +620,11 @@ class WorkflowHelper:"hitl_queue_name": hitl_queue_name,"hitl_packet_id": hitl_packet_id,"custom_data": custom_data,- "transport": transport,+ # Wire field the workers still branch on. PG is the only+ # transport now; the field goes when those branches do.+ "transport": WorkflowTransport.PG_QUEUE.value,}- # Orchestrator transport (9e PR A / 2d): dispatch async_execute_bin on- # the resolved transport (PG enqueue vs Celery). Extracted to a helper- # so this method stays simple and the fork is unit-testable.dispatch_handle = cls._dispatch_orchestrator_task(- transport=transport,queue=queue,args=dispatch_args,kwargs=dispatch_kwargs,@@ -691,26 +659,24 @@ class WorkflowHelper:workflow_execution: WorkflowExecution = WorkflowExecution.objects.get(id=execution_id)- # Record the dispatch handle (Celery task_id or PG msg_id) on the row.- # Best-effort, in its OWN try/except: it must NEVER abort the- # synchronous timeout wait below. (Writing a bigint msg_id into the- # UUID task_id used to raise ValueError inside update_execution_task- # (UUID coercion on save), which bubbled to the post-dispatch handler- # and silently skipped the wait — so every PG-routed API deployment- # ignored `timeout`.)+ # Record the PG msg_id on the row. Best-effort, in its OWN+ # try/except: it must NEVER abort the synchronous timeout wait below.+ # (Writing a bigint msg_id into the UUID task_id used to raise+ # ValueError inside update_execution_task (UUID coercion on save),+ # which bubbled to the post-dispatch handler and silently skipped the+ # wait — so every PG-routed API deployment ignored `timeout`.)try:cls._record_dispatch_handle(execution_id=execution_id,- transport=transport,dispatch_handle=dispatch_handle,org_schema=org_schema or "",file_count=len(hash_values_of_files),)except Exception:logger.exception(- f"[{org_schema}] Failed to record dispatch handle "- f"(transport={transport}) for execution '{execution_id}'; "- "continuing — the orchestrator is already running"+ f"[{org_schema}] Failed to record dispatch handle for "+ f"execution '{execution_id}'; continuing — the orchestrator "+ "is already running")execution_status = workflow_execution.status@@ -747,8 +713,8 @@ class WorkflowHelper:# it never raised; this generic handler covers any stray case.)logger.exception(f"[{org_schema}] Post-dispatch bookkeeping failed for execution "- f"'{execution_id}' (orchestrator already dispatched on "- f"{transport}); not marking ERROR"+ f"'{execution_id}' (orchestrator already enqueued on the PG "+ f"queue); not marking ERROR")return ExecutionResponse(workflow_id, execution_id, ExecutionStatus.EXECUTING.value@@ -1054,43 +1020,29 @@ class WorkflowHelper:workflow_id=workflow.id, single_step=True)try:- workflow_execution = WorkflowExecution.objects.get(pk=execution_id)+ # Existence probe only — the branch below raises unconditionally,+ # so the result is deliberately not bound. The query is still+ # load-bearing: its DoesNotExist selects the create-a-new-execution+ # path in the except.+ WorkflowExecution.objects.get(pk=execution_id)# Single-step is the one entry path that never moved onto PG. Its# only fan-out is the Celery chord in `process_input_files` — the- # normal path's PG fan-out lives in the general worker instead, so- # nothing here is transport-gated. With the flag on and the Celery- # file_processing workers scaled to zero (the epic's acceptance- # gate), those batches would sit unconsumed and the execution would- # hang in EXECUTING, invisible to the PG reaper. Fail fast instead:- # a 500 with a stated cause beats a silent forever-EXECUTING row.+ # normal path's PG fan-out lives in the general worker instead —+ # so with no Celery file_processing workers those batches would+ # sit unconsumed and the execution would hang in EXECUTING,+ # invisible to the PG reaper. Fail fast instead: a 500 with a+ # stated cause beats a silent forever-EXECUTING row.#- # Unreachable from the UI — the step buttons are gone and both live- # call sites pass isStepExecution=false, so `execution_action` is- # never sent (frontend Agency.jsx). This guards a direct API call.+ # Deprecated and unreachable from the UI — the step buttons are+ # gone and both live call sites pass isStepExecution=false, so+ # `execution_action` is never sent (frontend Agency.jsx). This+ # guards a direct API call. Removing the endpoint outright is+ # tracked separately.#- # Resolved here rather than in `process_input_files` so the Celery- # path stays byte-identical and an already-resolved execution is- # never re-resolved. Step executions are created by- # `create_and_make_execution_response`, which resolves no transport,- # so this is their first and only resolution.- transport = resolve_transport(- execution_id=execution_id,- organization_id=workflow.organization.organization_id,- workflow_id=workflow.id,- )- if is_pg_transport(transport):- raise WorkflowExecutionError(- "Single-step execution is not supported on the Postgres "- "queue transport: it has no PG fan-out, and its Celery "- "batches would never be consumed. Run the full workflow "- "instead, or disable pg_queue_enabled for this "- "organization."- )- return WorkflowHelper.run_workflow(- workflow=workflow,- single_step=True,- workflow_execution=workflow_execution,- hash_values_of_files=hash_values_of_files,+ raise WorkflowExecutionError(+ "Single-step execution is not supported: it has no PG "+ "fan-out, and its Celery batches would never be consumed. "+ "Run the full workflow instead.")except WorkflowExecution.DoesNotExist:return WorkflowHelper.create_and_make_execution_response(diff --git a/docker/README.md b/docker/README.mdindex 4990ffe31..be71603c8 100644--- a/docker/README.md+++ b/docker/README.md@@ -104,13 +104,13 @@ Debug ports per service:| platform-service | 5680 || prompt-service | 5681 || **V2 Workers** | |-| worker-file-processing-v2 | 5682 |-| worker-callback-v2 | 5683 |-| worker-api-deployment-v2 | 5684 |-| worker-general-v2 | 5685 |-| worker-notification-v2 | 5686 |-| worker-log-consumer-v2 | 5687 |-| worker-scheduler-v2 | 5688 |+| worker-pg-fileproc | 5682 |+| worker-pg-callback | 5683 |+| worker-pg-orchestrator-api | 5684 |+| worker-pg-orchestrator-general | 5685 |+| worker-pg-notification | 5686 |+| worker-log-stream-consumer | 5687 |+| worker-pg-scheduler | 5688 |### VSCode Configurationdiff --git a/docker/compose.debug.yaml b/docker/compose.debug.yamlindex 3522902b7..6028d2722 100644--- a/docker/compose.debug.yaml+++ b/docker/compose.debug.yaml@@ -7,13 +7,13 @@# 5678 - backend# 5679 - runner# 5680 - platform-service-# 5682 - worker-file-processing-v2-# 5683 - worker-callback-v2-# 5684 - worker-api-deployment-v2-# 5685 - worker-general-v2-# 5686 - worker-notification-v2-# 5687 - worker-log-consumer-v2-# 5688 - worker-scheduler-v2+# 5682 - worker-pg-fileproc+# 5683 - worker-pg-callback+# 5684 - worker-pg-orchestrator-api+# 5685 - worker-pg-orchestrator-general+# 5686 - worker-pg-notification+# 5687 - worker-log-stream-consumer+# 5688 - worker-pg-schedulerservices:#########################################################################################################@@ -68,128 +68,121 @@ services:# V2 Workers with debugpy# Using --pool=threads for debugpy compatibility (prefork doesn't work well with debugpy)- worker-file-processing-v2:+ worker-pg-fileproc:user: rootports:- "5682:5682"entrypoint: ["bash", "-c"]+ # Wrap the PG consumer, NOT `celery -A worker worker` (UN-4046). This+ # override REPLACES the base file's command, so a Celery invocation here+ # turns the service into a RabbitMQ worker while the backend enqueues to+ # Postgres — the debug stack then accepts work and runs none of it.+ #+ # Queue set and worker type are deliberately NOT repeated: the module reads+ # WORKER_PG_QUEUE_CONSUMER_QUEUE / _WORKER_TYPE from the base file's+ # environment. Restating them here is what let this file drift out of sync.command: [- "uv run python -m debugpy --listen 0.0.0.0:5682 -m celery- -A worker worker- --queues=file_processing,api_file_processing,file_processing_priority- --loglevel=INFO- --pool=threads- --concurrency=2- --prefetch-multiplier=1- --without-gossip- --without-mingle- --without-heartbeat"+ "uv run python -m debugpy --listen 0.0.0.0:5682 -m pg_queue_consumer"]- worker-callback-v2:+ worker-pg-callback:user: rootports:- "5683:5683"entrypoint: ["bash", "-c"]+ # Wrap the PG consumer, NOT `celery -A worker worker` (UN-4046). This+ # override REPLACES the base file's command, so a Celery invocation here+ # turns the service into a RabbitMQ worker while the backend enqueues to+ # Postgres — the debug stack then accepts work and runs none of it.+ #+ # Queue set and worker type are deliberately NOT repeated: the module reads+ # WORKER_PG_QUEUE_CONSUMER_QUEUE / _WORKER_TYPE from the base file's+ # environment. Restating them here is what let this file drift out of sync.command: [- "uv run python -m debugpy --listen 0.0.0.0:5683 -m celery- -A worker worker- --queues=file_processing_callback,api_file_processing_callback- --loglevel=INFO- --pool=threads- --concurrency=2- --prefetch-multiplier=1- --without-gossip- --without-mingle- --without-heartbeat"+ "uv run python -m debugpy --listen 0.0.0.0:5683 -m pg_queue_consumer"]- worker-api-deployment-v2:+ worker-pg-orchestrator-api:user: rootports:- "5684:5684"entrypoint: ["bash", "-c"]+ # Wrap the PG consumer, NOT `celery -A worker worker` (UN-4046). This+ # override REPLACES the base file's command, so a Celery invocation here+ # turns the service into a RabbitMQ worker while the backend enqueues to+ # Postgres — the debug stack then accepts work and runs none of it.+ #+ # Queue set and worker type are deliberately NOT repeated: the module reads+ # WORKER_PG_QUEUE_CONSUMER_QUEUE / _WORKER_TYPE from the base file's+ # environment. Restating them here is what let this file drift out of sync.command: [- "uv run python -m debugpy --listen 0.0.0.0:5684 -m celery- -A worker worker- --queues=celery_api_deployments- --loglevel=INFO- --pool=threads- --concurrency=2- --prefetch-multiplier=1- --without-gossip- --without-mingle- --without-heartbeat"+ "uv run python -m debugpy --listen 0.0.0.0:5684 -m pg_queue_consumer"]- worker-general-v2:+ worker-pg-orchestrator-general:user: rootports:- "5685:5685"entrypoint: ["bash", "-c"]+ # Wrap the PG consumer, NOT `celery -A worker worker` (UN-4046). This+ # override REPLACES the base file's command, so a Celery invocation here+ # turns the service into a RabbitMQ worker while the backend enqueues to+ # Postgres — the debug stack then accepts work and runs none of it.+ #+ # Queue set and worker type are deliberately NOT repeated: the module reads+ # WORKER_PG_QUEUE_CONSUMER_QUEUE / _WORKER_TYPE from the base file's+ # environment. Restating them here is what let this file drift out of sync.command: [- "uv run python -m debugpy --listen 0.0.0.0:5685 -m celery- -A worker worker- --queues=celery- --loglevel=INFO- --pool=threads- --concurrency=2- --prefetch-multiplier=1- --without-gossip- --without-mingle- --without-heartbeat"+ "uv run python -m debugpy --listen 0.0.0.0:5685 -m pg_queue_consumer"]- worker-notification-v2:+ worker-pg-notification:user: rootports:- "5686:5686"entrypoint: ["bash", "-c"]+ # Wrap the PG consumer, NOT `celery -A worker worker` (UN-4046). This+ # override REPLACES the base file's command, so a Celery invocation here+ # turns the service into a RabbitMQ worker while the backend enqueues to+ # Postgres — the debug stack then accepts work and runs none of it.+ #+ # Queue set and worker type are deliberately NOT repeated: the module reads+ # WORKER_PG_QUEUE_CONSUMER_QUEUE / _WORKER_TYPE from the base file's+ # environment. Restating them here is what let this file drift out of sync.command: [- "uv run python -m debugpy --listen 0.0.0.0:5686 -m celery- -A worker worker- --queues=notifications,notifications_webhook,notifications_email,notifications_sms,notifications_priority- --loglevel=INFO- --pool=threads- --concurrency=2- --prefetch-multiplier=1- --without-gossip- --without-mingle- --without-heartbeat"+ "uv run python -m debugpy --listen 0.0.0.0:5686 -m pg_queue_consumer"]- worker-log-consumer-v2:+ worker-log-stream-consumer:user: rootports:- "5687:5687"entrypoint: ["bash", "-c"]+ # Wrap the PG consumer, NOT `celery -A worker worker` (UN-4046). This+ # override REPLACES the base file's command, so a Celery invocation here+ # turns the service into a RabbitMQ worker while the backend enqueues to+ # Postgres — the debug stack then accepts work and runs none of it.+ #+ # Queue set and worker type are deliberately NOT repeated: the module reads+ # WORKER_PG_QUEUE_CONSUMER_QUEUE / _WORKER_TYPE from the base file's+ # environment. Restating them here is what let this file drift out of sync.command: [- "uv run python -m debugpy --listen 0.0.0.0:5687 -m celery- -A worker worker- --queues=celery_log_task_queue,celery_periodic_logs- --loglevel=INFO- --pool=threads- --concurrency=2- --prefetch-multiplier=1- --without-gossip- --without-mingle- --without-heartbeat"+ "uv run python -m debugpy --listen 0.0.0.0:5687 -m log_consumer.redis_stream_consumer"]- worker-scheduler-v2:+ worker-pg-scheduler:user: rootports:- "5688:5688"entrypoint: ["bash", "-c"]+ # Wrap the PG consumer, NOT `celery -A worker worker` (UN-4046). This+ # override REPLACES the base file's command, so a Celery invocation here+ # turns the service into a RabbitMQ worker while the backend enqueues to+ # Postgres — the debug stack then accepts work and runs none of it.+ #+ # Queue set and worker type are deliberately NOT repeated: the module reads+ # WORKER_PG_QUEUE_CONSUMER_QUEUE / _WORKER_TYPE from the base file's+ # environment. Restating them here is what let this file drift out of sync.command: [- "uv run python -m debugpy --listen 0.0.0.0:5688 -m celery- -A worker worker- --queues=scheduler- --loglevel=INFO- --pool=threads- --concurrency=2- --prefetch-multiplier=1- --without-gossip- --without-mingle- --without-heartbeat"+ "uv run python -m debugpy --listen 0.0.0.0:5688 -m pg_queue_consumer"]diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yamlindex dda8f27a2..9f6ca7789 100644--- a/docker/docker-compose.yaml+++ b/docker/docker-compose.yaml@@ -50,54 +50,30 @@ services:- ${TOOL_REGISTRY_CONFIG_SRC_PATH}:/data/tool_registry_configenvironment:- APPLICATION_NAME=unstract-backend+ # Redis log transport (UN-4046) — set on EVERY publisher: backend, runner+ # and all twelve workers. worker-log-consumer-v2 is gone from this stack, so+ # the "celery" default publishes execution logs to a RabbitMQ queue nothing+ # drains; LogPublisher swallows its errors, so the only symptom is execution+ # logs that stay empty while the queue grows.+ #+ # Deliberately here and not in essentials.env: that file is git-ignored and+ # generated, and copy_or_merge_envs only merges new keys on --only-env or+ # --update, so an existing checkout would silently keep the old default.+ - LOG_TRANSPORT=redis+ # Adopt the non-pipeline periodics (dashboard_metrics.*) onto the PG+ # scheduler (UN-4046). converge_pg_scheduler only adopts them with+ # --periodics, which entrypoint.sh derives from this variable — and it was+ # set NOWHERE. This stack deletes celery-beat and worker-metrics, so+ # without it those rows stay Beat-owned (pg_owned=false) with Beat gone:+ # nothing fires them, worker-pg-metrics idles on an empty queue, and there+ # is no error, no queue depth and no failed pod. The cloud chart refuses+ # this combination at render time; compose has no equivalent guard.+ - PG_SCHEDULER_ADOPT_PERIODICS=truelabels:- traefik.enable=true- traefik.http.routers.backend.rule=Host(`frontend.unstract.localhost`) && (PathPrefix(`/api/v1`) || PathPrefix(`/deployment`) || PathPrefix(`/public`))- traefik.http.services.backend.loadbalancer.server.port=8000- # Celery worker for dashboard metrics processing- worker-metrics:- <<: *host_gateway- image: unstract/backend:${VERSION}- container_name: unstract-worker-metrics- restart: unless-stopped- entrypoint: .venv/bin/celery- command: "-A backend worker --loglevel=info -Q dashboard_metric_events --autoscale=${WORKER_METRICS_AUTOSCALE:-4,1}"- env_file:- - ../backend/.env- depends_on:- - rabbitmq- - db- environment:- - APPLICATION_NAME=unstract-worker-metrics-- labels:- - traefik.enable=false-- # Celery worker for Prompt Studio IDE callbacks.- # Processes post-execution callbacks via InternalAPIClient (no Django).- # Handles: ide_index_complete/error, ide_prompt_complete/error.- worker-ide-callback:- <<: *host_gateway- image: unstract/worker-unified:${VERSION}- container_name: unstract-worker-ide-callback- restart: unless-stopped- command: ["ide-callback"]- env_file:- - ../workers/.env- - ./essentials.env- depends_on:- - db- - redis- - rabbitmq- environment:- - APPLICATION_NAME=unstract-worker-ide-callback- - WORKER_TYPE=ide_callback- - WORKER_NAME=ide-callback-worker- labels:- - traefik.enable=false-- # Celery Flowercelery-flower:<<: *host_gatewayimage: unstract/backend:${VERSION}@@ -120,24 +96,6 @@ services:profiles:- optional- # Celery Beat- celery-beat:- <<: *host_gateway- image: unstract/backend:${VERSION}- container_name: unstract-celery-beat- restart: unless-stopped- entrypoint: .venv/bin/celery- command: "-A backend beat --scheduler django_celery_beat.schedulers:DatabaseScheduler -l INFO"- env_file:- - ../backend/.env- - ./essentials.env- depends_on:- - db- - rabbitmq- environment:- - APPLICATION_NAME=unstract-celery-beat-- # Frontend React appfrontend:image: unstract/frontend:${VERSION}container_name: unstract-frontend@@ -193,6 +151,11 @@ services:- 5002:5002env_file:- ../runner/.env+ environment:+ # Publisher side of the Redis log transport (UN-4046) — see the note on the+ # backend service. runner matters most of the three: it forwards this to every+ # tool sidecar it spawns, so a stale value here strands the tool logs too.+ - LOG_TRANSPORT=redisvolumes:- ./workflow_data:/data# Docker socket bind mount to spawn tool containers@@ -203,244 +166,6 @@ services:labels:- traefik.enable=false- # ====================================================================- # WORKER SERVICES- # ====================================================================-- worker-api-deployment-v2:- <<: *host_gateway- image: unstract/worker-unified:${VERSION}- container_name: unstract-worker-api-deployment-v2- restart: unless-stopped- command: ["api-deployment"]- ports:- - "8085:8090"- env_file:- - ../workers/.env- - ./essentials.env- depends_on:- - db- - redis- - rabbitmq- environment:- - APPLICATION_NAME=unstract-worker-api-deployment-v2- - WORKER_TYPE=api_deployment- - CELERY_QUEUES_API_DEPLOYMENT=${CELERY_QUEUES_API_DEPLOYMENT:-celery_api_deployments}- - CELERY_POOL=${WORKER_API_DEPLOYMENT_POOL:-threads}- - CELERY_PREFETCH_MULTIPLIER=${WORKER_API_DEPLOYMENT_PREFETCH_MULTIPLIER:-1}- - CELERY_CONCURRENCY=${WORKER_API_DEPLOYMENT_CONCURRENCY:-4}- - CELERY_EXTRA_ARGS=${WORKER_API_DEPLOYMENT_EXTRA_ARGS:-}- - WORKER_NAME=api-deployment-worker-v2- - API_DEPLOYMENT_METRICS_PORT=8090- - HEALTH_PORT=8090- labels:- - traefik.enable=false- volumes:- - ./workflow_data:/data- - ${TOOL_REGISTRY_CONFIG_SRC_PATH}:/data/tool_registry_config-- worker-callback-v2:- <<: *host_gateway- image: unstract/worker-unified:${VERSION}- container_name: unstract-worker-callback-v2- restart: unless-stopped- command: ["callback"]- ports:- - "8086:8083"- env_file:- - ../workers/.env- - ./essentials.env- depends_on:- - db- - redis- - rabbitmq- environment:- - APPLICATION_NAME=unstract-worker-callback-v2- - WORKER_TYPE=callback- - WORKER_NAME=callback-worker-v2- - CALLBACK_METRICS_PORT=8083- labels:- - traefik.enable=false- volumes:- - ./workflow_data:/data- - ${TOOL_REGISTRY_CONFIG_SRC_PATH}:/data/tool_registry_config-- worker-file-processing-v2:- <<: *host_gateway- image: unstract/worker-unified:${VERSION}- container_name: unstract-worker-file-processing-v2- restart: unless-stopped- # command: ["file-processing"]- command:- [- ".venv/bin/celery",- "-A",- "worker",- "worker",- "--queues=file_processing,api_file_processing,file_processing_priority",- "--loglevel=INFO",- "--pool=prefork",- "--concurrency=4",- "--prefetch-multiplier=1",- "--without-gossip",- "--without-mingle",- "--without-heartbeat",- ]- ports:- - "8087:8082"- env_file:- - ../workers/.env- - ./essentials.env- depends_on:- - db- - redis- - rabbitmq- environment:- - APPLICATION_NAME=unstract-worker-file-processing-v2- - WORKER_TYPE=file_processing- - WORKER_MODE=oss- - WORKER_NAME=file-processing-worker-v2- - FILE_PROCESSING_METRICS_PORT=8082- # OSS Configuration - Enterprise features disabled- - MANUAL_REVIEW_ENABLED=false- - ENTERPRISE_FEATURES_ENABLED=false- - PLUGIN_REGISTRY_MODE=oss- # Configurable Celery options- - CELERY_QUEUES_FILE_PROCESSING=${CELERY_QUEUES_FILE_PROCESSING:-file_processing,api_file_processing}- - CELERY_POOL=${WORKER_FILE_PROCESSING_POOL:-threads}- - CELERY_PREFETCH_MULTIPLIER=${WORKER_FILE_PROCESSING_PREFETCH_MULTIPLIER:-1}- - CELERY_CONCURRENCY=${WORKER_FILE_PROCESSING_CONCURRENCY:-4}- - CELERY_EXTRA_ARGS=${WORKER_FILE_PROCESSING_EXTRA_ARGS:-}- labels:- - traefik.enable=false- volumes:- - ./workflow_data:/data- - ${TOOL_REGISTRY_CONFIG_SRC_PATH}:/data/tool_registry_config-- worker-general-v2:- <<: *host_gateway- image: unstract/worker-unified:${VERSION}- container_name: unstract-worker-general-v2- restart: unless-stopped- command: ["general"]- ports:- - "8088:8082"- env_file:- - ../workers/.env- - ./essentials.env- depends_on:- - db- - redis- - rabbitmq- environment:- - APPLICATION_NAME=unstract-worker-general-v2- - WORKER_TYPE=general- - WORKER_NAME=general-worker-v2- - GENERAL_METRICS_PORT=8081- - HEALTH_PORT=8082- labels:- - traefik.enable=false- volumes:- - ./workflow_data:/data- - ${TOOL_REGISTRY_CONFIG_SRC_PATH}:/data/tool_registry_config-- worker-notification-v2:- <<: *host_gateway- image: unstract/worker-unified:${VERSION}- container_name: unstract-worker-notification-v2- restart: unless-stopped- command: ["notification"]- ports:- - "8089:8085"- env_file:- - ../workers/.env- - ./essentials.env- depends_on:- - db- - redis- - rabbitmq- environment:- - APPLICATION_NAME=unstract-worker-notification-v2- - WORKER_TYPE=notification- - WORKER_NAME=notification-worker-v2- - NOTIFICATION_METRICS_PORT=8085- - HEALTH_PORT=8085- # Notification specific configs- - NOTIFICATION_QUEUE_NAME=notifications- - WEBHOOK_QUEUE_NAME=notifications_webhook- - EMAIL_QUEUE_NAME=notifications_email- - SMS_QUEUE_NAME=notifications_sms- - PRIORITY_QUEUE_NAME=notifications_priority- # Configurable Celery options- - CELERY_QUEUES_NOTIFICATION=${CELERY_QUEUES_NOTIFICATION:-notifications,notifications_webhook,notifications_email,notifications_sms,notifications_priority}- - CELERY_POOL=${WORKER_NOTIFICATION_POOL:-prefork}- - CELERY_PREFETCH_MULTIPLIER=${WORKER_NOTIFICATION_PREFETCH_MULTIPLIER:-1}- - CELERY_CONCURRENCY=${WORKER_NOTIFICATION_CONCURRENCY:-4}- - CELERY_EXTRA_ARGS=${WORKER_NOTIFICATION_EXTRA_ARGS:-}- # Complete command override (if set, ignores all other options)- - CELERY_COMMAND_OVERRIDE=${WORKER_NOTIFICATION_COMMAND_OVERRIDE:-}- # Individual argument overrides- - CELERY_APP_MODULE=${WORKER_NOTIFICATION_APP_MODULE:-worker}- - CELERY_LOG_LEVEL=${WORKER_NOTIFICATION_LOG_LEVEL:-INFO}- - CELERY_HOSTNAME=${WORKER_NOTIFICATION_HOSTNAME:-}- - CELERY_MAX_TASKS_PER_CHILD=${WORKER_NOTIFICATION_MAX_TASKS_PER_CHILD:-}- - CELERY_TIME_LIMIT=${WORKER_NOTIFICATION_TIME_LIMIT:-}- - CELERY_SOFT_TIME_LIMIT=${WORKER_NOTIFICATION_SOFT_TIME_LIMIT:-}- labels:- - traefik.enable=false- volumes:- - ./workflow_data:/data- - ${TOOL_REGISTRY_CONFIG_SRC_PATH}:/data/tool_registry_config-- worker-log-consumer-v2:- <<: *host_gateway- image: unstract/worker-unified:${VERSION}- container_name: unstract-worker-log-consumer-v2- restart: unless-stopped- command: ["log-consumer"]- ports:- - "8090:8084"- env_file:- - ../workers/.env- - ./essentials.env- depends_on:- - db- - redis- - rabbitmq- environment:- - APPLICATION_NAME=unstract-worker-log-consumer-v2- - WORKER_TYPE=log_consumer- - WORKER_NAME=log-consumer-worker-v2- - LOG_CONSUMER_METRICS_PORT=8084- - HEALTH_PORT=8084- # Log consumer specific configs- - LOG_CONSUMER_QUEUE_NAME=celery_log_task_queue- # Multiple queue support - supports comma-separated queue names- - CELERY_QUEUES_LOG_CONSUMER=${CELERY_QUEUES_LOG_CONSUMER:-celery_log_task_queue,celery_periodic_logs}- - PERIODIC_LOGS_QUEUE_NAME=${PERIODIC_LOGS_QUEUE_NAME:-celery_periodic_logs}- # Log history configuration- - LOG_HISTORY_QUEUE_NAME=${LOG_HISTORY_QUEUE_NAME:-log_history_queue}- - LOGS_BATCH_LIMIT=${LOGS_BATCH_LIMIT:-100}- - ENABLE_LOG_HISTORY=${ENABLE_LOG_HISTORY:-true}- - CELERY_POOL=${WORKER_LOG_CONSUMER_POOL:-prefork}- - CELERY_PREFETCH_MULTIPLIER=${WORKER_LOG_CONSUMER_PREFETCH_MULTIPLIER:-1}- - CELERY_CONCURRENCY=${WORKER_LOG_CONSUMER_CONCURRENCY:-2}- - CELERY_EXTRA_ARGS=${WORKER_LOG_CONSUMER_EXTRA_ARGS:-}- # Complete command override (if set, ignores all other options)- - CELERY_COMMAND_OVERRIDE=${WORKER_LOG_CONSUMER_COMMAND_OVERRIDE:-}- # Individual argument overrides- - CELERY_APP_MODULE=${WORKER_LOG_CONSUMER_APP_MODULE:-worker}- - CELERY_LOG_LEVEL=${WORKER_LOG_CONSUMER_LOG_LEVEL:-INFO}- - CELERY_HOSTNAME=${WORKER_LOG_CONSUMER_HOSTNAME:-}- - CELERY_MAX_TASKS_PER_CHILD=${WORKER_LOG_CONSUMER_MAX_TASKS_PER_CHILD:-}- - CELERY_TIME_LIMIT=${WORKER_LOG_CONSUMER_TIME_LIMIT:-}- - CELERY_SOFT_TIME_LIMIT=${WORKER_LOG_CONSUMER_SOFT_TIME_LIMIT:-}- labels:- - traefik.enable=false- volumes:- - ./workflow_data:/data- - ${TOOL_REGISTRY_CONFIG_SRC_PATH}:/data/tool_registry_config-worker-log-history-scheduler-v2:<<: *host_gatewayimage: unstract/worker-unified:${VERSION}@@ -457,6 +182,8 @@ services:- rabbitmqenvironment:- APPLICATION_NAME=unstract-worker-log-history-scheduler-v2+ # Log publisher — see the note on the backend service (UN-4046).+ - LOG_TRANSPORT=redis# Scheduler interval in seconds- LOG_HISTORY_CONSUMER_INTERVAL=${LOG_HISTORY_CONSUMER_INTERVAL:-5}# Override example: TASK_TRIGGER_COMMAND=/custom/trigger/script.sh@@ -464,102 +191,6 @@ services:labels:- traefik.enable=false- worker-scheduler-v2:- <<: *host_gateway- image: unstract/worker-unified:${VERSION}- container_name: unstract-worker-scheduler-v2- restart: unless-stopped- command: ["scheduler"]- ports:- - "8091:8087"- env_file:- - ../workers/.env- - ./essentials.env- depends_on:- - db- - redis- - rabbitmq- environment:- - APPLICATION_NAME=unstract-worker-scheduler-v2- - WORKER_TYPE=scheduler- - WORKER_NAME=scheduler-worker-v2- - SCHEDULER_METRICS_PORT=8087- - HEALTH_PORT=8087- # Scheduler specific configs- - SCHEDULER_QUEUE_NAME=scheduler- # Configurable Celery options- - CELERY_QUEUES_SCHEDULER=${CELERY_QUEUES_SCHEDULER:-scheduler}- - CELERY_POOL=${WORKER_SCHEDULER_POOL:-prefork}- - CELERY_PREFETCH_MULTIPLIER=${WORKER_SCHEDULER_PREFETCH_MULTIPLIER:-1}- - CELERY_CONCURRENCY=${WORKER_SCHEDULER_CONCURRENCY:-2}- - CELERY_EXTRA_ARGS=${WORKER_SCHEDULER_EXTRA_ARGS:-}- # Complete command override (if set, ignores all other options)- - CELERY_COMMAND_OVERRIDE=${WORKER_SCHEDULER_COMMAND_OVERRIDE:-}- # Individual argument overrides- - CELERY_APP_MODULE=${WORKER_SCHEDULER_APP_MODULE:-worker}- - CELERY_LOG_LEVEL=${WORKER_SCHEDULER_LOG_LEVEL:-INFO}- - CELERY_HOSTNAME=${WORKER_SCHEDULER_HOSTNAME:-}- - CELERY_MAX_TASKS_PER_CHILD=${WORKER_SCHEDULER_MAX_TASKS_PER_CHILD:-}- - CELERY_TIME_LIMIT=${WORKER_SCHEDULER_TIME_LIMIT:-}- - CELERY_SOFT_TIME_LIMIT=${WORKER_SCHEDULER_SOFT_TIME_LIMIT:-}- labels:- - traefik.enable=false- volumes:- - ./workflow_data:/data- - ${TOOL_REGISTRY_CONFIG_SRC_PATH}:/data/tool_registry_config-- worker-executor-v2:- <<: *host_gateway- image: unstract/worker-unified:${VERSION}- container_name: unstract-worker-executor-v2- restart: unless-stopped- command: ["executor"]- ports:- - "8092:8088"- env_file:- - ../workers/.env- - ./essentials.env- depends_on:- - db- - redis- - rabbitmq- - platform-service- environment:- - APPLICATION_NAME=unstract-worker-executor-v2- - WORKER_TYPE=executor- - WORKER_NAME=executor-worker-v2- - EXECUTOR_METRICS_PORT=8088- - HEALTH_PORT=8088- # Configurable Celery options- - CELERY_QUEUES_EXECUTOR=${CELERY_QUEUES_EXECUTOR:-celery_executor_legacy,celery_executor_agentic,celery_executor_agentic_table}- - CELERY_POOL=${WORKER_EXECUTOR_POOL:-prefork}- - CELERY_PREFETCH_MULTIPLIER=${WORKER_EXECUTOR_PREFETCH_MULTIPLIER:-1}- - CELERY_CONCURRENCY=${WORKER_EXECUTOR_CONCURRENCY:-2}- - CELERY_EXTRA_ARGS=${WORKER_EXECUTOR_EXTRA_ARGS:-}- labels:- - traefik.enable=false- volumes:- - ./workflow_data:/data- - ${TOOL_REGISTRY_CONFIG_SRC_PATH}:/data/tool_registry_config- - prompt_studio_data:/app/prompt-studio-data-- # ===========================================================================- # PG-queue services (Postgres-backed transport) — gated behind the `pg-queue`- # compose profile, so a plain `docker compose up` starts NONE of them and- # behaviour is identical to today. Bring them up with:- # docker compose --profile pg-queue up -d- # They consume from Postgres, not the broker. The orchestrator consumers and- # the reaper are broker-free for PG-routed executions (fan-out honors the- # carried transport; a missing transport fails closed to Celery, which these- # consumers aren't expected to receive). The fan-out and callback consumers- # still hand off to the Celery executor (tool-execution RPC) / Celery- # notifications, so they keep a rabbitmq dependency until the executor and- # notifications themselves move to PG. Each maps 1:1 to a future K8s Deployment.- # Executions still route to Celery until the single `pg_queue_enabled` Flipt- # flag is ramped — that is the rollout, not this change.- # ===========================================================================-- # Orchestrator consumer for API-deployment executions (async_execute_bin_api).worker-pg-orchestrator-api:image: unstract/worker-unified:${VERSION}container_name: unstract-worker-pg-orchestrator-api@@ -580,6 +211,8 @@ services:- WORKER_PG_QUEUE_CONSUMER_WORKER_TYPE=api_deployment- WORKER_PG_QUEUE_CONSUMER_QUEUE=celery_api_deployments- WORKER_PG_QUEUE_CONSUMER_HEALTH_PORT=8090+ # Log publisher — see the note on the backend service (UN-4046).+ - LOG_TRANSPORT=redis# async_execute_bin arms the barrier + dispatches batches, then returns — a# bounded-minutes orchestration, and idempotency-guarded (UN-3671) so a vt# overrun can't double-orchestrate. Without an explicit vt the consumer@@ -592,8 +225,6 @@ services:volumes:- ./workflow_data:/data- ${TOOL_REGISTRY_CONFIG_SRC_PATH}:/data/tool_registry_config- profiles:- - pg-queue# Orchestrator consumer for general / scheduled-ETL executions (async_execute_bin).worker-pg-orchestrator-general:@@ -616,6 +247,8 @@ services:- WORKER_PG_QUEUE_CONSUMER_WORKER_TYPE=general- WORKER_PG_QUEUE_CONSUMER_QUEUE=celery- WORKER_PG_QUEUE_CONSUMER_HEALTH_PORT=8090+ # Log publisher — see the note on the backend service (UN-4046).+ - LOG_TRANSPORT=redis# Orchestration is bounded-minutes + claim-guarded (see orchestrator-api).# Without an explicit vt it defaults to 30s. 600/660 matches our k8s deployment defaults (workerPg* values, separate chart repo).- WORKER_PG_QUEUE_CONSUMER_VT_SECONDS=${WORKER_PG_ORCHESTRATOR_VT_SECONDS:-600}@@ -625,8 +258,6 @@ services:volumes:- ./workflow_data:/data- ${TOOL_REGISTRY_CONFIG_SRC_PATH}:/data/tool_registry_config- profiles:- - pg-queue# Fan-out consumer (process_file_batch) — both the ETL and API file queues.worker-pg-fileproc:@@ -648,8 +279,10 @@ services:- APPLICATION_NAME=unstract-worker-pg-fileproc- WORKER_BARRIER_BACKEND=pg- WORKER_PG_QUEUE_CONSUMER_WORKER_TYPE=file_processing- - WORKER_PG_QUEUE_CONSUMER_QUEUE=file_processing,api_file_processing+ - WORKER_PG_QUEUE_CONSUMER_QUEUE=file_processing,file_processing_priority,api_file_processing- WORKER_PG_QUEUE_CONSUMER_HEALTH_PORT=8090+ # Log publisher — see the note on the backend service (UN-4046).+ - LOG_TRANSPORT=redis# UN-3606: prefork N consumer children so file batches run in parallel (the# PG analogue of the Celery worker's --concurrency). SKIP LOCKED distributes# batches across children + replicas; a single execution is still capped by@@ -666,8 +299,6 @@ services:volumes:- ./workflow_data:/data- ${TOOL_REGISTRY_CONFIG_SRC_PATH}:/data/tool_registry_config- profiles:- - pg-queue# Fan-in callback consumer (process_batch_callback / _api) — both callback queues.worker-pg-callback:@@ -689,8 +320,10 @@ services:- APPLICATION_NAME=unstract-worker-pg-callback- WORKER_BARRIER_BACKEND=pg- WORKER_PG_QUEUE_CONSUMER_WORKER_TYPE=callback- - WORKER_PG_QUEUE_CONSUMER_QUEUE=file_processing_callback,api_file_processing_callback+ - WORKER_PG_QUEUE_CONSUMER_QUEUE=file_processing_callback,celery_callback,api_file_processing_callback- WORKER_PG_QUEUE_CONSUMER_HEALTH_PORT=8090+ # Log publisher — see the note on the backend service (UN-4046).+ - LOG_TRANSPORT=redis# The aggregating callback (fan-in) runs after all batches: aggregate results,# update status, push destination, notify. Without an explicit vt it defaults# to 30s and a >30s callback redelivers — re-firing webhooks + subscription@@ -702,8 +335,6 @@ services:volumes:- ./workflow_data:/data- ${TOOL_REGISTRY_CONFIG_SRC_PATH}:/data/tool_registry_config- profiles:- - pg-queue# Scheduler consumer — runs scheduler.tasks.execute_pipeline_task fired by the# orchestrator's PG scheduler tick (Beat replacement). Distinct from the Celery@@ -730,6 +361,8 @@ services:- WORKER_PG_QUEUE_CONSUMER_WORKER_TYPE=scheduler- WORKER_PG_QUEUE_CONSUMER_QUEUE=scheduler- WORKER_PG_QUEUE_CONSUMER_HEALTH_PORT=8090+ # Log publisher — see the note on the backend service (UN-4046).+ - LOG_TRANSPORT=redis# Scheduled-trigger dispatch is quick (enqueue the execution). Without an# explicit vt it defaults to 30s. 180/240 matches our k8s deployment defaults (separate chart repo).- WORKER_PG_QUEUE_CONSUMER_VT_SECONDS=${WORKER_PG_SCHEDULER_VT_SECONDS:-180}@@ -739,8 +372,6 @@ services:volumes:- ./workflow_data:/data- ${TOOL_REGISTRY_CONFIG_SRC_PATH}:/data/tool_registry_config- profiles:- - pg-queue# Dashboard-metrics periodics over PG (UN-3796) — the execution half of the Beat# replacement. The PG scheduler fires dashboard_metrics.* onto dashboard_metric_events@@ -783,6 +414,8 @@ services:- WORKER_PG_QUEUE_CONSUMER_WORKER_TYPE=scheduler- WORKER_PG_QUEUE_CONSUMER_QUEUE=dashboard_metric_events- WORKER_PG_QUEUE_CONSUMER_HEALTH_PORT=8090+ # Log publisher — see the note on the backend service (UN-4046).+ - LOG_TRANSPORT=redis# Sized for a minutes-long aggregation: stale > vt > the task's own ceiling.- WORKER_PG_QUEUE_CONSUMER_VT_SECONDS=${WORKER_PG_METRICS_VT_SECONDS:-900}- WORKER_PG_QUEUE_CONSUMER_HEALTH_STALE_SECONDS=${WORKER_PG_METRICS_HEALTH_STALE_SECONDS:-960}@@ -791,19 +424,21 @@ services:- WORKER_PG_QUEUE_CONSUMER_MAX_ATTEMPTS=1labels:- traefik.enable=false- profiles:- - pg-queue# Log streaming without Celery (UN-3755) — the flag-on replacement for- # worker-log-consumer-v2. Note the transport here is a REDIS LIST, not+ # the retired Celery log consumer. Note the transport here is a REDIS LIST, not# pg_queue_message: the log consumer's own sinks are already Redis (log_history_queue# + the Socket.IO emit), so routing through Postgres would detour a lossy, high-rate# stream through the platform's most contended resource to buy durability that logs- # explicitly discard at their 10k cap. It sits in the pg-queue profile because it is- # part of the same flag-on fleet, not because it speaks to Postgres.+ # explicitly discard at their 10k cap. Despite the name it speaks to Redis, not+ # Postgres; it is part of the PG-era fleet by association, not by transport.#- # LOG_TRANSPORT must be set to "redis" on every PUBLISHER too (backend, workers,- # runner) or this drains an empty list while logs keep going to RabbitMQ.+ # LOG_TRANSPORT must be set to "redis" on every PUBLISHER too or this drains an+ # empty list while logs keep going to RabbitMQ — where, since UN-4046 removed+ # worker-log-consumer-v2 from this stack, nothing reads them. Set on backend and+ # runner in their own environment blocks, and in each worker's own environment+ # block too — deliberately NOT essentials.env (that file is git-ignored and+ # generated, so an existing checkout only picks up new keys on --update).worker-log-stream-consumer:image: unstract/worker-unified:${VERSION}container_name: unstract-worker-log-stream-consumer@@ -826,15 +461,13 @@ services:- ENABLE_LOG_HISTORY=${ENABLE_LOG_HISTORY:-true}labels:- traefik.enable=false- profiles:- - pg-queue# Executor RPC over PG — runs execute_extraction as a request-reply: claims# work from Postgres and writes the ExecutionResult to pg_task_result for the- # blocking caller. Same heavy executor runtime as worker-executor-v2 (tool+ # blocking caller. Same heavy executor runtime as the file-processing consumer (tool# execution → platform-service / adapters / file storage), but broker-free: it- # is the terminal of the RPC and dispatches nothing onward to Celery. Dark- # until the backend pg_queue_enabled gate is flipped.+ # is the terminal of the RPC and dispatches nothing onward to Celery. It is the+ # ONLY executor since UN-4046 — scaling it to zero stops all tool execution.worker-pg-executor:image: unstract/worker-unified:${VERSION}container_name: unstract-worker-pg-executor@@ -854,8 +487,10 @@ services:- APPLICATION_NAME=unstract-worker-pg-executor- WORKER_BARRIER_BACKEND=pg- WORKER_PG_QUEUE_CONSUMER_WORKER_TYPE=executor- - WORKER_PG_QUEUE_CONSUMER_QUEUE=celery_executor_legacy,celery_executor_agentic,celery_executor_agentic_table+ - WORKER_PG_QUEUE_CONSUMER_QUEUE=celery_executor_legacy,celery_executor_agentic,celery_executor_table,celery_executor_smart_table,celery_executor_simple_prompt_studio,celery_executor_agentic_table,celery_executor_lookup_test- WORKER_PG_QUEUE_CONSUMER_HEALTH_PORT=8090+ # Log publisher — see the note on the backend service (UN-4046).+ - LOG_TRANSPORT=redis# The executor runs LLM inference — far longer than the consumer's 30s/60s# defaults. Set the visibility timeout AND the liveness stale-threshold# ABOVE the executor's hard per-task limit (EXECUTOR_TASK_TIME_LIMIT=3600s),@@ -876,8 +511,6 @@ services:- ./workflow_data:/data- ${TOOL_REGISTRY_CONFIG_SRC_PATH}:/data/tool_registry_config- prompt_studio_data:/app/prompt-studio-data- profiles:- - pg-queue# IDE callback consumer (③c) — drains the ``ide_callback`` queue the executor# self-chains onto for async/callback dispatch (Prompt Studio run/index/extract,@@ -909,6 +542,8 @@ services:- WORKER_PG_QUEUE_CONSUMER_QUEUE=ide_callback- WORKER_PG_QUEUE_CONSUMER_HEALTH_PORT=8090- WORKER_PG_QUEUE_CONSUMER_CONCURRENCY=${PG_IDE_CALLBACK_CONCURRENCY:-2}+ # Log publisher — see the note on the backend service (UN-4046).+ - LOG_TRANSPORT=redis# Callbacks are sub-second (ws emit + small API writes), so nowhere near the# executor's 1h limit — but set the visibility timeout above a slow internal# API write (backend under load) so a sibling replica can't re-claim mid-run@@ -921,8 +556,46 @@ services:- ./workflow_data:/data- ${TOOL_REGISTRY_CONFIG_SRC_PATH}:/data/tool_registry_config- prompt_studio_data:/app/prompt-studio-data- profiles:- - pg-queue++ # Buffered webhook notifications. The backend's notification seam enqueues+ # ``send_webhook_notification`` onto these queues unconditionally (UN-4046), so+ # without this consumer every buffered webhook would sit undrained — nothing+ # sweeps pg_queue_message.+ worker-pg-notification:+ image: unstract/worker-unified:${VERSION}+ container_name: unstract-worker-pg-notification+ restart: unless-stopped+ command: ["pg-queue-consumer"]+ ports:+ - "8104:8090"+ env_file:+ - ../workers/.env+ - ./essentials.env+ depends_on:+ - db+ - redis+ - platform-service+ environment:+ - ENVIRONMENT=development+ - APPLICATION_NAME=unstract-worker-pg-notification+ - WORKER_BARRIER_BACKEND=pg+ - WORKER_PG_QUEUE_CONSUMER_WORKER_TYPE=notification+ - WORKER_PG_QUEUE_CONSUMER_QUEUE=notifications,notifications_webhook,notifications_email,notifications_sms,notifications_priority+ - WORKER_PG_QUEUE_CONSUMER_HEALTH_PORT=8090+ - WORKER_PG_QUEUE_CONSUMER_CONCURRENCY=${PG_NOTIFICATION_CONCURRENCY:-2}+ # Log publisher — see the note on the backend service (UN-4046).+ - LOG_TRANSPORT=redis+ # One outbound HTTP POST per task. VT sits above a slow subscriber so a+ # sibling replica cannot re-claim mid-POST and double-deliver; health-stale+ # above that. Mirrors the chart's workerPgNotification bounds.+ - WORKER_PG_QUEUE_CONSUMER_VT_SECONDS=${PG_NOTIFICATION_VT_SECONDS:-300}+ - WORKER_PG_QUEUE_CONSUMER_HEALTH_STALE_SECONDS=${PG_NOTIFICATION_HEALTH_STALE_SECONDS:-360}+ labels:+ - traefik.enable=false+ volumes:+ - ./workflow_data:/data+ - ${TOOL_REGISTRY_CONFIG_SRC_PATH}:/data/tool_registry_config+ - prompt_studio_data:/app/prompt-studio-data# Reaper / orchestrator — leader-elected loop. Run exactly ONE instance (it# elects a single leader via pg_orchestrator_lock; extra replicas idle as@@ -946,6 +619,8 @@ services:- APPLICATION_NAME=unstract-worker-pg-reaper- WORKER_PG_REAPER_INTERVAL_SECONDS=${WORKER_PG_REAPER_INTERVAL_SECONDS:-5}- WORKER_PG_REAPER_HEALTH_PORT=8086+ # Log publisher — see the note on the backend service (UN-4046).+ - LOG_TRANSPORT=redis# Retention sweep (cadence-gated inside the leader's tick): drop expired# pg_task_result rows + orphaned pg_batch_dedup markers so neither grows# unbounded as the gate ramps. The dedup retention below (default 24h) should@@ -958,8 +633,6 @@ services:volumes:- ./workflow_data:/data- ${TOOL_REGISTRY_CONFIG_SRC_PATH}:/data/tool_registry_config- profiles:- - pg-queuevolumes:prompt_studio_data:diff --git a/docker/sample.compose.override.yaml b/docker/sample.compose.override.yamlindex b0e98d8b1..bdf379713 100644--- a/docker/sample.compose.override.yaml+++ b/docker/sample.compose.override.yaml@@ -162,7 +162,7 @@ services:# V2 Workers - Build configurations with memory-optimized settings# All use --pool=threads and --concurrency=2 for lower memory footprint- worker-api-deployment-v2:+ worker-pg-orchestrator-api:build:dockerfile: docker/dockerfiles/worker-unified.Dockerfilecontext: ..@@ -179,7 +179,7 @@ services:- action: rebuildpath: ../workers/uv.lock- worker-callback-v2:+ worker-pg-callback:build:dockerfile: docker/dockerfiles/worker-unified.Dockerfilecontext: ..@@ -196,7 +196,7 @@ services:- action: rebuildpath: ../workers/uv.lock- worker-file-processing-v2:+ worker-pg-fileproc:build:dockerfile: docker/dockerfiles/worker-unified.Dockerfilecontext: ..@@ -213,7 +213,7 @@ services:- action: rebuildpath: ../workers/uv.lock- worker-general-v2:+ worker-pg-orchestrator-general:build:dockerfile: docker/dockerfiles/worker-unified.Dockerfilecontext: ..@@ -230,7 +230,7 @@ services:- action: rebuildpath: ../workers/uv.lock- worker-notification-v2:+ worker-pg-notification:build:dockerfile: docker/dockerfiles/worker-unified.Dockerfilecontext: ..@@ -247,7 +247,7 @@ services:- action: rebuildpath: ../workers/uv.lock- worker-log-consumer-v2:+ worker-log-stream-consumer:build:dockerfile: docker/dockerfiles/worker-unified.Dockerfilecontext: ..@@ -270,7 +270,7 @@ services:dockerfile: docker/dockerfiles/worker-unified.Dockerfilecontext: ..- worker-scheduler-v2:+ worker-pg-scheduler:build:dockerfile: docker/dockerfiles/worker-unified.Dockerfilecontext: ..@@ -289,16 +289,20 @@ services:########################################################################################################## IDE callback worker (workers image, processes ide_callback queue)- worker-ide-callback:+ worker-pg-ide-callback:build:dockerfile: docker/dockerfiles/worker-unified.Dockerfilecontext: ..- # Celery worker for dashboard metrics (memory optimized: reduced autoscale)- worker-metrics:+ # PG consumer for the dashboard_metrics.* periodics fired by the PG scheduler+ # tick. Must build the WORKER image: the base file runs `pg-queue-consumer`,+ # which the backend image's entrypoint rejects outright ("Unknown argument"),+ # so building backend.Dockerfile here crash-loops the pod and leaves+ # dashboard_metric_events with no consumer. No `command:` override either —+ # the base file's is correct.+ worker-pg-metrics:build:- dockerfile: docker/dockerfiles/backend.Dockerfile+ dockerfile: docker/dockerfiles/worker-unified.Dockerfilecontext: ..- command: "-A backend worker --loglevel=info -Q dashboard_metric_events --autoscale=2,1"develop:watch:- action: sync+restartdiff --git a/docker/sample.env b/docker/sample.envindex 6c96821a4..47905773d 100644--- a/docker/sample.env+++ b/docker/sample.env@@ -144,7 +144,6 @@ WORKER_PG_REAPER_INTERVAL_SECONDS=5 # Reaper sweep interval (seconds)# above the vt, which must stay above the task itself.# above the vt, which must stay above the task itself.#-# NOTE: routing executions to PG is a SEPARATE, later step. Running these-# services does NOT move any traffic — the single `pg_queue_enabled` Flipt flag-# (default off, fail-closed) decides per-execution transport, and stays off until-# the rollout ramp.+# NOTE: since UN-4046 these services ARE the execution path — there is no flag+# and no Celery fallback. Every dispatch goes to PG, so a queue whose consumer is+# not running accepts work and never executes it, with no error at the producer.diff --git a/tests/README.md b/tests/README.mdindex bdd20fe47..0d2724e3f 100644--- a/tests/README.md+++ b/tests/README.md@@ -221,7 +221,7 @@ Only `LLM` completions are mocked, not embeddings: `provisioned_workflow` pins `### Fan-out (`MAX_PARALLEL_FILE_BATCHES`)-Defaults to `1`, meaning every file of a multi-file run lands in one batch and is processed serially — so a fan-out test would pass without any fan-out happening. The overlay defaults it to `3` on `backend`, which is what normally takes effect: workers ask the backend for this value and fall back to their own env only when they can't reach it or it carries no value (the overlay sets it on `worker-api-deployment-v2` too, to keep the fallback in step). Batches are `min(MAX_PARALLEL_FILE_BATCHES, num_files)`, so N files with the same N gives one batch each.+Defaults to `1`, meaning every file of a multi-file run lands in one batch and is processed serially — so a fan-out test would pass without any fan-out happening. The overlay defaults it to `3` on `backend`, which is what normally takes effect: workers ask the backend for this value and fall back to their own env only when they can't reach it or it carries no value (the overlay sets it on `worker-pg-orchestrator-api` too, to keep the fallback in step). Batches are `min(MAX_PARALLEL_FILE_BATCHES, num_files)`, so N files with the same N gives one batch each.**The fan-out half is an open gap** (`workflow-execution-fan-out` in `critical_paths.yaml`), and closing it needs a product change, not a cleverer test. Nothing persists a batch or task id: the batch index is a discarded loop local, and the celery task id only ever reaches worker stdout. That leaves per-file timing as the only proxy, and timing does not work here — measured on CI, three files genuinely fanned out finished *further* apart (durations `6.6 / 18.1 / 22.9`) than the ~2s steps seen when they share a batch, because three concurrent tool containers on a loaded runner cost more in contention than they gain in overlap. No threshold separates those two distributions, and no delay value fixes it: the contention scales with the parallelism being measured.diff --git a/tests/compose/docker-compose.test.yaml b/tests/compose/docker-compose.test.yamlindex 78713fefa..742d6b2b5 100644--- a/tests/compose/docker-compose.test.yaml+++ b/tests/compose/docker-compose.test.yaml@@ -17,18 +17,25 @@ services:runner:image: unstract/runner:${UNSTRACT_TEST_VERSION:-latest}+ # Retargeted onto the PG consumers (UN-4046). The Celery worker services these+ # used to name — worker-executor-v2, worker-file-processing-v2,+ # worker-api-deployment-v2 — are gone from docker/docker-compose.yaml. An+ # override for a service that does not exist is NOT a no-op: compose creates it+ # with neither image nor build context and rejects the whole project with+ # "invalid compose project", so e2e could not bring the stack up at all.+# Execute-path e2e must never reach a real provider.- worker-executor-v2:+ worker-pg-executor:environment:- UNSTRACT_LLM_MOCK_RESPONSE=${UNSTRACT_LLM_MOCK_RESPONSE:-}- worker-file-processing-v2:+ worker-pg-fileproc:environment:- UNSTRACT_LLM_MOCK_RESPONSE=${UNSTRACT_LLM_MOCK_RESPONSE:-}# Only the worker's fallback if it can't reach the backend; kept in step so it# can't quietly serialise the fan-out test.- worker-api-deployment-v2:+ worker-pg-orchestrator-api:environment:- MAX_PARALLEL_FILE_BATCHES=${MAX_PARALLEL_FILE_BATCHES:-3}diff --git a/unstract/core/src/unstract/core/data_models.py b/unstract/core/src/unstract/core/data_models.pyindex bc2fd6dbc..00073ee2a 100644--- a/unstract/core/src/unstract/core/data_models.py+++ b/unstract/core/src/unstract/core/data_models.py@@ -214,8 +214,32 @@ class WorkflowTransport(str, Enum):PG_QUEUE = "pg_queue"-# Default transport when none is resolved/carried — keeps every pre-existing-# payload and caller on the legacy path until a transport is explicitly chosen.+# Default transport when none is carried on the payload.+#+# ⚠️ KNOWN HAZARD, DELIBERATELY NOT FIXED HERE — the safe direction has inverted.+#+# CELERY was the safe fallback while Celery had consumers: a payload that lost the+# field degraded onto a working substrate. With the Celery fleet at zero that is no+# longer true. A payload defaulting to "celery" now selects CeleryChordBarrier in+# `_barrier_for_transport`, skips the PG orchestration-claim and terminal-redelivery+# guards, and publishes the fan-out to a broker nothing drains — and because no PG+# rows are written, neither the reaper nor the undispatched sweep can see the+# stranded execution. Reachable on a rolling deploy, where a not-yet-upgraded+# backend still emits `transport: "celery"`.+#+# Flipping this constant to PG_QUEUE is the right fix and it is a one-line change,+# but it is NOT a one-line consequence: it breaks 19 tests across 6 files+# (test_pg_barrier, test_chord_sites_characterisation, test_barrier,+# test_dispatch_with_callback, test_dispatch_sites_characterisation,+# test_workflow_context_transport), every one of which relies on the implicit+# Celery default — most of them characterising PgBarrier's Celery-link branch,+# which is deferred-deletion code. Rewriting those to state their transport+# explicitly is correct, but it is a behaviour change that deserves its own review+# rather than riding along in a documentation sweep.+#+# Until then the mitigation is that every producer sets the field explicitly+# (workflow_helper.py and internal_api_views.py both hardcode PG_QUEUE), so the+# default is only reachable via version skew.DEFAULT_WORKFLOW_TRANSPORT = WorkflowTransport.CELERY.value@@ -227,14 +251,18 @@ def normalize_transport(value: object, *, logger: Any = None, context: str = "")route an execution onto an unknown substrate. This **fails closed**: anunrecognized value (``None``, ``"celary"``, ``""``) logs a warning (when a``logger`` is given) and falls back to :data:`DEFAULT_WORKFLOW_TRANSPORT`- (Celery). A recognized value passes through as its canonical string.+ — see the hazard note on that constant. A recognized value passes through as+ its canonical string.Coerce-not-raise is deliberate, and differs from how``WorkflowContextData.workflow_type`` is validated: ``transport`` has an- explicit safe default by design (the whole point of the seam is reversible- fallback to Celery), so a bad value should degrade to Celery, not crash the- execution. ``context`` is an optional suffix for the log line (e.g. an- ``exec:<id>`` tag) to make a version-skew warning traceable.+ explicit safe default by design, so a bad value should degrade rather than+ crash the execution. What changed with UN-4046 is WHICH default is safe — see+ the note on the constant. A rolling deploy where an older backend still emits+ ``"celery"`` is the live case: that value is recognized, so it passes through+ here untouched and the guards keyed on it are skipped. This function only+ protects the missing/garbage case. ``context`` is an optional suffix for the+ log line (e.g. an ``exec:<id>`` tag) to make a version-skew warning traceable."""try:return WorkflowTransport(value).valuediff --git a/unstract/workflow-execution/src/unstract/workflow_execution/executor_rpc.py b/unstract/workflow-execution/src/unstract/workflow_execution/executor_rpc.pyindex 14dcc6e04..3fc3a2017 100644--- a/unstract/workflow-execution/src/unstract/workflow_execution/executor_rpc.py+++ b/unstract/workflow-execution/src/unstract/workflow_execution/executor_rpc.py@@ -1,13 +1,12 @@-"""Shared executor-RPC dispatch for the PG path — the gate + reply_key orchestration.+"""Shared executor-RPC dispatch — the reply_key/timeout orchestration.The executor "RPC" is a synchronous request-reply: a caller sends an``ExecutionContext`` to the executor worker and blocks for the ``ExecutionResult``.-The legacy transport is Celery (the SDK ``ExecutionDispatcher``); the PG path adds a-parallel Postgres transport. Backend (Django/prompt-studio) and workers-(``structure_tool``) both need it, and used to carry **byte-for-byte mirrors** of-this logic — the only thing that genuinely differs between them is the *transport-primitive*: the backend enqueues via the Django ORM (``enqueue_task`` +-``PgTaskResult``), the workers via psycopg2 (``PgQueueClient`` + ``PgResultBackend``).+Backend (Django/prompt-studio) and workers (``structure_tool``) both need it, and+used to carry **byte-for-byte mirrors** of this logic — the only thing that+genuinely differs between them is the *transport primitive*: the backend enqueues+via the Django ORM (``enqueue_task`` + ``PgTaskResult``), the workers via psycopg2+(``PgQueueClient`` + ``PgResultBackend``).So this module owns everything transport-agnostic exactly once, and the differingprimitive is **injected** (composition, not inheritance) via :class:`QueueTransport`:@@ -16,12 +15,10 @@ primitive is **injected** (composition, not inheritance) via :class:`QueueTransp``dispatch_with_callback`` + the reply_key/timeout orchestration and thenever-raises contract (timeout/failure → ``ExecutionResult.failure``). It calls``transport.enqueue(...)`` and ``transport.wait_for_result(...)``.-- :func:`resolve_pg_transport` — the gate: the single ``pg_queue_enabled`` Flipt- flag (bucketed per org) is the sole control. Fails closed to Celery on a- blind/unreachable Flipt or any error.-- :class:`RoutingExecutionDispatcher` — picks PG-vs-Celery per call (instant- rollout/rollback) for every mode; the Celery dispatcher, the PG dispatcher and the- per-side ``resolve`` are all injected.++PG is the only transport. This module used to also hold ``resolve_pg_transport``+(the ``pg_queue_enabled`` Flipt gate) and ``RoutingExecutionDispatcher`` (which+picked PG-vs-Celery per call); both went with the flag in UN-4046.It lives in ``unstract-workflow-execution`` (which both backend and workers alreadydepend on) rather than ``unstract.core`` because it needs ``unstract.sdk1`` and@@ -35,25 +32,19 @@ import loggingimport osimport uuidfrom dataclasses import dataclass-from typing import TYPE_CHECKING, Any, Protocol+from typing import TYPE_CHECKING, Protocolfrom unstract.core.data_models import PgTaskStatusfrom unstract.core.execution_dispatch import DispatchHandle, signature_to_continuation-from unstract.flags.feature_flag import check_feature_flag_statusfrom unstract.sdk1.execution.result import ExecutionResultif TYPE_CHECKING:- from collections.abc import Callable-from unstract.core.data_models import ContinuationSpecfrom unstract.core.execution_dispatch import CallbackSignaturefrom unstract.sdk1.execution.context import ExecutionContextlogger = logging.getLogger(__name__)-# The single PG-queue rollout flag — the same key execution and the scheduler read,-# so one flip turns the whole PG-queue feature on/off.-PG_QUEUE_FLAG_KEY = "pg_queue_enabled"EXECUTE_TASK = "execute_extraction"# Mirror the SDK's queue-per-executor convention so the PG executor queue name# matches the Celery one (the worker-pg-executor consumer subscribes to these).@@ -64,32 +55,6 @@ DEFAULT_TIMEOUT_ENV = "EXECUTOR_RESULT_TIMEOUT"DEFAULT_TIMEOUT = 3600-class _CeleryDispatcher(Protocol):- """The subset of the SDK ``ExecutionDispatcher`` the router delegates to on the- Celery path — so ``RoutingExecutionDispatcher`` holds it typed, not as ``Any``.- """-- def dispatch(- self,- context: ExecutionContext,- timeout: int | None = ...,- headers: dict[str, Any] | None = ...,- ) -> ExecutionResult: ...-- def dispatch_async(- self, context: ExecutionContext, headers: dict[str, Any] | None = ...- ) -> str: ...-- def dispatch_with_callback(- self,- context: ExecutionContext,- on_success: CallbackSignature | None = ...,- on_error: CallbackSignature | None = ...,- task_id: str | None = ...,- headers: dict[str, Any] | None = ...,- ) -> Any: ...--@dataclassclass ExecResultRow:"""Normalised executor-RPC result row — the transport-agnostic shape@@ -131,42 +96,6 @@ class QueueTransport(Protocol):def wait_for_result(self, reply_key: str, timeout: float) -> ExecResultRow | None: ...-def resolve_pg_transport(- context: ExecutionContext,- *,- flag_key: str = PG_QUEUE_FLAG_KEY,-) -> bool:- """True → route this executor dispatch over PG; False → Celery (default).-- Gated by the single ``pg_queue_enabled`` Flipt flag, bucketed per org.- **Fails closed to Celery** on a blind Flipt or any error — so the executor- never silently loses its transport.- """- if os.environ.get("FLIPT_SERVICE_AVAILABLE", "false").lower() != "true":- logger.warning(- "resolve_pg_transport: FLIPT_SERVICE_AVAILABLE != true "- "(Flipt blind); using Celery"- )- return False- org = getattr(context, "organization_id", None)- # %-bucket keyed on org; fall back to run_id so a context without an org still- # resolves deterministically.- entity_id = str(org or getattr(context, "run_id", "") or "default")- flag_context = {"executor_name": str(context.executor_name)}- if org:- flag_context["organization_id"] = str(org)- try:- enabled = check_feature_flag_status(- flag_key=flag_key, entity_id=entity_id, context=flag_context- )- except Exception:- logger.warning(- "resolve_pg_transport: Flipt check failed; using Celery", exc_info=True- )- return False- return bool(enabled)--def _resolve_timeout(timeout: int | None) -> int:"""Caller timeout, defaulting to ``EXECUTOR_RESULT_TIMEOUT`` env then 3600s.@@ -215,8 +144,11 @@ class PgExecutionDispatcher:callers branch on ``result.success`` identically on either transport.No ``headers`` on any PG dispatch method: the PG path carries org/routing in- the enqueue payload, not Celery headers, so the ``RoutingExecutionDispatcher``- does not forward fairness headers to the PG path.+ the enqueue payload (``transport.enqueue(..., org_id=...)``), not Celery+ headers. This method takes no ``headers`` argument: callers must not pass+ one. The routing dispatcher that used to absorb a ``headers=`` kwarg went+ with the flag in UN-4046, and three call sites kept passing it — every+ extraction raised ``TypeError`` until they were fixed."""timeout = _resolve_timeout(timeout)reply_key = str(uuid.uuid4())@@ -393,69 +325,3 @@ class PgExecutionDispatcher:error_spec["task_name"] if error_spec else None,)return DispatchHandle(task_id)---class RoutingExecutionDispatcher:- """Gate-routed executor dispatcher: every mode picks PG vs Celery per call- (read at dispatch time → flipping the flag is an instant, no-redeploy- rollout/rollback). Duck-typed against the SDK ``ExecutionDispatcher`` so call- sites are unchanged.-- Composition-injected: the Celery dispatcher (``celery``), the PG dispatcher- (``pg``) and the per-side gate (``resolve(context) -> bool``).- """-- def __init__(- self,- *,- celery: _CeleryDispatcher,- pg: PgExecutionDispatcher,- resolve: Callable[[ExecutionContext], bool],- ) -> None:- self._celery = celery- self._pg = pg- self._resolve = resolve-- def dispatch(- self,- context: ExecutionContext,- timeout: int | None = None,- headers: dict[str, Any] | None = None,- ) -> ExecutionResult:- if self._resolve(context):- logger.info(- "Executor RPC → PG transport (executor=%s run_id=%s)",- context.executor_name,- context.run_id,- )- # PG carries org/routing via the enqueue payload, not Celery headers, so- # the fairness headers are intentionally not forwarded here.- return self._pg.dispatch(context, timeout=timeout)- return self._celery.dispatch(context, timeout=timeout, headers=headers)-- def dispatch_async(- self, context: ExecutionContext, headers: dict[str, Any] | None = None- ) -> str:- if self._resolve(context):- return self._pg.dispatch_async(context)- return self._celery.dispatch_async(context, headers=headers)-- def dispatch_with_callback(- self,- context: ExecutionContext,- on_success: CallbackSignature | None = None,- on_error: CallbackSignature | None = None,- task_id: str | None = None,- headers: dict[str, Any] | None = None,- ) -> Any:- if self._resolve(context):- return self._pg.dispatch_with_callback(- context, on_success=on_success, on_error=on_error, task_id=task_id- )- return self._celery.dispatch_with_callback(- context,- on_success=on_success,- on_error=on_error,- task_id=task_id,- headers=headers,- )diff --git a/workers/file_processing/structure_tool_task.py b/workers/file_processing/structure_tool_task.pyindex 971a78398..9e6353e65 100644--- a/workers/file_processing/structure_tool_task.py+++ b/workers/file_processing/structure_tool_task.py@@ -22,11 +22,9 @@ import timefrom pathlib import Pathfrom typing import Any-from file_processing.worker import app-from queue_backend import FairnessKey, worker_task-from queue_backend.fairness import WorkloadType+from queue_backend import worker_taskfrom queue_backend.pg_queue.executor_rpc import (- RoutingExecutionDispatcher,+ PgExecutionDispatcher,get_executor_dispatcher,)from shared.enums.task_enums import TaskName@@ -43,25 +41,13 @@ logger = logging.getLogger(__name__)EXECUTOR_TIMEOUT = int(os.environ.get("EXECUTOR_RESULT_TIMEOUT", 3600))-def _fairness_headers(- organization_id: str,-) -> dict[str, dict[str, str | int | None]]:- """Fairness header for executor dispatches.-- Structure-tool dispatches are workflow-execution work — both ETL- and API workflows route through here. ``NON_API`` is the safe- default (API traffic preempting is a strictly weaker mistake than- the inverse).-- TODO(UN-3504): propagate the caller's WorkloadType (API vs ETL)- instead of hard-coding ``NON_API``. Requires the chord-lift work- so the workflow type flows from the chord caller down to here.- """- return FairnessKey(- org_id=organization_id,- workload_type=WorkloadType.NON_API,- ).as_header()-+# NOTE: `_fairness_headers()` lived here and was passed as `headers=` to every+# executor dispatch. It is gone with the routing dispatcher (UN-4046):+# `PgExecutionDispatcher.dispatch()` takes no `headers`, and the router that used+# to absorb them never forwarded them to the PG path anyway — PG carries org+# routing in the enqueue payload (`transport.enqueue(..., org_id=...)`).+# UN-3504 (propagating the caller's WorkloadType) is unaffected: it was always+# about the PG payload, not these headers.# -----------------------------------------------------------------------# Constants mirrored from tools/structure/src/constants.py@@ -279,9 +265,9 @@ def _execute_structure_tool_impl(params: dict) -> dict:)platform_helper = _create_platform_helper(shim, file_execution_id)- # Gate-routed: PG executor RPC when pg_queue_enabled is on, else the unchanged- # Celery ExecutionDispatcher (zero-regression by construction — see executor_rpc).- dispatcher = get_executor_dispatcher(celery_app=app)+ # PG executor RPC. ``celery_app`` is accepted and ignored by the factory —+ # see executor_rpc.+ dispatcher = get_executor_dispatcher()fs = _get_file_storage()# ---- Step 2: Fetch tool metadata ----@@ -498,7 +484,6 @@ def _execute_structure_tool_impl(params: dict) -> dict:at_result = dispatcher.dispatch(at_ctx,timeout=EXECUTOR_TIMEOUT,- headers=_fairness_headers(organization_id),)if not at_result.success:return at_result.to_dict()@@ -541,7 +526,6 @@ def _execute_structure_tool_impl(params: dict) -> dict:pipeline_result = dispatcher.dispatch(pipeline_ctx,timeout=EXECUTOR_TIMEOUT,- headers=_fairness_headers(organization_id),)pipeline_elapsed = time.monotonic() - pipeline_start@@ -684,7 +668,7 @@ def _run_agentic_extraction(input_file_path: str,output_dir_path: str,tool_instance_metadata: dict,- dispatcher: RoutingExecutionDispatcher,+ dispatcher: PgExecutionDispatcher,shim: Any,file_execution_id: str,execution_id: str,@@ -758,7 +742,6 @@ def _run_agentic_extraction(agentic_result = dispatcher.dispatch(agentic_ctx,timeout=EXECUTOR_TIMEOUT,- headers=_fairness_headers(organization_id),)if not agentic_result.success:diff --git a/workers/notification/providers/slack_webhook.py b/workers/notification/providers/slack_webhook.pyindex 6c97cde02..6d8856412 100644--- a/workers/notification/providers/slack_webhook.py+++ b/workers/notification/providers/slack_webhook.py@@ -55,9 +55,9 @@ class SlackWebhook(WebhookProvider):Two input shapes are accepted:- Backend-rendered ``{"text": "<mrkdwn>"}`` (any backend dispatchthrough ``clubbed_renderer``) — passed through.- - Flat per-event dict from the generic internal webhook-send endpoints —- wrapped in a single-event envelope and rendered to the canonical- Slack mrkdwn body.+ - Flat per-event dict from a direct ``send_webhook_notification``+ dispatch — wrapped in a single-event envelope and rendered to the+ canonical Slack mrkdwn body."""if "text" in payload and "events" not in payload:return {"text": payload["text"]}diff --git a/workers/queue_backend/__init__.py b/workers/queue_backend/__init__.pyindex d1c9154aa..e148708a6 100644--- a/workers/queue_backend/__init__.py+++ b/workers/queue_backend/__init__.py@@ -25,13 +25,14 @@ time rather than silently falling back, which would mask a typo'dflag in a production env.**Queue-transport routing (separate axis).** ``dispatch()`` consults-:func:`~queue_backend.routing.select_backend`, which reads the-PG-queue routing table (``WORKER_PG_QUEUE_ENABLED_TASKS``) to decide-Celery-vs-PG per task. This is orthogonal to the barrier choice above:-the barrier is *how a batch's fan-in fires the callback*; the transport-is *how messages travel*. Both default to Celery. The routing table is-a scaffold today — PG-selected tasks still ride Celery (no PG consumer-yet) — so it is observable but inert.+:func:`~queue_backend.routing.select_backend`, which returns PG+unconditionally since UN-4046. This is orthogonal to the barrier choice+above: the barrier is *how a batch's fan-in fires the callback*; the+transport is *how messages travel*. It is emphatically NOT inert — every+dispatch rides PG and needs its consumer running, or the message is+durably enqueued and never executed, with no error at the producer.+(It used to read a ``WORKER_PG_QUEUE_ENABLED_TASKS`` allow-list that was+set nowhere, so everything defaulted to Celery; both are gone.)"""import osdiff --git a/workers/queue_backend/dispatch.py b/workers/queue_backend/dispatch.pyindex 3af477dd2..88bfd9860 100644--- a/workers/queue_backend/dispatch.py+++ b/workers/queue_backend/dispatch.py@@ -1,21 +1,25 @@-"""Transport-agnostic task dispatch.+"""Task dispatch onto the PG queue.-Routes each task to its transport via :func:`resolve_backend` (which applies a-per-call ``backend`` override, else defers to :func:`select_backend`):+Routes each task via :func:`resolve_backend` (a per-call ``backend`` override,+else :func:`select_backend`):-- **Celery** (default) — a thin pass-through to ``current_app.send_task``.-- **PG Queue** — when a task is opted into ``WORKER_PG_QUEUE_ENABLED_TASKS``- (or pinned via a ``backend=`` override), the task is serialised and enqueued- to ``pg_queue_message`` (9b); the PG consumer (9c) drains and runs it.+- **PG Queue** (the only transport :func:`select_backend` returns since UN-4046)+ — the task is serialised and enqueued to ``pg_queue_message``; the PG consumer+ drains and runs it.+- **Celery** — a thin pass-through to ``current_app.send_task``, reachable only+ by an explicit ``backend=QueueBackend.CELERY`` override. Nothing passes one;+ the branch goes with the rest of the Celery transport.-The default (empty allow-list) routes everything to Celery, so dispatch is-unchanged unless an operator explicitly opts a task in.+This used to default to Celery and route to PG only for task names opted into+``WORKER_PG_QUEUE_ENABLED_TASKS``. That allow-list was set nowhere, so the+default sent everything to Celery; UN-4046 removed it along with the+``pg_queue_enabled`` flag... warning::- A task opted into the PG queue **requires the PG consumer to be running**- — otherwise the message is durably enqueued but never executed. Only opt- in tasks once the consumer is deployed (and, per the migration-coherence- decision, only *leaf* tasks until execution-level routing exists).+ Every PG queue **requires its consumer to be running** — otherwise the+ message is durably enqueued and never executed, with no error at the+ producer. ``validate-pg-worker-fleet.yaml`` in the chart refuses a partial+ fleet at render time for exactly this reason."""from __future__ import annotations@@ -88,16 +92,11 @@ def dispatch(``backend`` is a per-call transport override (see:func:`~queue_backend.routing.resolve_backend` for the precedence). When- ``None`` (the default, and every call site today) the transport is the env- allow-list decision via ``select_backend`` — behaviour is unchanged. When- set, it wins over the allow-list: this is the seam the execution-level PG- pipeline uses to route a whole execution's header/callback- dispatches onto PG without opting their task *names* into- ``WORKER_PG_QUEUE_ENABLED_TASKS``. (The allow-list is for *leaf* tasks; the- coupled pipeline's migration unit is the whole execution — its transport is- resolved once at creation and travels on the execution's task kwargs onto- ``WorkflowContextData.transport``.) The override only forces the *transport*; it does not- bypass ``_enqueue_pg``'s no-silent-fallback contract.+ ``None`` — the default, and every call site today — ``select_backend`` decides,+ and since UN-4046 it returns PG unconditionally. The override is therefore the+ only remaining dial, and nothing in production passes one. It forces the+ *transport* only; it does not bypass ``_enqueue_pg``'s no-silent-fallback+ contract."""if resolve_backend(task_name, backend) is QueueBackend.PG:return _enqueue_pg(task_name, args, kwargs, queue, fairness)@@ -130,12 +129,6 @@ def _enqueue_pg(# true regardless of send outcome, so a first-dispatch failure# (DB down / unmigrated) must not suppress the one announcement.# INFO so it survives a default log config; once-per-task bounds it.- # NOTE: keyed on task name only, not on *why* it routed to PG. If a name- # is first PG-routed via a ``backend=`` override and the operator later- # adds it to the allow-list, the allow-list cutover won't re-announce- # (already in the set). Benign given the usage split (override = pipeline- # headers, allow-list = leaf tasks, so no overlap expected); the- # allow-list config itself is still announced by _log_allow_list_once._pg_routing_logged.add(task_name)logger.info("PG-queue: routing task=%r to Postgres (queue=%r). Requires the "diff --git a/workers/queue_backend/pg_barrier.py b/workers/queue_backend/pg_barrier.pyindex b044b4438..d841ae0bc 100644--- a/workers/queue_backend/pg_barrier.py+++ b/workers/queue_backend/pg_barrier.py@@ -472,8 +472,8 @@ def try_claim_orchestration(execution_id: str, organization_id: str) -> bool:raise RuntimeError("pg_orchestration_claim schema is out of date — migrations ""0012_pgorchestrationclaim / 0013_pgorchestrationclaim_organization_id "- "have not been fully applied. Run backend migrations before enabling "- "PG transport (pg_queue_enabled)."+ "have not been fully applied. Run backend migrations — PG is the only "+ "execution transport, so this blocks all orchestration.") from excdiff --git a/workers/queue_backend/pg_queue/README.md b/workers/queue_backend/pg_queue/README.mdindex bd303c0f2..8ae3f8fea 100644--- a/workers/queue_backend/pg_queue/README.md+++ b/workers/queue_backend/pg_queue/README.md@@ -3,8 +3,9 @@A quick-reference dictionary for the bespoke Postgres-backed work queue that canstand in for Celery/RabbitMQ. This file is the **terminology + config lookup**.-The transport is chosen **per org** by the Flipt flag `pg_queue_enabled`-(fail-closed to Celery), so the system runs **with and without** PG unchanged.+PG is the **only** transport since UN-4046. There is no per-org flag and no Celery+fallback: every dispatch is enqueued here, so a queue whose consumer is not running+accepts work and never executes it — durably, and with no error at the producer.---@@ -52,11 +53,11 @@ integers.| `WORKER_PG_DEDUP_RETENTION_SECONDS` | Retention for `pg_batch_dedup` rows || `WORKER_PG_QUEUE_CONNECT_RETRIES` / `_BACKOFF` | Reconnect attempts / backoff on a stale or broken DB connection |-### Routing / flag-| Env / flag | One-line |-|---|---|-| `pg_queue_enabled` (Flipt, `PG_QUEUE_FLAG_KEY`) | Per-org gate that routes dispatch to PG; **fail-closed to Celery** on any Flipt error |-| `WORKER_PG_QUEUE_ENABLED_TASKS` | Comma-separated task names eligible for PG routing (empty → everything stays on Celery) |+### Routing+There are no routing knobs left. `select_backend()` returns PG unconditionally; the+`pg_queue_enabled` Flipt flag and the `WORKER_PG_QUEUE_ENABLED_TASKS` allow-list were+both removed in UN-4046. Which work reaches a given consumer is decided entirely by+`WORKER_PG_QUEUE_CONSUMER_QUEUE` above.---diff --git a/workers/queue_backend/pg_queue/__init__.py b/workers/queue_backend/pg_queue/__init__.pyindex 303fefd9d..01573a225 100644--- a/workers/queue_backend/pg_queue/__init__.py+++ b/workers/queue_backend/pg_queue/__init__.py@@ -16,8 +16,8 @@ consumer poll loop that drains and runs claimed tasks, the leader-electedreaper, and the result backend. :func:`queue_backend.dispatch.dispatch` enqueuesa PG-selected task here as a :class:`TaskPayload` instead of sending it to Celery.-Default-empty ``WORKER_PG_QUEUE_ENABLED_TASKS`` → everything still routes to-Celery, so this is inert unless a task is explicitly opted in.+Every dispatch routes here since UN-4046 — this subpackage is on the critical+path, not inert. A queue without its consumer running strands work silently."""from .client import PgQueueClient, QueueMessagediff --git a/workers/queue_backend/pg_queue/executor_rpc.py b/workers/queue_backend/pg_queue/executor_rpc.pyindex 373a331f7..fadad7d04 100644--- a/workers/queue_backend/pg_queue/executor_rpc.py+++ b/workers/queue_backend/pg_queue/executor_rpc.py@@ -1,29 +1,26 @@"""Executor-RPC for the PG path — workers (psycopg2) transport adapter.-The gate + reply_key/timeout orchestration + routing live ONCE in+The reply_key/timeout orchestration lives ONCE in``unstract.workflow_execution.executor_rpc`` (shared with the backend). This moduleis the thin psycopg2 half: a :class:`PgClientQueueTransport` that enqueues via:class:`~queue_backend.pg_queue.client.PgQueueClient` and polls via:class:`~queue_backend.pg_queue.result_backend.PgResultBackend`, plus the:func:`get_executor_dispatcher` factory.-Zero-regression: with the ``pg_queue_enabled`` Flipt flag off the routing dispatcher-delegates every mode to the unchanged Celery ``ExecutionDispatcher`` and no-``pg_task_result`` row is created.+Since UN-4046 the factory returns the PG dispatcher unconditionally: there is no+gate, no routing dispatcher and no Celery fall-through, so a ``pg_task_result``+row is written on every request-reply dispatch."""from __future__ import annotationsfrom typing import TYPE_CHECKING-from unstract.sdk1.execution.dispatcher import ExecutionDispatcherfrom unstract.workflow_execution.executor_rpc import (EXECUTE_TASK,ExecResultRow,PgExecutionDispatcher,QueueTransport,- RoutingExecutionDispatcher,- resolve_pg_transport,)from .client import PgQueueClient@@ -39,20 +36,10 @@ if TYPE_CHECKING:__all__ = ["PgClientQueueTransport","PgExecutionDispatcher",- "RoutingExecutionDispatcher","get_executor_dispatcher",- "resolve_executor_transport",]-def resolve_executor_transport(context: ExecutionContext) -> bool:- """True → route this executor dispatch over PG; False → Celery (default).-- The single ``pg_queue_enabled`` Flipt flag (fail-closed).- """- return resolve_pg_transport(context)--class PgClientQueueTransport(QueueTransport):""":class:`QueueTransport` over psycopg2 (the workers half).@@ -107,12 +94,15 @@ class PgClientQueueTransport(QueueTransport):)-def get_executor_dispatcher(- celery_app: object | None = None,-) -> RoutingExecutionDispatcher:- """Factory: the gate-routed executor dispatcher (PG when enabled, else Celery)."""- return RoutingExecutionDispatcher(- celery=ExecutionDispatcher(celery_app=celery_app),- pg=PgExecutionDispatcher(PgClientQueueTransport()),- resolve=resolve_executor_transport,- )+def get_executor_dispatcher() -> PgExecutionDispatcher:+ """Factory: the executor dispatcher.++ Takes no arguments. It used to accept a ``celery_app`` that fed the Celery+ branch of the routing dispatcher; that branch went with the ``pg_queue_enabled``+ flag (UN-4046), and the parameter was kept for a while so the call sites did not+ all need editing at once. Keeping an ignored parameter made the signature+ decoration rather than a contract — and it read as "this dispatcher may use+ Celery", which is exactly the belief that left a ``headers=`` argument at three+ call sites and broke every extraction. Removed.+ """+ return PgExecutionDispatcher(PgClientQueueTransport())diff --git a/workers/queue_backend/routing.py b/workers/queue_backend/routing.pyindex 0cf9d9d5d..b38c0e0b2 100644--- a/workers/queue_backend/routing.py+++ b/workers/queue_backend/routing.py@@ -1,85 +1,42 @@-"""Queue-transport routing gate (Strangler-Fig).--Decides whether a given dispatch should ride the **Celery** transport-(today's only real path) or the future **PG Queue** transport,-based on a per-task-type opt-in allow-list read from env:--- ``WORKER_PG_QUEUE_ENABLED_TASKS`` — comma-separated task names routed- to PG. Empty / unset → everything routes to Celery.--**Why an allow-list, not a global boolean.** A single-``WORKER_QUEUE_BACKEND=pg`` switch would move *all* traffic at once —-the big-bang migration the PG Queue rollout explicitly forbids. The-allow-list lets an operator migrate one task type at a time-(drain-and-cutover) and roll back instantly by removing an entry.--(Per-org granularity is intentionally out of scope here. When tenant-level-canarying is needed it slots in behind :func:`select_backend` as a small-additive change — no call site touches the routing decision directly.)--**Migration coherence — what the allow-list may and may not split.**-Per-task routing is the right granularity for *independent / leaf* tasks-(e.g. ``process_log_history``, ``send_webhook_notification``): no chain,-no barrier, so one can ride PG while the rest stay on Celery — these are-the safe first migration candidates. It is **not** valid to split the-*coupled execution pipeline* (``async_execute_bin`` → file processing →-callback, with the barrier fan-in) across substrates: a chain hands off-stage→stage and the barrier coordinates a batch, so a single execution-must run **entirely on one transport**. The migration unit for the-pipeline is therefore the *execution*, not the task — the next phase will-resolve the transport once at kickoff and carry it in ``ExecutionContext``-(like the fairness key), and downstream dispatches will honour that-carried choice over the per-task env. ``select_backend`` then becomes:-"if the dispatch carries an execution-transport marker, use it; otherwise-consult this allow-list." Until then, only enable *leaf* tasks here.--**Routing posture.** This module only makes the routing *decision*.-When the flag and the ``WORKER_PG_QUEUE_ENABLED_TASKS`` allow-list select-PG, ``dispatch()`` really enqueues the task to Postgres (a durable INSERT-via ``_enqueue_pg``) and the PG consumer runs it — this moves live traffic-onto the PG transport, it is not a no-op. ``resolve_backend()`` wraps the-``select_backend()`` allow-list with the per-call execution-transport-override used to pin a whole coupled pipeline to one substrate.--**Observability.** Routing is logged at INFO so the signals survive a-default log config: the configured allow-list is logged once per process-(:func:`_log_allow_list_once`, so a typo is visible even if it never-matches a real task), and the first time each task is routed to PG-(``dispatch()``, log-once per task name). A task not in the allow-list-still routes to the safe ``CELERY`` default.--**Fail-safe parsing.** Unlike ``get_barrier()`` (which raises on an-unrecognised value, because a typo'd substrate must not silently fall-back), the routing table is a membership set with no "invalid value"-concept: blanks and stray whitespace are simply dropped. Malformed or-empty config can only ever resolve to the safe ``CELERY`` default —-``select_backend()`` never raises.+"""Queue-transport routing.++PG is the only transport. :func:`select_backend` returns ``PG``+unconditionally and :func:`resolve_backend` applies the per-call override on+top of it.++**History (UN-4046).** This was the Strangler-Fig gate: a per-task opt-in+allow-list, ``WORKER_PG_QUEUE_ENABLED_TASKS``, with everything not listed+routed to Celery. That allow-list was never set in any environment, so every+dispatch lacking an explicit ``backend=`` override went to Celery — harmless+while Celery consumers existed, and a queue nothing drains once they were+scaled to zero. The allow-list and the Celery default went with the+``pg_queue_enabled`` flag.++**The per-call override survives** (``resolve_backend(task_name, override)``).+It is how the coupled execution pipeline (``async_execute_bin`` → file+processing → callback, with the barrier fan-in) pins a whole execution to one+substrate. That mattered when two substrates existed; it is kept because+stating the transport at the dispatch site reads as intent rather than as+reliance on a module default, and because the barrier's fan-in still threads it.++**Observability.** The first dispatch of each task name is logged at INFO by+``dispatch()``, so the signal survives a default log config."""from __future__ import annotationsimport logging-import osfrom enum import StrEnumlogger = logging.getLogger(__name__)-# Env var name kept as a module literal (mirrors how the sibling-# ``WORKER_BARRIER_BACKEND`` flag is read in ``queue_backend.__init__``)-# so the queue-backend seam stays self-contained.-_ENABLED_TASKS_ENV_VAR = "WORKER_PG_QUEUE_ENABLED_TASKS"--# One-shot guard so the configured allow-list is logged once per process-# (see ``_log_allow_list_once``). Module-level → per prefork child.-_allow_list_logged = False-class QueueBackend(StrEnum):"""Transport a dispatch is routed to.``StrEnum`` (3.11+) is inherited for symmetry with ``BarrierBackend``,but unlike that enum this one is never read from / written to env —- only the task-name allow-list is. So callers MUST compare by identity+ it is never read from env at all. So callers MUST compare by identity(``backend is QueueBackend.PG``), never ``== "pg"``: ``StrEnum`` makes atypo'd ``== "cellery"`` a silent ``False`` rather than an error."""@@ -88,69 +45,34 @@ class QueueBackend(StrEnum):PG = "pg"-def _parse_allow_list() -> frozenset[str]:- """Parse ``WORKER_PG_QUEUE_ENABLED_TASKS`` into a set of task names.-- Trims surrounding whitespace per entry and drops blanks, so- ``"a, b ,, c"`` → ``{"a", "b", "c"}`` and ``""`` / unset → ``frozenset()``.- Read at call time (not import) so test harnesses that- ``monkeypatch.setenv`` flip the table per-test without a reload.- """- stripped = (- entry.strip() for entry in os.getenv(_ENABLED_TASKS_ENV_VAR, "").split(",")- )- return frozenset(entry for entry in stripped if entry)---def _log_allow_list_once(allow_list: frozenset[str]) -> None:- """Log the configured allow-list once per process, at INFO.-- Makes a misconfiguration eyeballable at boot: a typo'd task name- (``async_execute_bni``) silently routes everything to Celery, but it- still shows up here so an operator can spot it. Only fires for a- *non-empty* allow-list — the default (feature off) stays silent so- the scaffold is truly inert when unused.- """- global _allow_list_logged- if _allow_list_logged or not allow_list:- return- _allow_list_logged = True- logger.info(- "PG-queue routing enabled for tasks: %s (all others → Celery)",- sorted(allow_list),- )---def select_backend(task_name: str) -> QueueBackend:- """Return the transport a dispatch should ride.+def select_backend() -> QueueBackend:+ """Return the transport a dispatch should ride: always ``PG``.- ``PG`` if ``task_name`` is in ``WORKER_PG_QUEUE_ENABLED_TASKS``;- otherwise ``CELERY``. Empty / unset allow-list → always ``CELERY``.+ This used to consult a ``WORKER_PG_QUEUE_ENABLED_TASKS`` allow-list and fall+ back to ``CELERY`` for anything not opted in. The allow-list was set nowhere,+ so every dispatch without an explicit ``backend=`` override published to+ RabbitMQ — which, with no Celery consumers, is a queue nothing drains+ (UN-4046). PG is the only transport now, so there is nothing to select.- Never raises — the worst case is the safe ``CELERY`` default.+ Takes no argument: the answer no longer depends on the task. It used to."""- allow_list = _parse_allow_list()- _log_allow_list_once(allow_list)- if task_name in allow_list:- return QueueBackend.PG- return QueueBackend.CELERY+ return QueueBackend.PGdef resolve_backend(task_name: str, override: QueueBackend | None) -> QueueBackend:"""Resolve the transport for a dispatch, applying the per-call override.- The single home for the override-wins-else-allow-list precedence so the+ The single home for the override-wins-else-``select_backend`` precedence so therule reads in one place (and ``dispatch()`` plus the call sitesshare it):- - ``override`` is ``None`` → defer to :func:`select_backend` (the env- allow-list) — the behaviour of every call site today.- - ``override`` is a :class:`QueueBackend` → it wins. This is how the- execution-level PG pipeline pins a whole execution's header/callback- dispatches to one transport regardless of the per-task allow-list (the- allow-list is for *leaf* tasks; the coupled pipeline's migration unit is- the execution).+ - ``override`` is ``None`` → defer to :func:`select_backend`, which is now+ unconditionally ``PG``.+ - ``override`` is a :class:`QueueBackend` → it wins. Retained because the+ execution-level pipeline pins a whole execution's header/callback+ dispatches explicitly, which reads at the call site as intent rather than+ as reliance on a default.Never raises — both branches resolve to a valid :class:`QueueBackend`."""- return override if override is not None else select_backend(task_name)+ return override if override is not None else select_backend()diff --git a/workers/run-worker.sh b/workers/run-worker.shindex 5c37466bf..5bd41dfbe 100755--- a/workers/run-worker.sh+++ b/workers/run-worker.sh@@ -58,11 +58,13 @@ readonly PG_ROLE_CALLBACK="pg-callback"readonly PG_ROLE_SCHEDULER="pg-scheduler"readonly PG_ROLE_EXECUTOR="pg-executor"readonly PG_ROLE_METRICS="pg-metrics"+readonly PG_ROLE_NOTIFICATION="pg-notification"+readonly PG_ROLE_IDE_CALLBACK="pg-ide-callback"declare -rA PG_CONSUMER_ROLES=(["$PG_ROLE_ORCH_API"]="api_deployment;celery_api_deployments"["$PG_ROLE_ORCH_GENERAL"]="general;celery"- ["$PG_ROLE_FILEPROC"]="file_processing;file_processing,api_file_processing"- ["$PG_ROLE_CALLBACK"]="callback;file_processing_callback,api_file_processing_callback"+ ["$PG_ROLE_FILEPROC"]="file_processing;file_processing,file_processing_priority,api_file_processing"+ ["$PG_ROLE_CALLBACK"]="callback;file_processing_callback,celery_callback,api_file_processing_callback"# Runs scheduler.tasks.execute_pipeline_task fired by the orchestrator's PG# scheduler tick (Beat replacement). Distinct from the Celery 'scheduler'# worker (which Beat fires onto RabbitMQ).@@ -70,7 +72,7 @@ declare -rA PG_CONSUMER_ROLES=(# Runs execute_extraction (the executor RPC) over PG — request-reply: writes# the result to pg_task_result for the blocking caller. Queues mirror the# Celery executor's CELERY_QUEUES_EXECUTOR.- ["$PG_ROLE_EXECUTOR"]="executor;celery_executor_legacy,celery_executor_agentic,celery_executor_agentic_table"+ ["$PG_ROLE_EXECUTOR"]="executor;celery_executor_legacy,celery_executor_agentic,celery_executor_table,celery_executor_smart_table,celery_executor_simple_prompt_studio,celery_executor_agentic_table,celery_executor_lookup_test"# Runs the dashboard_metrics.* periodics fired by the PG scheduler tick,# replacing the Celery 'workerMetrics' (-Q dashboard_metric_events). Its own# role rather than a queue bolted onto pg-scheduler: the consumer's health@@ -79,6 +81,13 @@ declare -rA PG_CONSUMER_ROLES=(# pg-scheduler's tight 240s bound would restart that pod and take in-flight# pipeline triggers down with it. Sized in docker-compose.yaml.["$PG_ROLE_METRICS"]="scheduler;dashboard_metric_events"+ # The backend notification seam enqueues send_webhook_notification onto+ # these queues unconditionally (UN-4046). Without this role a host-run PG+ # fleet accepts every buffered webhook and drains none — enqueue_task+ # succeeds, so nothing errors at the producer.+ ["$PG_ROLE_NOTIFICATION"]="notification;notifications,notifications_webhook,notifications_email,notifications_sms,notifications_priority"+ # Prompt Studio IDE callbacks (ide_index_*/ide_prompt_*/extraction_*).+ ["$PG_ROLE_IDE_CALLBACK"]="ide_callback;ide_callback")declare -rA PG_QUEUE_MEMBERS=(["$PG_QUEUE_CONSUMER_TYPE"]=1@@ -90,6 +99,8 @@ declare -rA PG_QUEUE_MEMBERS=(["$PG_ROLE_SCHEDULER"]=1["$PG_ROLE_EXECUTOR"]=1["$PG_ROLE_METRICS"]=1+ ["$PG_ROLE_NOTIFICATION"]=1+ ["$PG_ROLE_IDE_CALLBACK"]=1)# The Celery transport set: every worker EXCEPT the PG-queue members — the# *complement* of the 'pg-queue' set, so the two transports' logs can be taileddiff --git a/workers/sample.env b/workers/sample.envindex 6a80eb455..abb09255f 100644--- a/workers/sample.env+++ b/workers/sample.env@@ -131,34 +131,6 @@ WORKER_BARRIER_BACKEND=chord# workflows can plausibly span >6h.WORKER_BARRIER_KEY_TTL_SECONDS=21600-# =============================================================================-# PG Queue Routing (Strangler-Fig)-# =============================================================================-# Selects the queue *transport* per task — Celery (default) or PG Queue-# (PG Queue). Separate axis from the barrier above: the barrier is how a-# batch's fan-in fires the callback; this is how messages travel.-#-# A comma-separated allow-list drives a gradual, per-task-type migration-# (NOT a global on/off switch). A task routes to PG if its name is listed.-# Unset / empty (the default) → everything routes to Celery, zero-# behaviour change. No Flipt server required.-#-# ⚠️ A listed task is enqueued to Postgres instead of Celery. It will only-# execute if the PG consumer is running — otherwise the message is durably-# enqueued but never run. Do NOT opt a task in unless the consumer is-# deployed.-#-# Roll back instantly by removing an entry; flips take effect on worker-# restart (same posture as every other WORKER_* env).-#-# Only list *leaf* tasks dispatched via queue_backend.dispatch() — i.e. no-# downstream chain or barrier fan-in (e.g. send_webhook_notification). Do-# NOT list coupled pipeline tasks like async_execute_bin here: one execution-# must run wholly on one transport, so the pipeline migrates per-execution-# (via ExecutionContext) in a later phase, not per-task. See the routing.py-# module docstring for the full constraint.-# WORKER_PG_QUEUE_ENABLED_TASKS=send_webhook_notification-# Database URL (for fallback usage)DATABASE_URL=postgresql://unstract_dev:unstract_pass@unstract-db:5432/unstract_dbdiff --git a/workers/shared/api/internal_client.py b/workers/shared/api/internal_client.pyindex bba4af529..ee9fbca7a 100644--- a/workers/shared/api/internal_client.py+++ b/workers/shared/api/internal_client.py@@ -711,12 +711,6 @@ class InternalAPIClient(CachedAPIClientMixin):workflow_id, execution_id, organization_id)- def submit_file_batch_for_processing(- self, batch_data: dict[str, Any]- ) -> dict[str, Any]:- """Submit file batch for processing."""- return self.execution_client.submit_file_batch_for_processing(batch_data)-def execute_workflow_async(self, execution_data: dict[str, Any]) -> dict[str, Any]:"""Execute workflow asynchronously for scheduler.diff --git a/workers/shared/clients/base_client.py b/workers/shared/clients/base_client.pyindex 017a1d62e..585c0683c 100644--- a/workers/shared/clients/base_client.py+++ b/workers/shared/clients/base_client.py@@ -19,8 +19,6 @@ from celery.exceptions import SoftTimeLimitExceededfrom requests.adapters import HTTPAdapterfrom urllib3.util.retry import Retry-from unstract.flags.feature_flag import check_feature_flag_status-from ..data.models import APIResponsefrom ..enums import HTTPMethodfrom ..infrastructure.config.worker_config import WorkerConfig@@ -32,27 +30,8 @@ logger = WorkerLogger.get_logger(__name__)APPLICATION_JSON = "application/json"-# Single PG-queue rollout flag (same key as pg_queue.flags / executor_rpc).-_PG_QUEUE_FLAG_KEY = "pg_queue_enabled"---def _pg_queue_jitter_enabled() -> bool:- """Whether retry jitter is on — gated by the single ``pg_queue_enabled`` Flipt- flag (fail-closed to False, like ``check_feature_flag_status`` itself), so the- Celery flow's retry cadence is unchanged and the jitter tracks the PG rollout.-- The retry path carries no per-execution/org context, so the flag is evaluated- globally (entity ``"default"``) — a coarse "is PG being rolled out" gate, which- is all a transport-agnostic HTTP-retry hardening needs.- """- try:- return check_feature_flag_status(flag_key=_PG_QUEUE_FLAG_KEY, entity_id="default")- except Exception:- return False--def _backoff_with_jitter(backoff_factor: float, attempt: int) -> float:- """Exponential backoff, with equal jitter when the PG-queue flag is on.+ """Exponential backoff with equal jitter.Without jitter, every concurrent worker retrying a saturated backend sleeps onthe identical 1s/2s/4s schedule and their retries arrive in synchronized waves@@ -60,16 +39,14 @@ def _backoff_with_jitter(backoff_factor: float, attempt: int) -> float:finalization-write failures under high concurrency. Equal jitter spreads eachretry across [base/2, base] so the waves de-correlate.- Gated by the ``pg_queue_enabled`` Flipt flag (fail-closed): OFF keeps the exact- exponential backoff for the Celery flow; evaluated at call time on the rare- retry path (behind a sleep + network round-trip), so the Flipt cost is- irrelevant and it stays runtime-togglable via the flag.+ Previously gated on the ``pg_queue_enabled`` Flipt flag so the Celery flow kept+ the exact exponential cadence. With that flag removed the jitter is+ unconditional — it is transport-agnostic HTTP-retry hardening, and the jittered+ cadence is the one integration has run under load."""base = backoff_factor * (2**attempt)if base <= 0:return 0.0- if not _pg_queue_jitter_enabled():- return base # exact exponential backoff (unchanged for the Celery flow)return base / 2 + random.uniform(0, base / 2)diff --git a/workers/shared/clients/execution_client.py b/workers/shared/clients/execution_client.pyindex 22bc9b355..ef98f8dae 100644--- a/workers/shared/clients/execution_client.py+++ b/workers/shared/clients/execution_client.py@@ -712,23 +712,6 @@ class ExecutionAPIClient(BaseAPIClient):organization_id=organization_id,)- def submit_file_batch_for_processing(- self, batch_data: dict[str, Any]- ) -> dict[str, Any]:- """Submit file batch for processing.-- Args:- batch_data: File batch data-- Returns:- Submission result- """- return self.post(- "v1/workflow-manager/file-batch/submit/",- batch_data,- organization_id=batch_data.get("organization_id"),- )-# Pipeline Management Methods for Scheduler Workerdef get_pipeline_data(diff --git a/workers/shared/constants/api_endpoints.py b/workers/shared/constants/api_endpoints.pyindex 0737c202c..8d88ee845 100644--- a/workers/shared/constants/api_endpoints.py+++ b/workers/shared/constants/api_endpoints.py@@ -57,7 +57,6 @@ class APIEndpoints:)# Notification endpoints- WEBHOOK_SEND = build_internal_endpoint("webhook/send/")NOTIFICATION_SEND = build_internal_endpoint("notification/send/")# Health and monitoring endpointsdiff --git a/workers/shared/patterns/worker_patterns.py b/workers/shared/patterns/worker_patterns.pyindex 77acb32dd..fbd915efe 100644--- a/workers/shared/patterns/worker_patterns.py+++ b/workers/shared/patterns/worker_patterns.py@@ -152,7 +152,6 @@ class APIEndpoints:"workflow-file-execution/create/")PIPELINE_STATUS = build_internal_endpoint("pipeline/{pipeline_id}/status/")- WEBHOOK_SEND = build_internal_endpoint("webhook/send/")class WorkerConfig:diff --git a/workers/tests/test_dispatch_pg.py b/workers/tests/test_dispatch_pg.pyindex c97a2664f..65eb378c4 100644--- a/workers/tests/test_dispatch_pg.py+++ b/workers/tests/test_dispatch_pg.py@@ -18,7 +18,6 @@ import pytestfrom queue_backend import dispatchfrom queue_backend.fairness import FairnessKey, WorkloadTypefrom queue_backend.pg_queue import to_payload-from queue_backend.routing import _ENABLED_TASKS_ENV_VAR as ENABLED_TASKS_ENVfrom queue_backend.routing import QueueBackend# ``queue_backend.dispatch`` the attribute is the function (shadows the@@ -28,7 +27,6 @@ dispatch_mod = importlib.import_module("queue_backend.dispatch")@pytest.fixture(autouse=True)def _reset(monkeypatch):- monkeypatch.delenv(ENABLED_TASKS_ENV, raising=False)dispatch_mod._pg_routing_logged.clear()dispatch_mod._pg_local.client = None@@ -83,7 +81,6 @@ class TestToPayload:class TestDispatchEnqueueIntegration:def test_dispatch_lands_decodable_row(self, monkeypatch, pg_client):queue_name = f"test_dispatch_{os.getpid()}"- monkeypatch.setenv(ENABLED_TASKS_ENV, "leaf_task")monkeypatch.setattr(dispatch_mod, "_get_pg_client", lambda: pg_client)fairness = FairnessKey(org_id="org-x", workload_type=WorkloadType.API)try:@@ -110,27 +107,17 @@ class TestDispatchEnqueueIntegration:)pg_client.conn.commit()- def test_celery_dispatch_unaffected(self):- # No opt-in → Celery path; never touches the PG client.- with (- patch("queue_backend.dispatch.current_app") as mock_app,- patch("queue_backend.dispatch._get_pg_client") as mock_get,- ):- dispatch("some_task", args=["x"], queue="general")- mock_app.send_task.assert_called_once()- mock_get.assert_not_called()-class TestDispatchBackendOverride:"""The per-call ``backend=`` override (9e PR 2a inert foundation).- When set it wins over the ``WORKER_PG_QUEUE_ENABLED_TASKS`` allow-list, so- the execution-level PG pipeline can route a whole execution's headers /+ Retained after the allow-list went (UN-4046), so the execution-level PG+ pipeline can state a whole execution's headers /callback onto PG without opting their task *names* in. ``None`` (default)preserves the allow-list decision exactly — every call site today."""- def test_override_pg_forces_pg_without_allow_list(self, monkeypatch):+ def test_override_pg_forces_pg(self, monkeypatch):# Task name NOT in the (empty) allow-list → select_backend says Celery;# the explicit override must still route it to PG.captured: dict = {}@@ -142,16 +129,13 @@ class TestDispatchBackendOverride:monkeypatch.setattr(dispatch_mod, "_get_pg_client", lambda: _Client())with patch("queue_backend.dispatch.current_app") as mock_app:- handle = dispatch(- "pipeline_header", queue="general", backend=QueueBackend.PG- )+ handle = dispatch("pipeline_header", queue="general", backend=QueueBackend.PG)mock_app.send_task.assert_not_called()assert handle.id == "11"assert captured["payload"]["task_name"] == "pipeline_header"- def test_override_celery_forces_celery_despite_allow_list(self, monkeypatch):+ def test_override_celery_forces_celery(self, monkeypatch):# Task name IS opted into PG, but the override pins it back to Celery.- monkeypatch.setenv(ENABLED_TASKS_ENV, "pipeline_header")with (patch("queue_backend.dispatch.current_app") as mock_app,patch("queue_backend.dispatch._get_pg_client") as mock_get,@@ -160,7 +144,7 @@ class TestDispatchBackendOverride:mock_app.send_task.assert_called_once()mock_get.assert_not_called()- def test_default_none_preserves_allow_list_decision(self, monkeypatch):+ def test_default_none_resolves_to_pg(self, monkeypatch):# No override → allow-list decides. Opted-in name → PG.captured: dict = {}@@ -169,7 +153,6 @@ class TestDispatchBackendOverride:captured.update(queue=queue_name)return 5- monkeypatch.setenv(ENABLED_TASKS_ENV, "leaf_task")monkeypatch.setattr(dispatch_mod, "_get_pg_client", lambda: _Client())with patch("queue_backend.dispatch.current_app") as mock_app:handle = dispatch("leaf_task", queue="general")@@ -215,7 +198,6 @@ class TestDispatchPriorityWiring:captured.update(queue=queue_name, **kwargs)return 7- monkeypatch.setenv(ENABLED_TASKS_ENV, "leaf_task")monkeypatch.setattr(dispatch_mod, "_get_pg_client", lambda: _Client())return captureddiff --git a/workers/tests/test_dispatch_with_callback.py b/workers/tests/test_dispatch_with_callback.pyindex 1449e395b..dce8b73f9 100644--- a/workers/tests/test_dispatch_with_callback.py+++ b/workers/tests/test_dispatch_with_callback.py@@ -928,17 +928,15 @@ class TestStructureToolSingleDispatch:assert "answer_params" in ctx.executor_paramsassert "pipeline_options" in ctx.executor_params- # Fairness header lands on the dispatch call. A regression that- # drops ``headers=`` or flips ``NON_API``→``API`` here would- # otherwise stay green — keep this assertion paired with the- # ``_fairness_headers`` unit test in test_structure_tool_task.py.- assert dispatcher.dispatch.call_args.kwargs["headers"] == {- "x-fairness-key": {- "org_id": "org-1",- "workload_type": "non_api",- "pipeline_priority": 5,- }- }+ # No ``headers`` on the dispatch call, and that is the assertion now+ # (UN-4046). This used to pin the fairness header; the routing dispatcher+ # that accepted ``headers=`` is gone, ``PgExecutionDispatcher.dispatch``+ # takes no such argument, and passing one raises TypeError — which is+ # exactly what shipped and broke every extraction until it was caught.+ # Fairness is not lost: PG carries org routing in the enqueue payload+ # (``transport.enqueue(..., org_id=...)``), which is queue_backend's to+ # cover, not this call site's.+ assert "headers" not in dispatcher.dispatch.call_args.kwargs# ---------------------------------------------------------------------------diff --git a/workers/tests/test_executor_rpc.py b/workers/tests/test_executor_rpc.pyindex 97822e9ba..99fc1b261 100644--- a/workers/tests/test_executor_rpc.py+++ b/workers/tests/test_executor_rpc.py@@ -14,15 +14,13 @@ from unittest.mock import MagicMock, patchimport pytestfrom queue_backend.pg_queue.executor_rpc import (PgClientQueueTransport,- RoutingExecutionDispatcher,get_executor_dispatcher,- resolve_executor_transport,)+from unstract.core.execution_dispatch import DispatchHandle, signature_to_continuationfrom unstract.workflow_execution.executor_rpc import (ExecResultRow,PgExecutionDispatcher,- resolve_pg_transport,)_WMOD = "queue_backend.pg_queue.executor_rpc"@@ -96,20 +94,31 @@ class TestSharedDispatchContract:def test_failed_row_returns_error(self):row = ExecResultRow(status="failed", result=None, error="boom")- res = PgExecutionDispatcher(_FakeTransport(wait_return=row)).dispatch(_ctx(), timeout=5)+ res = PgExecutionDispatcher(_FakeTransport(wait_return=row)).dispatch(+ _ctx(), timeout=5+ )assert res.success is False and res.error == "boom"def test_failed_row_empty_error_falls_back(self):row = ExecResultRow(status="failed", result=None, error="")- res = PgExecutionDispatcher(_FakeTransport(wait_return=row)).dispatch(_ctx(), timeout=5)+ res = PgExecutionDispatcher(_FakeTransport(wait_return=row)).dispatch(+ _ctx(), timeout=5+ )assert res.success is False and "executor task failed" in res.errordef test_completed_but_result_none_is_failure(self):row = ExecResultRow(status="completed", result=None, error="")- assert PgExecutionDispatcher(_FakeTransport(wait_return=row)).dispatch(_ctx(), timeout=5).success is False+ assert (+ PgExecutionDispatcher(_FakeTransport(wait_return=row))+ .dispatch(_ctx(), timeout=5)+ .success+ is False+ )def test_malformed_completed_row_is_failure_not_raise(self):- d = PgExecutionDispatcher(_FakeTransport(wait_return=_completed({"bad": "shape"})))+ d = PgExecutionDispatcher(+ _FakeTransport(wait_return=_completed({"bad": "shape"}))+ )res = d.dispatch(_ctx(), timeout=5)assert res.success is False and "Malformed" in res.error@@ -152,15 +161,19 @@ class TestSharedDispatchContract:@staticmethoddef _sig(task: str):return MagicMock(- task=task, args=(), kwargs={"callback_kwargs": {"room": "r1"}},+ task=task,+ args=(),+ kwargs={"callback_kwargs": {"room": "r1"}},options={"queue": "ide_callback"},)def test_dispatch_with_callback_translates_both_signatures(self):t = _FakeTransport()handle = PgExecutionDispatcher(t).dispatch_with_callback(- _ctx(), on_success=self._sig("ide_prompt_complete"),- on_error=self._sig("ide_prompt_error"), task_id="tid-7",+ _ctx(),+ on_success=self._sig("ide_prompt_complete"),+ on_error=self._sig("ide_prompt_error"),+ task_id="tid-7",)assert handle.id == "tid-7"(kw,) = t.enqueue_calls@@ -185,85 +198,9 @@ class TestSharedDispatchContract:# --- Shared gate: resolve_pg_transport (single Flipt flag, fail-closed) ----class TestResolvePgTransport:- def test_flipt_unavailable_is_celery(self, monkeypatch):- monkeypatch.setenv("FLIPT_SERVICE_AVAILABLE", "false")- with patch(f"{_SMOD}.check_feature_flag_status") as flag:- assert resolve_pg_transport(_ctx()) is False- flag.assert_not_called()-- def test_flag_true_is_pg_keyed_on_org(self, monkeypatch):- monkeypatch.setenv("FLIPT_SERVICE_AVAILABLE", "true")- with patch(f"{_SMOD}.check_feature_flag_status", return_value=True) as flag:- assert resolve_pg_transport(_ctx("orgX")) is True- assert flag.call_args.kwargs["entity_id"] == "orgX"- assert flag.call_args.kwargs["flag_key"] == "pg_queue_enabled"-- def test_flag_false_is_celery(self, monkeypatch):- monkeypatch.setenv("FLIPT_SERVICE_AVAILABLE", "true")- with patch(f"{_SMOD}.check_feature_flag_status", return_value=False):- assert resolve_pg_transport(_ctx()) is False-- def test_flipt_error_fails_closed(self, monkeypatch):- monkeypatch.setenv("FLIPT_SERVICE_AVAILABLE", "true")- with patch(f"{_SMOD}.check_feature_flag_status", side_effect=RuntimeError("x")):- assert resolve_pg_transport(_ctx()) is False-- def test_org_less_context_buckets_on_run_id(self, monkeypatch):- monkeypatch.setenv("FLIPT_SERVICE_AVAILABLE", "true")- with patch(f"{_SMOD}.check_feature_flag_status", return_value=True) as flag:- assert resolve_pg_transport(_ctx(org=None)) is True- assert flag.call_args.kwargs["entity_id"] == "run-1"- assert "organization_id" not in flag.call_args.kwargs["context"]--# --- Shared routing: RoutingExecutionDispatcher (zero-regression) ----class TestRoutingDispatch:- @staticmethod- def _build(route_to_pg: bool):- celery, pg = MagicMock(), MagicMock()- d = RoutingExecutionDispatcher(- celery=celery, pg=pg, resolve=lambda _ctx: route_to_pg- )- return d, celery, pg-- def test_gate_off_forwards_timeout_and_headers_to_celery(self):- d, celery, pg = self._build(route_to_pg=False)- ctx = _ctx()- hdrs = {"x-fairness-key": {"org_id": "o"}}- d.dispatch(ctx, timeout=9, headers=hdrs)- celery.dispatch.assert_called_once_with(ctx, timeout=9, headers=hdrs)- pg.dispatch.assert_not_called()-- def test_gate_on_passes_timeout_to_pg_and_drops_headers(self):- d, celery, pg = self._build(route_to_pg=True)- ctx = _ctx()- d.dispatch(ctx, timeout=7, headers={"x-fairness-key": {"o": 1}})- pg.dispatch.assert_called_once_with(ctx, timeout=7) # headers dropped- celery.dispatch.assert_not_called()-- def test_async_and_callback_stay_celery_when_gate_off(self):- d, celery, pg = self._build(route_to_pg=False)- d.dispatch_async(_ctx(), headers={"h": 1})- d.dispatch_with_callback(_ctx(), on_success="s", on_error="e")- celery.dispatch_async.assert_called_once()- celery.dispatch_with_callback.assert_called_once()- pg.dispatch_async.assert_not_called()- pg.dispatch_with_callback.assert_not_called()-- def test_async_and_callback_route_to_pg_when_gated(self):- d, celery, pg = self._build(route_to_pg=True)- d.dispatch_async(_ctx())- d.dispatch_with_callback(_ctx(), on_success="s", on_error="e", task_id="t")- pg.dispatch_async.assert_called_once()- pg.dispatch_with_callback.assert_called_once()- assert "headers" not in pg.dispatch_with_callback.call_args.kwargs- celery.dispatch_async.assert_not_called()- celery.dispatch_with_callback.assert_not_called()--# --- Workers adapter: PgClientQueueTransport + env gate + factory wiring ---@@ -286,8 +223,10 @@ class TestWorkersAdapter:client = self._client()with patch(f"{_WMOD}.PgQueueClient", return_value=client):PgClientQueueTransport().enqueue(- queue="celery_executor_legacy", context=self._ctx(),- org_id="org9", reply_key="rk1",+ queue="celery_executor_legacy",+ context=self._ctx(),+ org_id="org9",+ reply_key="rk1",)client.send.assert_called_once()queue_arg, payload_arg = client.send.call_args.args[:2]@@ -302,8 +241,11 @@ class TestWorkersAdapter:spec = {"task_name": "ide_prompt_complete", "kwargs": {}, "queue": "ide_callback"}with patch(f"{_WMOD}.PgQueueClient", return_value=client):PgClientQueueTransport().enqueue(- queue="celery_executor_legacy", context=self._ctx(), org_id="o",- on_success=spec, task_id="tid-7",+ queue="celery_executor_legacy",+ context=self._ctx(),+ org_id="o",+ on_success=spec,+ task_id="tid-7",)payload = client.send.call_args.args[1]assert payload["on_success"] == spec and payload["task_id"] == "tid-7"@@ -312,7 +254,11 @@ class TestWorkersAdapter:def test_wait_for_result_folds_dict_to_row(self):rb = MagicMock()rb.__enter__.return_value = rb- rb.wait_for_result.return_value = {"status": "completed", "result": {"a": 1}, "error": ""}+ rb.wait_for_result.return_value = {+ "status": "completed",+ "result": {"a": 1},+ "error": "",+ }with patch(f"{_WMOD}.PgResultBackend", return_value=rb):row = PgClientQueueTransport().wait_for_result("rk", 5)assert isinstance(row, ExecResultRow)@@ -354,24 +300,12 @@ class TestWorkersAdapter:PgClientQueueTransport().wait_for_result("rk", 5)rb.forget.assert_not_called()- def test_resolve_delegates_to_shared_flipt_resolver_true(self):- with patch(f"{_WMOD}.resolve_pg_transport", return_value=True) as r:- assert resolve_executor_transport(self._ctx()) is True- r.assert_called_once()- # No env master-gate is threaded any more — Flipt is the sole gate.- assert "master_gate_enabled" not in r.call_args.kwargs-- def test_resolve_delegates_to_shared_flipt_resolver_false(self):- with patch(f"{_WMOD}.resolve_pg_transport", return_value=False):- assert resolve_executor_transport(self._ctx()) is False-- def test_factory_wires_routing_with_workers_transport(self):- d = get_executor_dispatcher(celery_app="app")- assert isinstance(d, RoutingExecutionDispatcher)- # The PG dispatcher is wired with the workers psycopg2 transport, and the gate- # is the workers' Flipt resolver.- assert isinstance(d._pg._transport, PgClientQueueTransport)- assert d._resolve is resolve_executor_transport+ def test_factory_wires_the_workers_transport(self):+ # The factory takes no arguments (UN-4046) and returns the PG dispatcher+ # wired with the workers psycopg2 transport.+ d = get_executor_dispatcher()+ assert isinstance(d, PgExecutionDispatcher)+ assert isinstance(d._transport, PgClientQueueTransport)# --- Core helpers (unchanged; the shared signature/handle primitives) ---@@ -382,9 +316,12 @@ class TestSharedDispatchHelpers:assert signature_to_continuation(None) is Nonedef test_signature_translates_task_kwargs_and_queue(self):- sig = MagicMock(task="ide_prompt_complete", args=(),- kwargs={"callback_kwargs": {"room": "r1"}},- options={"queue": "ide_callback"})+ sig = MagicMock(+ task="ide_prompt_complete",+ args=(),+ kwargs={"callback_kwargs": {"room": "r1"}},+ options={"queue": "ide_callback"},+ )assert signature_to_continuation(sig) == {"task_name": "ide_prompt_complete","kwargs": {"callback_kwargs": {"room": "r1"}},@@ -402,8 +339,12 @@ class TestSharedDispatchHelpers:signature_to_continuation(sig)def test_signature_with_positional_args_fails_fast(self):- sig = MagicMock(task="ide_prompt_complete", args=("pos",), kwargs={},- options={"queue": "ide_callback"})+ sig = MagicMock(+ task="ide_prompt_complete",+ args=("pos",),+ kwargs={},+ options={"queue": "ide_callback"},+ )with pytest.raises(ValueError, match="positional args"):signature_to_continuation(sig)diff --git a/workers/tests/test_fairness_key.py b/workers/tests/test_fairness_key.pyindex 0ef3e53a5..c720be4ed 100644--- a/workers/tests/test_fairness_key.py+++ b/workers/tests/test_fairness_key.py@@ -11,6 +11,7 @@ import pytestfrom celery import Celeryfrom queue_backend import FairnessKey, dispatch+from queue_backend.routing import QueueBackendfrom queue_backend.fairness import (DEFAULT_PRIORITY,FAIRNESS_HEADER_NAME,@@ -153,25 +154,40 @@ class TestFairnessKey:class TestDispatchAttachesFairness:- def test_omitted_fairness_no_header_sent(self):- with patch("queue_backend.dispatch.current_app") as mock_app:+ """UN-4046: these assert the PG path, which is the only one dispatch takes.++ They used to patch ``current_app`` and read ``send_task.call_args`` — the+ Celery branch. ``select_backend`` returns PG unconditionally now, so that+ branch is unreachable without an explicit ``backend=`` override and the+ assertions failed on a mock that was never called. The property under test is+ unchanged: fairness must reach the transport WITHOUT being folded into the+ business kwargs, since tasks without ``**kwargs`` would break. On PG that+ means the ``fairness`` payload slot plus the ``org_id``/``priority`` columns+ the dequeue orders by.+ """++ def test_omitted_fairness_writes_neutral_defaults(self):+ with patch("queue_backend.dispatch._get_pg_client") as mock_client:dispatch("any_task", kwargs={"foo": "bar"})- call_kwargs = mock_app.send_task.call_args.kwargs- assert call_kwargs["headers"] is None- assert call_kwargs["kwargs"] == {"foo": "bar"}+ _, payload = mock_client.return_value.send.call_args.args+ send_kwargs = mock_client.return_value.send.call_args.kwargs+ assert payload["fairness"] is None+ assert payload["kwargs"] == {"foo": "bar"}+ assert send_kwargs["org_id"] is None+ assert send_kwargs["priority"] == DEFAULT_PRIORITY- def test_explicit_fairness_none_no_header_sent(self):+ def test_explicit_fairness_none_writes_neutral_defaults(self):# Documented opt-out for non-workflow dispatches.- with patch("queue_backend.dispatch.current_app") as mock_app:+ with patch("queue_backend.dispatch._get_pg_client") as mock_client:dispatch("send_webhook_notification", kwargs={"x": 1}, fairness=None)- call_kwargs = mock_app.send_task.call_args.kwargs- assert call_kwargs["headers"] is None- assert call_kwargs["kwargs"] == {"x": 1}+ _, payload = mock_client.return_value.send.call_args.args+ assert payload["fairness"] is None+ assert payload["kwargs"] == {"x": 1}- def test_provided_fairness_attached_as_message_header(self):- with patch("queue_backend.dispatch.current_app") as mock_app:+ def test_provided_fairness_rides_the_payload_and_the_columns(self):+ with patch("queue_backend.dispatch._get_pg_client") as mock_client:dispatch("any_task",kwargs={"foo": "bar"},@@ -180,40 +196,43 @@ class TestDispatchAttachesFairness:),)- call_kwargs = mock_app.send_task.call_args.kwargs- assert call_kwargs["headers"] == {- FAIRNESS_HEADER_NAME: {- "org_id": "org-1",- "workload_type": "api",- "pipeline_priority": 9,- }+ _, payload = mock_client.return_value.send.call_args.args+ send_kwargs = mock_client.return_value.send.call_args.kwargs+ assert payload["fairness"] == {+ "org_id": "org-1",+ "workload_type": "api",+ "pipeline_priority": 9,}+ # org_id + priority are separate COLUMNS, not just payload keys: the+ # dequeue orders by them, so a payload-only fairness would be inert.+ assert send_kwargs["org_id"] == "org-1"+ assert send_kwargs["priority"] == 9# Business kwargs must NOT contain the fairness slot — tasks# without **kwargs would break.- sent_kwargs = call_kwargs["kwargs"]- assert sent_kwargs == {"foo": "bar"}- assert FAIRNESS_HEADER_NAME not in sent_kwargs+ assert payload["kwargs"] == {"foo": "bar"}+ assert FAIRNESS_HEADER_NAME not in payload["kwargs"]def test_fairness_with_no_business_kwargs(self):- with patch("queue_backend.dispatch.current_app") as mock_app:+ with patch("queue_backend.dispatch._get_pg_client") as mock_client:dispatch("any_task",fairness=FairnessKey(org_id=None, workload_type=WorkloadType.NON_API),)- call_kwargs = mock_app.send_task.call_args.kwargs- assert call_kwargs["kwargs"] is None- assert call_kwargs["headers"] == {- FAIRNESS_HEADER_NAME: {- "org_id": None,- "workload_type": "non_api",- "pipeline_priority": DEFAULT_PRIORITY,- }+ _, payload = mock_client.return_value.send.call_args.args+ send_kwargs = mock_client.return_value.send.call_args.kwargs+ assert payload["kwargs"] == {}+ assert payload["fairness"] == {+ "org_id": None,+ "workload_type": "non_api",+ "pipeline_priority": DEFAULT_PRIORITY,}+ assert send_kwargs["org_id"] is None+ assert send_kwargs["priority"] == DEFAULT_PRIORITYdef test_caller_kwargs_not_mutated_in_place(self):caller_kwargs = {"foo": "bar"}- with patch("queue_backend.dispatch.current_app"):+ with patch("queue_backend.dispatch._get_pg_client"):dispatch("any_task",kwargs=caller_kwargs,@@ -223,6 +242,33 @@ class TestDispatchAttachesFairness:assert caller_kwargs == {"foo": "bar"}assert FAIRNESS_HEADER_NAME not in caller_kwargs+ def test_explicit_celery_override_still_attaches_the_header(self):+ """The Celery branch survives only as a ``backend=`` override target.++ Nothing selects it — ``select_backend`` returns PG unconditionally — and+ it is slated for deletion with the rest of the Celery transport. Until+ then it must not rot silently: an override is the one way to reach it.+ """+ with patch("queue_backend.dispatch.current_app") as mock_app:+ dispatch(+ "any_task",+ kwargs={"foo": "bar"},+ fairness=FairnessKey(+ org_id="org-1", workload_type=WorkloadType.API, pipeline_priority=9+ ),+ backend=QueueBackend.CELERY,+ )++ call_kwargs = mock_app.send_task.call_args.kwargs+ assert call_kwargs["headers"] == {+ FAIRNESS_HEADER_NAME: {+ "org_id": "org-1",+ "workload_type": "api",+ "pipeline_priority": 9,+ }+ }+ assert call_kwargs["kwargs"] == {"foo": "bar"}+class TestDispatchCallSitesPassFairness:"""AST audit: every production ``dispatch(...)`` declares fairness."""@@ -279,7 +325,12 @@ class TestNoConsumerYet:class TestHeaderSurvivesCeleryPipeline:- """End-to-end: header survives Celery's real send_task code path."""+ """End-to-end: header survives Celery's real send_task code path.++ Reached via an explicit ``backend=`` override since UN-4046 — see+ ``test_explicit_celery_override_still_attaches_the_header``. Kept as long as+ the Celery branch exists, and deleted with it.+ """def test_header_present_on_outbound_message(self):app = Celery(@@ -296,6 +347,7 @@ class TestHeaderSurvivesCeleryPipeline:workload_type=WorkloadType.NON_API,pipeline_priority=9,),+ backend=QueueBackend.CELERY,)call_headers = wrapped_send.call_args.kwargs["headers"]diff --git a/workers/tests/test_pg_finalization_fixes.py b/workers/tests/test_pg_finalization_fixes.pyindex 064524459..b1daad5c2 100644--- a/workers/tests/test_pg_finalization_fixes.py+++ b/workers/tests/test_pg_finalization_fixes.py@@ -3,51 +3,45 @@- L2: ``_update_execution_status_unified`` re-raises on the PG path (so the failedfinalization vt-redelivers / poison-drops) but swallows on the Celery path.- L3: ``_backoff_with_jitter`` spreads retries across [base/2, base] (equal jitter)- so concurrent workers don't retry in synchronized waves.+ so concurrent workers don't retry in synchronized waves. Unconditional since+ UN-4046 — it was gated on ``pg_queue_enabled`` to keep the Celery flow on the+ exact exponential cadence."""-from unittest.mock import MagicMock, patch+from unittest.mock import MagicMockimport pytest-_JITTER_FLAG = "shared.clients.base_client.check_feature_flag_status"-class TestBackoffJitter:- def test_flag_off_returns_exact_base(self):- # Celery flow (pg_queue_enabled off): exact exponential backoff, no jitter.- from shared.clients.base_client import _backoff_with_jitter-- with patch(_JITTER_FLAG, return_value=False):- assert _backoff_with_jitter(1.0, 2) == pytest.approx(4.0)-- def test_flipt_error_fails_closed_to_no_jitter(self):- from shared.clients.base_client import _backoff_with_jitter-- with patch(_JITTER_FLAG, side_effect=RuntimeError("flipt down")):- assert _backoff_with_jitter(1.0, 2) == pytest.approx(4.0)+ """UN-4046: jitter is unconditional, so there is no flag to patch.++ The two flag-off cases (``flag off returns exact base``, ``Flipt error fails+ closed to no jitter``) are GONE — they asserted the exact exponential cadence+ the Celery flow kept, and there is no Celery flow and no Flipt call left. What+ they were really protecting is that the RESULT stays a sane backoff, which the+ band assertion below already pins from both sides. The remaining three drop+ their patch and assert the same properties directly.+ """- def test_within_equal_jitter_band_when_flag_on(self):+ def test_within_equal_jitter_band(self):from shared.clients.base_client import _backoff_with_jitter# base = backoff_factor(1.0) * 2**attempt(2) = 4.0 → equal jitter [2.0, 4.0]- with patch(_JITTER_FLAG, return_value=True):- for _ in range(500):- assert 2.0 <= _backoff_with_jitter(1.0, 2) <= 4.0+ for _ in range(500):+ assert 2.0 <= _backoff_with_jitter(1.0, 2) <= 4.0def test_zero_base_returns_zero(self):from shared.clients.base_client import _backoff_with_jitter- with patch(_JITTER_FLAG, return_value=True):- assert _backoff_with_jitter(0.0, 3) == pytest.approx(0.0)+ assert _backoff_with_jitter(0.0, 3) == pytest.approx(0.0)- def test_is_not_constant_when_flag_on(self):+ def test_is_not_constant(self):from shared.clients.base_client import _backoff_with_jitter# De-correlation: repeated calls must NOT all land on the same value# (that synchronized schedule is exactly the thundering herd we fix).- with patch(_JITTER_FLAG, return_value=True):- assert len({_backoff_with_jitter(1.0, 3) for _ in range(64)}) > 1+ assert len({_backoff_with_jitter(1.0, 3) for _ in range(64)}) > 1class TestPGFinalizationReraise:diff --git a/workers/tests/test_queue_backend_seam.py b/workers/tests/test_queue_backend_seam.pyindex abc0ac154..12e85f9fd 100644--- a/workers/tests/test_queue_backend_seam.py+++ b/workers/tests/test_queue_backend_seam.py@@ -1,163 +1,67 @@-"""Equivalence tests for the queue-backend seam.+"""Contract tests for the queue-backend seam.-Today the seam is a transparent pass-through to Celery. These tests-lock that in so future PRs adding per-task routing (PG Queue) can be-proved to preserve the Celery default path.+``dispatch()`` enqueues to Postgres — PG is the only transport (UN-4046). Two+layers:-Two layers:+1. **No dispatch may reach Celery.** The canary below asserts that no call site+ can resolve to ``CELERY``, because a producer without a consumer does not just+ lose the work: the messages accumulate in RabbitMQ until it hits its memory+ high-watermark and starts blocking publishers.-1. **dispatch()** must produce the same ``current_app.send_task`` call- as the raw Celery idiom — checked against representative- ``send_webhook_notification`` and ``async_execute_bin``- (``scheduler/tasks.py::_execute_scheduled_workflow``) shapes.+2. **@worker_task** must still register a task with the Celery app. That is not+ vestigial — the PG consumer resolves work via ``self._app.tasks.get(name)``, so+ the Celery *registry* stays load-bearing even though the Celery *transport* is+ gone.-2. **@worker_task** must register a task with the Celery app so that- functions decorated with it behave identically to ones decorated- with ``@shared_task``.+The old layer 1 was the mirror image of this: a suite proving ``dispatch()``+produced exactly the same ``current_app.send_task`` call as the raw Celery idiom,+so that adding PG routing could be shown to preserve the Celery default. That+default is what we have now removed, so those cases are gone."""from __future__ import annotations-from unittest.mock import patch+from unittest.mock import MagicMock, patchimport pytestfrom celery import current_app+from queue_backend import QueueBackend, dispatch, select_backend+from queue_backend.routing import resolve_backend-# --- dispatch() equivalence ---+# --- no Celery producer ----class TestDispatchEquivalence:- """dispatch() forwards to current_app.send_task with the same shape."""+class TestNoCeleryProducer:+ """The zero-producer canary (UN-4046).- def test_dispatches_task_name(self):- from queue_backend import dispatch-- with patch("queue_backend.dispatch.current_app") as mock_app:- dispatch("send_webhook_notification")-- mock_app.send_task.assert_called_once()- assert mock_app.send_task.call_args.args[0] == "send_webhook_notification"-- def test_forwards_args(self):- from queue_backend import dispatch-- with patch("queue_backend.dispatch.current_app") as mock_app:- dispatch("any_task", args=["a", "b", 42])-- assert mock_app.send_task.call_args.kwargs["args"] == ["a", "b", 42]-- def test_forwards_kwargs(self):- from queue_backend import dispatch-- with patch("queue_backend.dispatch.current_app") as mock_app:- dispatch("any_task", kwargs={"max_retries": 5, "platform": "SLACK"})-- assert mock_app.send_task.call_args.kwargs["kwargs"] == {- "max_retries": 5,- "platform": "SLACK",- }-- def test_forwards_queue(self):- from queue_backend import dispatch-- with patch("queue_backend.dispatch.current_app") as mock_app:- dispatch("any_task", queue="notifications")-- assert mock_app.send_task.call_args.kwargs["queue"] == "notifications"-- def test_omitted_args_forwarded_as_none(self):- """Omitted ``args`` is forwarded verbatim as ``None``.-- Celery's own ``send_task`` normalises ``None`` to its native default- (a tuple) internally — the seam doesn't coerce, so any third-party- router that checks ``isinstance(args, tuple)`` sees Celery's default.- """- from queue_backend import dispatch-- with patch("queue_backend.dispatch.current_app") as mock_app:- dispatch("any_task")-- assert mock_app.send_task.call_args.kwargs["args"] is None-- def test_omitted_kwargs_forwarded_as_none(self):- """Same as args — omitted kwargs reaches Celery as ``None``."""- from queue_backend import dispatch-- with patch("queue_backend.dispatch.current_app") as mock_app:- dispatch("any_task")-- assert mock_app.send_task.call_args.kwargs["kwargs"] is None-- def test_returns_underlying_handle(self):- """Whatever send_task returns is what dispatch returns."""- from queue_backend import dispatch-- sentinel = object()- with patch("queue_backend.dispatch.current_app") as mock_app:- mock_app.send_task.return_value = sentinel- result = dispatch("any_task")-- assert result is sentinel+ Any dispatch that resolves to CELERY publishes to RabbitMQ, where — with the+ Celery fleet scaled to zero — nothing drains it. Left unnoticed the queue grows+ until the broker blocks publishers, so this is asserted rather than assumed.+ """- def test_matches_notification_helper_shape(self):- """Output mirrors the raw ``send_task`` shape for a notification task.+ def test_select_backend_never_resolves_to_celery(self):+ # `select_backend()` takes no argument since UN-4046: the answer cannot+ # depend on the task name, so enumerating names proved nothing. (Sonar+ # flagged the unused parameter; removing it makes that structural.)+ assert select_backend() is QueueBackend.PG- The callback notification path itself migrated to an HTTP- buffer-enqueue in UN-3056; this keeps a representative- ``send_webhook_notification`` call as a seam-equivalence reference.+ def test_resolve_backend_defaults_to_pg_without_an_override(self):+ assert resolve_backend("any_task", None) is QueueBackend.PG- Reference call shape (paraphrased):+ def test_an_explicit_pg_override_is_honoured(self):+ assert resolve_backend("any_task", QueueBackend.PG) is QueueBackend.PG- current_app.send_task(- "send_webhook_notification",- args=[url, payload_dict, headers, timeout],- kwargs={"max_retries": ..., "retry_delay": ..., "platform": ...},- queue="notifications",- )- """- from queue_backend import dispatch-- with patch("queue_backend.dispatch.current_app") as mock_app:- dispatch(- "send_webhook_notification",- args=["https://example.com/hook", {"event": "x"}, {}, 10],- kwargs={"max_retries": 3, "retry_delay": 10, "platform": "API"},- queue="notifications",- )-- call = mock_app.send_task.call_args- assert call.args[0] == "send_webhook_notification"- assert call.kwargs["args"] == ["https://example.com/hook", {"event": "x"}, {}, 10]- assert call.kwargs["kwargs"] == {- "max_retries": 3,- "retry_delay": 10,- "platform": "API",- }- assert call.kwargs["queue"] == "notifications"-- def test_matches_scheduler_dispatch_shape(self):- """Output mirrors the raw send_task at ``_execute_scheduled_workflow``."""- from queue_backend import dispatch-- with patch("queue_backend.dispatch.current_app") as mock_app:- dispatch(- "async_execute_bin",- args=["org-1", "wf-1", "exec-1", {}, True],- kwargs={"use_file_history": False, "pipeline_id": "pipe-1"},- queue="general",- )-- call = mock_app.send_task.call_args- assert call.args[0] == "async_execute_bin"- assert call.kwargs["args"] == ["org-1", "wf-1", "exec-1", {}, True]- assert call.kwargs["kwargs"] == {- "use_file_history": False,- "pipeline_id": "pipe-1",- }- assert call.kwargs["queue"] == "general"---# --- @worker_task equivalence ---+ def test_dispatch_never_calls_send_task(self):+ client = MagicMock()+ client.send.return_value = 1+ with (+ patch("queue_backend.dispatch.current_app") as mock_app,+ patch("queue_backend.dispatch._get_pg_client", return_value=client),+ ):+ dispatch("send_webhook_notification", args=["a"], queue="notifications")+ dispatch("async_execute_bin", args=["b"], queue="celery")+ mock_app.send_task.assert_not_called()+ assert client.send.call_count == 2class TestWorkerTaskEquivalence:diff --git a/workers/tests/test_routing.py b/workers/tests/test_routing.pyindex 7c3e6447e..f9c46abcd 100644--- a/workers/tests/test_routing.py+++ b/workers/tests/test_routing.py@@ -1,19 +1,13 @@-"""Tests for the PG-queue routing gate (``queue_backend.routing``).+"""Tests for queue-transport routing and ``dispatch()`` (UN-4046).-Three layers:+PG is the only transport: ``select_backend()`` returns ``PG`` unconditionally and+``dispatch()`` enqueues to Postgres. What these pin is the *payload* the PG+consumer receives, the no-silent-fallback contract, and the per-thread client+reuse.-1. **select_backend()** resolves the per-task allow-list correctly,- defaults to ``CELERY`` when unset, and never raises on malformed- input.--2. **Observability** — the configured allow-list is logged once, and a- PG cutover emits a log. These are the gate's only observable effect,- so they're asserted explicitly (a future "the gate is inert, delete- it" refactor must fail a test, not pass silently).--3. **dispatch() is inert under routing** — the ``current_app.send_task``- call is byte-identical whether the routing table selects ``CELERY``- or ``PG``.+The allow-list suites are gone with ``WORKER_PG_QUEUE_ENABLED_TASKS`` — the+parsing, the tolerant-whitespace cases, the "logged once when configured"+observability and every "celery path" case describe a gate that no longer exists."""from __future__ import annotations@@ -24,9 +18,7 @@ from unittest.mock import MagicMock, patchimport pytestfrom queue_backend import QueueBackend, dispatch, select_backend-from queue_backend import routing as routing_modfrom queue_backend.fairness import FairnessKey, WorkloadType-from queue_backend.routing import _ENABLED_TASKS_ENV_VAR as ENABLED_TASKS_ENV# ``queue_backend.__init__`` binds ``dispatch`` to the *function*, shadowing# the submodule attribute — import the module explicitly to reach its globals.@@ -34,62 +26,30 @@ dispatch_mod = importlib.import_module("queue_backend.dispatch")@pytest.fixture(autouse=True)-def _reset_routing_state(monkeypatch):- """Empty allow-list + cleared log-once guards before each test.+def _reset_routing_state():+ """Clear the log-once guard and the per-thread PG client before each test.- The allow-list-logged flag and the per-task routing-logged set are- process-global one-shot guards; reset them so caplog assertions are- deterministic regardless of test order.+ The per-task routing-logged set is a process-global one-shot guard; reset it+ so caplog assertions are deterministic regardless of test order."""- monkeypatch.delenv(ENABLED_TASKS_ENV, raising=False)- routing_mod._allow_list_logged = Falsedispatch_mod._pg_routing_logged.clear()- dispatch_mod._pg_local.client = None # drop the per-thread PG client+ dispatch_mod._pg_local.client = None-# --- select_backend() resolution ---+def _mock_pg_client(monkeypatch, *, msg_id=99):+ client = MagicMock()+ client.send.return_value = msg_id+ # Patch on the module object (string target would navigate the shadowing+ # ``dispatch`` *function*, not the submodule).+ monkeypatch.setattr(dispatch_mod, "_get_pg_client", lambda: client)+ return clientclass TestSelectBackend:- def test_empty_table_routes_to_celery(self):- assert select_backend("any_task") is QueueBackend.CELERY-- def test_task_opted_in_routes_to_pg(self, monkeypatch):- monkeypatch.setenv(ENABLED_TASKS_ENV, "process_log_history")- assert select_backend("process_log_history") is QueueBackend.PG-- def test_task_not_opted_in_routes_to_celery(self, monkeypatch):- monkeypatch.setenv(ENABLED_TASKS_ENV, "process_log_history")- assert select_backend("some_other_task") is QueueBackend.CELERY-- def test_multiple_entries_membership(self, monkeypatch):- monkeypatch.setenv(ENABLED_TASKS_ENV, "a,b,c")- assert select_backend("b") is QueueBackend.PG- assert select_backend("d") is QueueBackend.CELERY---class TestSelectBackendTolerantParsing:- """Malformed / whitespace input resolves to the safe default, never raises."""-- def test_whitespace_around_entries_trimmed(self, monkeypatch):- monkeypatch.setenv(ENABLED_TASKS_ENV, " a , b , c ")- assert select_backend("b") is QueueBackend.PG-- def test_empty_and_blank_segments_ignored(self, monkeypatch):- monkeypatch.setenv(ENABLED_TASKS_ENV, "a,, ,b")- assert select_backend("a") is QueueBackend.PG- # A blank segment must not become a matchable empty-string entry.- assert select_backend("") is QueueBackend.CELERY-- def test_empty_string_env_routes_to_celery(self, monkeypatch):- monkeypatch.setenv(ENABLED_TASKS_ENV, " ")- assert select_backend("any_task") is QueueBackend.CELERY-- def test_does_not_raise_on_any_input(self, monkeypatch):- for value in ("", " ", ",,,", "a,b", "\t\n", ",a,"):- monkeypatch.setenv(ENABLED_TASKS_ENV, value)- # Must resolve to an enum member, never raise.- assert select_backend("a") in (QueueBackend.CELERY, QueueBackend.PG)+ def test_always_pg(self):+ # No argument since UN-4046 — the task name never mattered once the+ # allow-list went, and Sonar flagged it as an unused parameter.+ assert select_backend() is QueueBackend.PGclass TestQueueBackendEnum:@@ -101,52 +61,10 @@ class TestQueueBackendEnum:assert {b.value for b in QueueBackend} == {"celery", "pg"}-# --- Observability (the gate's only observable effect) ------class TestObservability:- def test_allow_list_logged_once_when_configured(self, monkeypatch, caplog):- monkeypatch.setenv(ENABLED_TASKS_ENV, "async_execute_bin")- with caplog.at_level(logging.INFO, logger="queue_backend.routing"):- select_backend("x")- select_backend("y") # second call must NOT re-log- hits = [r for r in caplog.records if "PG-queue routing enabled" in r.getMessage()]- assert len(hits) == 1- assert "async_execute_bin" in hits[0].getMessage()-- def test_empty_allow_list_logs_nothing(self, caplog):- """Default (feature off) stays silent — truly inert."""- with caplog.at_level(logging.INFO, logger="queue_backend.routing"):- select_backend("x")- assert "PG-queue routing enabled" not in caplog.text---# --- dispatch() routing behaviour (9b: PG-selected enqueues to Postgres) ------def _mock_pg_client(monkeypatch, *, msg_id=99):- client = MagicMock()- client.send.return_value = msg_id- # Patch on the module object (string target would navigate the shadowing- # ``dispatch`` *function*, not the submodule).- monkeypatch.setattr(dispatch_mod, "_get_pg_client", lambda: client)- return client--class TestDispatchRouting:- """Celery path unchanged; a PG-selected task enqueues to Postgres, not Celery."""-- def test_celery_path_sends_to_celery_and_never_touches_pg(self):- with (- patch("queue_backend.dispatch.current_app") as mock_app,- patch("queue_backend.dispatch._get_pg_client") as mock_get,- ):- dispatch("t1", args=["a"], kwargs={"k": "v"}, queue="general")- mock_app.send_task.assert_called_once()- mock_get.assert_not_called()+ """A dispatch enqueues to Postgres and never touches Celery."""- def test_pg_selected_enqueues_to_pg_not_celery(self, monkeypatch):- monkeypatch.setenv(ENABLED_TASKS_ENV, "t1")+ def test_enqueues_to_pg_not_celery(self, monkeypatch):client = _mock_pg_client(monkeypatch, msg_id=99)fairness = FairnessKey(org_id="org-1", workload_type=WorkloadType.API)with patch("queue_backend.dispatch.current_app") as mock_app:@@ -166,17 +84,15 @@ class TestDispatchRouting:# Handle satisfies TaskHandle (.id) and carries the msg_id.assert handle.id == "99"- def test_pg_default_queue_name_when_unset(self, monkeypatch):- monkeypatch.setenv(ENABLED_TASKS_ENV, "t1")+ def test_default_queue_name_when_unset(self, monkeypatch):client = _mock_pg_client(monkeypatch)dispatch("t1")assert client.send.call_args.args[0] == "default"# No fairness → org_id None (client coerces to "").assert client.send.call_args.kwargs["org_id"] is None- def test_pg_enqueue_failure_propagates_and_does_not_fall_back(self, monkeypatch):+ def test_enqueue_failure_propagates_and_does_not_fall_back(self, monkeypatch):"""A PG enqueue failure raises — no silent Celery fallback."""- monkeypatch.setenv(ENABLED_TASKS_ENV, "t1")client = MagicMock()client.send.side_effect = RuntimeError("db down")monkeypatch.setattr(dispatch_mod, "_get_pg_client", lambda: client)@@ -202,28 +118,17 @@ class TestDispatchRouting:class TestCutoverLog:- """The PG cutover log: visible (INFO), once per task name."""+ """The PG routing log: visible (INFO), once per task name."""def test_pg_enqueue_logs_once_per_task(self, monkeypatch, caplog):- monkeypatch.setenv(ENABLED_TASKS_ENV, "t1")_mock_pg_client(monkeypatch)with caplog.at_level(logging.INFO, logger="queue_backend.dispatch"):dispatch("t1")dispatch("t1")dispatch("t1")- hits = [- r for r in caplog.records if "routing task=" in r.getMessage()- ]+ hits = [r for r in caplog.records if "routing task=" in r.getMessage()]assert len(hits) == 1- def test_celery_path_emits_no_pg_log(self, caplog):- with (- patch("queue_backend.dispatch.current_app"),- caplog.at_level(logging.INFO, logger="queue_backend.dispatch"),- ):- dispatch("t1") # empty allow-list → Celery- assert "routing task=" not in caplog.text-if __name__ == "__main__":pytest.main([__file__, "-v"])diff --git a/workers/tests/test_structure_tool_pipeline.py b/workers/tests/test_structure_tool_pipeline.pyindex c565621da..4c2812bf6 100644--- a/workers/tests/test_structure_tool_pipeline.py+++ b/workers/tests/test_structure_tool_pipeline.py@@ -7,7 +7,15 @@ single ``structure_pipeline`` operation to the executor worker instead ofare assembled and the result is written to filesystem."""-from unittest.mock import MagicMock, patch+from unittest.mock import MagicMock, create_autospec, patch++# create_autospec, NOT MagicMock(spec=...): only autospec enforces the method+# SIGNATURE. A bare MagicMock accepts any keyword, which is how a `headers=`+# argument the real dispatcher does not take (UN-4046) reached production and+# made every extraction raise TypeError while this suite stayed green.+# `spec=` alone does not close that hole — it restricts attribute names only,+# and was verified to still accept `headers=`.+from unstract.workflow_execution.executor_rpc import PgExecutionDispatcherimport pytestfrom shared.enums.task_enums import TaskName@@ -174,7 +182,7 @@ class TestStructureToolPipeline:"tool_metadata": tool_metadata_regular,}- dispatcher_instance = MagicMock()+ dispatcher_instance = create_autospec(PgExecutionDispatcher, instance=True)mock_get_executor_dispatcher.return_value = dispatcher_instancepipeline_result = _make_pipeline_result(@@ -225,7 +233,7 @@ class TestStructureToolPipeline:"tool_metadata": tool_metadata_regular,}- dispatcher_instance = MagicMock()+ dispatcher_instance = create_autospec(PgExecutionDispatcher, instance=True)mock_get_executor_dispatcher.return_value = dispatcher_instancedispatcher_instance.dispatch.return_value = _make_pipeline_result()@@ -289,7 +297,7 @@ class TestStructureToolSinglePass:base_params["tool_instance_metadata"]["single_pass_extraction_mode"] = True- dispatcher_instance = MagicMock()+ dispatcher_instance = create_autospec(PgExecutionDispatcher, instance=True)mock_get_executor_dispatcher.return_value = dispatcher_instancedispatcher_instance.dispatch.return_value = _make_pipeline_result(output={"field_a": "answer"},@@ -337,7 +345,7 @@ class TestStructureToolSummarize:tool_metadata_regular["tool_settings"]["summarize_prompt"] = "Summarize this doc"base_params["tool_instance_metadata"]["summarize_as_source"] = True- dispatcher_instance = MagicMock()+ dispatcher_instance = create_autospec(PgExecutionDispatcher, instance=True)mock_get_executor_dispatcher.return_value = dispatcher_instancedispatcher_instance.dispatch.return_value = _make_pipeline_result(output={"field_a": "answer"},@@ -396,7 +404,7 @@ class TestStructureToolSmartTable:"tool_metadata": tool_metadata_regular,}- dispatcher_instance = MagicMock()+ dispatcher_instance = create_autospec(PgExecutionDispatcher, instance=True)mock_get_executor_dispatcher.return_value = dispatcher_instancedispatcher_instance.dispatch.return_value = _make_pipeline_result(output={"field_a": "table_answer"},@@ -451,7 +459,7 @@ class TestStructureToolAgentic:"tool_metadata": agentic_metadata,}- dispatcher_instance = MagicMock()+ dispatcher_instance = create_autospec(PgExecutionDispatcher, instance=True)mock_get_executor_dispatcher.return_value = dispatcher_instance# Mock X2Text extraction@@ -511,7 +519,7 @@ class TestStructureToolProfileOverrides:"llm_id": "llm-override",}- dispatcher_instance = MagicMock()+ dispatcher_instance = create_autospec(PgExecutionDispatcher, instance=True)mock_get_executor_dispatcher.return_value = dispatcher_instancedispatcher_instance.dispatch.return_value = _make_pipeline_result(output={"field_a": "answer"},@@ -552,7 +560,7 @@ class TestStructureToolPipelineFailure:"tool_metadata": tool_metadata_regular,}- dispatcher_instance = MagicMock()+ dispatcher_instance = create_autospec(PgExecutionDispatcher, instance=True)mock_get_executor_dispatcher.return_value = dispatcher_instancepipeline_failure = ExecutionResult.failure(@@ -601,7 +609,7 @@ class TestStructureToolMultipleOutputs:"tool_metadata": tool_metadata_regular,}- dispatcher_instance = MagicMock()+ dispatcher_instance = create_autospec(PgExecutionDispatcher, instance=True)mock_get_executor_dispatcher.return_value = dispatcher_instancedispatcher_instance.dispatch.return_value = _make_pipeline_result(output={"field_a": "a", "field_b": "b"},@@ -647,7 +655,7 @@ class TestStructureToolOutputWritten:"tool_metadata": tool_metadata_regular,}- dispatcher_instance = MagicMock()+ dispatcher_instance = create_autospec(PgExecutionDispatcher, instance=True)mock_get_executor_dispatcher.return_value = dispatcher_instancedispatcher_instance.dispatch.return_value = _make_pipeline_result(output={"field_a": "answer"},@@ -717,7 +725,7 @@ class TestStructureToolMetadataFileName:"tool_metadata": tool_metadata_regular,}- dispatcher_instance = MagicMock()+ dispatcher_instance = create_autospec(PgExecutionDispatcher, instance=True)mock_get_executor_dispatcher.return_value = dispatcher_instancedispatcher_instance.dispatch.return_value = _make_pipeline_result(output={"field_a": "answer"},@@ -758,7 +766,7 @@ class TestStructureToolNoSummarize:"tool_metadata": tool_metadata_regular,}- dispatcher_instance = MagicMock()+ dispatcher_instance = create_autospec(PgExecutionDispatcher, instance=True)mock_get_executor_dispatcher.return_value = dispatcher_instancedispatcher_instance.dispatch.return_value = _make_pipeline_result()diff --git a/workers/tests/test_structure_tool_task.py b/workers/tests/test_structure_tool_task.pyindex dcef6f0b1..969ef9eed 100644--- a/workers/tests/test_structure_tool_task.py+++ b/workers/tests/test_structure_tool_task.py@@ -1,18 +1,20 @@"""Direct unit tests for ``file_processing.structure_tool_task`` helpers.-Locks contracts that the integration test in ``test_sanity_phase5.py``-exercises indirectly. Specifically guards ``_fairness_headers``:--* The hard-coded ``WorkloadType.NON_API`` default. Per the docstring,- ``API`` is "strictly worse" here — flipping it would silently- preempt ETL work under API traffic.-* The wire shape consumers see at the dispatcher. A regression in- ``FairnessKey.as_header()``'s serialisation surfaces here too.--Paired with the ``headers=`` assertion in-``test_sanity_phase5.TestStructureToolSingleDispatch`` — together-they cover both "the helper returns the right thing" and "the call-site forwards it".+UN-4046 removed ``TestFairnessHeaders``. It pinned the wire shape of+``_fairness_headers()``, which was passed as ``headers=`` to every executor+dispatch — but the routing dispatcher never forwarded headers to the PG path+(see ``PgExecutionDispatcher.dispatch``'s own docstring), so the value was+inert on the only transport that runs. With the router gone the helper is+deleted and the three tests describe a function that no longer exists.++Fairness on PG is not lost and is not tested here: it rides the enqueue payload+(``transport.enqueue(..., org_id=...)``), and ``queue_backend`` owns its+coverage. The paired "call site forwards it" assertion referenced by the old+docstring lived in a ``test_sanity_phase5.py`` that no longer exists either.++What remains is ``TestDispatcherFactory``, which guards the UN-3779 regression:+the impl must build its dispatcher via the factory, not a raw+``ExecutionDispatcher``."""from __future__ import annotations@@ -20,43 +22,13 @@ from __future__ import annotationsfrom unittest.mock import patchimport pytest-from file_processing import structure_tool_task as st-from file_processing.structure_tool_task import _fairness_headers-from queue_backend.fairness import WorkloadType---class TestFairnessHeaders:- def test_returns_non_api_wire_shape(self):- """``_fairness_headers`` emits the canonical ``x-fairness-key``- envelope with ``workload_type='non_api'`` and the default- ``pipeline_priority=5``."""- assert _fairness_headers("org-1") == {- "x-fairness-key": {- "org_id": "org-1",- "workload_type": "non_api",- "pipeline_priority": 5,- }- }-- def test_org_id_is_propagated_verbatim(self):- wire = _fairness_headers("another-org")- assert wire["x-fairness-key"]["org_id"] == "another-org"-- def test_workload_type_is_non_api_not_api(self):- """Pinned because the docstring calls ``NON_API`` "the safe- default" and ``API`` "strictly worse" — a regression that- flips this would silently let API traffic preempt ETL work.- """- wire = _fairness_headers("org-1")- assert wire["x-fairness-key"]["workload_type"] == WorkloadType.NON_API.value- assert wire["x-fairness-key"]["workload_type"] != WorkloadType.API.valueclass TestDispatcherFactory:"""Pin the call-site swap this PR makes: the impl builds its dispatcher via- ``get_executor_dispatcher(celery_app=app)`` (the gate-routed dispatcher), not- the raw SDK ``ExecutionDispatcher``. A mis-import or wrong arg would otherwise+ ``get_executor_dispatcher()`` (the shared factory), not+ the raw SDK ``ExecutionDispatcher``. A mis-import would otherwisesilently bypass the PG routing with nothing failing."""@@ -73,7 +45,7 @@ class TestDispatcherFactory:"execution_data_dir": "/data",}- def test_impl_builds_dispatcher_via_factory_with_app(self):+ def test_impl_builds_dispatcher_via_factory(self):# Stub everything up to (and just past) the dispatcher construction, then# raise to stop before the heavy tool-metadata work runs.with (@@ -86,7 +58,9 @@ class TestDispatcherFactory:params = self._params()with pytest.raises(RuntimeError, match="stop"):st._execute_structure_tool_impl(params)- factory.assert_called_once_with(celery_app=st.app)+ # No arguments since UN-4046 — the factory's accepted-and-ignored+ # `celery_app` was removed once Sonar flagged it as an unused parameter.+ factory.assert_called_once_with()if __name__ == "__main__":