Skip to content

LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows

UN-4046 [MISC] Roll the PG queue out of the pg_queue_enabled feature flag (#2262)

alia8cab8c17406
diff --git a/backend/backend/internal_api_constants.py b/backend/backend/internal_api_constants.py
index 52c0838db..37b19f578 100644
--- a/backend/backend/internal_api_constants.py
+++ b/backend/backend/internal_api_constants.py
@@ -64,11 +64,6 @@ class InternalEndpoints:
"""File execution status endpoint."""
return build_internal_endpoint(f"file-execution/{file_execution_id}/status")
- @staticmethod
- def webhook_send() -> str:
- """Webhook send endpoint."""
- return build_internal_endpoint("webhook/send")
-
@staticmethod
def organization(org_id: str = "{org_id}") -> str:
"""Organization endpoint."""
diff --git a/backend/entrypoint.sh b/backend/entrypoint.sh
index 9766b01ad..db6ef7298 100755
--- a/backend/entrypoint.sh
+++ b/backend/entrypoint.sh
@@ -35,11 +35,11 @@ if [ "$migrate" = true ]; then
#
# READ THIS BEFORE ASSUMING A RESTART IS INERT. This is NOT the earlier
# `--mirror-only` invocation, and it does NOT leave Beat alone: with
- # PG_SCHEDULER_ENABLED=true it flips pg_owned and DISABLES the Beat PeriodicTask
- # row for every mirrored pipeline THE pg_queue_enabled FLAG SELECTS — ownership is
- # additionally per-schedule Flipt-gated and fails closed, so with Flipt blind or the
- # flag at 0% this moves nothing and Beat keeps every row. A rolling restart therefore can move schedules
- # off Beat. (A comment here used to describe --mirror-only and state that
+ # PG_SCHEDULER_ENABLED it flips pg_owned and DISABLES the Beat PeriodicTask row
+ # for EVERY mirrored pipeline. There is no second gate any more: UN-4046 removed
+ # the pg_queue_enabled flag, and PG_SCHEDULER_ENABLED now DEFAULTS TO TRUE, so the
+ # first start after an upgrade moves every schedule off Beat unless you set it to
+ # false. Do not read this comment as "a restart is inert" — it no longer is. (A comment here used to describe --mirror-only and state that
# "ownership hand-over stays an explicit operator action" — it survived the switch
# to converge_pg_scheduler and told an SRE auditing exactly this question the
# opposite of the truth.)
diff --git a/backend/notification_v2/internal_api_views.py b/backend/notification_v2/internal_api_views.py
index 42d0eeac9..4b24cf98d 100644
--- a/backend/notification_v2/internal_api_views.py
+++ b/backend/notification_v2/internal_api_views.py
@@ -28,7 +28,6 @@ from pipeline_v2.models import Pipeline
from utils.organization_utils import filter_queryset_by_organization
from workflow_manager.workflow_v2.models.execution import WorkflowExecution
-from backend.celery_service import app as celery_app
from notification_v2.clubbed_renderer import MAX_BATCH_SIZE, render_clubbed_message
from notification_v2.enums import BufferStatus
from notification_v2.helper import (
@@ -530,16 +529,16 @@ def _reclaim_stale_sending() -> int:
def _org_identifier(org_pk: int) -> str | None:
"""Resolve the string ``Organization.organization_id`` from the buffer's org pk.
- ``resolve_transport`` keys its Flipt decision on the org's string identifier,
- but the buffer stores/uses the Organization pk. One indexed pk lookup per
- dispatch group (post-commit) — negligible relative to the downstream webhook
- dispatch.
+ The PG queue row records the org's string identifier (used for per-org
+ fairness), but the buffer stores/uses the Organization pk. One indexed pk
+ lookup per dispatch group (post-commit) — negligible relative to the
+ downstream webhook dispatch.
Data-anomaly guard: ``NotificationBuffer.organization`` is
``on_delete=CASCADE``, so a live buffer row with a missing org is unreachable
in normal operation. A ``None`` here therefore signals a dangling FK — we log
- it (the only org-traceable breadcrumb; resolve_transport's own warning is keyed
- on the random dispatch uuid) and fail closed to Celery in resolve_transport.
+ it (the only org-traceable breadcrumb) and enqueue with an empty org id, which
+ dispatches correctly but without fairness attribution.
"""
org_string_id = (
Organization.objects.filter(pk=org_pk)
@@ -549,11 +548,11 @@ def _org_identifier(org_pk: int) -> str | None:
if org_string_id is None:
# Sentry-routed (logger.error): a live buffer row with no org is a data
# anomaly (dangling FK / corruption) that shouldn't happen under the
- # CASCADE constraint, not routine noise. Routing still fails closed to
- # Celery in resolve_transport.
+ # CASCADE constraint, not routine noise. The enqueue still succeeds — it
+ # just carries an empty org id, so the row loses fairness attribution.
logger.error(
"metric=notification_org_identifier_missing_total org_pk=%s "
- "(dangling FK; notification routing falls back to Celery)",
+ "(dangling FK; enqueued without fairness attribution)",
org_pk,
)
return org_string_id
@@ -586,12 +585,10 @@ def _send_clubbed(
``buffer_row_ids`` + ``organization_id`` to the worker so it can mark them.
"""
try:
- # Flag-gated transport (UN-3753): PG queue when pg_queue_enabled for this
- # org, else Celery (byte-identical to the prior send_task). resolve_transport
- # keys on the org STRING id, but the buffer/worker contract below uses the
- # org pk — hence _org_identifier(org_id) for routing, org_id in kwargs.
- dispatched = dispatch_webhook_notification(
- celery_app=celery_app,
+ # The queue row records the org STRING id, but the buffer/worker contract
+ # below uses the org pk — hence _org_identifier(org_id) on the row,
+ # org_id in kwargs. The two must not be conflated.
+ dispatch_task_id = dispatch_webhook_notification(
args=[url, body, headers, settings.NOTIFICATION_TIMEOUT],
kwargs={
"max_retries": max_retries,
@@ -609,18 +606,14 @@ def _send_clubbed(
queue="notifications",
org_string_id=_org_identifier(org_id),
)
- # transport= makes the rollout answerable from the logs: during a percentage
- # ramp the question is "are PG-routed notifications succeeding at the same
- # rate as Celery-routed ones?", which result=success alone cannot answer.
logger.info(
"metric=notification_batch_dispatched_total platform=%s result=success "
- "transport=%s org_id=%s webhook_url_hash=%s rows=%d task_id=%s",
+ "org_id=%s webhook_url_hash=%s rows=%d task_id=%s",
platform,
- dispatched.transport,
org_id,
webhook_url_hash(url),
len(buffer_ids),
- dispatched.task_id,
+ dispatch_task_id,
)
except PermanentDispatchError:
# PG-ONLY permanent failure. From this path the only reachable cause is
diff --git a/backend/notification_v2/internal_serializers.py b/backend/notification_v2/internal_serializers.py
index 94669d64a..ab2b55711 100644
--- a/backend/notification_v2/internal_serializers.py
+++ b/backend/notification_v2/internal_serializers.py
@@ -30,66 +30,6 @@ class NotificationSerializer(serializers.ModelSerializer):
]
-class WebhookNotificationRequestSerializer(serializers.Serializer):
- """Serializer for webhook notification requests."""
-
- notification_id = serializers.UUIDField(required=False)
- url = serializers.URLField(required=True)
- payload = serializers.JSONField(required=True)
- authorization_type = serializers.ChoiceField(
- choices=AuthorizationType.choices(), default=AuthorizationType.NONE.value
- )
- authorization_key = serializers.CharField(required=False, allow_blank=True)
- authorization_header = serializers.CharField(required=False, allow_blank=True)
- headers = serializers.DictField(required=False, default=dict)
- timeout = serializers.IntegerField(default=30, min_value=1, max_value=300)
- max_retries = serializers.IntegerField(default=3, min_value=0, max_value=10)
- retry_delay = serializers.IntegerField(default=60, min_value=1, max_value=3600)
-
-
-class WebhookNotificationResponseSerializer(serializers.Serializer):
- """Serializer for webhook notification responses."""
-
- task_id = serializers.CharField()
- notification_id = serializers.UUIDField(required=False)
- url = serializers.URLField()
- status = serializers.CharField()
- queued_at = serializers.DateTimeField()
-
-
-class WebhookStatusSerializer(serializers.Serializer):
- """Serializer for webhook delivery status."""
-
- task_id = serializers.CharField()
- status = serializers.CharField()
- url = serializers.CharField()
- attempts = serializers.IntegerField()
- success = serializers.BooleanField()
- error_message = serializers.CharField(required=False, allow_null=True)
-
-
-class WebhookBatchRequestSerializer(serializers.Serializer):
- """Serializer for batch webhook requests."""
-
- batch_name = serializers.CharField(required=False, max_length=255)
- webhooks = serializers.ListField(
- child=WebhookNotificationRequestSerializer(), min_length=1, max_length=100
- )
- delay_between_requests = serializers.IntegerField(
- default=0, min_value=0, max_value=60
- )
-
-
-class WebhookBatchResponseSerializer(serializers.Serializer):
- """Serializer for batch webhook responses."""
-
- batch_id = serializers.CharField()
- batch_name = serializers.CharField()
- total_webhooks = serializers.IntegerField()
- queued_webhooks = serializers.ListField(child=WebhookNotificationResponseSerializer())
- failed_webhooks = serializers.ListField(child=serializers.DictField())
-
-
class WebhookConfigurationSerializer(serializers.Serializer):
"""Serializer for webhook configuration."""
diff --git a/backend/notification_v2/internal_urls.py b/backend/notification_v2/internal_urls.py
index 90da468fb..2a35dcaf3 100644
--- a/backend/notification_v2/internal_urls.py
+++ b/backend/notification_v2/internal_urls.py
@@ -7,12 +7,8 @@ from rest_framework.routers import DefaultRouter
from . import internal_api_views
from .internal_views import (
- WebhookBatchAPIView,
- WebhookBatchStatusAPIView,
WebhookInternalViewSet,
WebhookMetricsAPIView,
- WebhookSendAPIView,
- WebhookStatusAPIView,
WebhookTestAPIView,
)
@@ -66,13 +62,7 @@ urlpatterns = [
name="get_api_data",
),
# Webhook operation endpoints
- path("send/", WebhookSendAPIView.as_view(), name="webhook-send"),
- path("batch/", WebhookBatchAPIView.as_view(), name="webhook-batch"),
path("test/", WebhookTestAPIView.as_view(), name="webhook-test"),
- path("status/<str:task_id>/", WebhookStatusAPIView.as_view(), name="webhook-status"),
- path(
- "batch-status/", WebhookBatchStatusAPIView.as_view(), name="webhook-batch-status"
- ),
path("metrics/", WebhookMetricsAPIView.as_view(), name="webhook-metrics"),
# Webhook configuration CRUD (via router)
path("", include(router.urls)),
diff --git a/backend/notification_v2/internal_views.py b/backend/notification_v2/internal_views.py
index e352f56db..596945d01 100644
--- a/backend/notification_v2/internal_views.py
+++ b/backend/notification_v2/internal_views.py
@@ -3,12 +3,8 @@ Handles webhook notification related endpoints for internal services.
"""
import logging
-import uuid
from typing import Any
-from celery import current_app as celery_app
-from celery.result import AsyncResult
-from django.utils import timezone
from rest_framework import status, viewsets
from rest_framework.decorators import action
from rest_framework.response import Response
@@ -21,12 +17,7 @@ from notification_v2.enums import AuthorizationType, NotificationType, PlatformT
from notification_v2.internal_serializers import (
NotificationListSerializer,
NotificationSerializer,
- WebhookBatchRequestSerializer,
- WebhookBatchResponseSerializer,
WebhookConfigurationSerializer,
- WebhookNotificationRequestSerializer,
- WebhookNotificationResponseSerializer,
- WebhookStatusSerializer,
WebhookTestSerializer,
)
from notification_v2.models import Notification
@@ -110,219 +101,6 @@ class WebhookInternalViewSet(viewsets.ReadOnlyModelViewSet):
)
-class WebhookSendAPIView(APIView):
- """Internal API endpoint for sending webhook notifications."""
-
- def post(self, request):
- """Send a webhook notification."""
- try:
- serializer = WebhookNotificationRequestSerializer(data=request.data)
-
- if not serializer.is_valid():
- return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST)
-
- validated_data = serializer.validated_data
-
- # Build headers based on authorization type
- headers = self._build_headers(validated_data)
-
- # Send webhook notification task
- task = celery_app.send_task(
- "send_webhook_notification",
- args=[
- validated_data["url"],
- validated_data["payload"],
- headers,
- validated_data["timeout"],
- ],
- kwargs={
- "max_retries": validated_data["max_retries"],
- "retry_delay": validated_data["retry_delay"],
- },
- )
-
- # Prepare response
- response_data = {
- "task_id": task.id,
- "notification_id": validated_data.get("notification_id"),
- "url": validated_data["url"],
- "status": "queued",
- "queued_at": timezone.now(),
- }
-
- response_serializer = WebhookNotificationResponseSerializer(response_data)
-
- logger.info(
- f"Queued webhook notification task {task.id} for URL {validated_data['url']}"
- )
-
- return Response(response_serializer.data, status=status.HTTP_202_ACCEPTED)
-
- except Exception as e:
- logger.error(f"Failed to send webhook notification: {str(e)}")
- return Response(
- {"error": "Failed to send webhook notification", "detail": str(e)},
- status=status.HTTP_500_INTERNAL_SERVER_ERROR,
- )
-
- def _build_headers(self, validated_data: dict[str, Any]) -> dict[str, str]:
- """Build headers based on authorization configuration."""
- headers = {"Content-Type": APPLICATION_JSON}
-
- auth_type = validated_data.get("authorization_type", AuthorizationType.NONE.value)
- auth_key = validated_data.get("authorization_key")
- auth_header = validated_data.get("authorization_header")
-
- if validated_data.get("headers"):
- headers.update(validated_data["headers"])
-
- if auth_type == AuthorizationType.BEARER.value and auth_key:
- headers["Authorization"] = f"Bearer {auth_key}"
- elif auth_type == AuthorizationType.API_KEY.value and auth_key:
- headers["Authorization"] = auth_key
- elif (
- auth_type == AuthorizationType.CUSTOM_HEADER.value
- and auth_header
- and auth_key
- ):
- headers[auth_header] = auth_key
-
- return headers
-
-
-class WebhookStatusAPIView(APIView):
- """Internal API endpoint for checking webhook delivery status."""
-
- def get(self, request, task_id):
- """Get webhook delivery status by task ID."""
- try:
- task_result = AsyncResult(task_id, app=celery_app)
-
- status_data = {
- "task_id": task_id,
- "status": task_result.status,
- "url": "unknown",
- "attempts": 0,
- "success": task_result.successful(),
- "error_message": None,
- }
-
- if task_result.failed():
- status_data["error_message"] = str(task_result.result)
- elif task_result.successful():
- status_data["attempts"] = getattr(task_result.result, "attempts", 1)
-
- serializer = WebhookStatusSerializer(status_data)
- return Response(serializer.data)
-
- except Exception as e:
- logger.error(f"Failed to get webhook status for task {task_id}: {str(e)}")
- return Response(
- {"error": "Failed to get webhook status", "detail": str(e)},
- status=status.HTTP_500_INTERNAL_SERVER_ERROR,
- )
-
-
-class WebhookBatchAPIView(APIView):
- """Internal API endpoint for sending batch webhook notifications."""
-
- def post(self, request):
- """Send multiple webhook notifications in batch."""
- try:
- serializer = WebhookBatchRequestSerializer(data=request.data)
-
- if not serializer.is_valid():
- return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST)
-
- validated_data = serializer.validated_data
- webhooks = validated_data["webhooks"]
- delay_between = validated_data.get("delay_between_requests", 0)
-
- batch_id = str(uuid.uuid4())
- queued_webhooks = []
- failed_webhooks = []
-
- for i, webhook_data in enumerate(webhooks):
- try:
- headers = self._build_headers(webhook_data)
- countdown = i * delay_between if delay_between > 0 else 0
-
- task = celery_app.send_task(
- "send_webhook_notification",
- args=[
- webhook_data["url"],
- webhook_data["payload"],
- headers,
- webhook_data["timeout"],
- ],
- kwargs={
- "max_retries": webhook_data["max_retries"],
- "retry_delay": webhook_data["retry_delay"],
- },
- countdown=countdown,
- )
-
- queued_webhooks.append(
- {
- "task_id": task.id,
- "notification_id": webhook_data.get("notification_id"),
- "url": webhook_data["url"],
- "status": "queued",
- "queued_at": timezone.now(),
- }
- )
-
- except Exception as e:
- failed_webhooks.append({"url": webhook_data["url"], "error": str(e)})
-
- response_data = {
- "batch_id": batch_id,
- "batch_name": validated_data.get("batch_name", f"Batch-{batch_id[:8]}"),
- "total_webhooks": len(webhooks),
- "queued_webhooks": queued_webhooks,
- "failed_webhooks": failed_webhooks,
- }
-
- response_serializer = WebhookBatchResponseSerializer(response_data)
-
- logger.info(
- f"Queued batch {batch_id} with {len(queued_webhooks)} webhooks, {len(failed_webhooks)} failed"
- )
-
- return Response(response_serializer.data, status=status.HTTP_202_ACCEPTED)
-
- except Exception as e:
- logger.error(f"Failed to send webhook batch: {str(e)}")
- return Response(
- {"error": "Failed to send webhook batch", "detail": str(e)},
- status=status.HTTP_500_INTERNAL_SERVER_ERROR,
- )
-
- def _build_headers(self, webhook_data: dict[str, Any]) -> dict[str, str]:
- """Build headers for webhook request."""
- headers = {"Content-Type": APPLICATION_JSON}
-
- auth_type = webhook_data.get("authorization_type", AuthorizationType.NONE.value)
- auth_key = webhook_data.get("authorization_key")
- auth_header = webhook_data.get("authorization_header")
-
- if webhook_data.get("headers"):
- headers.update(webhook_data["headers"])
-
- if auth_type == AuthorizationType.BEARER.value and auth_key:
- headers["Authorization"] = f"Bearer {auth_key}"
- elif auth_type == AuthorizationType.API_KEY.value and auth_key:
- headers["Authorization"] = auth_key
- elif (
- auth_type == AuthorizationType.CUSTOM_HEADER.value
- and auth_header
- and auth_key
- ):
- headers[auth_header] = auth_key
-
- return headers
-
-
class WebhookTestAPIView(APIView):
"""Internal API endpoint for testing webhook configurations."""
@@ -406,73 +184,6 @@ class WebhookTestAPIView(APIView):
return headers
-class WebhookBatchStatusAPIView(APIView):
- """Internal API endpoint for checking batch webhook delivery status."""
-
- def get(self, request):
- """Get batch webhook delivery status."""
- try:
- batch_id = request.query_params.get("batch_id")
- task_ids = request.query_params.get("task_ids", "").split(",")
-
- if not batch_id and not task_ids:
- return Response(
- {"error": "Either batch_id or task_ids parameter is required"},
- status=status.HTTP_400_BAD_REQUEST,
- )
-
- batch_results = []
-
- if task_ids and task_ids[0]: # task_ids is not empty
- for task_id in task_ids:
- if task_id.strip():
- try:
- task_result = AsyncResult(task_id.strip(), app=celery_app)
-
- batch_results.append(
- {
- "task_id": task_id.strip(),
- "status": task_result.status,
- "success": task_result.successful(),
- "error_message": str(task_result.result)
- if task_result.failed()
- else None,
- }
- )
- except Exception as e:
- batch_results.append(
- {
- "task_id": task_id.strip(),
- "status": "ERROR",
- "success": False,
- "error_message": f"Failed to get task status: {str(e)}",
- }
- )
-
- response_data = {
- "batch_id": batch_id,
- "total_tasks": len(batch_results),
- "results": batch_results,
- "summary": {
- "completed": sum(
- 1 for r in batch_results if r["status"] == "SUCCESS"
- ),
- "failed": sum(1 for r in batch_results if r["status"] == "FAILURE"),
- "pending": sum(1 for r in batch_results if r["status"] == "PENDING"),
- "running": sum(1 for r in batch_results if r["status"] == "STARTED"),
- },
- }
-
- return Response(response_data)
-
- except Exception as e:
- logger.error(f"Failed to get batch webhook status: {str(e)}")
- return Response(
- {"error": "Failed to get batch webhook status", "detail": str(e)},
- status=status.HTTP_500_INTERNAL_SERVER_ERROR,
- )
-
-
class WebhookMetricsAPIView(APIView):
"""Internal API endpoint for webhook delivery metrics."""
diff --git a/backend/notification_v2/notification_dispatch.py b/backend/notification_v2/notification_dispatch.py
index 4b027182e..535450931 100644
--- a/backend/notification_v2/notification_dispatch.py
+++ b/backend/notification_v2/notification_dispatch.py
@@ -1,37 +1,25 @@
-"""Transport-routed dispatch for buffered webhook notifications (UN-3753).
+"""PG-queue dispatch for buffered webhook notifications (UN-3753, UN-4046).
-Routes the ``send_webhook_notification`` task through the same
-:func:`resolve_transport` flag as the execution path: the PG queue when
-``pg_queue_enabled`` for this org, else Celery. **Fail-closed** — with the gate
-off (the production default) it resolves to Celery, behaving exactly like the
-prior unconditional ``celery_app.send_task`` (zero regression).
+Enqueues ``send_webhook_notification`` onto the PG queue the notification
+consumer polls (``WORKER_PG_QUEUE_CONSUMER_WORKER_TYPE=notification``).
-DEPLOYMENT PREREQUISITE (PG path): on PG the task lands on the ``notifications``
-queue and requires a pg-queue consumer configured with
-``WORKER_PG_QUEUE_CONSUMER_WORKER_TYPE=notification`` /
-``WORKER_PG_QUEUE_CONSUMER_QUEUE=notifications``. **No such service exists in this
-repo's compose yet** — none of the ``pg-queue-consumer`` services in
-``docker/docker-compose.yaml`` polls ``notifications``, and ``run-worker.sh``'s
-``PG_CONSUMER_ROLES`` has no notification role. Until one is deployed the flag must
-stay off for the org: enqueued rows would sit undrained (no TTL sweep covers
-``pg_queue_message``) and every buffered webhook for that org would be lost.
+This used to route through ``resolve_transport``/``pg_queue_enabled`` and fall
+back to ``celery_app.send_task``. The flag is gone (UN-4046) and PG is the only
+transport, so the Celery branch went with it — along with the ``DispatchResult``
+transport label, which existed only to tell the two apart during a ramp.
-``args`` and ``queue`` are forwarded verbatim on both paths. ``kwargs`` differ in
-exactly two keys, forced on the PG branch only (see the inline note at the branch):
-``raise_on_final_failure`` → ``False`` and ``max_retries`` → ``0``. On PG the
-payload is additionally JSON-normalized by ``enqueue_task`` (UUIDs/datetimes → str).
+``args`` and ``queue`` are forwarded verbatim; the payload is JSON-normalized by
+``enqueue_task`` (UUIDs/datetimes → str). Two ``kwargs`` are forced — see the
+inline note at the enqueue.
"""
from __future__ import annotations
import logging
import uuid
-from typing import Any, NamedTuple
+from typing import Any
from pg_queue.producer import enqueue_task
-from workflow_manager.workflow_v2.transport import resolve_transport
-
-from unstract.core.data_models import is_pg_transport
logger = logging.getLogger(__name__)
@@ -40,141 +28,94 @@ logger = logging.getLogger(__name__)
WEBHOOK_NOTIFICATION_TASK = "send_webhook_notification"
-# Transport labels for the dispatch metric. During a percentage ramp the one thing
-# you need from the logs is "are PG-routed notifications succeeding at the same rate
-# as Celery-routed ones?" — a bare task id can't answer that, since a minted PG id
-# and a Celery AsyncResult id are indistinguishable to the caller.
-PG_TRANSPORT = "pg_queue"
-CELERY_TRANSPORT = "celery"
-
-
-class DispatchResult(NamedTuple):
- """Which transport actually took the dispatch, plus the resulting task id."""
-
- transport: str
- task_id: str
-
-
class PermanentDispatchError(Exception):
"""A dispatch failure that would fail identically on every retry.
- Raised ONLY on the PG path, when ``enqueue_task`` rejects the message for a
- permanent reason (priority range / reply_key+callback exclusivity validation,
- or a payload that can't be JSON-serialized). The Celery path never raises it,
- so a caller can dead-letter on this exception without altering the flag-off
- (Celery) error flow — a Celery ``send_task`` failure stays an ordinary
- ``Exception`` the caller's transient handler owns, exactly as before.
+ Raised when ``enqueue_task`` rejects the message for a permanent reason
+ (priority range / reply_key+callback exclusivity validation, or a payload that
+ can't be JSON-serialized), so a caller can dead-letter on it. A transient PG
+ error (DB down) is deliberately NOT wrapped — it propagates as an ordinary
+ ``Exception`` into the caller's retry path.
"""
def dispatch_webhook_notification(
*,
- celery_app: Any,
args: list[Any],
kwargs: dict[str, Any],
queue: str,
org_string_id: str | None,
-) -> DispatchResult:
- """Dispatch ``send_webhook_notification`` on the resolved transport.
-
- ``args``/``queue`` are forwarded unchanged on both paths, and on the Celery
- branch ``kwargs`` too — so the flag-off path is byte-identical to the legacy
- ``send_task`` call. The PG branch overrides two retry-semantics kwargs (see
- below); nothing else differs.
+) -> str:
+ """Enqueue ``send_webhook_notification`` on the PG queue.
Args:
- celery_app: Injected Celery app (the backend's ``celery_service.app``);
- passed in rather than imported so this seam stays trivially testable.
args: Positional task args, forwarded verbatim.
- kwargs: Keyword task args. Forwarded as-is on Celery; on PG,
- ``raise_on_final_failure`` is overridden to ``False`` and
- ``max_retries`` to ``0`` (the in-task retry loop is a no-op under the
- consumer's eager ``apply()`` — see the inline note). For the buffered
- path this
- carries ``organization_id`` = the buffer's org **pk** (the worker's
- buffer-mark contract) — deliberately a DIFFERENT identifier from the
- ``org_string_id`` param below (the two must not be conflated).
+ kwargs: Keyword task args. ``raise_on_final_failure`` is overridden to
+ ``False`` and ``max_retries`` to ``0`` (the in-task retry loop is a
+ no-op under the consumer's eager ``apply()`` — see the inline note).
+ For the buffered path this carries ``organization_id`` = the buffer's
+ org **pk** (the worker's buffer-mark contract) — deliberately a
+ DIFFERENT identifier from ``org_string_id`` (the two must not be
+ conflated).
queue: Target queue name, forwarded verbatim.
org_string_id: The org's **string** identifier
- (``Organization.organization_id``), used solely for the Flipt
- transport decision — NOT the org pk carried in ``kwargs``. ``None`` (or
- empty) fails closed to Celery.
+ (``Organization.organization_id``), recorded on the queue row for
+ fairness/routing.
Returns:
- A :class:`DispatchResult` carrying which transport took the dispatch and
- the resulting task id (the Celery ``AsyncResult`` id, or the minted PG task
- id). The transport is what makes the caller's dispatch metric answerable
- during a ramp — the two id flavours are otherwise indistinguishable.
+ The minted PG task id.
"""
# A buffered notification is a single fire-and-forget task with no natural
- # sticky entity, so mint a fresh id to drive Flipt's percentage bucketing and
- # to serve as the PG task id.
+ # sticky entity, so mint a fresh id to serve as the PG task id.
dispatch_id = str(uuid.uuid4())
- # resolve_transport already normalizes falsy input to Celery, so pass the id
- # straight through (no `or None` needed).
- transport = resolve_transport(
- execution_id=dispatch_id,
- organization_id=org_string_id,
- )
- # Use the shared is_pg_transport() — the single source for "what counts as PG
- # transport" — rather than opening a second comparison site.
- if is_pg_transport(transport):
- # Two PG-only kwarg overrides, both about the SAME thing: the consumer runs
- # the task eagerly via ``task.apply(..., throw=True)``, where Celery's
- # in-task retry loop does not work and its terminal branch is what we need.
- #
- # 1. ``max_retries`` → 0. Under ``apply()`` ``self.request.retries`` is ALWAYS
- # 0, so the worker's ``if self.request.retries < max_retries`` guard
- # (workers/notification/tasks.py) is true on the FIRST failure for any
- # max_retries >= 1 and calls ``self.retry(...)``, which raises ``Retry``;
- # with ``throw=True`` that propagates straight out of ``apply()`` and the
- # terminal branch is never reached — so the buffers are never marked
- # DEAD_LETTER and ``raise_on_final_failure`` is never even read. The
- # consumer's ``except Exception`` then leaves the row for vt-expiry
- # redelivery, re-POSTing the subscriber every time. Forcing 0 sends the
- # task down the terminal branch on the first failure instead.
- # 2. ``raise_on_final_failure`` → False. On BOTH transports the worker marks
- # the buffers DEAD_LETTER over the internal API *before* it re-raises, so
- # the re-raise is only a FAILURE-state signal for Celery monitoring (no
- # redelivery there). On the PG consumer that same raise is treated as a
- # failure and leaves the row for redelivery, so it must not raise.
- #
- # Together: one POST, buffers dead-lettered, task returns None → the consumer
- # acks (deletes) the row. Retry spacing belongs to the PG layer, not the task.
- # (The Celery branch below keeps kwargs verbatim — byte-identical.)
- pg_kwargs = {**kwargs, "raise_on_final_failure": False, "max_retries": 0}
- try:
- msg_id = enqueue_task(
- task_name=WEBHOOK_NOTIFICATION_TASK,
- queue=queue,
- args=args,
- kwargs=pg_kwargs,
- # enqueue_task types org_id as str; None→"" here satisfies that
- # type only, not runtime — enqueue_task itself re-coerces
- # ``org_id or ""`` at insert (producer.py), so this has no runtime
- # effect.
- org_id=org_string_id or "",
- task_id=dispatch_id,
- )
- except (ValueError, TypeError) as exc:
- # PG-only permanent failure (enqueue_task validation / JSON encode):
- # re-raise as PermanentDispatchError so the caller dead-letters it.
- # A transient PG error (DB down) is NOT wrapped — it propagates as an
- # ordinary Exception into the caller's retry (revert-to-PENDING) path.
- raise PermanentDispatchError(str(exc)) from exc
- # msg_id correlates this line with the producer's own ``msg_id=`` log, so a
- # dropped notification can be traced across the seam/producer boundary.
- logger.info(
- "Webhook notification enqueued on PG '%s' queue (task_id=%s msg_id=%s)",
- queue,
- dispatch_id,
- msg_id,
+ # Two kwarg overrides, both about the SAME thing: the consumer runs the task
+ # eagerly via ``task.apply(..., throw=True)``, where Celery's in-task retry
+ # loop does not work and its terminal branch is what we need.
+ #
+ # 1. ``max_retries`` → 0. Under ``apply()`` ``self.request.retries`` is ALWAYS
+ # 0, so the worker's ``if self.request.retries < max_retries`` guard
+ # (workers/notification/tasks.py) is true on the FIRST failure for any
+ # max_retries >= 1 and calls ``self.retry(...)``, which raises ``Retry``;
+ # with ``throw=True`` that propagates straight out of ``apply()`` and the
+ # terminal branch is never reached — so the buffers are never marked
+ # DEAD_LETTER and ``raise_on_final_failure`` is never even read. The
+ # consumer's ``except Exception`` then leaves the row for vt-expiry
+ # redelivery, re-POSTing the subscriber every time. Forcing 0 sends the
+ # task down the terminal branch on the first failure instead.
+ # 2. ``raise_on_final_failure`` → False. The worker marks the buffers
+ # DEAD_LETTER over the internal API *before* it re-raises, so the re-raise
+ # carries no state the caller needs — and on the PG consumer that raise is
+ # treated as a failure and leaves the row for redelivery, so it must not
+ # raise.
+ #
+ # Together: one POST, buffers dead-lettered, task returns None → the consumer
+ # acks (deletes) the row. Retry spacing belongs to the PG layer, not the task.
+ pg_kwargs = {**kwargs, "raise_on_final_failure": False, "max_retries": 0}
+ try:
+ msg_id = enqueue_task(
+ task_name=WEBHOOK_NOTIFICATION_TASK,
+ queue=queue,
+ args=args,
+ kwargs=pg_kwargs,
+ # enqueue_task types org_id as str; None→"" here satisfies that
+ # type only, not runtime — enqueue_task itself re-coerces
+ # ``org_id or ""`` at insert (producer.py), so this has no runtime
+ # effect.
+ org_id=org_string_id or "",
+ task_id=dispatch_id,
)
- return DispatchResult(transport=PG_TRANSPORT, task_id=dispatch_id)
- result = celery_app.send_task(
- WEBHOOK_NOTIFICATION_TASK,
- args=args,
- kwargs=kwargs,
- queue=queue,
+ except (ValueError, TypeError) as exc:
+ # PG-only permanent failure (enqueue_task validation / JSON encode):
+ # re-raise as PermanentDispatchError so the caller dead-letters it.
+ # A transient PG error (DB down) is NOT wrapped — it propagates as an
+ # ordinary Exception into the caller's retry (revert-to-PENDING) path.
+ raise PermanentDispatchError(str(exc)) from exc
+ # msg_id correlates this line with the producer's own ``msg_id=`` log, so a
+ # dropped notification can be traced across the seam/producer boundary.
+ logger.info(
+ "Webhook notification enqueued on PG '%s' queue (task_id=%s msg_id=%s)",
+ queue,
+ dispatch_id,
+ msg_id,
)
- return DispatchResult(transport=CELERY_TRANSPORT, task_id=result.id)
+ return dispatch_id
diff --git a/backend/notification_v2/tests/test_notification_dispatch.py b/backend/notification_v2/tests/test_notification_dispatch.py
index e08bf55ff..450287a94 100644
--- a/backend/notification_v2/tests/test_notification_dispatch.py
+++ b/backend/notification_v2/tests/test_notification_dispatch.py
@@ -1,17 +1,20 @@
-"""Unit tests for the buffered-webhook transport routing (UN-3753).
-
-``resolve_transport`` + ``enqueue_task`` are patched on the module, so no Flipt /
-DB is needed — these pin the routing contract: PG when the flag resolves PG,
-Celery otherwise (fail-closed), with identical args/queue on both paths. The
-kwargs are identical EXCEPT two retry-semantics keys forced on the PG branch
-(``raise_on_final_failure`` -> False, ``max_retries`` -> 0), because the consumer
-runs tasks eagerly via ``apply()`` where the in-task retry loop cannot work — see
-``test_pg_forces_terminal_branch_kwargs``.
+"""Unit tests for the buffered-webhook PG dispatch (UN-3753, UN-4046).
+
+``enqueue_task`` is patched on the module, so no DB is needed. These pin the
+payload contract the notification consumer depends on, the permanent-vs-transient
+error split the caller branches on, and the two retry-semantics kwargs the seam
+forces because the consumer runs tasks eagerly via ``apply()``.
+
+The transport-routing tests went with the ``pg_queue_enabled`` flag (UN-4046) —
+"routes to celery", "fails closed to celery on a missing org", "buckets by minted
+dispatch id" and "identical on both paths" no longer describe anything. What the
+missing-org case still asserts (the row carries an empty org id rather than
+``None``) is kept below, because ``enqueue_task`` types ``org_id`` as ``str``.
"""
from __future__ import annotations
-from unittest.mock import MagicMock, patch
+from unittest.mock import patch
import pytest
@@ -29,9 +32,8 @@ _KWARGS = {
_QUEUE = "notifications"
-def _dispatch(celery_app, org_string_id="org_x"):
+def _dispatch(org_string_id="org_x"):
return nd.dispatch_webhook_notification(
- celery_app=celery_app,
args=_ARGS,
kwargs=_KWARGS,
queue=_QUEUE,
@@ -40,22 +42,16 @@ def _dispatch(celery_app, org_string_id="org_x"):
class TestDispatchWebhookNotification:
- def test_routes_to_pg_when_flag_resolves_pg(self):
- celery = MagicMock()
- with (
- patch.object(nd, "resolve_transport", return_value="pg_queue"),
- patch.object(nd, "enqueue_task", return_value=42) as enqueue,
- ):
- result = _dispatch(celery)
- task_id = result.task_id
- assert result.transport == nd.PG_TRANSPORT # rollout metric can tell them apart
+ def test_enqueues_with_the_consumer_payload_contract(self):
+ with patch.object(nd, "enqueue_task", return_value=42) as enqueue:
+ task_id = _dispatch()
enqueue.assert_called_once()
kwargs = enqueue.call_args.kwargs
assert kwargs["task_name"] == "send_webhook_notification"
assert kwargs["queue"] == _QUEUE
assert kwargs["args"] == _ARGS
- # Every kwarg is forwarded verbatim EXCEPT the two retry-semantics keys the
- # PG branch forces (see test_pg_forces_terminal_branch_kwargs).
+ # Every kwarg forwarded verbatim EXCEPT the two retry-semantics keys the
+ # seam forces (see test_forces_terminal_branch_kwargs).
assert kwargs["kwargs"] == {
**_KWARGS,
"raise_on_final_failure": False,
@@ -64,98 +60,31 @@ class TestDispatchWebhookNotification:
assert kwargs["org_id"] == "org_x"
# The minted PG task id is returned and threaded into the enqueue row.
assert kwargs["task_id"] == task_id
- celery.send_task.assert_not_called()
-
- def test_routes_to_celery_when_flag_resolves_celery(self):
- celery = MagicMock()
- with (
- patch.object(nd, "resolve_transport", return_value="celery"),
- patch.object(nd, "enqueue_task") as enqueue,
- ):
- result = _dispatch(celery)
- celery.send_task.assert_called_once_with(
- "send_webhook_notification", args=_ARGS, kwargs=_KWARGS, queue=_QUEUE
- )
- enqueue.assert_not_called()
- assert result.task_id is celery.send_task.return_value.id
- assert result.transport == nd.CELERY_TRANSPORT
- def test_pg_enqueue_failure_propagates_with_no_celery_fallback(self):
- # No silent Celery fallback on a TRANSIENT PG failure — it propagates raw
- # (not wrapped), so the caller (_send_clubbed) reverts rows to PENDING.
- celery = MagicMock()
- with (
- patch.object(nd, "resolve_transport", return_value="pg_queue"),
- patch.object(nd, "enqueue_task", side_effect=RuntimeError("pg down")),
- ):
+ def test_transient_enqueue_failure_propagates_raw(self):
+ # A TRANSIENT PG failure propagates unwrapped, so the caller (_send_clubbed)
+ # reverts the rows to PENDING rather than dead-lettering them.
+ with patch.object(nd, "enqueue_task", side_effect=RuntimeError("pg down")):
with pytest.raises(RuntimeError, match="pg down"):
- _dispatch(celery)
- celery.send_task.assert_not_called()
+ _dispatch()
- def test_pg_permanent_enqueue_error_is_wrapped(self):
+ def test_permanent_enqueue_error_is_wrapped(self):
# A PERMANENT enqueue error (ValueError/TypeError: validation / JSON encode)
- # is re-raised as PermanentDispatchError so the caller dead-letters it —
- # raised ONLY on the PG path, so the Celery flow is never affected.
- celery = MagicMock()
+ # is re-raised as PermanentDispatchError so the caller dead-letters it
+ # instead of retrying something that will fail identically every time.
for exc in (ValueError("priority out of range"), TypeError("not serializable")):
- with (
- patch.object(nd, "resolve_transport", return_value="pg_queue"),
- patch.object(nd, "enqueue_task", side_effect=exc),
- ):
+ with patch.object(nd, "enqueue_task", side_effect=exc):
with pytest.raises(nd.PermanentDispatchError):
- _dispatch(celery)
- celery.send_task.assert_not_called()
-
- def test_none_org_fails_closed_to_celery(self):
- # A missing org string (org deleted) must not route to PG — resolve_transport
- # fails closed, and we pass organization_id=None straight through to it.
- celery = MagicMock()
- with (
- patch.object(nd, "resolve_transport", return_value="celery") as resolve,
- patch.object(nd, "enqueue_task") as enqueue,
- ):
- _dispatch(celery, org_string_id=None)
- assert resolve.call_args.kwargs["organization_id"] is None
- celery.send_task.assert_called_once()
- enqueue.assert_not_called()
+ _dispatch()
- def test_resolve_transport_buckets_by_minted_dispatch_id(self):
- # Fire-and-forget: entity_id is a freshly minted uuid (str), and it equals
- # the PG task_id so the row and the Flipt bucket agree.
- celery = MagicMock()
- with (
- patch.object(nd, "resolve_transport", return_value="pg_queue") as resolve,
- patch.object(nd, "enqueue_task", return_value=1) as enqueue,
- ):
- task_id = _dispatch(celery).task_id
- assert resolve.call_args.kwargs["execution_id"] == task_id
- assert enqueue.call_args.kwargs["task_id"] == task_id
+ def test_none_org_is_coerced_to_empty_string_on_the_row(self):
+ # enqueue_task types org_id as str; a deleted org must not put None on the
+ # row. (The producer re-coerces too, so this pins the seam's own contract.)
+ with patch.object(nd, "enqueue_task", return_value=1) as enqueue:
+ _dispatch(org_string_id=None)
+ assert enqueue.call_args.kwargs["org_id"] == ""
- def test_args_and_queue_identical_on_both_paths(self):
- # The consumer must behave the same regardless of transport — args/queue are
- # byte-identical, and kwargs match apart from the transport-specific
- # raise_on_final_failure override (asserted separately below).
- celery = MagicMock()
- with patch.object(nd, "resolve_transport", return_value="celery"):
- _dispatch(celery)
- celery_call = celery.send_task.call_args.kwargs
- celery2 = MagicMock()
- with (
- patch.object(nd, "resolve_transport", return_value="pg_queue"),
- patch.object(nd, "enqueue_task", return_value=1) as enqueue,
- ):
- _dispatch(celery2)
- assert enqueue.call_args.kwargs["args"] == celery_call["args"]
- assert enqueue.call_args.kwargs["queue"] == celery_call["queue"]
- # kwargs differ ONLY by the two PG-forced retry-semantics keys.
- pg_kwargs = enqueue.call_args.kwargs["kwargs"]
- assert pg_kwargs == {
- **celery_call["kwargs"],
- "raise_on_final_failure": False,
- "max_retries": 0,
- }
-
- def test_pg_forces_terminal_branch_kwargs(self):
+ def test_forces_terminal_branch_kwargs(self):
# Regression (UN-3753): the PG consumer runs the task eagerly via
# ``task.apply(..., throw=True)``, where the in-task retry loop cannot work:
# * max_retries >= 1 -> the worker's ``request.retries < max_retries`` guard
@@ -164,29 +93,17 @@ class TestDispatchWebhookNotification:
# (mark DEAD_LETTER + honour raise_on_final_failure) never runs and the row
# is left for vt-expiry redelivery, re-POSTing the subscriber each time.
# * a terminal re-raise is likewise treated as failure -> redelivery.
- # So the seam must force BOTH max_retries=0 and raise_on_final_failure=False on
- # the PG branch. The Celery branch keeps the caller's values verbatim.
+ # So the seam must force BOTH max_retries=0 and raise_on_final_failure=False.
# (The end-to-end behaviour these kwargs buy — one POST + one dead-letter mark
# through a real ``task.apply()`` — is asserted in
# workers/tests/test_notification_pg_terminal.py.)
assert _KWARGS["raise_on_final_failure"] is True # guard the fixture premise
assert _KWARGS["max_retries"] == 3 # ... and that retries are configured
- celery = MagicMock()
- with (
- patch.object(nd, "resolve_transport", return_value="pg_queue"),
- patch.object(nd, "enqueue_task", return_value=1) as enqueue,
- ):
- _dispatch(celery)
+ with patch.object(nd, "enqueue_task", return_value=1) as enqueue:
+ _dispatch()
pg_kwargs = enqueue.call_args.kwargs["kwargs"]
assert pg_kwargs["raise_on_final_failure"] is False
assert pg_kwargs["max_retries"] == 0
# The caller's dict is not mutated in place (a fresh dict is enqueued).
assert _KWARGS["raise_on_final_failure"] is True
assert _KWARGS["max_retries"] == 3
-
- celery2 = MagicMock()
- with patch.object(nd, "resolve_transport", return_value="celery"):
- _dispatch(celery2)
- sent_kwargs = celery2.send_task.call_args.kwargs["kwargs"]
- assert sent_kwargs["raise_on_final_failure"] is True
- assert sent_kwargs["max_retries"] == 3
diff --git a/backend/pg_queue/executor_rpc.py b/backend/pg_queue/executor_rpc.py
index 1bed4a03d..b76b05787 100644
--- a/backend/pg_queue/executor_rpc.py
+++ b/backend/pg_queue/executor_rpc.py
@@ -1,14 +1,14 @@
"""Executor-RPC for the PG path — backend (Django) transport adapter.
-The gate + reply_key/timeout orchestration + routing live ONCE in
+The reply_key/timeout orchestration lives ONCE in
``unstract.workflow_execution.executor_rpc`` (shared with the workers). This module
is the thin Django half: a :class:`DjangoQueueTransport` that enqueues via the ORM
(``enqueue_task``) and polls ``PgTaskResult``, plus the :func:`get_executor_dispatcher`
factory that wires them together.
-Zero-regression: with the ``pg_queue_enabled`` Flipt flag off the routing dispatcher
-delegates every mode to the unchanged Celery ``ExecutionDispatcher`` and no
-``pg_task_result`` row is created.
+Since UN-4046 the factory returns the PG dispatcher unconditionally: there is no
+gate, no routing dispatcher and no Celery fall-through, so a ``pg_task_result``
+row is written on every request-reply dispatch.
"""
from __future__ import annotations
@@ -21,14 +21,11 @@ from django.db import close_old_connections
from pg_queue.models import PgTaskResult
from pg_queue.producer import enqueue_task
from unstract.core.polling import poll_for_row
-from unstract.sdk1.execution.dispatcher import ExecutionDispatcher
from unstract.workflow_execution.executor_rpc import (
EXECUTE_TASK,
ExecResultRow,
PgExecutionDispatcher,
QueueTransport,
- RoutingExecutionDispatcher,
- resolve_pg_transport,
)
if TYPE_CHECKING:
@@ -41,20 +38,10 @@ logger = logging.getLogger(__name__)
__all__ = [
"DjangoQueueTransport",
"PgExecutionDispatcher",
- "RoutingExecutionDispatcher",
"get_executor_dispatcher",
- "resolve_executor_transport",
]
-def resolve_executor_transport(context: ExecutionContext) -> bool:
- """True → route this executor dispatch over PG; False → Celery (default).
-
- The single ``pg_queue_enabled`` Flipt flag (fail-closed).
- """
- return resolve_pg_transport(context)
-
-
class DjangoQueueTransport(QueueTransport):
""":class:`QueueTransport` over the Django ORM (the backend half).
@@ -122,12 +109,15 @@ class DjangoQueueTransport(QueueTransport):
return row
-def get_executor_dispatcher(
- celery_app: object | None = None,
-) -> RoutingExecutionDispatcher:
- """Factory: the gate-routed executor dispatcher (PG when enabled, else Celery)."""
- return RoutingExecutionDispatcher(
- celery=ExecutionDispatcher(celery_app=celery_app),
- pg=PgExecutionDispatcher(DjangoQueueTransport()),
- resolve=resolve_executor_transport,
- )
+def get_executor_dispatcher() -> PgExecutionDispatcher:
+ """Factory: the executor dispatcher.
+
+ Takes no arguments. It used to accept a ``celery_app`` that fed the Celery
+ branch of the routing dispatcher; that branch went with the ``pg_queue_enabled``
+ flag (UN-4046), and the parameter was kept for a while so the call sites did not
+ all need editing at once. Keeping an ignored parameter made the signature
+ decoration rather than a contract — and it read as "this dispatcher may use
+ Celery", which is exactly the belief that left a ``headers=`` argument at three
+ call sites and broke every extraction. Removed.
+ """
+ return PgExecutionDispatcher(DjangoQueueTransport())
diff --git a/backend/pg_queue/flags.py b/backend/pg_queue/flags.py
deleted file mode 100644
index cf190134c..000000000
--- a/backend/pg_queue/flags.py
+++ /dev/null
@@ -1,12 +0,0 @@
-"""Single source of truth for the PG-queue rollout flag key.
-
-One Flipt flag gates the whole PG-queue feature — execution
-(``workflow_v2/transport.py``), scheduler (``scheduler/ownership.py``), and
-executor (``pg_queue/executor_rpc.py``) all read this one key. Kept in a neutral
-leaf module so the three resolvers import a single constant instead of
-duplicating the literal (a grep on ``PG_QUEUE_FLAG_KEY`` finds every use), making
-"one flag" a structural guarantee. This flag is the **sole** rollout control —
-fail-closed to Celery on a blind/unreachable Flipt or any error.
-"""
-
-PG_QUEUE_FLAG_KEY = "pg_queue_enabled"
diff --git a/backend/pg_queue/management/commands/reconcile_pg_schedules.py b/backend/pg_queue/management/commands/reconcile_pg_schedules.py
index f5e9cd6f4..af7e163fb 100644
--- a/backend/pg_queue/management/commands/reconcile_pg_schedules.py
+++ b/backend/pg_queue/management/commands/reconcile_pg_schedules.py
@@ -260,7 +260,7 @@ class Command(BaseCommand):
# Preview only — read the would-be owner (no DB write) so an
# operator can see how many a ramp change would hand to PG.
reconciled += 1
- if resolve_schedule_owner(str(row.pipeline_id), row.organization_id):
+ if resolve_schedule_owner():
pg_owned += 1
continue
# mirror.enabled tracks pipeline.active (dual-write); use it as the
diff --git a/backend/pg_queue/tests/test_converge_pg_scheduler.py b/backend/pg_queue/tests/test_converge_pg_scheduler.py
index f0edae1fe..b66ba222e 100644
--- a/backend/pg_queue/tests/test_converge_pg_scheduler.py
+++ b/backend/pg_queue/tests/test_converge_pg_scheduler.py
@@ -48,15 +48,17 @@ class TestDirection:
assert kwargs["mirror_only"] is True
assert kwargs["release_stale"] is True
- def test_an_unset_gate_converges_to_beat(self, monkeypatch):
- """Absent must mean Beat, not "do nothing" — the default has to be the
- safe direction, since that is what an environment that never opted in has.
+ def test_an_unset_gate_converges_to_pg(self, monkeypatch):
+ """Absent must mean PG, not "do nothing" — the default has to be the
+ direction the deployment actually runs, and PG_SCHEDULER_ENABLED now
+ defaults on (UN-4046).
"""
monkeypatch.delenv("PG_SCHEDULER_ENABLED", raising=False)
with patch(f"{_CMD}.call_command") as sub:
call_command("converge_pg_scheduler")
kwargs = {c.args[0]: c.kwargs for c in sub.call_args_list}
- assert kwargs["reconcile_pg_schedules"]["release_stale"] is True
+ # Hand-over direction: adopt, not release.
+ assert "release_stale" not in kwargs["reconcile_pg_schedules"]
class TestPeriodicsAreOptIn:
diff --git a/backend/pg_queue/tests/test_executor_rpc.py b/backend/pg_queue/tests/test_executor_rpc.py
index 697d53d46..0a51ecb91 100644
--- a/backend/pg_queue/tests/test_executor_rpc.py
+++ b/backend/pg_queue/tests/test_executor_rpc.py
@@ -10,14 +10,11 @@ single-Flipt-flag gate, and the factory wiring.
from unittest.mock import MagicMock, patch
-from unstract.workflow_execution.executor_rpc import ExecResultRow
-
from pg_queue.executor_rpc import (
DjangoQueueTransport,
- RoutingExecutionDispatcher,
get_executor_dispatcher,
- resolve_executor_transport,
)
+from unstract.workflow_execution.executor_rpc import ExecResultRow, PgExecutionDispatcher
_MOD = "pg_queue.executor_rpc"
@@ -34,7 +31,9 @@ class TestDjangoQueueTransportEnqueue:
def test_enqueue_calls_enqueue_task_with_request_reply_fields(self):
with patch(f"{_MOD}.enqueue_task") as enq:
DjangoQueueTransport().enqueue(
- queue="celery_executor_legacy", context=_ctx(), org_id="org9",
+ queue="celery_executor_legacy",
+ context=_ctx(),
+ org_id="org9",
reply_key="rk1",
)
kw = enq.call_args.kwargs
@@ -49,8 +48,11 @@ class TestDjangoQueueTransportEnqueue:
spec = {"task_name": "ide_prompt_complete", "kwargs": {}, "queue": "ide_callback"}
with patch(f"{_MOD}.enqueue_task") as enq:
DjangoQueueTransport().enqueue(
- queue="celery_executor_legacy", context=_ctx(), org_id="o",
- on_success=spec, task_id="tid-7",
+ queue="celery_executor_legacy",
+ context=_ctx(),
+ org_id="o",
+ on_success=spec,
+ task_id="tid-7",
)
kw = enq.call_args.kwargs
assert kw["on_success"] == spec and kw["task_id"] == "tid-7"
@@ -116,23 +118,10 @@ class TestDjangoQueueTransportWait:
qs.filter.return_value.update.assert_not_called()
-class TestResolveExecutorTransport:
- def test_delegates_to_shared_flipt_resolver_true(self):
- with patch(f"{_MOD}.resolve_pg_transport", return_value=True) as r:
- assert resolve_executor_transport(_ctx()) is True
- r.assert_called_once()
- # No master-gate is threaded any more — Flipt is the sole gate.
- assert "master_gate_enabled" not in r.call_args.kwargs
-
- def test_delegates_to_shared_flipt_resolver_false(self):
- with patch(f"{_MOD}.resolve_pg_transport", return_value=False):
- assert resolve_executor_transport(_ctx()) is False
-
-
class TestFactoryWiring:
- def test_factory_wires_routing_with_django_transport(self):
- d = get_executor_dispatcher(celery_app="app")
- assert isinstance(d, RoutingExecutionDispatcher)
- # PG dispatcher uses the backend ORM transport; gate = the Flipt resolver.
- assert isinstance(d._pg._transport, DjangoQueueTransport)
- assert d._resolve is resolve_executor_transport
+ def test_factory_wires_the_django_transport(self):
+ # The factory takes no arguments (UN-4046) and returns the PG dispatcher
+ # wired with the backend ORM transport.
+ d = get_executor_dispatcher()
+ assert isinstance(d, PgExecutionDispatcher)
+ assert isinstance(d._transport, DjangoQueueTransport)
diff --git a/backend/pipeline_v2/piepline_api_execution_views.py b/backend/pipeline_v2/piepline_api_execution_views.py
index ca0ba8fa3..5b78e020e 100644
--- a/backend/pipeline_v2/piepline_api_execution_views.py
+++ b/backend/pipeline_v2/piepline_api_execution_views.py
@@ -5,7 +5,6 @@ from rest_framework import status, views
from rest_framework.request import Request
from rest_framework.response import Response
-from backend.celery_service import app as celery_app
from pipeline_v2.deployment_helper import DeploymentHelper
from pipeline_v2.models import Pipeline
from pipeline_v2.pipeline_dispatch import dispatch_pipeline_trigger
@@ -30,10 +29,7 @@ class PipelineApiExecution(views.APIView):
def post(
self, request: Request, org_name: str, pipeline_id: str, pipeline: Pipeline
) -> Response:
- # Route through the transport flag instead of hardcoding Celery
- # (see dispatch_pipeline_trigger).
dispatch_pipeline_trigger(
- celery_app=celery_app,
org_id=org_name,
pipeline_id=pipeline_id,
pipeline_name=pipeline.pipeline_name,
diff --git a/backend/pipeline_v2/pipeline_dispatch.py b/backend/pipeline_v2/pipeline_dispatch.py
index c8d8002d0..7fc39f35e 100644
--- a/backend/pipeline_v2/pipeline_dispatch.py
+++ b/backend/pipeline_v2/pipeline_dispatch.py
@@ -1,29 +1,24 @@
-"""Transport-routed dispatch for the API-triggered pipeline execution trigger.
+"""PG-queue dispatch for the API-triggered pipeline execution trigger.
-Routes ``scheduler.tasks.execute_pipeline_task`` through the same
-:func:`resolve_transport` flag as the rest of the execution path: the PG queue
-when enabled for this pipeline/org, else Celery. **Fail-closed** — with the
-master gate off (the production default) it resolves to Celery, behaving exactly
-like the prior unconditional ``send_task`` (zero regression). The
-scheduled-pipeline path already enqueues this task onto the PG ``scheduler``
-queue (``pg_scheduler.dispatch_due_schedules``, run by ``worker-pg-scheduler``);
-this brings the API-trigger view to parity.
+Enqueues ``scheduler.tasks.execute_pipeline_task`` onto the PG ``scheduler``
+queue, where ``worker-pg-scheduler`` polls it — the same queue and task the
+scheduled-pipeline path uses (``pg_scheduler.dispatch_due_schedules``), so the
+API trigger and the schedule trigger are at parity.
-The same ``args`` are sent on both paths; on PG they're JSON-normalized by
-``enqueue_task`` (UUIDs/datetimes → str), a no-op for these string/bool values,
-so the consumer sees the same payload.
+This used to route through ``resolve_transport``/``pg_queue_enabled`` and fall
+back to ``celery_app.send_task``. The flag is gone (UN-4046) and PG is the only
+transport, so the Celery branch went with it.
+
+``args`` are JSON-normalized by ``enqueue_task`` (UUIDs/datetimes → str), a no-op
+for these string/bool values.
"""
from __future__ import annotations
import logging
-from typing import Any
from uuid import UUID
from pg_queue.producer import enqueue_task
-from workflow_manager.workflow_v2.transport import resolve_transport
-
-from unstract.core.data_models import is_pg_transport
logger = logging.getLogger(__name__)
@@ -36,40 +31,26 @@ SCHEDULER_QUEUE = "scheduler"
def dispatch_pipeline_trigger(
*,
- celery_app: Any,
org_id: str | UUID,
pipeline_id: str | UUID,
pipeline_name: str,
) -> None:
- """Dispatch the pipeline-trigger task on the resolved transport.
+ """Enqueue the pipeline-trigger task on the PG scheduler queue.
- The positional args match ``execute_pipeline_task``'s signature on **both**
- paths: ``(workflow_id, org_schema, execution_action, execution_id,
- pipeline_id, with_logs, name)``. ``org_id`` / ``pipeline_id`` accept ``UUID``
- (what ``resolve_transport`` takes) and are str-coerced into the task args.
+ The positional args match ``execute_pipeline_task``'s signature:
+ ``(workflow_id, org_schema, execution_action, execution_id, pipeline_id,
+ with_logs, name)``. ``org_id`` / ``pipeline_id`` accept ``UUID`` and are
+ str-coerced into the task args.
"""
args = ["", str(org_id), "", "", str(pipeline_id), True, pipeline_name]
- # No execution exists yet (it's created inside the task), so the trigger
- # buckets the flag by pipeline_id as the sticky entity.
- transport = resolve_transport(
- execution_id=pipeline_id,
- organization_id=org_id,
- pipeline_id=pipeline_id,
+ msg_id = enqueue_task(
+ task_name=PIPELINE_TRIGGER_TASK,
+ queue=SCHEDULER_QUEUE,
+ args=args,
+ org_id=str(org_id or ""),
+ )
+ logger.info(
+ "Pipeline %s trigger enqueued on PG scheduler queue (msg_id=%s)",
+ pipeline_id,
+ msg_id,
)
- # Use the shared is_pg_transport() — the single source for "what counts as PG
- # transport" — rather than opening a second comparison site.
- if is_pg_transport(transport):
- msg_id = enqueue_task(
- task_name=PIPELINE_TRIGGER_TASK,
- queue=SCHEDULER_QUEUE,
- args=args,
- org_id=str(org_id or ""),
- )
- logger.info(
- "Pipeline %s trigger enqueued on PG scheduler queue (msg_id=%s)",
- pipeline_id,
- msg_id,
- )
- else:
- celery_app.send_task(PIPELINE_TRIGGER_TASK, args=args)
- logger.info("Pipeline %s trigger dispatched on Celery", pipeline_id)
diff --git a/backend/pipeline_v2/tests/test_pipeline_dispatch.py b/backend/pipeline_v2/tests/test_pipeline_dispatch.py
index ac8bd63e0..64ad6b7d2 100644
--- a/backend/pipeline_v2/tests/test_pipeline_dispatch.py
+++ b/backend/pipeline_v2/tests/test_pipeline_dispatch.py
@@ -1,14 +1,19 @@
-"""Unit tests for the API-triggered pipeline-trigger transport routing (UN-3616).
+"""Unit tests for the API-triggered pipeline-trigger dispatch (UN-3616, UN-4046).
-`resolve_transport` + `enqueue_task` are patched on the module, so no Flipt / DB
-is needed — these pin the routing contract: PG when the flag resolves PG, Celery
-otherwise (fail-closed), with byte-identical args on both paths.
+``enqueue_task`` is patched on the module, so no DB is needed. These pin the
+payload contract the ``scheduler`` consumer depends on, and the no-silent-fallback
+property.
+
+The transport-routing tests are gone with the ``pg_queue_enabled`` flag (UN-4046):
+there is no Celery branch left to choose, so "routes to celery", "resolve_transport
+called with the pipeline as entity" and "args identical on both paths" no longer
+describe anything. What survives is what the consumer actually sees.
"""
from __future__ import annotations
import uuid
-from unittest.mock import MagicMock, patch
+from unittest.mock import patch
import pytest
@@ -17,9 +22,8 @@ import pipeline_v2.pipeline_dispatch as pd
_EXPECTED_ARGS = ["", "org_x", "", "", "pid-1", True, "My Pipeline"]
-def _dispatch(celery_app):
+def _dispatch():
return pd.dispatch_pipeline_trigger(
- celery_app=celery_app,
org_id="org_x",
pipeline_id="pid-1",
pipeline_name="My Pipeline",
@@ -27,83 +31,29 @@ def _dispatch(celery_app):
class TestDispatchPipelineTrigger:
- def test_routes_to_pg_when_flag_resolves_pg(self):
- celery = MagicMock()
- with (
- patch.object(pd, "resolve_transport", return_value="pg_queue"),
- patch.object(pd, "enqueue_task", return_value=42) as enqueue,
- ):
- _dispatch(celery)
+ def test_enqueues_on_the_pg_scheduler_queue(self):
+ with patch.object(pd, "enqueue_task", return_value=42) as enqueue:
+ _dispatch()
enqueue.assert_called_once()
kwargs = enqueue.call_args.kwargs
assert kwargs["task_name"] == "scheduler.tasks.execute_pipeline_task"
assert kwargs["queue"] == "scheduler"
assert kwargs["args"] == _EXPECTED_ARGS
assert kwargs["org_id"] == "org_x"
- celery.send_task.assert_not_called()
- def test_pg_enqueue_failure_propagates_with_no_celery_fallback(self):
- # The dispatcher has no try/except → a PG enqueue failure must surface
- # (no silent Celery fallback, which would risk a double-dispatch).
- celery = MagicMock()
- with (
- patch.object(pd, "resolve_transport", return_value="pg_queue"),
- patch.object(pd, "enqueue_task", side_effect=RuntimeError("pg down")),
- ):
+ def test_pg_enqueue_failure_propagates(self):
+ # The dispatcher has no try/except → a PG enqueue failure must surface to
+ # the caller rather than being swallowed into a silent no-dispatch.
+ with patch.object(pd, "enqueue_task", side_effect=RuntimeError("pg down")):
with pytest.raises(RuntimeError, match="pg down"):
- _dispatch(celery)
- celery.send_task.assert_not_called()
+ _dispatch()
def test_uuid_pipeline_id_is_coerced_to_str_in_args(self):
- # resolve_transport accepts a UUID, but the task args must carry strings.
+ # The signature accepts a UUID, but the task args must carry strings —
+ # enqueue_task JSON-normalizes, and the consumer unpacks positionally.
pid = uuid.UUID("b1f16024-45f2-4e39-8756-d40e24148e30")
- celery = MagicMock()
- with (
- patch.object(pd, "resolve_transport", return_value="pg_queue") as resolve,
- patch.object(pd, "enqueue_task", return_value=1) as enqueue,
- ):
+ with patch.object(pd, "enqueue_task", return_value=1) as enqueue:
pd.dispatch_pipeline_trigger(
- celery_app=celery, org_id="org_x", pipeline_id=pid, pipeline_name="P"
+ org_id="org_x", pipeline_id=pid, pipeline_name="P"
)
assert enqueue.call_args.kwargs["args"][4] == str(pid)
- # The raw UUID is passed to resolve_transport as the sticky entity.
- assert resolve.call_args.kwargs["execution_id"] == pid
-
- def test_routes_to_celery_when_flag_resolves_celery(self):
- celery = MagicMock()
- with (
- patch.object(pd, "resolve_transport", return_value="celery"),
- patch.object(pd, "enqueue_task") as enqueue,
- ):
- _dispatch(celery)
- celery.send_task.assert_called_once_with(
- "scheduler.tasks.execute_pipeline_task", args=_EXPECTED_ARGS
- )
- enqueue.assert_not_called()
-
- def test_resolve_transport_called_with_pipeline_as_entity(self):
- celery = MagicMock()
- with (
- patch.object(pd, "resolve_transport", return_value="celery") as resolve,
- patch.object(pd, "enqueue_task"),
- ):
- _dispatch(celery)
- kwargs = resolve.call_args.kwargs
- assert kwargs["execution_id"] == "pid-1" # buckets by pipeline_id
- assert kwargs["organization_id"] == "org_x"
- assert kwargs["pipeline_id"] == "pid-1"
-
- def test_args_identical_on_both_paths(self):
- # The consumer must behave the same regardless of transport — assert the
- # PG-path args equal the Celery-path args.
- celery = MagicMock()
- with patch.object(pd, "resolve_transport", return_value="celery"):
- _dispatch(celery)
- celery_args = celery.send_task.call_args.kwargs["args"]
- celery2 = MagicMock()
- with (
- patch.object(pd, "resolve_transport", return_value="pg_queue"),
- patch.object(pd, "enqueue_task", return_value=1) as enqueue,
- ):
- _dispatch(celery2)
- assert enqueue.call_args.kwargs["args"] == celery_args
diff --git a/backend/prompt_studio/prompt_studio_core_v2/prompt_studio_helper.py b/backend/prompt_studio/prompt_studio_core_v2/prompt_studio_helper.py
index ba157801c..94f3850c6 100644
--- a/backend/prompt_studio/prompt_studio_core_v2/prompt_studio_helper.py
+++ b/backend/prompt_studio/prompt_studio_core_v2/prompt_studio_helper.py
@@ -26,7 +26,6 @@ from utils.file_storage.constants import FileStorageKeys
from utils.file_storage.helpers.prompt_studio_file_helper import PromptStudioFileHelper
from utils.local_context import StateStore
-from backend.celery_service import app as celery_app
from prompt_studio.lookup_utils import (
get_lookup_config,
get_lookup_configs_for_tool,
@@ -327,15 +326,17 @@ class PromptStudioHelper:
def _get_dispatcher():
"""Executor dispatcher for the executor worker.
- Gate-routed: when ``pg_queue_enabled`` is on the blocking
- ``dispatch()`` rides the PG request-reply transport; otherwise — and for
- all async/callback dispatches — it is the unchanged Celery
- ``ExecutionDispatcher``. The decision is read per dispatch, so flipping
- the flag is an instant, no-redeploy rollout/rollback.
+ Always the PG request-reply dispatcher since UN-4046. This used to be
+ gate-routed per dispatch on ``pg_queue_enabled``, falling back to the
+ Celery ``ExecutionDispatcher``; with the flag gone the factory returns
+ the PG dispatcher directly. It is kept as a factory call rather than a
+ direct construction on purpose — UN-3779 was a hardcoded
+ ``ExecutionDispatcher`` here that published to RabbitMQ and hung for its
+ full 1200s timeout with no consumer.
"""
from pg_queue.executor_rpc import get_executor_dispatcher
- return get_executor_dispatcher(celery_app=celery_app)
+ return get_executor_dispatcher()
@staticmethod
def _get_platform_api_key(org_id: str) -> str:
diff --git a/backend/prompt_studio/prompt_studio_core_v2/tests/test_task_status.py b/backend/prompt_studio/prompt_studio_core_v2/tests/test_task_status.py
index ab0754ee1..64864bc2c 100644
--- a/backend/prompt_studio/prompt_studio_core_v2/tests/test_task_status.py
+++ b/backend/prompt_studio/prompt_studio_core_v2/tests/test_task_status.py
@@ -1,11 +1,13 @@
-"""UN-3693: PromptStudioCoreView.task_status — flag-gated.
+"""UN-3693 / UN-4046: PromptStudioCoreView.task_status.
-Gate on the per-org ``pg_queue_enabled`` flag (the same the executor dispatch uses):
-flag ON → read ``pg_task_result`` (completed / failed / none→processing); flag OFF
-(default) → Celery ``AsyncResult`` unchanged, and the PG table is never queried on
-the hot poll path. ``check_feature_flag_status`` already fails closed to ``False`` on
-any Flipt error, so "flag off / Flipt down" is one path. A transient PG DB error
-degrades to "processing", not a bare 500.
+The task rode PG, so status is read from ``pg_task_result``: completed / failed /
+no row → processing. A transient PG DB error degrades to "processing", not a bare
+500 — a status-code-keyed client would misread a 500 as a terminal task failure.
+
+This used to be gated on the per-org ``pg_queue_enabled`` flag, with a Celery
+``AsyncResult`` fall-through when it was off. The flag is gone (UN-4046), so the
+flag-off suite went with it — there is no Celery result backend to poll (the eager
+PG executor never wrote one, which is why the gate existed at all).
NOTE: run via a Django-bootstrapped harness (no pytest-django in this repo yet — the
CI-gating of these view tests is tracked in UN-3692). Verified locally green.
@@ -15,7 +17,6 @@ from unittest.mock import MagicMock, patch
from prompt_studio.prompt_studio_core_v2.views import PromptStudioCoreView
-_VIEWS = "prompt_studio.prompt_studio_core_v2.views"
_TID = "task-abc"
@@ -25,14 +26,6 @@ def _view():
return view
-def _flag(enabled):
- return patch(f"{_VIEWS}.check_feature_flag_status", return_value=enabled)
-
-
-def _org():
- return patch(f"{_VIEWS}.UserSessionUtils.get_organization_id", return_value="org1")
-
-
def _stub_pg(pg_row=None, *, error=None):
"""Patch pg_task_result read: .filter().values().first() → pg_row, or raise."""
m = MagicMock()
@@ -43,72 +36,30 @@ def _stub_pg(pg_row=None, *, error=None):
return patch("pg_queue.models.PgTaskResult", m), m
-def _async(ready, successful=True, result=""):
- ar = MagicMock()
- ar.return_value.ready.return_value = ready
- ar.return_value.successful.return_value = successful
- ar.return_value.result = result
- return ar
-
-
-class TestTaskStatusFlagOn:
- def test_pg_completed(self):
+class TestTaskStatus:
+ def test_completed(self):
p_pg, _ = _stub_pg({"status": "completed", "error": ""})
- with _org(), _flag(True), p_pg:
+ with p_pg:
resp = _view().task_status(MagicMock(), task_id=_TID)
assert resp.data == {"task_id": _TID, "status": "completed"}
- def test_pg_failed(self):
+ def test_failed(self):
p_pg, _ = _stub_pg({"status": "failed", "error": "extraction blew up"})
- with _org(), _flag(True), p_pg:
+ with p_pg:
resp = _view().task_status(MagicMock(), task_id=_TID)
assert resp.data["status"] == "failed"
assert resp.data["error"] == "extraction blew up"
assert resp.status_code == 500
- def test_no_pg_row_is_processing(self):
+ def test_no_row_is_processing(self):
p_pg, _ = _stub_pg(None) # PG task not done yet — no "pending" row
- with _org(), _flag(True), p_pg:
+ with p_pg:
resp = _view().task_status(MagicMock(), task_id=_TID)
assert resp.data == {"task_id": _TID, "status": "processing"}
- def test_pg_db_error_degrades_to_processing(self):
+ def test_db_error_degrades_to_processing(self):
# A transient PG read error must not 500; degrade to "processing".
p_pg, _ = _stub_pg(error=RuntimeError("db blip"))
- with _org(), _flag(True), p_pg:
- resp = _view().task_status(MagicMock(), task_id=_TID)
- assert resp.data == {"task_id": _TID, "status": "processing"}
-
-
-class TestTaskStatusFlagOff:
- def test_celery_completed_never_queries_pg(self):
- # flag off (also the Flipt-error fail-closed path): Celery + PG untouched.
- p_pg, pg = _stub_pg({"status": "completed"}) # would resolve, if queried
- with (
- _org(),
- _flag(False),
- p_pg,
- patch(f"{_VIEWS}.AsyncResult", _async(ready=True)),
- ):
- resp = _view().task_status(MagicMock(), task_id=_TID)
- assert resp.data == {"task_id": _TID, "status": "completed"}
- pg.objects.filter.assert_not_called() # default path never touches PG
-
- def test_celery_failed(self):
- with (
- _org(),
- _flag(False),
- patch(
- f"{_VIEWS}.AsyncResult",
- _async(ready=True, successful=False, result="celery boom"),
- ),
- ):
- resp = _view().task_status(MagicMock(), task_id=_TID)
- assert resp.data["status"] == "failed"
- assert "celery boom" in resp.data["error"]
- assert resp.status_code == 500
-
- def test_pending(self):
- with _org(), _flag(False), patch(f"{_VIEWS}.AsyncResult", _async(ready=False)):
+ with p_pg:
resp = _view().task_status(MagicMock(), task_id=_TID)
assert resp.data == {"task_id": _TID, "status": "processing"}
diff --git a/backend/prompt_studio/prompt_studio_core_v2/views.py b/backend/prompt_studio/prompt_studio_core_v2/views.py
index b3d42cf5f..8990e870e 100644
--- a/backend/prompt_studio/prompt_studio_core_v2/views.py
+++ b/backend/prompt_studio/prompt_studio_core_v2/views.py
@@ -9,7 +9,6 @@ from typing import Any
import magic
from account_v2.custom_exceptions import DuplicateData
from celery import signature
-from celery.result import AsyncResult
from django.db import IntegrityError
from django.db.models import Count, OuterRef, QuerySet, Subquery
from django.http import HttpRequest, HttpResponse
@@ -20,7 +19,6 @@ from permissions.membership_views import OwnerManagementMixin
from permissions.permission import IsOwner, IsOwnerOrSharedUserOrSharedToOrg
from permissions.resource_share_views import ResourceShareManagementMixin
from permissions.roles import ResourceRole
-from pg_queue.flags import PG_QUEUE_FLAG_KEY
from plugins import get_plugin
from rest_framework import status, viewsets
from rest_framework.decorators import action
@@ -33,7 +31,6 @@ from utils.pagination import OptionalPagination
from utils.user_context import UserContext
from utils.user_session import UserSessionUtils
-from backend.celery_service import app as celery_app
from prompt_studio.lookup_utils import (
get_latest_lookup_mutation_for_tool,
get_lookup_validation_for_tool,
@@ -86,7 +83,6 @@ from prompt_studio.tool_usage import (
join_deployment_types,
)
from unstract.core.data_models import PgTaskStatus
-from unstract.flags.feature_flag import check_feature_flag_status
from unstract.sdk1.utils.common import Utils as CommonUtils
from .models import CustomTool
@@ -865,57 +861,34 @@ class PromptStudioCoreView(
# Verify the user has access to this tool (triggers permission check)
self.get_object()
- # Gate on the same per-org ``pg_queue_enabled`` flag the executor dispatch
- # uses (``resolve_pg_transport`` — bucketed per org): flag ON → the task rode
- # PG, which records its terminal state in ``pg_task_result`` (the eager PG
- # executor never writes a Celery result backend, so ``AsyncResult`` alone
- # would poll "processing" forever — UN-3693); flag OFF (default) → Celery,
- # unchanged, and PG is not touched. ``entity_id``/``context`` match
- # ``resolve_pg_transport`` so the per-org bucket agrees with the dispatch.
- # No wrapper: ``check_feature_flag_status`` already fails closed to ``False``
- # (its own try/except + warning) on any Flipt error → reads as "off" → Celery.
- org_id = str(UserSessionUtils.get_organization_id(request) or "")
- pg_enabled = check_feature_flag_status(
- flag_key=PG_QUEUE_FLAG_KEY,
- entity_id=org_id or "default",
- context={"organization_id": org_id},
- )
- if pg_enabled:
- from pg_queue.models import PgTaskResult
+ # The task rode PG, which records its terminal state in ``pg_task_result``.
+ # (The eager PG executor never writes a Celery result backend, so the old
+ # ``AsyncResult`` poll would have said "processing" forever — UN-3693.)
+ from pg_queue.models import PgTaskResult
- try:
- row = (
- PgTaskResult.objects.filter(task_id=task_id)
- .values("status", "error")
- .first()
- )
- except Exception:
- # A transient DB blip degrades to "processing" (a poll retries) rather
- # than a bare 500 that a status-code-keyed client would misread as a
- # terminal task failure.
- logger.exception(
- "task_status: pg_task_result read failed for %s; treating as pending",
- task_id,
- )
- row = None
- # No row → not finished yet (there is no "pending" row). Status compared
- # against PgTaskStatus (the writer's source of truth), not a bare literal.
- if row is None:
- return Response({"task_id": task_id, "status": "processing"})
- if row["status"] == PgTaskStatus.COMPLETED.value:
- return Response({"task_id": task_id, "status": "completed"})
- return Response(
- {"task_id": task_id, "status": "failed", "error": row["error"] or ""},
- status=status.HTTP_500_INTERNAL_SERVER_ERROR,
+ try:
+ row = (
+ PgTaskResult.objects.filter(task_id=task_id)
+ .values("status", "error")
+ .first()
)
-
- result = AsyncResult(task_id, app=celery_app)
- if not result.ready():
+ except Exception:
+ # A transient DB blip degrades to "processing" (a poll retries) rather
+ # than a bare 500 that a status-code-keyed client would misread as a
+ # terminal task failure.
+ logger.exception(
+ "task_status: pg_task_result read failed for %s; treating as pending",
+ task_id,
+ )
+ row = None
+ # No row → not finished yet (there is no "pending" row). Status compared
+ # against PgTaskStatus (the writer's source of truth), not a bare literal.
+ if row is None:
return Response({"task_id": task_id, "status": "processing"})
- if result.successful():
+ if row["status"] == PgTaskStatus.COMPLETED.value:
return Response({"task_id": task_id, "status": "completed"})
return Response(
- {"task_id": task_id, "status": "failed", "error": str(result.result)},
+ {"task_id": task_id, "status": "failed", "error": row["error"] or ""},
status=status.HTTP_500_INTERNAL_SERVER_ERROR,
)
diff --git a/backend/scheduler/helper.py b/backend/scheduler/helper.py
index d4ea878c4..dc3421c2b 100644
--- a/backend/scheduler/helper.py
+++ b/backend/scheduler/helper.py
@@ -75,11 +75,10 @@ class SchedulerHelper:
cron_string=cron_string,
enabled=pipeline.active,
)
- # Ramp control (Phase 9, ②c): align this schedule's firer (Beat vs PG)
- # with the rollout. Inert until the gate + pg_scheduler_enabled flag are
- # on (fails closed to Beat), so this is a no-op during normal operation;
- # when a schedule is owned by PG it disables the Beat PeriodicTask in the
- # same transaction so the two never both fire. Best-effort (never raises).
+ # Align this schedule's firer (Beat vs PG). NOT a no-op: since UN-4046
+ # PG_SCHEDULER_ENABLED defaults to true, so the normal path here hands the
+ # schedule to PG and disables the Beat PeriodicTask in the same transaction
+ # (which is what stops the two both firing). Best-effort (never raises).
# organization_id is the org identifier string (what the mirror stores).
reconcile_ownership_for(str(pipeline.pk), organization_id, active=pipeline.active)
diff --git a/backend/scheduler/ownership.py b/backend/scheduler/ownership.py
index c7c9c2817..a53c3d21e 100644
--- a/backend/scheduler/ownership.py
+++ b/backend/scheduler/ownership.py
@@ -4,24 +4,23 @@ from Celery Beat to the Postgres scheduler, per-schedule and reversibly.
A schedule is owned by exactly one firer. ``reconcile_ownership_for``
applies that decision atomically:
- pg_periodic_schedule.pg_owned = resolve_schedule_owner(...) # PG fires it
+ pg_periodic_schedule.pg_owned = resolve_schedule_owner() # PG fires it
PeriodicTask.enabled = active AND NOT pg_owned # Beat fires it
Doing both in one transaction is what makes "never double-fires" real (it was
*conditional* on this slice): a ``pg_owned`` row always has its Beat
``PeriodicTask`` disabled, so the two can't both fire.
-**Two gates, and both must be on.** ``PG_SCHEDULER_ENABLED`` (env, default off) comes
-first; the ``pg_queue_enabled`` Flipt flag then decides per schedule, failing closed to
-Beat on a blind Flipt or any error.
+**One gate, and it defaults ON.** ``PG_SCHEDULER_ENABLED`` (env) is the whole decision.
+UN-4046 removed the second one — the per-schedule ``pg_queue_enabled`` Flipt flag that
+used to fail closed to Beat — and flipped this env's default from ``false`` to ``true``.
-They are separate because Beat stays the **sole scheduler for the whole PG rollout**:
-pipelines already reach PG without this module (``execute_pipeline_task_v2`` →
-``complete_execution`` → ``execute_workflow_async`` → ``resolve_transport``), so the PG
-scheduler exists only to retire the Beat *deployment* — a later, deliberate step. While
-the env gate is off this module writes **nothing at all**, so Beat's tables are untouched
-for the entire rollout and turning the flag off again is a pure Flipt flip with nothing
-to restore.
+The consequence is the opposite of what this docstring said before, so it is worth
+stating plainly: on a default deployment this module DOES write, on every schedule save
+and on every ``--migrate`` start, handing pipelines to PG and disabling Beat's
+``PeriodicTask`` rows. Rolling back is not a flag flip with nothing to restore — it is
+``PG_SCHEDULER_ENABLED=false`` **plus** a ``converge_pg_scheduler`` run to release the
+rows already adopted. Beat coming back does not re-enable rows this module disabled.
"""
from __future__ import annotations
@@ -32,89 +31,46 @@ import os
from django.db import transaction
from django.utils import timezone
from django_celery_beat.models import PeriodicTask, PeriodicTasks
-from pg_queue.flags import PG_QUEUE_FLAG_KEY
from pg_queue.models import PgPeriodicSchedule
-from unstract.flags.feature_flag import check_feature_flag_status
-
logger = logging.getLogger(__name__)
-# Gating uses the single shared PG-queue flag (pg_queue.flags.PG_QUEUE_FLAG_KEY,
-# imported above) — one flip gates execution + scheduler + executor. The scheduler
-# buckets the %-rollout on pipeline_id (each subsystem keys on its own entity).
-
-
-# Second gate, ahead of the Flipt flag: hand-over to the PG scheduler happens only
-# when this is explicitly on. Referenced as "pg_scheduler_enabled" by this module's
-# callers' comments (scheduler/helper.py, reconcile_pg_schedules) since the ramp was
-# designed, but never actually implemented — resolve_schedule_owner gated on
-# pg_queue_enabled alone, so the two flips were welded together.
-#
-# They must be separable, because Beat stays the SOLE scheduler for the whole PG
-# rollout: pipelines already reach PG without it (execute_pipeline_task_v2 →
-# complete_execution → execute_workflow_async → resolve_transport), so the PG
-# scheduler exists only to retire the Beat *deployment* — a later, deliberate step.
-#
-# Without this gate, turning pg_queue_enabled on would hand schedules to a PG
-# scheduler that is not running: reconcile_ownership_for disables the Beat
-# PeriodicTask, nothing polls the PG side, and the pipeline has no firer at all. It
-# runs in the backend on every schedule save, so scaling Beat to zero does not avoid
-# it and no ramp command is needed to trigger it — a user saving a schedule suffices.
+# The sole gate on schedule hand-over. It used to sit ahead of the
+# ``pg_queue_enabled`` Flipt flag; the flag is gone (UN-4046) and this remains,
+# because it guards a failure mode of its own: ``reconcile_ownership_for``
+# disables the Beat ``PeriodicTask`` whenever a schedule is PG-owned, so handing
+# schedules over while no PG scheduler is running leaves the pipeline with no
+# firer at all. It runs in the backend on every schedule save, so scaling Beat to
+# zero does not avoid it — a user saving a schedule suffices.
_PG_SCHEDULER_ENABLED_ENV = "PG_SCHEDULER_ENABLED"
def pg_scheduler_enabled() -> bool:
- """Whether schedule hand-over to the PG scheduler is switched on at all.
+ """Whether schedule hand-over to the PG scheduler is switched on.
- Defaults **off**. Flip it only alongside deploying the PG scheduler worker, as
- part of retiring Celery Beat.
+ Defaults **on** (UN-4046): PG is the only transport and the PG scheduler ships
+ with the fleet. Set ``PG_SCHEDULER_ENABLED=false`` only to keep Beat as the
+ firer in a deployment that deliberately runs without ``worker-pg-scheduler``.
"""
- return os.environ.get(_PG_SCHEDULER_ENABLED_ENV, "false").strip().lower() == "true"
+ return os.environ.get(_PG_SCHEDULER_ENABLED_ENV, "true").strip().lower() == "true"
-def resolve_schedule_owner(pipeline_id: str, organization_id: str | None) -> bool:
+def resolve_schedule_owner() -> bool:
"""True → the PG scheduler owns this schedule; False → Celery Beat does.
- Mirrors ``resolve_transport``: gated by the single ``pg_queue_enabled`` Flipt
- flag, keyed on ``pipeline_id`` for a stable percentage bucket. **Fails closed to
- Beat** on a blind Flipt or any error — so a schedule never silently loses its
- firer.
+ Gated solely by ``PG_SCHEDULER_ENABLED``. This used to *also* require the
+ ``pg_queue_enabled`` Flipt flag, because during the rollout the flag could be
+ on while the PG scheduler was still dark and a schedule handed to a scheduler
+ that is not running would simply stop firing. The flag is gone (UN-4046), so
+ the env gate is the sole dial — and it still has to be, for the same reason:
+ ``reconcile_ownership_for`` disables the Beat ``PeriodicTask`` when this
+ returns True, so turning it on without a running PG scheduler leaves the
+ pipeline with no firer at all.
- Requires ``PG_SCHEDULER_ENABLED`` **as well as** the flag: during the PG rollout
- the flag is on while the PG scheduler is still dark, and a schedule handed to a
- scheduler that is not running would simply stop firing.
+ ``organization_id`` is retained for call-site compatibility; it fed the Flipt
+ context and is now unused.
"""
- if not pg_scheduler_enabled():
- return False
- if os.environ.get("FLIPT_SERVICE_AVAILABLE", "false").lower() != "true":
- logger.warning(
- "resolve_schedule_owner: FLIPT_SERVICE_AVAILABLE != true "
- "(Flipt blind) for pipeline %s; leaving on Beat",
- pipeline_id,
- )
- return False
-
- # Flipt context is a gRPC map<string,string>; coerce values to str (a non-str
- # makes the client swallow it as False). entity_id str-coerced too so the
- # %-rollout bucket is stable across str/UUID call sites.
- context = {"pipeline_id": str(pipeline_id)}
- if organization_id:
- context["organization_id"] = str(organization_id)
- try:
- owned = check_feature_flag_status(
- flag_key=PG_QUEUE_FLAG_KEY, entity_id=str(pipeline_id), context=context
- )
- except Exception:
- # Expected, recoverable (fail-closed to Beat) and runs on every schedule
- # edit — warn with traceback rather than logger.exception so a persistently
- # down Flipt doesn't bury real errors as a per-edit Sentry exception.
- logger.warning(
- "resolve_schedule_owner: Flipt check failed for pipeline %s; leaving on Beat",
- pipeline_id,
- exc_info=True,
- )
- return False
- return bool(owned)
+ return pg_scheduler_enabled()
def _has_stale_pg_ownership(pipeline_id: str) -> bool:
@@ -149,7 +105,7 @@ def reconcile_ownership_for(
window where both fire. On rollback (``pg_owned`` → False) ``next_run_at`` is
also cleared so a later re-hand-over re-enters the PG tick's NULL baseline
(no burst). ``organization_id`` is the org *identifier* string (what the
- mirror stores), used for Flipt per-org segmenting. ``active`` is keyword-only
+ mirror stores), carried for logging/traceability. ``active`` is keyword-only
(a fire/don't-fire boolean trap otherwise).
Best-effort: a DB failure is logged and swallowed so it can never break the
@@ -195,7 +151,7 @@ def reconcile_ownership_for(
)
pg_owned = False
else:
- pg_owned = resolve_schedule_owner(pipeline_id, organization_id)
+ pg_owned = resolve_schedule_owner()
try:
with transaction.atomic():
was_pg_owned = (
diff --git a/backend/scheduler/tests/test_pg_schedule_ownership.py b/backend/scheduler/tests/test_pg_schedule_ownership.py
index 4f05e8d15..0b6e5ee18 100644
--- a/backend/scheduler/tests/test_pg_schedule_ownership.py
+++ b/backend/scheduler/tests/test_pg_schedule_ownership.py
@@ -18,47 +18,25 @@ _ORG = "org_abc"
class TestResolveScheduleOwner:
- @pytest.fixture(autouse=True)
- def _scheduler_gate_on(self, monkeypatch):
- """These pin the FINAL-phase behaviour, when hand-over is switched on.
+ """Ownership is gated solely by PG_SCHEDULER_ENABLED (UN-4046).
- PG_SCHEDULER_ENABLED defaults off for the whole rollout (Beat stays the sole
- scheduler), so without this every case below would short-circuit to Beat and
- assert nothing about the logic it was written for.
- """
- monkeypatch.setenv("PG_SCHEDULER_ENABLED", "true")
+ The ``pg_queue_enabled`` Flipt cases (unavailable / true / false / error) are
+ gone with the flag. The env gate still has to hold in both directions:
+ ``reconcile_ownership_for`` disables the Beat PeriodicTask whenever this says
+ PG, so a deployment without a running PG scheduler must be able to keep Beat.
+ """
- def test_flipt_unavailable_is_beat(self, monkeypatch):
- monkeypatch.setenv("FLIPT_SERVICE_AVAILABLE", "false")
- with patch("scheduler.ownership.check_feature_flag_status") as flag:
- assert ownership.resolve_schedule_owner(_PID, _ORG) is False
- flag.assert_not_called()
-
- def test_flag_true_is_pg(self, monkeypatch):
- monkeypatch.setenv("FLIPT_SERVICE_AVAILABLE", "true")
- with patch(
- "scheduler.ownership.check_feature_flag_status", return_value=True
- ) as flag:
- assert ownership.resolve_schedule_owner(_PID, _ORG) is True
- # entity_id = pipeline_id (stable %-bucket); org in context.
- assert flag.call_args.kwargs["entity_id"] == _PID
- assert flag.call_args.kwargs["context"]["pipeline_id"] == _PID
- assert flag.call_args.kwargs["context"]["organization_id"] == _ORG
-
- def test_flag_false_is_beat(self, monkeypatch):
- monkeypatch.setenv("FLIPT_SERVICE_AVAILABLE", "true")
- with patch(
- "scheduler.ownership.check_feature_flag_status", return_value=False
- ):
- assert ownership.resolve_schedule_owner(_PID, _ORG) is False
+ def test_defaults_to_pg(self, monkeypatch):
+ monkeypatch.delenv("PG_SCHEDULER_ENABLED", raising=False)
+ assert ownership.resolve_schedule_owner() is True
- def test_flipt_error_fails_closed_to_beat(self, monkeypatch):
- monkeypatch.setenv("FLIPT_SERVICE_AVAILABLE", "true")
- with patch(
- "scheduler.ownership.check_feature_flag_status",
- side_effect=RuntimeError("flipt down"),
- ):
- assert ownership.resolve_schedule_owner(_PID, _ORG) is False
+ def test_env_gate_off_keeps_beat(self, monkeypatch):
+ monkeypatch.setenv("PG_SCHEDULER_ENABLED", "false")
+ assert ownership.resolve_schedule_owner() is False
+
+ def test_env_gate_on_is_pg(self, monkeypatch):
+ monkeypatch.setenv("PG_SCHEDULER_ENABLED", "true")
+ assert ownership.resolve_schedule_owner() is True
class TestReconcileOwnership:
@@ -66,9 +44,9 @@ class TestReconcileOwnership:
def _scheduler_gate_on(self, monkeypatch):
"""These pin the FINAL-phase behaviour, when hand-over is switched on.
- PG_SCHEDULER_ENABLED defaults off for the whole rollout (Beat stays the sole
- scheduler), so without this every case below would short-circuit to Beat and
- assert nothing about the logic it was written for.
+ PG_SCHEDULER_ENABLED is the sole ownership gate. It now defaults ON, so
+ this is belt-and-braces: it keeps these cases independent of the default
+ should a future change flip it.
"""
monkeypatch.setenv("PG_SCHEDULER_ENABLED", "true")
@@ -82,9 +60,7 @@ class TestReconcileOwnership:
def _patches(self, *, owner: bool, rows_matched: int = 1):
sched = patch("scheduler.ownership.PgPeriodicSchedule")
pt = patch("scheduler.ownership.PeriodicTask")
- resolve = patch(
- "scheduler.ownership.resolve_schedule_owner", return_value=owner
- )
+ resolve = patch("scheduler.ownership.resolve_schedule_owner", return_value=owner)
# transaction.atomic() as a no-op context manager.
txn = patch(
"scheduler.ownership.transaction.atomic",
@@ -101,14 +77,11 @@ class TestReconcileOwnership:
assert result is True
# mirror pg_owned set True
assert (
- Sched.objects.filter.return_value.update.call_args.kwargs["pg_owned"]
- is True
+ Sched.objects.filter.return_value.update.call_args.kwargs["pg_owned"] is True
)
# Beat PeriodicTask disabled (active AND NOT pg_owned == False)
PT.objects.filter.assert_called_once_with(name=_PID)
- assert (
- PT.objects.filter.return_value.update.call_args.kwargs["enabled"] is False
- )
+ assert PT.objects.filter.return_value.update.call_args.kwargs["enabled"] is False
def test_not_pg_owned_enables_beat_and_clears_next_run(self):
sched, pt, resolve, txn = self._patches(owner=False)
@@ -120,9 +93,7 @@ class TestReconcileOwnership:
assert update_kwargs["pg_owned"] is False
# Rollback to Beat clears next_run_at so a re-hand-over re-baselines.
assert update_kwargs["next_run_at"] is None
- assert (
- PT.objects.filter.return_value.update.call_args.kwargs["enabled"] is True
- )
+ assert PT.objects.filter.return_value.update.call_args.kwargs["enabled"] is True
def test_an_ALREADY_pg_owned_schedule_does_not_clear_next_run(self):
"""Narrowed: this is the True→True case, not "adopt never clears".
@@ -152,9 +123,7 @@ class TestReconcileOwnership:
ownership.reconcile_ownership_for(_PID, _ORG, active=False)
# active=False → Beat stays disabled regardless of ownership.
- assert (
- PT.objects.filter.return_value.update.call_args.kwargs["enabled"] is False
- )
+ assert PT.objects.filter.return_value.update.call_args.kwargs["enabled"] is False
def test_missing_mirror_row_skips_and_reports_beat(self):
sched, pt, resolve, txn = self._patches(owner=True)
@@ -178,7 +147,8 @@ class TestReconcileOwnership:
clobbering a concurrent reconcile), which bypasses django-celery-beat's
post_save signal. Without an explicit PeriodicTasks.update_changed() bump,
DatabaseScheduler never reloads and Beat keeps firing the handed-over
- schedule from its stale in-memory copy (breaking no-double-fire)."""
+ schedule from its stale in-memory copy (breaking no-double-fire).
+ """
sched, pt, resolve, txn = self._patches(owner=True)
with sched as Sched, pt, resolve, txn:
Sched.objects.filter.return_value.update.return_value = 1
@@ -187,7 +157,8 @@ class TestReconcileOwnership:
def test_beat_reload_not_signalled_when_no_mirror_row(self, _mock_periodic_tasks):
"""No mirror row → the method returns before touching the PeriodicTask, so
- no reload should be signalled either."""
+ no reload should be signalled either.
+ """
sched, pt, resolve, txn = self._patches(owner=True)
with sched as Sched, pt, resolve, txn:
Sched.objects.filter.return_value.update.return_value = 0
@@ -196,34 +167,30 @@ class TestReconcileOwnership:
class TestPgSchedulerGate:
- """The gate that keeps Beat the sole scheduler for the whole PG rollout.
+ """The gate deciding whether the PG scheduler or Beat owns a schedule.
- Pipelines already reach PG without the PG scheduler (execute_pipeline_task_v2 →
- complete_execution → execute_workflow_async → resolve_transport), so the PG
- scheduler exists only to retire the Beat *deployment* — a later, deliberate step.
- Handing a schedule over before then leaves it with NO firer: Beat's PeriodicTask
- disabled, nothing polling the PG side.
+ It defaults ON (UN-4046) — PG is the only transport and the PG scheduler ships
+ with the fleet. It remains a gate because handing a schedule over while no PG
+ scheduler is running leaves it with NO firer: Beat's PeriodicTask disabled,
+ nothing polling the PG side.
"""
- def test_defaults_off(self, monkeypatch):
+ def test_defaults_on(self, monkeypatch):
monkeypatch.delenv("PG_SCHEDULER_ENABLED", raising=False)
- assert ownership.pg_scheduler_enabled() is False
+ assert ownership.pg_scheduler_enabled() is True
@pytest.mark.parametrize("value", ["false", "False", "0", "", "yes", "TRUE "])
- def test_only_an_exact_true_opts_in(self, monkeypatch, value):
+ def test_only_an_exact_true_stays_on(self, monkeypatch, value):
+ """Opting OUT is the deliberate act now — anything but an exact ``true``
+ (case/whitespace-insensitive) turns the gate off.
+ """
monkeypatch.setenv("PG_SCHEDULER_ENABLED", value)
assert ownership.pg_scheduler_enabled() is (value.strip().lower() == "true")
- def test_owner_stays_beat_even_with_the_flag_on(self, monkeypatch):
- """The rollout state: pg_queue_enabled on, PG scheduler still dark."""
- monkeypatch.delenv("PG_SCHEDULER_ENABLED", raising=False)
- monkeypatch.setenv("FLIPT_SERVICE_AVAILABLE", "true")
- with patch(
- "scheduler.ownership.check_feature_flag_status", return_value=True
- ) as flag:
- assert ownership.resolve_schedule_owner(_PID, _ORG) is False
- # Short-circuits before Flipt — no evaluation on every schedule save.
- flag.assert_not_called()
+ def test_owner_stays_beat_when_the_gate_is_off(self, monkeypatch):
+ """A deployment that deliberately runs without worker-pg-scheduler."""
+ monkeypatch.setenv("PG_SCHEDULER_ENABLED", "false")
+ assert ownership.resolve_schedule_owner() is False
def test_reconcile_writes_NEITHER_beat_table(self, monkeypatch):
"""The regression this gate exists to prevent.
@@ -233,13 +200,11 @@ class TestPgSchedulerGate:
on every schedule save — writing back a value Beat already had and forcing a
reload, on a table we promised not to touch.
"""
- monkeypatch.delenv("PG_SCHEDULER_ENABLED", raising=False)
- monkeypatch.setenv("FLIPT_SERVICE_AVAILABLE", "true")
+ monkeypatch.setenv("PG_SCHEDULER_ENABLED", "false")
with (
patch("scheduler.ownership.PeriodicTask") as PT,
patch("scheduler.ownership.PeriodicTasks") as PTs,
patch("scheduler.ownership.PgPeriodicSchedule") as Sched,
- patch("scheduler.ownership.check_feature_flag_status", return_value=True),
):
# A CLEAN row: not pg_owned, so there is nothing to repair and the
# write-free path must be taken. Must be set explicitly — a bare
@@ -286,7 +251,7 @@ class TestStalePgOwnershipIsReleased:
patch.stopall()
def test_a_stale_row_is_handed_back_to_beat(self, monkeypatch):
- monkeypatch.delenv("PG_SCHEDULER_ENABLED", raising=False)
+ monkeypatch.setenv("PG_SCHEDULER_ENABLED", "false")
sched = self._mocks(stale=True)
with (
patch("scheduler.ownership.PeriodicTask") as PT,
@@ -315,7 +280,7 @@ class TestStalePgOwnershipIsReleased:
def test_a_paused_pipeline_is_not_resurrected_by_the_repair(self, monkeypatch):
"""Repair restores the FIRER, never the on/off state the user chose."""
- monkeypatch.delenv("PG_SCHEDULER_ENABLED", raising=False)
+ monkeypatch.setenv("PG_SCHEDULER_ENABLED", "false")
self._mocks(stale=True)
with (
patch("scheduler.ownership.PeriodicTask") as PT,
@@ -334,22 +299,17 @@ class TestStalePgOwnershipIsReleased:
Asking Flipt could return True and re-hand the schedule to a PG scheduler
that is not running — turning a double-fire into no firer at all.
"""
- monkeypatch.delenv("PG_SCHEDULER_ENABLED", raising=False)
- monkeypatch.setenv("FLIPT_SERVICE_AVAILABLE", "true")
+ monkeypatch.setenv("PG_SCHEDULER_ENABLED", "false")
self._mocks(stale=True)
with (
patch("scheduler.ownership.PeriodicTask"),
patch("scheduler.ownership.PeriodicTasks"),
- patch(
- "scheduler.ownership.check_feature_flag_status", return_value=True
- ) as flag,
):
ownership.reconcile_ownership_for(_PID, _ORG, active=True)
- flag.assert_not_called()
def test_a_db_error_on_the_check_falls_back_to_writing_nothing(self, monkeypatch):
"""A repair that cannot confirm it is needed must not run."""
- monkeypatch.delenv("PG_SCHEDULER_ENABLED", raising=False)
+ monkeypatch.setenv("PG_SCHEDULER_ENABLED", "false")
sched = patch("scheduler.ownership.PgPeriodicSchedule").start()
sched.objects.filter.side_effect = Exception("db down")
with (
@@ -412,9 +372,7 @@ class TestReconcileAtomicityRealDB:
patch("scheduler.ownership.resolve_schedule_owner", return_value=True),
patch("scheduler.ownership.PeriodicTask", failing_pt),
):
- result = ownership.reconcile_ownership_for(
- pid, "org_atomic", active=True
- )
+ result = ownership.reconcile_ownership_for(pid, "org_atomic", active=True)
assert result is None # failure signalled
# The pg_owned=True write was rolled back with the failed PT update.
assert PgPeriodicSchedule.objects.get(pipeline_id=pid).pg_owned is False
@@ -445,7 +403,6 @@ class TestNextRunBaselineOnTransition:
def _reconcile(self, monkeypatch, *, was_pg_owned, now_pg_owned):
monkeypatch.setenv("PG_SCHEDULER_ENABLED", "true")
- monkeypatch.setenv("FLIPT_SERVICE_AVAILABLE", "true")
with (
patch("scheduler.ownership.PgPeriodicSchedule") as Sched,
patch("scheduler.ownership.PeriodicTask"),
@@ -455,7 +412,7 @@ class TestNextRunBaselineOnTransition:
return_value=contextlib.nullcontext(),
),
patch(
- "scheduler.ownership.check_feature_flag_status",
+ "scheduler.ownership.resolve_schedule_owner",
return_value=now_pg_owned,
),
):
@@ -503,7 +460,6 @@ class TestBeatClockBaselineOnRelease:
def _reconcile(self, monkeypatch, *, was_pg_owned, now_pg_owned, active=True):
monkeypatch.setenv("PG_SCHEDULER_ENABLED", "true")
- monkeypatch.setenv("FLIPT_SERVICE_AVAILABLE", "true")
with (
patch("scheduler.ownership.PgPeriodicSchedule") as Sched,
patch("scheduler.ownership.PeriodicTask") as Beat,
@@ -513,7 +469,7 @@ class TestBeatClockBaselineOnRelease:
return_value=contextlib.nullcontext(),
),
patch(
- "scheduler.ownership.check_feature_flag_status",
+ "scheduler.ownership.resolve_schedule_owner",
return_value=now_pg_owned,
),
):
@@ -534,7 +490,8 @@ class TestBeatClockBaselineOnRelease:
def test_handing_over_to_pg_does_NOT_touch_beats_clock(self, monkeypatch):
"""Adoption switches Beat OFF, so its clock is irrelevant — and overwriting it
- would destroy the value the eventual release needs to restore from."""
+ would destroy the value the eventual release needs to restore from.
+ """
beat = self._reconcile(monkeypatch, was_pg_owned=False, now_pg_owned=True)
assert beat["enabled"] is False
assert "last_run_at" not in beat
@@ -542,7 +499,8 @@ class TestBeatClockBaselineOnRelease:
def test_an_unchanged_owner_is_NOT_re_stamped(self, monkeypatch):
"""Same transition-scoping reason as next_run_at: reconcile runs on every
pipeline save, and stamping unconditionally would push Beat's clock forward
- on an ordinary edit and silently skip a due fire."""
+ on an ordinary edit and silently skip a due fire.
+ """
beat = self._reconcile(monkeypatch, was_pg_owned=False, now_pg_owned=False)
assert "last_run_at" not in beat
@@ -550,7 +508,8 @@ class TestBeatClockBaselineOnRelease:
self, monkeypatch
):
"""A paused schedule comes back paused — but if it is later resumed, Beat must
- not then replay the backlog it accrued while PG owned it."""
+ not then replay the backlog it accrued while PG owned it.
+ """
beat = self._reconcile(
monkeypatch, was_pg_owned=True, now_pg_owned=False, active=False
)
diff --git a/backend/workflow_manager/internal_api_views.py b/backend/workflow_manager/internal_api_views.py
index bdc09a0a5..c54199f16 100644
--- a/backend/workflow_manager/internal_api_views.py
+++ b/backend/workflow_manager/internal_api_views.py
@@ -18,9 +18,9 @@ from rest_framework.decorators import api_view
from rest_framework.response import Response
from tool_instance_v2.models import ToolInstance
+from unstract.core.data_models import WorkflowTransport
from workflow_manager.workflow_v2.enums import ExecutionStatus
from workflow_manager.workflow_v2.models import Workflow, WorkflowExecution
-from workflow_manager.workflow_v2.transport import resolve_transport
logger = logging.getLogger(__name__)
@@ -321,23 +321,14 @@ def create_workflow_execution(request):
# Handle tags logic if needed
pass
- # Resolve the transport this execution rides (9e). Decided once here, at
- # the creation chokepoint, and returned so the caller carries it in the
- # dispatched task's payload — not persisted on the row. Flipt (gated by
- # the env master-switch) decides; fails closed to "celery".
- transport = resolve_transport(
- execution_id=str(execution.id),
- organization_id=org_id,
- workflow_id=str(workflow.id),
- pipeline_id=data.get("pipeline_id"),
- )
-
return Response(
{
"execution_id": str(execution.id),
"status": execution.status,
"execution_log_id": execution.execution_log_id, # Return for workers to use
- "transport": transport,
+ # Wire field the workers still branch on. PG is the only
+ # transport now; the field goes when those branches do.
+ "transport": WorkflowTransport.PG_QUEUE.value,
}
)
@@ -389,72 +380,3 @@ def compile_workflow(request):
{"error": "Internal server error"},
status=status.HTTP_500_INTERNAL_SERVER_ERROR,
)
-
-
-@csrf_exempt # Safe: Internal API with Bearer token auth, no session/cookies
-@api_view(["POST"])
-def submit_file_batch_for_processing(request):
- """Submit file batch for processing by workers.
-
- This endpoint receives batch data and returns immediately,
- as actual processing is handled by Celery workers.
-
- Returns:
- JSON response with batch submission status
- """
- try:
- batch_data = request.data
-
- # Get organization from header
- org_id = request.headers.get("X-Organization-ID")
- if not org_id:
- return Response(
- {"error": "X-Organization-ID header is required"},
- status=status.HTTP_400_BAD_REQUEST,
- )
-
- # Add organization ID to file_data where WorkerFileData expects it
- if "file_data" in batch_data:
- batch_data["file_data"]["organization_id"] = org_id
- else:
- # Fallback: add at top level for backward compatibility
- batch_data["organization_id"] = org_id
-
- # Submit to file processing worker queue using Celery
- try:
- from backend.celery_service import app as celery_app
-
- # Submit the batch data to the file processing worker using send_task
- # This calls the task by name without needing to import it
- task_result = celery_app.send_task(
- "process_file_batch", # Task name as defined in workers/file_processing/tasks.py
- args=[batch_data], # Pass batch_data as first argument
- queue="file_processing", # Send to file processing queue
- )
-
- logger.info(
- f"Successfully submitted file batch {batch_data.get('batch_id')} to worker queue (task: {task_result.id})"
- )
-
- return Response(
- {
- "success": True,
- "batch_id": batch_data.get("batch_id"),
- "celery_task_id": task_result.id,
- "message": "Batch submitted for processing",
- }
- )
-
- except Exception as e:
- logger.error(f"Failed to submit batch to worker queue: {e}")
- return Response(
- {"error": f"Failed to submit batch for processing: {str(e)}"},
- status=status.HTTP_500_INTERNAL_SERVER_ERROR,
- )
-
- except Exception as e:
- logger.error(f"Error submitting file batch: {e}")
- return Response(
- {"error": "Internal server error"},
- status=status.HTTP_500_INTERNAL_SERVER_ERROR,
- )
diff --git a/backend/workflow_manager/internal_urls.py b/backend/workflow_manager/internal_urls.py
index 6fac0be62..35db8645f 100644
--- a/backend/workflow_manager/internal_urls.py
+++ b/backend/workflow_manager/internal_urls.py
@@ -34,12 +34,6 @@ urlpatterns = [
internal_api_views.compile_workflow,
name="compile_workflow",
),
- # File batch processing
- path(
- "file-batch/submit/",
- internal_api_views.submit_file_batch_for_processing,
- name="submit_file_batch_for_processing",
- ),
# Workflow definition and type detection (using sophisticated class-based views)
path(
"workflow/<str:workflow_id>/",
diff --git a/backend/workflow_manager/workflow_v2/execution_log_internal_views.py b/backend/workflow_manager/workflow_v2/execution_log_internal_views.py
index 0c07b0474..7a3d63fc6 100644
--- a/backend/workflow_manager/workflow_v2/execution_log_internal_views.py
+++ b/backend/workflow_manager/workflow_v2/execution_log_internal_views.py
@@ -190,7 +190,8 @@ class SweepUndispatchedExecutionsAPIView(APIView):
sweep also needs the rate limiter — both backend-side.
Transport-agnostic on purpose: the create-then-dispatch window sits upstream of
- ``resolve_transport``, so this recovers Celery-path strands too.
+ transport selection, so this still recovers strands left by the pre-UN-4046
+ Celery path.
"""
def post(self, request: Request) -> Response:
diff --git a/backend/workflow_manager/workflow_v2/tests/test_dispatch_orchestrator.py b/backend/workflow_manager/workflow_v2/tests/test_dispatch_orchestrator.py
index 6b0a03553..3ee117480 100644
--- a/backend/workflow_manager/workflow_v2/tests/test_dispatch_orchestrator.py
+++ b/backend/workflow_manager/workflow_v2/tests/test_dispatch_orchestrator.py
@@ -6,7 +6,7 @@ decision (PG enqueue vs Celery send), the bare-msg_id task-id contract, the
WorkloadType value, and the two org sentinels.
"""
-from unittest.mock import MagicMock, patch
+from unittest.mock import patch
from utils.constants import CeleryQueue
from workflow_manager.workflow_v2.workflow_helper import WorkflowHelper
@@ -16,10 +16,9 @@ _CELERY = "workflow_manager.workflow_v2.workflow_helper.celery_app"
class TestDispatchOrchestratorTask:
- def test_pg_transport_enqueues_to_pg_not_celery(self):
+ def test_enqueues_to_the_pg_queue(self):
with patch(_PG, return_value=4242) as pg, patch(_CELERY) as celery:
task_id = WorkflowHelper._dispatch_orchestrator_task(
- transport="pg_queue",
queue=CeleryQueue.CELERY_API_DEPLOYMENTS,
args=["a"],
kwargs={"transport": "pg_queue"},
@@ -37,7 +36,7 @@ class TestDispatchOrchestratorTask:
def test_pg_general_queue_uses_non_api_workload(self):
with patch(_PG, return_value=7) as pg, patch(_CELERY):
WorkflowHelper._dispatch_orchestrator_task(
- transport="pg_queue", queue=None, args=[], kwargs={}, org_schema="org1"
+ queue=None, args=[], kwargs={}, org_schema="org1"
)
assert pg.call_args.kwargs["fairness"]["workload_type"] == "non_api"
@@ -45,18 +44,8 @@ class TestDispatchOrchestratorTask:
"""Row org_id column is NOT NULL → ""; fairness org_id is str|None → None."""
with patch(_PG, return_value=1) as pg, patch(_CELERY):
WorkflowHelper._dispatch_orchestrator_task(
- transport="pg_queue", queue=None, args=[], kwargs={}, org_schema=""
+ queue=None, args=[], kwargs={}, org_schema=""
)
kw = pg.call_args.kwargs
assert kw["org_id"] == ""
assert kw["fairness"]["org_id"] is None
-
- def test_celery_transport_uses_send_task_not_pg(self):
- with patch(_PG) as pg, patch(_CELERY) as celery:
- celery.send_task.return_value = MagicMock(id="celery-task-1")
- task_id = WorkflowHelper._dispatch_orchestrator_task(
- transport="celery", queue=None, args=[], kwargs={}, org_schema="org1"
- )
- assert task_id == "celery-task-1"
- pg.assert_not_called()
- celery.send_task.assert_called_once()
diff --git a/backend/workflow_manager/workflow_v2/tests/test_execute_workflow_async_wait.py b/backend/workflow_manager/workflow_v2/tests/test_execute_workflow_async_wait.py
index 74586ab53..91a5e1bcb 100644
--- a/backend/workflow_manager/workflow_v2/tests/test_execute_workflow_async_wait.py
+++ b/backend/workflow_manager/workflow_v2/tests/test_execute_workflow_async_wait.py
@@ -27,14 +27,11 @@ class TestTimeoutWaitSurvivesBookkeepingFailure:
exec_row = MagicMock(status=ExecutionStatus.EXECUTING.value)
with (
- patch(f"{_MOD}.resolve_transport", return_value="pg_queue"),
patch(f"{_MOD}.UserContext") as user_ctx,
patch(f"{_MOD}.StateStore") as state_store,
patch(f"{_MOD}.time"), # no-op sleep
patch(f"{_MOD}.WorkflowExecution") as wf_exec,
- patch.object(
- WorkflowHelper, "_dispatch_orchestrator_task", return_value="1"
- ),
+ patch.object(WorkflowHelper, "_dispatch_orchestrator_task", return_value="1"),
patch.object(
WorkflowHelper,
"_record_dispatch_handle",
diff --git a/backend/workflow_manager/workflow_v2/tests/test_record_dispatch_handle.py b/backend/workflow_manager/workflow_v2/tests/test_record_dispatch_handle.py
index ec40a18bb..dcba6ac65 100644
--- a/backend/workflow_manager/workflow_v2/tests/test_record_dispatch_handle.py
+++ b/backend/workflow_manager/workflow_v2/tests/test_record_dispatch_handle.py
@@ -23,20 +23,17 @@ from workflow_manager.workflow_v2.workflow_helper import WorkflowHelper
_MODEL = "workflow_manager.workflow_v2.execution.WorkflowExecution"
-_HELPER = (
- "workflow_manager.workflow_v2.workflow_helper.WorkflowExecutionServiceHelper"
-)
+_HELPER = "workflow_manager.workflow_v2.workflow_helper.WorkflowExecutionServiceHelper"
_EXEC = "exec-123"
class TestRecordDispatchHandle:
- def test_pg_handle_goes_to_queue_message_id_as_int_not_task_id(self):
+ def test_handle_goes_to_queue_message_id_as_int(self):
"""The PG msg_id (bigint string) must be stored in queue_message_id as an
int — never in the UUID task_id (the bug that crashed the wait loop)."""
with patch(_HELPER) as helper:
WorkflowHelper._record_dispatch_handle(
execution_id=_EXEC,
- transport="pg_queue",
dispatch_handle="1527",
org_schema="org1",
file_count=3,
@@ -45,32 +42,15 @@ class TestRecordDispatchHandle:
execution_id=_EXEC, queue_message_id=1527
)
assert isinstance(
- helper.update_execution_queue_message_id.call_args.kwargs[
- "queue_message_id"
- ],
+ helper.update_execution_queue_message_id.call_args.kwargs["queue_message_id"],
int,
)
helper.update_execution_task.assert_not_called() # task_id stays NULL
- def test_celery_handle_goes_to_task_id_not_queue_message_id(self):
- with patch(_HELPER) as helper:
- WorkflowHelper._record_dispatch_handle(
- execution_id=_EXEC,
- transport="celery",
- dispatch_handle="b1b2c3d4-0000-0000-0000-000000000000",
- org_schema="org1",
- file_count=1,
- )
- helper.update_execution_task.assert_called_once_with(
- execution_id=_EXEC, task_id="b1b2c3d4-0000-0000-0000-000000000000"
- )
- helper.update_execution_queue_message_id.assert_not_called()
-
def test_empty_handle_records_nothing(self):
with patch(_HELPER) as helper:
WorkflowHelper._record_dispatch_handle(
execution_id=_EXEC,
- transport="celery",
dispatch_handle=None,
org_schema="org1",
file_count=0,
@@ -78,7 +58,7 @@ class TestRecordDispatchHandle:
helper.update_execution_task.assert_not_called()
helper.update_execution_queue_message_id.assert_not_called()
- def test_non_numeric_pg_handle_records_nothing(self):
+ def test_non_numeric_handle_records_nothing(self):
"""A malformed PG handle (not a bigint) must be parsed defensively — no
ValueError out of the helper, and nothing recorded. Today this can't
happen (``_dispatch_orchestrator_task`` always returns ``str(msg_id)``),
@@ -86,7 +66,6 @@ class TestRecordDispatchHandle:
with patch(_HELPER) as helper:
WorkflowHelper._record_dispatch_handle(
execution_id=_EXEC,
- transport="pg_queue",
dispatch_handle="not-a-number",
org_schema="org1",
file_count=1,
diff --git a/backend/workflow_manager/workflow_v2/tests/test_step_execution_transport.py b/backend/workflow_manager/workflow_v2/tests/test_step_execution_transport.py
index 2e179ad0e..38603d648 100644
--- a/backend/workflow_manager/workflow_v2/tests/test_step_execution_transport.py
+++ b/backend/workflow_manager/workflow_v2/tests/test_step_execution_transport.py
@@ -1,17 +1,17 @@
-"""Tests for the PG guard on single-step execution (UN-3445 acceptance gate).
+"""Tests for the single-step execution guard (UN-3445 acceptance gate, UN-4046).
-Single-step is the one entry path whose fan-out was never transport-gated: it
+Single-step is the one entry path whose fan-out was never moved onto PG: it
reaches the Celery chord in ``process_input_files`` directly, while the normal
-path's PG fan-out lives in the general worker. With the flag on and the Celery
-file_processing workers scaled to zero, those batches would never be consumed
-and the execution would hang in EXECUTING with no PG reaper aware of it.
+path's PG fan-out lives in the general worker. With no Celery file_processing
+workers those batches would never be consumed and the execution would hang in
+EXECUTING with no PG reaper aware of it — so it refuses loudly instead.
-So the guard has to hold in *both* directions, and that is what these pin:
-Celery must still run step execution unchanged (it is what staging and
-production run today), and PG must refuse it loudly rather than dispatch into a
-queue with no consumer.
+It used to refuse only when the ``pg_queue_enabled`` flag resolved to PG. The
+flag is gone (UN-4046) and PG is the only transport, so the refusal is
+unconditional and the "celery still runs it" / "transport is resolved on the
+execution id" cases no longer describe anything.
-DB-free: WorkflowExecution, resolve_transport and run_workflow are all mocked.
+DB-free: WorkflowExecution and run_workflow are mocked.
"""
from unittest.mock import MagicMock, patch
@@ -21,7 +21,6 @@ from workflow_manager.workflow_v2.exceptions import WorkflowExecutionError
from workflow_manager.workflow_v2.models import Workflow
from workflow_manager.workflow_v2.workflow_helper import WorkflowHelper
-_TRANSPORT = "workflow_manager.workflow_v2.workflow_helper.resolve_transport"
_EXECUTION = "workflow_manager.workflow_v2.workflow_helper.WorkflowExecution"
_RUN = "workflow_manager.workflow_v2.workflow_helper.WorkflowHelper.run_workflow"
@@ -49,11 +48,9 @@ def _patched_execution():
return model
-class TestStepExecutionTransportGuard:
- def test_pg_transport_refuses_instead_of_dispatching_to_celery(self):
- with patch(_EXECUTION, _patched_execution()), patch(
- _TRANSPORT, return_value="pg_queue"
- ), patch(_RUN) as run:
+class TestStepExecutionGuard:
+ def test_refuses_instead_of_dispatching_into_an_unconsumed_queue(self):
+ with patch(_EXECUTION, _patched_execution()), patch(_RUN) as run:
with pytest.raises(WorkflowExecutionError, match="not supported"):
WorkflowHelper.step_execution(
workflow=_workflow(),
@@ -63,58 +60,3 @@ class TestStepExecutionTransportGuard:
# The point of the guard: nothing reaches the chord. Asserting the raise
# alone would still pass if run_workflow had already fanned out.
run.assert_not_called()
-
- def test_celery_transport_still_runs_step_execution(self):
- """Flag-off is what staging and production run — it must be untouched."""
- with patch(_EXECUTION, _patched_execution()), patch(
- _TRANSPORT, return_value="celery"
- ), patch(_RUN, return_value="ran") as run:
- result = WorkflowHelper.step_execution(
- workflow=_workflow(),
- execution_action=_START,
- execution_id=_EXECUTION_ID,
- )
- assert result == "ran"
- assert run.call_args.kwargs["single_step"] is True
-
- def test_transport_is_resolved_on_the_execution_id(self):
- """entity_id must be the execution id — it is what Flipt buckets on, and
- what keeps one execution from re-bucketing across transports."""
- with patch(_EXECUTION, _patched_execution()), patch(
- _TRANSPORT, return_value="celery"
- ) as resolve, patch(_RUN):
- WorkflowHelper.step_execution(
- workflow=_workflow(),
- execution_action=_START,
- execution_id=_EXECUTION_ID,
- )
- assert resolve.call_args.kwargs["execution_id"] == _EXECUTION_ID
- assert resolve.call_args.kwargs["organization_id"] == "org_acme"
-
- def test_no_execution_id_creates_a_step_execution_without_resolving(self):
- """The START-without-id branch only mints a row; it dispatches nothing, so
- it must not consume a Flipt evaluation (and must not be blocked)."""
- with patch(_TRANSPORT) as resolve, patch.object(
- WorkflowHelper, "create_and_make_execution_response", return_value="created"
- ):
- result = WorkflowHelper.step_execution(
- workflow=_workflow(), execution_action=_START, execution_id=None
- )
- assert result == "created"
- resolve.assert_not_called()
-
- def test_missing_execution_falls_back_to_creating_one(self):
- """Pre-existing behaviour: a stale execution_id re-mints rather than 404s.
- Pinned so the guard's placement inside the try/except cannot change it."""
- model = _patched_execution()
- model.objects.get.side_effect = model.DoesNotExist
- with patch(_EXECUTION, model), patch(_TRANSPORT) as resolve, patch.object(
- WorkflowHelper, "create_and_make_execution_response", return_value="created"
- ):
- result = WorkflowHelper.step_execution(
- workflow=_workflow(),
- execution_action=_START,
- execution_id=_EXECUTION_ID,
- )
- assert result == "created"
- resolve.assert_not_called()
diff --git a/backend/workflow_manager/workflow_v2/tests/test_transport.py b/backend/workflow_manager/workflow_v2/tests/test_transport.py
deleted file mode 100644
index a93c926f4..000000000
--- a/backend/workflow_manager/workflow_v2/tests/test_transport.py
+++ /dev/null
@@ -1,161 +0,0 @@
-"""Tests for the 9e transport-resolution seam (PR 3 — Flipt canary wiring).
-
-``resolve_transport`` is the single chokepoint that decides whether a new
-execution rides the legacy Celery transport or the Postgres queue. It is gated by a
-single ``pg_queue_enabled`` Flipt boolean flag; it fails closed to Celery on any
-problem. These tests pin that contract.
-"""
-
-from unittest.mock import MagicMock, patch
-
-import pytest
-from unstract.core.data_models import (
- DEFAULT_WORKFLOW_TRANSPORT,
- WorkflowTransport,
- normalize_transport,
-)
-
-from workflow_manager.workflow_v2.transport import (
- PG_QUEUE_FLAG_KEY,
- resolve_transport,
-)
-
-# Where ``check_feature_flag_status`` is looked up (imported into the module).
-_FLIPT = "workflow_manager.workflow_v2.transport.check_feature_flag_status"
-
-
-class TestWorkflowTransportEnum:
- def test_values(self):
- assert WorkflowTransport.CELERY.value == "celery"
- assert WorkflowTransport.PG_QUEUE.value == "pg_queue"
-
- def test_default_is_celery(self):
- assert DEFAULT_WORKFLOW_TRANSPORT == WorkflowTransport.CELERY.value
-
-
-class TestResolveTransport:
- @pytest.fixture(autouse=True)
- def _flipt_available(self, monkeypatch):
- """resolve_transport short-circuits to celery when Flipt is marked
- unavailable; default it available so the flag-evaluation tests exercise
- the real path. Individual tests override as needed.
- """
- monkeypatch.setenv("FLIPT_SERVICE_AVAILABLE", "true")
-
- def test_missing_organization_forces_celery(self):
- """No org context → can't segment safely → fail closed; Flipt not called
- (str(None)/"" must never reach the Flipt org segment).
- """
- with patch(_FLIPT) as flipt:
- result = resolve_transport(execution_id="e1", organization_id="")
- assert result == WorkflowTransport.CELERY.value
- flipt.assert_not_called()
-
- def test_flipt_unavailable_forces_celery(self, monkeypatch):
- """Flipt service unavailable → celery, loudly — not a silent masquerade as a
- healthy 100%-celery rollout.
- """
- monkeypatch.setenv("FLIPT_SERVICE_AVAILABLE", "false")
- with patch(_FLIPT) as flipt:
- result = resolve_transport(execution_id="e1", organization_id="org1")
- assert result == WorkflowTransport.CELERY.value
- flipt.assert_not_called()
-
- def test_flipt_true_resolves_pg_queue(self):
- with patch(_FLIPT, return_value=True):
- result = resolve_transport(execution_id="e1", organization_id="org1")
- assert result == WorkflowTransport.PG_QUEUE.value
-
- def test_flipt_false_resolves_celery(self):
- with patch(_FLIPT, return_value=False):
- result = resolve_transport(execution_id="e1", organization_id="org1")
- assert result == WorkflowTransport.CELERY.value
-
- def test_flipt_exception_fails_closed_to_celery(self):
- """A Flipt outage must never break execution creation."""
- with patch(_FLIPT, side_effect=RuntimeError("flipt down")):
- result = resolve_transport(execution_id="e1", organization_id="org1")
- assert result == WorkflowTransport.CELERY.value
-
- def test_passes_execution_id_as_entity_and_builds_context(self):
- """entity_id = execution_id (sticky bucketing); context carries the
- org/workflow/pipeline for segment rules.
- """
- with patch(_FLIPT, return_value=True) as flipt:
- resolve_transport(
- execution_id="exec-42",
- organization_id="org1",
- workflow_id="wf1",
- pipeline_id="pl1",
- )
- flipt.assert_called_once_with(
- flag_key=PG_QUEUE_FLAG_KEY,
- entity_id="exec-42",
- context={
- "organization_id": "org1",
- "workflow_id": "wf1",
- "pipeline_id": "pl1",
- },
- )
-
- def test_non_string_ids_are_coerced_for_flipt(self):
- """Callers pass UUID objects for the ids. Flipt's context is a gRPC
- map<string,string> and entity_id must hash stably, so every value must
- reach check_feature_flag_status as a plain str — otherwise the client
- swallows the serialization error as False and silently forces celery.
- Regression test for that exact dev-test finding.
- """
- import uuid
-
- ex = uuid.UUID("8c091789-9dde-45e7-bb85-06f23fe120eb")
- wf = uuid.UUID("ebed2834-c9fb-4b6c-8df3-9dd841f616bb")
- pl = uuid.UUID("eaca3b0e-083a-4c75-8b25-85349d54145b")
- with patch(_FLIPT, return_value=True) as flipt:
- result = resolve_transport(
- execution_id=ex, organization_id="org1", workflow_id=wf, pipeline_id=pl
- )
- assert result == WorkflowTransport.PG_QUEUE.value
- _, kwargs = flipt.call_args
- assert kwargs["entity_id"] == str(ex)
- assert all(isinstance(v, str) for v in kwargs["context"].values())
- assert kwargs["context"]["workflow_id"] == str(wf)
- assert kwargs["context"]["pipeline_id"] == str(pl)
-
- def test_context_omits_unset_optional_ids(self):
- with patch(_FLIPT, return_value=True) as flipt:
- resolve_transport(execution_id="exec-42", organization_id="org1")
- _, kwargs = flipt.call_args
- assert kwargs["context"] == {"organization_id": "org1"}
-
- def test_result_is_a_valid_transport_value(self):
- valid = {t.value for t in WorkflowTransport}
- with patch(_FLIPT, return_value=True):
- assert (
- resolve_transport(execution_id="e1", organization_id="org1") in valid
- )
-
-
-class TestNormalizeTransport:
- """The fail-closed coercion used at every untrusted read boundary."""
-
- def test_recognized_values_pass_through(self):
- assert normalize_transport("celery") == "celery"
- assert normalize_transport("pg_queue") == "pg_queue"
-
- def test_unrecognized_value_falls_back_to_celery(self):
- assert normalize_transport("celary") == DEFAULT_WORKFLOW_TRANSPORT
- assert normalize_transport("pg-queue") == DEFAULT_WORKFLOW_TRANSPORT
-
- def test_none_and_empty_fall_back_to_celery(self):
- assert normalize_transport(None) == DEFAULT_WORKFLOW_TRANSPORT
- assert normalize_transport("") == DEFAULT_WORKFLOW_TRANSPORT
-
- def test_invalid_value_logs_a_warning_when_logger_given(self):
- log = MagicMock()
- assert normalize_transport("bogus", logger=log, context=" [exec:x]") == "celery"
- log.warning.assert_called_once()
-
- def test_valid_value_does_not_warn(self):
- log = MagicMock()
- normalize_transport("pg_queue", logger=log)
- log.warning.assert_not_called()
diff --git a/backend/workflow_manager/workflow_v2/transport.py b/backend/workflow_manager/workflow_v2/transport.py
deleted file mode 100644
index 08b2f31c2..000000000
--- a/backend/workflow_manager/workflow_v2/transport.py
+++ /dev/null
@@ -1,145 +0,0 @@
-"""Transport resolution for a workflow execution (9e).
-
-A workflow execution rides one transport end-to-end — legacy Celery/RabbitMQ
-or the bespoke Postgres queue. The choice is resolved **once**, at the
-execution-creation chokepoint, and returned to the caller so it can be carried
-in the dispatched task's payload (see ``workers/queue_backend/pg_queue/
-9e-design.md``). It is deliberately **not** persisted on the ``WorkflowExecution``
-row: the payload is the single carrier, durable for PG via the queue row's
-JSONB, and the giant shared table is never migrated for this work.
-
-The transport is resolved from a single Flipt evaluation:
-
- Flipt boolean (``pg_queue_enabled``) → transport
-
-Routing onto PG needs **both**: Flipt reachable (``FLIPT_SERVICE_AVAILABLE=true``)
-and the single ``pg_queue_enabled`` flag enabled for this execution. The flag is the
-sole rollout control — flip it to ramp PG, flip it off (or a Flipt outage) to fall
-back to Celery.
-
-- **Flipt** decides per-execution: ``entity_id = execution_id`` drives the
- percentage-rollout hashing (an execution resolves exactly once, so it can
- never re-bucket mid-flight); ``context`` carries org/workflow/pipeline for
- segment rules. The flag contract is fixed in 9e-design §2.
-- **Fail-closed to Celery**: a blind/unreachable Flipt, a missing org, or any error
- must never break execution creation — it resolves to Celery, mirroring
- ``normalize_transport`` on the read side. The decision is logged so a "flag on but
- still all Celery" situation (e.g. a blind Flipt) is visible rather than silent.
-"""
-
-from __future__ import annotations
-
-import logging
-import os
-from typing import TYPE_CHECKING
-
-from pg_queue.flags import PG_QUEUE_FLAG_KEY
-
-from unstract.core.data_models import WorkflowTransport
-from unstract.flags.feature_flag import check_feature_flag_status
-
-if TYPE_CHECKING:
- from uuid import UUID
-
-logger = logging.getLogger(__name__)
-
-# The single PG-queue rollout flag (Boolean, default false) is defined once in
-# pg_queue.flags and imported above; execution, scheduler, and executor all read
-# that one key. Re-exported here for callers/tests that import it from this module.
-
-
-def resolve_transport(
- *,
- execution_id: str | UUID,
- organization_id: str | UUID | None,
- workflow_id: str | UUID | None = None,
- pipeline_id: str | UUID | None = None,
-) -> str:
- """Resolve the transport for a new workflow execution.
-
- The id parameters are typed ``str | UUID`` because callers pass either
- (e.g. the view path pre-coerces with ``str(...)``, the helper path passes
- raw UUIDs). Coercion to the wire types Flipt needs is kept internal here so
- callers never have to know about it.
-
- Args:
- execution_id: The execution's id. Used as the Flipt ``entity_id`` so the
- percentage rollout buckets per execution and stays sticky (one
- execution is resolved exactly once).
- organization_id: The org's string identifier
- (``Organization.organization_id`` — the ``X-Organization-ID`` header
- value, *not* the DB pk). May be ``None`` on the helper path
- (``StateStore`` not populated) → resolves to celery. Carried in the
- Flipt ``context`` for per-org segment rollouts.
- workflow_id: Optional, carried in ``context`` for future segment rules.
- pipeline_id: Optional, carried in ``context`` for future segment rules.
-
- Returns:
- A :class:`WorkflowTransport` value string — ``"pg_queue"`` only when Flipt
- is reachable and says yes for this execution; ``"celery"`` otherwise
- (including any error — fail-closed).
- """
- celery = WorkflowTransport.CELERY.value
-
- # No org context → per-org segment matching can't be trusted (str(None) would
- # ship a bogus "None" org into the Flipt context and mis-segment). The view
- # path validates the header non-None, but the helper path reads it from
- # StateStore, which can be empty. Fail closed rather than mis-bucket.
- if not organization_id:
- logger.warning(
- "resolve_transport: no organization_id for execution %s; forcing celery",
- execution_id,
- )
- return celery
-
- # FliptClient returns False for ALL flags when the service is marked
- # unavailable — indistinguishable from "rollout says no". Surface it loudly so
- # a blind Flipt doesn't masquerade as a healthy 100%-Celery rollout. Parse
- # exactly as FliptClient does (``.lower()``, no ``.strip()``) so the two can
- # never disagree on a value like ``" true"``.
- if os.environ.get("FLIPT_SERVICE_AVAILABLE", "false").lower() != "true":
- logger.warning(
- "resolve_transport: FLIPT_SERVICE_AVAILABLE != true "
- "(Flipt is blind) for execution %s; forcing celery",
- execution_id,
- )
- return celery
-
- # Flipt's context is a gRPC map<string,string>; callers pass UUID objects for
- # the ids, so coerce every value to str. A non-str value makes the client's
- # serialization fail and check_feature_flag_status swallow it as False —
- # silently forcing celery. entity_id is str-coerced for the same reason (and
- # so the %-rollout hash is stable across str/UUID call sites).
- context = {"organization_id": str(organization_id)}
- if workflow_id:
- context["workflow_id"] = str(workflow_id)
- if pipeline_id:
- context["pipeline_id"] = str(pipeline_id)
-
- # Defense in depth: check_feature_flag_status already wraps its body in
- # try/except → False, but an explicit fail-closed wrap here keeps a future
- # change to that helper from ever letting a Flipt problem break execution
- # creation.
- try:
- enabled = check_feature_flag_status(
- flag_key=PG_QUEUE_FLAG_KEY,
- entity_id=str(execution_id),
- context=context,
- )
- except Exception:
- logger.warning(
- "resolve_transport: Flipt evaluation failed for execution %s; "
- "falling back to Celery",
- execution_id,
- exc_info=True,
- )
- return celery
-
- result = WorkflowTransport.PG_QUEUE.value if enabled else celery
- logger.info(
- "resolve_transport: execution %s resolved to %s (flipt enabled=%s)",
- execution_id,
- result,
- enabled,
- )
- return result
diff --git a/backend/workflow_manager/workflow_v2/undispatched_sweep.py b/backend/workflow_manager/workflow_v2/undispatched_sweep.py
index cc3d77f35..290f63522 100644
--- a/backend/workflow_manager/workflow_v2/undispatched_sweep.py
+++ b/backend/workflow_manager/workflow_v2/undispatched_sweep.py
@@ -21,9 +21,10 @@ executions were left PENDING** — 2914 execution rows from 2444 requests, becau
row outlived the request. ``pg_queue_message`` was completely empty, confirming the
work was never enqueued rather than enqueued-and-lost.
-**Not PG-specific.** The create-then-dispatch ordering sits *upstream* of
-``resolve_transport``, so the Celery path has the identical window. The predicate here
-is transport-aware for exactly that reason.
+**Not PG-specific.** The create-then-dispatch ordering sits *upstream* of transport
+selection, so the Celery path had the identical window (UN-4046 removed that path;
+the predicate stays transport-aware because rows dispatched before the upgrade carry
+``task_id`` rather than ``queue_message_id``).
**The predicate.** ``workflow_helper.py:566/570`` stamps ``queue_message_id`` (PG) or
``task_id`` (Celery) immediately after a successful dispatch, and the model documents
diff --git a/backend/workflow_manager/workflow_v2/workflow_helper.py b/backend/workflow_manager/workflow_v2/workflow_helper.py
index 971fb7a8c..da5ad380c 100644
--- a/backend/workflow_manager/workflow_v2/workflow_helper.py
+++ b/backend/workflow_manager/workflow_v2/workflow_helper.py
@@ -27,7 +27,7 @@ from utils.local_context import StateStore
from utils.user_context import UserContext
from backend.celery_service import app as celery_app
-from unstract.core.data_models import WorkloadType, is_pg_transport
+from unstract.core.data_models import WorkflowTransport, WorkloadType
from unstract.workflow_execution.enums import LogStage
from workflow_manager.endpoint_v2.destination import DestinationConnector
from workflow_manager.endpoint_v2.dto import FileHash
@@ -59,7 +59,6 @@ from workflow_manager.workflow_v2.execution import WorkflowExecutionServiceHelpe
from workflow_manager.workflow_v2.file_history_helper import FileHistoryHelper
from workflow_manager.workflow_v2.models.execution import WorkflowExecution
from workflow_manager.workflow_v2.models.workflow import Workflow
-from workflow_manager.workflow_v2.transport import resolve_transport
logger = logging.getLogger(__name__)
@@ -483,96 +482,80 @@ class WorkflowHelper:
@staticmethod
def _dispatch_orchestrator_task(
*,
- transport: str,
queue: str | None,
args: list[Any],
kwargs: dict[str, Any],
org_schema: str,
) -> str | None:
- """Dispatch ``async_execute_bin`` on the resolved transport; return its id.
+ """Enqueue ``async_execute_bin`` on the PG queue; return its id.
- ``pg_queue`` → enqueue to ``pg_queue_message`` (a PG consumer runs it);
- ``celery`` → ``celery_app.send_task``. The returned id is the **bare**
- ``str(msg_id)`` (PG) or the Celery task id — one format across entry
+ The returned id is the **bare** ``str(msg_id)`` — one format across entry
paths, matching the worker PG dispatch (``PgDispatchHandle.id``).
"""
- if is_pg_transport(transport):
- is_api_execution = queue == CeleryQueue.CELERY_API_DEPLOYMENTS
- msg_id = pg_enqueue_task(
- task_name="async_execute_bin",
- # None → "celery" (general); else celery_api_deployments.
- queue=queue,
- args=args,
- kwargs=kwargs,
- # Two sentinels, deliberately: the row's org_id column is NOT NULL
- # (wants ""), while the fairness payload's org_id is str | None (a
- # missing org means "no segment", not the literal "None").
- org_id=org_schema or "",
- priority=PG_DEFAULT_PRIORITY,
- fairness={
- "org_id": org_schema or None,
- "workload_type": (
- WorkloadType.API.value
- if is_api_execution
- else WorkloadType.NON_API.value
- ),
- "pipeline_priority": PG_DEFAULT_PRIORITY,
- },
- )
- return str(msg_id)
- async_execution = celery_app.send_task(
- "async_execute_bin", args=args, kwargs=kwargs, queue=queue
+ is_api_execution = queue == CeleryQueue.CELERY_API_DEPLOYMENTS
+ msg_id = pg_enqueue_task(
+ task_name="async_execute_bin",
+ # None → "celery" (general); else celery_api_deployments.
+ queue=queue,
+ args=args,
+ kwargs=kwargs,
+ # Two sentinels, deliberately: the row's org_id column is NOT NULL
+ # (wants ""), while the fairness payload's org_id is str | None (a
+ # missing org means "no segment", not the literal "None").
+ org_id=org_schema or "",
+ priority=PG_DEFAULT_PRIORITY,
+ fairness={
+ "org_id": org_schema or None,
+ "workload_type": (
+ WorkloadType.API.value
+ if is_api_execution
+ else WorkloadType.NON_API.value
+ ),
+ "pipeline_priority": PG_DEFAULT_PRIORITY,
+ },
)
- return async_execution.id
+ return str(msg_id)
@staticmethod
def _record_dispatch_handle(
*,
execution_id: str,
- transport: str,
dispatch_handle: str | None,
org_schema: str,
file_count: int,
) -> None:
- """Persist the transport's dispatch handle on the execution row.
+ """Persist the dispatch handle on the execution row.
- Celery → the Celery task UUID into ``task_id`` (UUIDField). PG → the
- ``pg_queue_message.msg_id`` into ``queue_message_id`` (BigIntegerField);
- ``task_id`` stays NULL because there is no Celery task on the PG path.
- Each id lives in its own correctly-typed column so a bigint msg_id is
- never forced into the UUID ``task_id``.
+ The ``pg_queue_message.msg_id`` goes into ``queue_message_id``
+ (BigIntegerField). ``task_id`` stays NULL — it held the Celery task UUID
+ and there is no Celery task on the PG path.
"""
if not dispatch_handle:
- # PG always yields a truthy msg_id, so an empty handle is Celery-only.
+ # PG always yields a truthy msg_id, so an empty handle means the
+ # enqueue did not return one — record it rather than silently
+ # leaving the execution without a handle.
logger.warning(
- f"[{org_schema}] Empty dispatch handle (transport={transport}) "
- f"for execution_id '{execution_id}'."
+ f"[{org_schema}] Empty dispatch handle for execution_id '{execution_id}'."
)
return
- if is_pg_transport(transport):
- # The PG handle is the bigint msg_id as a string. Parse defensively
- # so a malformed/future handle format surfaces its specific cause
- # here instead of being absorbed by the caller's generic guard.
- try:
- msg_id = int(dispatch_handle)
- except (TypeError, ValueError):
- logger.error(
- f"[{org_schema}] PG dispatch handle {dispatch_handle!r} is not "
- f"a valid bigint msg_id for execution_id '{execution_id}'; "
- "queue_message_id not recorded"
- )
- return
- WorkflowExecutionServiceHelper.update_execution_queue_message_id(
- execution_id=execution_id, queue_message_id=msg_id
- )
- else:
- WorkflowExecutionServiceHelper.update_execution_task(
- execution_id=execution_id, task_id=dispatch_handle
+ # The handle is the bigint msg_id as a string. Parse defensively so a
+ # malformed/future handle format surfaces its specific cause here
+ # instead of being absorbed by the caller's generic guard.
+ try:
+ msg_id = int(dispatch_handle)
+ except (TypeError, ValueError):
+ logger.error(
+ f"[{org_schema}] PG dispatch handle {dispatch_handle!r} is not "
+ f"a valid bigint msg_id for execution_id '{execution_id}'; "
+ "queue_message_id not recorded"
)
+ return
+ WorkflowExecutionServiceHelper.update_execution_queue_message_id(
+ execution_id=execution_id, queue_message_id=msg_id
+ )
logger.info(
- f"[{org_schema}] Job '{dispatch_handle}' enqueued "
- f"(transport={transport}) for execution_id '{execution_id}', "
- f"'{file_count}' files"
+ f"[{org_schema}] Job '{dispatch_handle}' enqueued for "
+ f"execution_id '{execution_id}', '{file_count}' files"
)
@classmethod
@@ -621,19 +604,6 @@ class WorkflowHelper:
# end-to-end. Flipt (gated by the env master-switch) decides; fails
# closed to "celery".
#
- # NOTE (deliberate, two resolution sites): transport is resolved here
- # for the API/manual/async paths and separately in
- # internal_api_views.create_workflow_execution for the scheduler
- # path. These are DISTINCT entry paths — never a double-resolution of
- # the same execution. entity_id is execution_id, so each execution
- # buckets exactly once and can't split across transports even if the
- # rollout % is re-rolled between the two sites.
- transport = resolve_transport(
- execution_id=execution_id,
- organization_id=org_schema,
- workflow_id=workflow_id,
- pipeline_id=pipeline_id,
- )
dispatch_args = [
org_schema, # schema_name
workflow_id, # workflow_id
@@ -650,13 +620,11 @@ class WorkflowHelper:
"hitl_queue_name": hitl_queue_name,
"hitl_packet_id": hitl_packet_id,
"custom_data": custom_data,
- "transport": transport,
+ # Wire field the workers still branch on. PG is the only
+ # transport now; the field goes when those branches do.
+ "transport": WorkflowTransport.PG_QUEUE.value,
}
- # Orchestrator transport (9e PR A / 2d): dispatch async_execute_bin on
- # the resolved transport (PG enqueue vs Celery). Extracted to a helper
- # so this method stays simple and the fork is unit-testable.
dispatch_handle = cls._dispatch_orchestrator_task(
- transport=transport,
queue=queue,
args=dispatch_args,
kwargs=dispatch_kwargs,
@@ -691,26 +659,24 @@ class WorkflowHelper:
workflow_execution: WorkflowExecution = WorkflowExecution.objects.get(
id=execution_id
)
- # Record the dispatch handle (Celery task_id or PG msg_id) on the row.
- # Best-effort, in its OWN try/except: it must NEVER abort the
- # synchronous timeout wait below. (Writing a bigint msg_id into the
- # UUID task_id used to raise ValueError inside update_execution_task
- # (UUID coercion on save), which bubbled to the post-dispatch handler
- # and silently skipped the wait — so every PG-routed API deployment
- # ignored `timeout`.)
+ # Record the PG msg_id on the row. Best-effort, in its OWN
+ # try/except: it must NEVER abort the synchronous timeout wait below.
+ # (Writing a bigint msg_id into the UUID task_id used to raise
+ # ValueError inside update_execution_task (UUID coercion on save),
+ # which bubbled to the post-dispatch handler and silently skipped the
+ # wait — so every PG-routed API deployment ignored `timeout`.)
try:
cls._record_dispatch_handle(
execution_id=execution_id,
- transport=transport,
dispatch_handle=dispatch_handle,
org_schema=org_schema or "",
file_count=len(hash_values_of_files),
)
except Exception:
logger.exception(
- f"[{org_schema}] Failed to record dispatch handle "
- f"(transport={transport}) for execution '{execution_id}'; "
- "continuing — the orchestrator is already running"
+ f"[{org_schema}] Failed to record dispatch handle for "
+ f"execution '{execution_id}'; continuing — the orchestrator "
+ "is already running"
)
execution_status = workflow_execution.status
@@ -747,8 +713,8 @@ class WorkflowHelper:
# it never raised; this generic handler covers any stray case.)
logger.exception(
f"[{org_schema}] Post-dispatch bookkeeping failed for execution "
- f"'{execution_id}' (orchestrator already dispatched on "
- f"{transport}); not marking ERROR"
+ f"'{execution_id}' (orchestrator already enqueued on the PG "
+ f"queue); not marking ERROR"
)
return ExecutionResponse(
workflow_id, execution_id, ExecutionStatus.EXECUTING.value
@@ -1054,43 +1020,29 @@ class WorkflowHelper:
workflow_id=workflow.id, single_step=True
)
try:
- workflow_execution = WorkflowExecution.objects.get(pk=execution_id)
+ # Existence probe only — the branch below raises unconditionally,
+ # so the result is deliberately not bound. The query is still
+ # load-bearing: its DoesNotExist selects the create-a-new-execution
+ # path in the except.
+ WorkflowExecution.objects.get(pk=execution_id)
# Single-step is the one entry path that never moved onto PG. Its
# only fan-out is the Celery chord in `process_input_files` — the
- # normal path's PG fan-out lives in the general worker instead, so
- # nothing here is transport-gated. With the flag on and the Celery
- # file_processing workers scaled to zero (the epic's acceptance
- # gate), those batches would sit unconsumed and the execution would
- # hang in EXECUTING, invisible to the PG reaper. Fail fast instead:
- # a 500 with a stated cause beats a silent forever-EXECUTING row.
+ # normal path's PG fan-out lives in the general worker instead —
+ # so with no Celery file_processing workers those batches would
+ # sit unconsumed and the execution would hang in EXECUTING,
+ # invisible to the PG reaper. Fail fast instead: a 500 with a
+ # stated cause beats a silent forever-EXECUTING row.
#
- # Unreachable from the UI — the step buttons are gone and both live
- # call sites pass isStepExecution=false, so `execution_action` is
- # never sent (frontend Agency.jsx). This guards a direct API call.
+ # Deprecated and unreachable from the UI — the step buttons are
+ # gone and both live call sites pass isStepExecution=false, so
+ # `execution_action` is never sent (frontend Agency.jsx). This
+ # guards a direct API call. Removing the endpoint outright is
+ # tracked separately.
#
- # Resolved here rather than in `process_input_files` so the Celery
- # path stays byte-identical and an already-resolved execution is
- # never re-resolved. Step executions are created by
- # `create_and_make_execution_response`, which resolves no transport,
- # so this is their first and only resolution.
- transport = resolve_transport(
- execution_id=execution_id,
- organization_id=workflow.organization.organization_id,
- workflow_id=workflow.id,
- )
- if is_pg_transport(transport):
- raise WorkflowExecutionError(
- "Single-step execution is not supported on the Postgres "
- "queue transport: it has no PG fan-out, and its Celery "
- "batches would never be consumed. Run the full workflow "
- "instead, or disable pg_queue_enabled for this "
- "organization."
- )
- return WorkflowHelper.run_workflow(
- workflow=workflow,
- single_step=True,
- workflow_execution=workflow_execution,
- hash_values_of_files=hash_values_of_files,
+ raise WorkflowExecutionError(
+ "Single-step execution is not supported: it has no PG "
+ "fan-out, and its Celery batches would never be consumed. "
+ "Run the full workflow instead."
)
except WorkflowExecution.DoesNotExist:
return WorkflowHelper.create_and_make_execution_response(
diff --git a/docker/README.md b/docker/README.md
index 4990ffe31..be71603c8 100644
--- a/docker/README.md
+++ b/docker/README.md
@@ -104,13 +104,13 @@ Debug ports per service:
| platform-service | 5680 |
| prompt-service | 5681 |
| **V2 Workers** | |
-| worker-file-processing-v2 | 5682 |
-| worker-callback-v2 | 5683 |
-| worker-api-deployment-v2 | 5684 |
-| worker-general-v2 | 5685 |
-| worker-notification-v2 | 5686 |
-| worker-log-consumer-v2 | 5687 |
-| worker-scheduler-v2 | 5688 |
+| worker-pg-fileproc | 5682 |
+| worker-pg-callback | 5683 |
+| worker-pg-orchestrator-api | 5684 |
+| worker-pg-orchestrator-general | 5685 |
+| worker-pg-notification | 5686 |
+| worker-log-stream-consumer | 5687 |
+| worker-pg-scheduler | 5688 |
### VSCode Configuration
diff --git a/docker/compose.debug.yaml b/docker/compose.debug.yaml
index 3522902b7..6028d2722 100644
--- a/docker/compose.debug.yaml
+++ b/docker/compose.debug.yaml
@@ -7,13 +7,13 @@
# 5678 - backend
# 5679 - runner
# 5680 - platform-service
-# 5682 - worker-file-processing-v2
-# 5683 - worker-callback-v2
-# 5684 - worker-api-deployment-v2
-# 5685 - worker-general-v2
-# 5686 - worker-notification-v2
-# 5687 - worker-log-consumer-v2
-# 5688 - worker-scheduler-v2
+# 5682 - worker-pg-fileproc
+# 5683 - worker-pg-callback
+# 5684 - worker-pg-orchestrator-api
+# 5685 - worker-pg-orchestrator-general
+# 5686 - worker-pg-notification
+# 5687 - worker-log-stream-consumer
+# 5688 - worker-pg-scheduler
services:
#########################################################################################################
@@ -68,128 +68,121 @@ services:
# V2 Workers with debugpy
# Using --pool=threads for debugpy compatibility (prefork doesn't work well with debugpy)
- worker-file-processing-v2:
+ worker-pg-fileproc:
user: root
ports:
- "5682:5682"
entrypoint: ["bash", "-c"]
+ # Wrap the PG consumer, NOT `celery -A worker worker` (UN-4046). This
+ # override REPLACES the base file's command, so a Celery invocation here
+ # turns the service into a RabbitMQ worker while the backend enqueues to
+ # Postgres — the debug stack then accepts work and runs none of it.
+ #
+ # Queue set and worker type are deliberately NOT repeated: the module reads
+ # WORKER_PG_QUEUE_CONSUMER_QUEUE / _WORKER_TYPE from the base file's
+ # environment. Restating them here is what let this file drift out of sync.
command: [
- "uv run python -m debugpy --listen 0.0.0.0:5682 -m celery
- -A worker worker
- --queues=file_processing,api_file_processing,file_processing_priority
- --loglevel=INFO
- --pool=threads
- --concurrency=2
- --prefetch-multiplier=1
- --without-gossip
- --without-mingle
- --without-heartbeat"
+ "uv run python -m debugpy --listen 0.0.0.0:5682 -m pg_queue_consumer"
]
- worker-callback-v2:
+ worker-pg-callback:
user: root
ports:
- "5683:5683"
entrypoint: ["bash", "-c"]
+ # Wrap the PG consumer, NOT `celery -A worker worker` (UN-4046). This
+ # override REPLACES the base file's command, so a Celery invocation here
+ # turns the service into a RabbitMQ worker while the backend enqueues to
+ # Postgres — the debug stack then accepts work and runs none of it.
+ #
+ # Queue set and worker type are deliberately NOT repeated: the module reads
+ # WORKER_PG_QUEUE_CONSUMER_QUEUE / _WORKER_TYPE from the base file's
+ # environment. Restating them here is what let this file drift out of sync.
command: [
- "uv run python -m debugpy --listen 0.0.0.0:5683 -m celery
- -A worker worker
- --queues=file_processing_callback,api_file_processing_callback
- --loglevel=INFO
- --pool=threads
- --concurrency=2
- --prefetch-multiplier=1
- --without-gossip
- --without-mingle
- --without-heartbeat"
+ "uv run python -m debugpy --listen 0.0.0.0:5683 -m pg_queue_consumer"
]
- worker-api-deployment-v2:
+ worker-pg-orchestrator-api:
user: root
ports:
- "5684:5684"
entrypoint: ["bash", "-c"]
+ # Wrap the PG consumer, NOT `celery -A worker worker` (UN-4046). This
+ # override REPLACES the base file's command, so a Celery invocation here
+ # turns the service into a RabbitMQ worker while the backend enqueues to
+ # Postgres — the debug stack then accepts work and runs none of it.
+ #
+ # Queue set and worker type are deliberately NOT repeated: the module reads
+ # WORKER_PG_QUEUE_CONSUMER_QUEUE / _WORKER_TYPE from the base file's
+ # environment. Restating them here is what let this file drift out of sync.
command: [
- "uv run python -m debugpy --listen 0.0.0.0:5684 -m celery
- -A worker worker
- --queues=celery_api_deployments
- --loglevel=INFO
- --pool=threads
- --concurrency=2
- --prefetch-multiplier=1
- --without-gossip
- --without-mingle
- --without-heartbeat"
+ "uv run python -m debugpy --listen 0.0.0.0:5684 -m pg_queue_consumer"
]
- worker-general-v2:
+ worker-pg-orchestrator-general:
user: root
ports:
- "5685:5685"
entrypoint: ["bash", "-c"]
+ # Wrap the PG consumer, NOT `celery -A worker worker` (UN-4046). This
+ # override REPLACES the base file's command, so a Celery invocation here
+ # turns the service into a RabbitMQ worker while the backend enqueues to
+ # Postgres — the debug stack then accepts work and runs none of it.
+ #
+ # Queue set and worker type are deliberately NOT repeated: the module reads
+ # WORKER_PG_QUEUE_CONSUMER_QUEUE / _WORKER_TYPE from the base file's
+ # environment. Restating them here is what let this file drift out of sync.
command: [
- "uv run python -m debugpy --listen 0.0.0.0:5685 -m celery
- -A worker worker
- --queues=celery
- --loglevel=INFO
- --pool=threads
- --concurrency=2
- --prefetch-multiplier=1
- --without-gossip
- --without-mingle
- --without-heartbeat"
+ "uv run python -m debugpy --listen 0.0.0.0:5685 -m pg_queue_consumer"
]
- worker-notification-v2:
+ worker-pg-notification:
user: root
ports:
- "5686:5686"
entrypoint: ["bash", "-c"]
+ # Wrap the PG consumer, NOT `celery -A worker worker` (UN-4046). This
+ # override REPLACES the base file's command, so a Celery invocation here
+ # turns the service into a RabbitMQ worker while the backend enqueues to
+ # Postgres — the debug stack then accepts work and runs none of it.
+ #
+ # Queue set and worker type are deliberately NOT repeated: the module reads
+ # WORKER_PG_QUEUE_CONSUMER_QUEUE / _WORKER_TYPE from the base file's
+ # environment. Restating them here is what let this file drift out of sync.
command: [
- "uv run python -m debugpy --listen 0.0.0.0:5686 -m celery
- -A worker worker
- --queues=notifications,notifications_webhook,notifications_email,notifications_sms,notifications_priority
- --loglevel=INFO
- --pool=threads
- --concurrency=2
- --prefetch-multiplier=1
- --without-gossip
- --without-mingle
- --without-heartbeat"
+ "uv run python -m debugpy --listen 0.0.0.0:5686 -m pg_queue_consumer"
]
- worker-log-consumer-v2:
+ worker-log-stream-consumer:
user: root
ports:
- "5687:5687"
entrypoint: ["bash", "-c"]
+ # Wrap the PG consumer, NOT `celery -A worker worker` (UN-4046). This
+ # override REPLACES the base file's command, so a Celery invocation here
+ # turns the service into a RabbitMQ worker while the backend enqueues to
+ # Postgres — the debug stack then accepts work and runs none of it.
+ #
+ # Queue set and worker type are deliberately NOT repeated: the module reads
+ # WORKER_PG_QUEUE_CONSUMER_QUEUE / _WORKER_TYPE from the base file's
+ # environment. Restating them here is what let this file drift out of sync.
command: [
- "uv run python -m debugpy --listen 0.0.0.0:5687 -m celery
- -A worker worker
- --queues=celery_log_task_queue,celery_periodic_logs
- --loglevel=INFO
- --pool=threads
- --concurrency=2
- --prefetch-multiplier=1
- --without-gossip
- --without-mingle
- --without-heartbeat"
+ "uv run python -m debugpy --listen 0.0.0.0:5687 -m log_consumer.redis_stream_consumer"
]
- worker-scheduler-v2:
+ worker-pg-scheduler:
user: root
ports:
- "5688:5688"
entrypoint: ["bash", "-c"]
+ # Wrap the PG consumer, NOT `celery -A worker worker` (UN-4046). This
+ # override REPLACES the base file's command, so a Celery invocation here
+ # turns the service into a RabbitMQ worker while the backend enqueues to
+ # Postgres — the debug stack then accepts work and runs none of it.
+ #
+ # Queue set and worker type are deliberately NOT repeated: the module reads
+ # WORKER_PG_QUEUE_CONSUMER_QUEUE / _WORKER_TYPE from the base file's
+ # environment. Restating them here is what let this file drift out of sync.
command: [
- "uv run python -m debugpy --listen 0.0.0.0:5688 -m celery
- -A worker worker
- --queues=scheduler
- --loglevel=INFO
- --pool=threads
- --concurrency=2
- --prefetch-multiplier=1
- --without-gossip
- --without-mingle
- --without-heartbeat"
+ "uv run python -m debugpy --listen 0.0.0.0:5688 -m pg_queue_consumer"
]
diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml
index dda8f27a2..9f6ca7789 100644
--- a/docker/docker-compose.yaml
+++ b/docker/docker-compose.yaml
@@ -50,54 +50,30 @@ services:
- ${TOOL_REGISTRY_CONFIG_SRC_PATH}:/data/tool_registry_config
environment:
- APPLICATION_NAME=unstract-backend
+ # Redis log transport (UN-4046) — set on EVERY publisher: backend, runner
+ # and all twelve workers. worker-log-consumer-v2 is gone from this stack, so
+ # the "celery" default publishes execution logs to a RabbitMQ queue nothing
+ # drains; LogPublisher swallows its errors, so the only symptom is execution
+ # logs that stay empty while the queue grows.
+ #
+ # Deliberately here and not in essentials.env: that file is git-ignored and
+ # generated, and copy_or_merge_envs only merges new keys on --only-env or
+ # --update, so an existing checkout would silently keep the old default.
+ - LOG_TRANSPORT=redis
+ # Adopt the non-pipeline periodics (dashboard_metrics.*) onto the PG
+ # scheduler (UN-4046). converge_pg_scheduler only adopts them with
+ # --periodics, which entrypoint.sh derives from this variable — and it was
+ # set NOWHERE. This stack deletes celery-beat and worker-metrics, so
+ # without it those rows stay Beat-owned (pg_owned=false) with Beat gone:
+ # nothing fires them, worker-pg-metrics idles on an empty queue, and there
+ # is no error, no queue depth and no failed pod. The cloud chart refuses
+ # this combination at render time; compose has no equivalent guard.
+ - PG_SCHEDULER_ADOPT_PERIODICS=true
labels:
- traefik.enable=true
- traefik.http.routers.backend.rule=Host(`frontend.unstract.localhost`) && (PathPrefix(`/api/v1`) || PathPrefix(`/deployment`) || PathPrefix(`/public`))
- traefik.http.services.backend.loadbalancer.server.port=8000
- # Celery worker for dashboard metrics processing
- worker-metrics:
- <<: *host_gateway
- image: unstract/backend:${VERSION}
- container_name: unstract-worker-metrics
- restart: unless-stopped
- entrypoint: .venv/bin/celery
- command: "-A backend worker --loglevel=info -Q dashboard_metric_events --autoscale=${WORKER_METRICS_AUTOSCALE:-4,1}"
- env_file:
- - ../backend/.env
- depends_on:
- - rabbitmq
- - db
- environment:
- - APPLICATION_NAME=unstract-worker-metrics
-
- labels:
- - traefik.enable=false
-
- # Celery worker for Prompt Studio IDE callbacks.
- # Processes post-execution callbacks via InternalAPIClient (no Django).
- # Handles: ide_index_complete/error, ide_prompt_complete/error.
- worker-ide-callback:
- <<: *host_gateway
- image: unstract/worker-unified:${VERSION}
- container_name: unstract-worker-ide-callback
- restart: unless-stopped
- command: ["ide-callback"]
- env_file:
- - ../workers/.env
- - ./essentials.env
- depends_on:
- - db
- - redis
- - rabbitmq
- environment:
- - APPLICATION_NAME=unstract-worker-ide-callback
- - WORKER_TYPE=ide_callback
- - WORKER_NAME=ide-callback-worker
- labels:
- - traefik.enable=false
-
- # Celery Flower
celery-flower:
<<: *host_gateway
image: unstract/backend:${VERSION}
@@ -120,24 +96,6 @@ services:
profiles:
- optional
- # Celery Beat
- celery-beat:
- <<: *host_gateway
- image: unstract/backend:${VERSION}
- container_name: unstract-celery-beat
- restart: unless-stopped
- entrypoint: .venv/bin/celery
- command: "-A backend beat --scheduler django_celery_beat.schedulers:DatabaseScheduler -l INFO"
- env_file:
- - ../backend/.env
- - ./essentials.env
- depends_on:
- - db
- - rabbitmq
- environment:
- - APPLICATION_NAME=unstract-celery-beat
-
- # Frontend React app
frontend:
image: unstract/frontend:${VERSION}
container_name: unstract-frontend
@@ -193,6 +151,11 @@ services:
- 5002:5002
env_file:
- ../runner/.env
+ environment:
+ # Publisher side of the Redis log transport (UN-4046) — see the note on the
+ # backend service. runner matters most of the three: it forwards this to every
+ # tool sidecar it spawns, so a stale value here strands the tool logs too.
+ - LOG_TRANSPORT=redis
volumes:
- ./workflow_data:/data
# Docker socket bind mount to spawn tool containers
@@ -203,244 +166,6 @@ services:
labels:
- traefik.enable=false
- # ====================================================================
- # WORKER SERVICES
- # ====================================================================
-
- worker-api-deployment-v2:
- <<: *host_gateway
- image: unstract/worker-unified:${VERSION}
- container_name: unstract-worker-api-deployment-v2
- restart: unless-stopped
- command: ["api-deployment"]
- ports:
- - "8085:8090"
- env_file:
- - ../workers/.env
- - ./essentials.env
- depends_on:
- - db
- - redis
- - rabbitmq
- environment:
- - APPLICATION_NAME=unstract-worker-api-deployment-v2
- - WORKER_TYPE=api_deployment
- - CELERY_QUEUES_API_DEPLOYMENT=${CELERY_QUEUES_API_DEPLOYMENT:-celery_api_deployments}
- - CELERY_POOL=${WORKER_API_DEPLOYMENT_POOL:-threads}
- - CELERY_PREFETCH_MULTIPLIER=${WORKER_API_DEPLOYMENT_PREFETCH_MULTIPLIER:-1}
- - CELERY_CONCURRENCY=${WORKER_API_DEPLOYMENT_CONCURRENCY:-4}
- - CELERY_EXTRA_ARGS=${WORKER_API_DEPLOYMENT_EXTRA_ARGS:-}
- - WORKER_NAME=api-deployment-worker-v2
- - API_DEPLOYMENT_METRICS_PORT=8090
- - HEALTH_PORT=8090
- labels:
- - traefik.enable=false
- volumes:
- - ./workflow_data:/data
- - ${TOOL_REGISTRY_CONFIG_SRC_PATH}:/data/tool_registry_config
-
- worker-callback-v2:
- <<: *host_gateway
- image: unstract/worker-unified:${VERSION}
- container_name: unstract-worker-callback-v2
- restart: unless-stopped
- command: ["callback"]
- ports:
- - "8086:8083"
- env_file:
- - ../workers/.env
- - ./essentials.env
- depends_on:
- - db
- - redis
- - rabbitmq
- environment:
- - APPLICATION_NAME=unstract-worker-callback-v2
- - WORKER_TYPE=callback
- - WORKER_NAME=callback-worker-v2
- - CALLBACK_METRICS_PORT=8083
- labels:
- - traefik.enable=false
- volumes:
- - ./workflow_data:/data
- - ${TOOL_REGISTRY_CONFIG_SRC_PATH}:/data/tool_registry_config
-
- worker-file-processing-v2:
- <<: *host_gateway
- image: unstract/worker-unified:${VERSION}
- container_name: unstract-worker-file-processing-v2
- restart: unless-stopped
- # command: ["file-processing"]
- command:
- [
- ".venv/bin/celery",
- "-A",
- "worker",
- "worker",
- "--queues=file_processing,api_file_processing,file_processing_priority",
- "--loglevel=INFO",
- "--pool=prefork",
- "--concurrency=4",
- "--prefetch-multiplier=1",
- "--without-gossip",
- "--without-mingle",
- "--without-heartbeat",
- ]
- ports:
- - "8087:8082"
- env_file:
- - ../workers/.env
- - ./essentials.env
- depends_on:
- - db
- - redis
- - rabbitmq
- environment:
- - APPLICATION_NAME=unstract-worker-file-processing-v2
- - WORKER_TYPE=file_processing
- - WORKER_MODE=oss
- - WORKER_NAME=file-processing-worker-v2
- - FILE_PROCESSING_METRICS_PORT=8082
- # OSS Configuration - Enterprise features disabled
- - MANUAL_REVIEW_ENABLED=false
- - ENTERPRISE_FEATURES_ENABLED=false
- - PLUGIN_REGISTRY_MODE=oss
- # Configurable Celery options
- - CELERY_QUEUES_FILE_PROCESSING=${CELERY_QUEUES_FILE_PROCESSING:-file_processing,api_file_processing}
- - CELERY_POOL=${WORKER_FILE_PROCESSING_POOL:-threads}
- - CELERY_PREFETCH_MULTIPLIER=${WORKER_FILE_PROCESSING_PREFETCH_MULTIPLIER:-1}
- - CELERY_CONCURRENCY=${WORKER_FILE_PROCESSING_CONCURRENCY:-4}
- - CELERY_EXTRA_ARGS=${WORKER_FILE_PROCESSING_EXTRA_ARGS:-}
- labels:
- - traefik.enable=false
- volumes:
- - ./workflow_data:/data
- - ${TOOL_REGISTRY_CONFIG_SRC_PATH}:/data/tool_registry_config
-
- worker-general-v2:
- <<: *host_gateway
- image: unstract/worker-unified:${VERSION}
- container_name: unstract-worker-general-v2
- restart: unless-stopped
- command: ["general"]
- ports:
- - "8088:8082"
- env_file:
- - ../workers/.env
- - ./essentials.env
- depends_on:
- - db
- - redis
- - rabbitmq
- environment:
- - APPLICATION_NAME=unstract-worker-general-v2
- - WORKER_TYPE=general
- - WORKER_NAME=general-worker-v2
- - GENERAL_METRICS_PORT=8081
- - HEALTH_PORT=8082
- labels:
- - traefik.enable=false
- volumes:
- - ./workflow_data:/data
- - ${TOOL_REGISTRY_CONFIG_SRC_PATH}:/data/tool_registry_config
-
- worker-notification-v2:
- <<: *host_gateway
- image: unstract/worker-unified:${VERSION}
- container_name: unstract-worker-notification-v2
- restart: unless-stopped
- command: ["notification"]
- ports:
- - "8089:8085"
- env_file:
- - ../workers/.env
- - ./essentials.env
- depends_on:
- - db
- - redis
- - rabbitmq
- environment:
- - APPLICATION_NAME=unstract-worker-notification-v2
- - WORKER_TYPE=notification
- - WORKER_NAME=notification-worker-v2
- - NOTIFICATION_METRICS_PORT=8085
- - HEALTH_PORT=8085
- # Notification specific configs
- - NOTIFICATION_QUEUE_NAME=notifications
- - WEBHOOK_QUEUE_NAME=notifications_webhook
- - EMAIL_QUEUE_NAME=notifications_email
- - SMS_QUEUE_NAME=notifications_sms
- - PRIORITY_QUEUE_NAME=notifications_priority
- # Configurable Celery options
- - CELERY_QUEUES_NOTIFICATION=${CELERY_QUEUES_NOTIFICATION:-notifications,notifications_webhook,notifications_email,notifications_sms,notifications_priority}
- - CELERY_POOL=${WORKER_NOTIFICATION_POOL:-prefork}
- - CELERY_PREFETCH_MULTIPLIER=${WORKER_NOTIFICATION_PREFETCH_MULTIPLIER:-1}
- - CELERY_CONCURRENCY=${WORKER_NOTIFICATION_CONCURRENCY:-4}
- - CELERY_EXTRA_ARGS=${WORKER_NOTIFICATION_EXTRA_ARGS:-}
- # Complete command override (if set, ignores all other options)
- - CELERY_COMMAND_OVERRIDE=${WORKER_NOTIFICATION_COMMAND_OVERRIDE:-}
- # Individual argument overrides
- - CELERY_APP_MODULE=${WORKER_NOTIFICATION_APP_MODULE:-worker}
- - CELERY_LOG_LEVEL=${WORKER_NOTIFICATION_LOG_LEVEL:-INFO}
- - CELERY_HOSTNAME=${WORKER_NOTIFICATION_HOSTNAME:-}
- - CELERY_MAX_TASKS_PER_CHILD=${WORKER_NOTIFICATION_MAX_TASKS_PER_CHILD:-}
- - CELERY_TIME_LIMIT=${WORKER_NOTIFICATION_TIME_LIMIT:-}
- - CELERY_SOFT_TIME_LIMIT=${WORKER_NOTIFICATION_SOFT_TIME_LIMIT:-}
- labels:
- - traefik.enable=false
- volumes:
- - ./workflow_data:/data
- - ${TOOL_REGISTRY_CONFIG_SRC_PATH}:/data/tool_registry_config
-
- worker-log-consumer-v2:
- <<: *host_gateway
- image: unstract/worker-unified:${VERSION}
- container_name: unstract-worker-log-consumer-v2
- restart: unless-stopped
- command: ["log-consumer"]
- ports:
- - "8090:8084"
- env_file:
- - ../workers/.env
- - ./essentials.env
- depends_on:
- - db
- - redis
- - rabbitmq
- environment:
- - APPLICATION_NAME=unstract-worker-log-consumer-v2
- - WORKER_TYPE=log_consumer
- - WORKER_NAME=log-consumer-worker-v2
- - LOG_CONSUMER_METRICS_PORT=8084
- - HEALTH_PORT=8084
- # Log consumer specific configs
- - LOG_CONSUMER_QUEUE_NAME=celery_log_task_queue
- # Multiple queue support - supports comma-separated queue names
- - CELERY_QUEUES_LOG_CONSUMER=${CELERY_QUEUES_LOG_CONSUMER:-celery_log_task_queue,celery_periodic_logs}
- - PERIODIC_LOGS_QUEUE_NAME=${PERIODIC_LOGS_QUEUE_NAME:-celery_periodic_logs}
- # Log history configuration
- - LOG_HISTORY_QUEUE_NAME=${LOG_HISTORY_QUEUE_NAME:-log_history_queue}
- - LOGS_BATCH_LIMIT=${LOGS_BATCH_LIMIT:-100}
- - ENABLE_LOG_HISTORY=${ENABLE_LOG_HISTORY:-true}
- - CELERY_POOL=${WORKER_LOG_CONSUMER_POOL:-prefork}
- - CELERY_PREFETCH_MULTIPLIER=${WORKER_LOG_CONSUMER_PREFETCH_MULTIPLIER:-1}
- - CELERY_CONCURRENCY=${WORKER_LOG_CONSUMER_CONCURRENCY:-2}
- - CELERY_EXTRA_ARGS=${WORKER_LOG_CONSUMER_EXTRA_ARGS:-}
- # Complete command override (if set, ignores all other options)
- - CELERY_COMMAND_OVERRIDE=${WORKER_LOG_CONSUMER_COMMAND_OVERRIDE:-}
- # Individual argument overrides
- - CELERY_APP_MODULE=${WORKER_LOG_CONSUMER_APP_MODULE:-worker}
- - CELERY_LOG_LEVEL=${WORKER_LOG_CONSUMER_LOG_LEVEL:-INFO}
- - CELERY_HOSTNAME=${WORKER_LOG_CONSUMER_HOSTNAME:-}
- - CELERY_MAX_TASKS_PER_CHILD=${WORKER_LOG_CONSUMER_MAX_TASKS_PER_CHILD:-}
- - CELERY_TIME_LIMIT=${WORKER_LOG_CONSUMER_TIME_LIMIT:-}
- - CELERY_SOFT_TIME_LIMIT=${WORKER_LOG_CONSUMER_SOFT_TIME_LIMIT:-}
- labels:
- - traefik.enable=false
- volumes:
- - ./workflow_data:/data
- - ${TOOL_REGISTRY_CONFIG_SRC_PATH}:/data/tool_registry_config
-
worker-log-history-scheduler-v2:
<<: *host_gateway
image: unstract/worker-unified:${VERSION}
@@ -457,6 +182,8 @@ services:
- rabbitmq
environment:
- APPLICATION_NAME=unstract-worker-log-history-scheduler-v2
+ # Log publisher — see the note on the backend service (UN-4046).
+ - LOG_TRANSPORT=redis
# Scheduler interval in seconds
- LOG_HISTORY_CONSUMER_INTERVAL=${LOG_HISTORY_CONSUMER_INTERVAL:-5}
# Override example: TASK_TRIGGER_COMMAND=/custom/trigger/script.sh
@@ -464,102 +191,6 @@ services:
labels:
- traefik.enable=false
- worker-scheduler-v2:
- <<: *host_gateway
- image: unstract/worker-unified:${VERSION}
- container_name: unstract-worker-scheduler-v2
- restart: unless-stopped
- command: ["scheduler"]
- ports:
- - "8091:8087"
- env_file:
- - ../workers/.env
- - ./essentials.env
- depends_on:
- - db
- - redis
- - rabbitmq
- environment:
- - APPLICATION_NAME=unstract-worker-scheduler-v2
- - WORKER_TYPE=scheduler
- - WORKER_NAME=scheduler-worker-v2
- - SCHEDULER_METRICS_PORT=8087
- - HEALTH_PORT=8087
- # Scheduler specific configs
- - SCHEDULER_QUEUE_NAME=scheduler
- # Configurable Celery options
- - CELERY_QUEUES_SCHEDULER=${CELERY_QUEUES_SCHEDULER:-scheduler}
- - CELERY_POOL=${WORKER_SCHEDULER_POOL:-prefork}
- - CELERY_PREFETCH_MULTIPLIER=${WORKER_SCHEDULER_PREFETCH_MULTIPLIER:-1}
- - CELERY_CONCURRENCY=${WORKER_SCHEDULER_CONCURRENCY:-2}
- - CELERY_EXTRA_ARGS=${WORKER_SCHEDULER_EXTRA_ARGS:-}
- # Complete command override (if set, ignores all other options)
- - CELERY_COMMAND_OVERRIDE=${WORKER_SCHEDULER_COMMAND_OVERRIDE:-}
- # Individual argument overrides
- - CELERY_APP_MODULE=${WORKER_SCHEDULER_APP_MODULE:-worker}
- - CELERY_LOG_LEVEL=${WORKER_SCHEDULER_LOG_LEVEL:-INFO}
- - CELERY_HOSTNAME=${WORKER_SCHEDULER_HOSTNAME:-}
- - CELERY_MAX_TASKS_PER_CHILD=${WORKER_SCHEDULER_MAX_TASKS_PER_CHILD:-}
- - CELERY_TIME_LIMIT=${WORKER_SCHEDULER_TIME_LIMIT:-}
- - CELERY_SOFT_TIME_LIMIT=${WORKER_SCHEDULER_SOFT_TIME_LIMIT:-}
- labels:
- - traefik.enable=false
- volumes:
- - ./workflow_data:/data
- - ${TOOL_REGISTRY_CONFIG_SRC_PATH}:/data/tool_registry_config
-
- worker-executor-v2:
- <<: *host_gateway
- image: unstract/worker-unified:${VERSION}
- container_name: unstract-worker-executor-v2
- restart: unless-stopped
- command: ["executor"]
- ports:
- - "8092:8088"
- env_file:
- - ../workers/.env
- - ./essentials.env
- depends_on:
- - db
- - redis
- - rabbitmq
- - platform-service
- environment:
- - APPLICATION_NAME=unstract-worker-executor-v2
- - WORKER_TYPE=executor
- - WORKER_NAME=executor-worker-v2
- - EXECUTOR_METRICS_PORT=8088
- - HEALTH_PORT=8088
- # Configurable Celery options
- - CELERY_QUEUES_EXECUTOR=${CELERY_QUEUES_EXECUTOR:-celery_executor_legacy,celery_executor_agentic,celery_executor_agentic_table}
- - CELERY_POOL=${WORKER_EXECUTOR_POOL:-prefork}
- - CELERY_PREFETCH_MULTIPLIER=${WORKER_EXECUTOR_PREFETCH_MULTIPLIER:-1}
- - CELERY_CONCURRENCY=${WORKER_EXECUTOR_CONCURRENCY:-2}
- - CELERY_EXTRA_ARGS=${WORKER_EXECUTOR_EXTRA_ARGS:-}
- labels:
- - traefik.enable=false
- volumes:
- - ./workflow_data:/data
- - ${TOOL_REGISTRY_CONFIG_SRC_PATH}:/data/tool_registry_config
- - prompt_studio_data:/app/prompt-studio-data
-
- # ===========================================================================
- # PG-queue services (Postgres-backed transport) — gated behind the `pg-queue`
- # compose profile, so a plain `docker compose up` starts NONE of them and
- # behaviour is identical to today. Bring them up with:
- # docker compose --profile pg-queue up -d
- # They consume from Postgres, not the broker. The orchestrator consumers and
- # the reaper are broker-free for PG-routed executions (fan-out honors the
- # carried transport; a missing transport fails closed to Celery, which these
- # consumers aren't expected to receive). The fan-out and callback consumers
- # still hand off to the Celery executor (tool-execution RPC) / Celery
- # notifications, so they keep a rabbitmq dependency until the executor and
- # notifications themselves move to PG. Each maps 1:1 to a future K8s Deployment.
- # Executions still route to Celery until the single `pg_queue_enabled` Flipt
- # flag is ramped — that is the rollout, not this change.
- # ===========================================================================
-
- # Orchestrator consumer for API-deployment executions (async_execute_bin_api).
worker-pg-orchestrator-api:
image: unstract/worker-unified:${VERSION}
container_name: unstract-worker-pg-orchestrator-api
@@ -580,6 +211,8 @@ services:
- WORKER_PG_QUEUE_CONSUMER_WORKER_TYPE=api_deployment
- WORKER_PG_QUEUE_CONSUMER_QUEUE=celery_api_deployments
- WORKER_PG_QUEUE_CONSUMER_HEALTH_PORT=8090
+ # Log publisher — see the note on the backend service (UN-4046).
+ - LOG_TRANSPORT=redis
# async_execute_bin arms the barrier + dispatches batches, then returns — a
# bounded-minutes orchestration, and idempotency-guarded (UN-3671) so a vt
# overrun can't double-orchestrate. Without an explicit vt the consumer
@@ -592,8 +225,6 @@ services:
volumes:
- ./workflow_data:/data
- ${TOOL_REGISTRY_CONFIG_SRC_PATH}:/data/tool_registry_config
- profiles:
- - pg-queue
# Orchestrator consumer for general / scheduled-ETL executions (async_execute_bin).
worker-pg-orchestrator-general:
@@ -616,6 +247,8 @@ services:
- WORKER_PG_QUEUE_CONSUMER_WORKER_TYPE=general
- WORKER_PG_QUEUE_CONSUMER_QUEUE=celery
- WORKER_PG_QUEUE_CONSUMER_HEALTH_PORT=8090
+ # Log publisher — see the note on the backend service (UN-4046).
+ - LOG_TRANSPORT=redis
# Orchestration is bounded-minutes + claim-guarded (see orchestrator-api).
# Without an explicit vt it defaults to 30s. 600/660 matches our k8s deployment defaults (workerPg* values, separate chart repo).
- WORKER_PG_QUEUE_CONSUMER_VT_SECONDS=${WORKER_PG_ORCHESTRATOR_VT_SECONDS:-600}
@@ -625,8 +258,6 @@ services:
volumes:
- ./workflow_data:/data
- ${TOOL_REGISTRY_CONFIG_SRC_PATH}:/data/tool_registry_config
- profiles:
- - pg-queue
# Fan-out consumer (process_file_batch) — both the ETL and API file queues.
worker-pg-fileproc:
@@ -648,8 +279,10 @@ services:
- APPLICATION_NAME=unstract-worker-pg-fileproc
- WORKER_BARRIER_BACKEND=pg
- WORKER_PG_QUEUE_CONSUMER_WORKER_TYPE=file_processing
- - WORKER_PG_QUEUE_CONSUMER_QUEUE=file_processing,api_file_processing
+ - WORKER_PG_QUEUE_CONSUMER_QUEUE=file_processing,file_processing_priority,api_file_processing
- WORKER_PG_QUEUE_CONSUMER_HEALTH_PORT=8090
+ # Log publisher — see the note on the backend service (UN-4046).
+ - LOG_TRANSPORT=redis
# UN-3606: prefork N consumer children so file batches run in parallel (the
# PG analogue of the Celery worker's --concurrency). SKIP LOCKED distributes
# batches across children + replicas; a single execution is still capped by
@@ -666,8 +299,6 @@ services:
volumes:
- ./workflow_data:/data
- ${TOOL_REGISTRY_CONFIG_SRC_PATH}:/data/tool_registry_config
- profiles:
- - pg-queue
# Fan-in callback consumer (process_batch_callback / _api) — both callback queues.
worker-pg-callback:
@@ -689,8 +320,10 @@ services:
- APPLICATION_NAME=unstract-worker-pg-callback
- WORKER_BARRIER_BACKEND=pg
- WORKER_PG_QUEUE_CONSUMER_WORKER_TYPE=callback
- - WORKER_PG_QUEUE_CONSUMER_QUEUE=file_processing_callback,api_file_processing_callback
+ - WORKER_PG_QUEUE_CONSUMER_QUEUE=file_processing_callback,celery_callback,api_file_processing_callback
- WORKER_PG_QUEUE_CONSUMER_HEALTH_PORT=8090
+ # Log publisher — see the note on the backend service (UN-4046).
+ - LOG_TRANSPORT=redis
# The aggregating callback (fan-in) runs after all batches: aggregate results,
# update status, push destination, notify. Without an explicit vt it defaults
# to 30s and a >30s callback redelivers — re-firing webhooks + subscription
@@ -702,8 +335,6 @@ services:
volumes:
- ./workflow_data:/data
- ${TOOL_REGISTRY_CONFIG_SRC_PATH}:/data/tool_registry_config
- profiles:
- - pg-queue
# Scheduler consumer — runs scheduler.tasks.execute_pipeline_task fired by the
# orchestrator's PG scheduler tick (Beat replacement). Distinct from the Celery
@@ -730,6 +361,8 @@ services:
- WORKER_PG_QUEUE_CONSUMER_WORKER_TYPE=scheduler
- WORKER_PG_QUEUE_CONSUMER_QUEUE=scheduler
- WORKER_PG_QUEUE_CONSUMER_HEALTH_PORT=8090
+ # Log publisher — see the note on the backend service (UN-4046).
+ - LOG_TRANSPORT=redis
# Scheduled-trigger dispatch is quick (enqueue the execution). Without an
# explicit vt it defaults to 30s. 180/240 matches our k8s deployment defaults (separate chart repo).
- WORKER_PG_QUEUE_CONSUMER_VT_SECONDS=${WORKER_PG_SCHEDULER_VT_SECONDS:-180}
@@ -739,8 +372,6 @@ services:
volumes:
- ./workflow_data:/data
- ${TOOL_REGISTRY_CONFIG_SRC_PATH}:/data/tool_registry_config
- profiles:
- - pg-queue
# Dashboard-metrics periodics over PG (UN-3796) — the execution half of the Beat
# replacement. The PG scheduler fires dashboard_metrics.* onto dashboard_metric_events
@@ -783,6 +414,8 @@ services:
- WORKER_PG_QUEUE_CONSUMER_WORKER_TYPE=scheduler
- WORKER_PG_QUEUE_CONSUMER_QUEUE=dashboard_metric_events
- WORKER_PG_QUEUE_CONSUMER_HEALTH_PORT=8090
+ # Log publisher — see the note on the backend service (UN-4046).
+ - LOG_TRANSPORT=redis
# Sized for a minutes-long aggregation: stale > vt > the task's own ceiling.
- WORKER_PG_QUEUE_CONSUMER_VT_SECONDS=${WORKER_PG_METRICS_VT_SECONDS:-900}
- WORKER_PG_QUEUE_CONSUMER_HEALTH_STALE_SECONDS=${WORKER_PG_METRICS_HEALTH_STALE_SECONDS:-960}
@@ -791,19 +424,21 @@ services:
- WORKER_PG_QUEUE_CONSUMER_MAX_ATTEMPTS=1
labels:
- traefik.enable=false
- profiles:
- - pg-queue
# Log streaming without Celery (UN-3755) — the flag-on replacement for
- # worker-log-consumer-v2. Note the transport here is a REDIS LIST, not
+ # the retired Celery log consumer. Note the transport here is a REDIS LIST, not
# pg_queue_message: the log consumer's own sinks are already Redis (log_history_queue
# + the Socket.IO emit), so routing through Postgres would detour a lossy, high-rate
# stream through the platform's most contended resource to buy durability that logs
- # explicitly discard at their 10k cap. It sits in the pg-queue profile because it is
- # part of the same flag-on fleet, not because it speaks to Postgres.
+ # explicitly discard at their 10k cap. Despite the name it speaks to Redis, not
+ # Postgres; it is part of the PG-era fleet by association, not by transport.
#
- # LOG_TRANSPORT must be set to "redis" on every PUBLISHER too (backend, workers,
- # runner) or this drains an empty list while logs keep going to RabbitMQ.
+ # LOG_TRANSPORT must be set to "redis" on every PUBLISHER too or this drains an
+ # empty list while logs keep going to RabbitMQ — where, since UN-4046 removed
+ # worker-log-consumer-v2 from this stack, nothing reads them. Set on backend and
+ # runner in their own environment blocks, and in each worker's own environment
+ # block too — deliberately NOT essentials.env (that file is git-ignored and
+ # generated, so an existing checkout only picks up new keys on --update).
worker-log-stream-consumer:
image: unstract/worker-unified:${VERSION}
container_name: unstract-worker-log-stream-consumer
@@ -826,15 +461,13 @@ services:
- ENABLE_LOG_HISTORY=${ENABLE_LOG_HISTORY:-true}
labels:
- traefik.enable=false
- profiles:
- - pg-queue
# Executor RPC over PG — runs execute_extraction as a request-reply: claims
# work from Postgres and writes the ExecutionResult to pg_task_result for the
- # blocking caller. Same heavy executor runtime as worker-executor-v2 (tool
+ # blocking caller. Same heavy executor runtime as the file-processing consumer (tool
# execution → platform-service / adapters / file storage), but broker-free: it
- # is the terminal of the RPC and dispatches nothing onward to Celery. Dark
- # until the backend pg_queue_enabled gate is flipped.
+ # is the terminal of the RPC and dispatches nothing onward to Celery. It is the
+ # ONLY executor since UN-4046 — scaling it to zero stops all tool execution.
worker-pg-executor:
image: unstract/worker-unified:${VERSION}
container_name: unstract-worker-pg-executor
@@ -854,8 +487,10 @@ services:
- APPLICATION_NAME=unstract-worker-pg-executor
- WORKER_BARRIER_BACKEND=pg
- WORKER_PG_QUEUE_CONSUMER_WORKER_TYPE=executor
- - WORKER_PG_QUEUE_CONSUMER_QUEUE=celery_executor_legacy,celery_executor_agentic,celery_executor_agentic_table
+ - WORKER_PG_QUEUE_CONSUMER_QUEUE=celery_executor_legacy,celery_executor_agentic,celery_executor_table,celery_executor_smart_table,celery_executor_simple_prompt_studio,celery_executor_agentic_table,celery_executor_lookup_test
- WORKER_PG_QUEUE_CONSUMER_HEALTH_PORT=8090
+ # Log publisher — see the note on the backend service (UN-4046).
+ - LOG_TRANSPORT=redis
# The executor runs LLM inference — far longer than the consumer's 30s/60s
# defaults. Set the visibility timeout AND the liveness stale-threshold
# ABOVE the executor's hard per-task limit (EXECUTOR_TASK_TIME_LIMIT=3600s),
@@ -876,8 +511,6 @@ services:
- ./workflow_data:/data
- ${TOOL_REGISTRY_CONFIG_SRC_PATH}:/data/tool_registry_config
- prompt_studio_data:/app/prompt-studio-data
- profiles:
- - pg-queue
# IDE callback consumer (③c) — drains the ``ide_callback`` queue the executor
# self-chains onto for async/callback dispatch (Prompt Studio run/index/extract,
@@ -909,6 +542,8 @@ services:
- WORKER_PG_QUEUE_CONSUMER_QUEUE=ide_callback
- WORKER_PG_QUEUE_CONSUMER_HEALTH_PORT=8090
- WORKER_PG_QUEUE_CONSUMER_CONCURRENCY=${PG_IDE_CALLBACK_CONCURRENCY:-2}
+ # Log publisher — see the note on the backend service (UN-4046).
+ - LOG_TRANSPORT=redis
# Callbacks are sub-second (ws emit + small API writes), so nowhere near the
# executor's 1h limit — but set the visibility timeout above a slow internal
# API write (backend under load) so a sibling replica can't re-claim mid-run
@@ -921,8 +556,46 @@ services:
- ./workflow_data:/data
- ${TOOL_REGISTRY_CONFIG_SRC_PATH}:/data/tool_registry_config
- prompt_studio_data:/app/prompt-studio-data
- profiles:
- - pg-queue
+
+ # Buffered webhook notifications. The backend's notification seam enqueues
+ # ``send_webhook_notification`` onto these queues unconditionally (UN-4046), so
+ # without this consumer every buffered webhook would sit undrained — nothing
+ # sweeps pg_queue_message.
+ worker-pg-notification:
+ image: unstract/worker-unified:${VERSION}
+ container_name: unstract-worker-pg-notification
+ restart: unless-stopped
+ command: ["pg-queue-consumer"]
+ ports:
+ - "8104:8090"
+ env_file:
+ - ../workers/.env
+ - ./essentials.env
+ depends_on:
+ - db
+ - redis
+ - platform-service
+ environment:
+ - ENVIRONMENT=development
+ - APPLICATION_NAME=unstract-worker-pg-notification
+ - WORKER_BARRIER_BACKEND=pg
+ - WORKER_PG_QUEUE_CONSUMER_WORKER_TYPE=notification
+ - WORKER_PG_QUEUE_CONSUMER_QUEUE=notifications,notifications_webhook,notifications_email,notifications_sms,notifications_priority
+ - WORKER_PG_QUEUE_CONSUMER_HEALTH_PORT=8090
+ - WORKER_PG_QUEUE_CONSUMER_CONCURRENCY=${PG_NOTIFICATION_CONCURRENCY:-2}
+ # Log publisher — see the note on the backend service (UN-4046).
+ - LOG_TRANSPORT=redis
+ # One outbound HTTP POST per task. VT sits above a slow subscriber so a
+ # sibling replica cannot re-claim mid-POST and double-deliver; health-stale
+ # above that. Mirrors the chart's workerPgNotification bounds.
+ - WORKER_PG_QUEUE_CONSUMER_VT_SECONDS=${PG_NOTIFICATION_VT_SECONDS:-300}
+ - WORKER_PG_QUEUE_CONSUMER_HEALTH_STALE_SECONDS=${PG_NOTIFICATION_HEALTH_STALE_SECONDS:-360}
+ labels:
+ - traefik.enable=false
+ volumes:
+ - ./workflow_data:/data
+ - ${TOOL_REGISTRY_CONFIG_SRC_PATH}:/data/tool_registry_config
+ - prompt_studio_data:/app/prompt-studio-data
# Reaper / orchestrator — leader-elected loop. Run exactly ONE instance (it
# elects a single leader via pg_orchestrator_lock; extra replicas idle as
@@ -946,6 +619,8 @@ services:
- APPLICATION_NAME=unstract-worker-pg-reaper
- WORKER_PG_REAPER_INTERVAL_SECONDS=${WORKER_PG_REAPER_INTERVAL_SECONDS:-5}
- WORKER_PG_REAPER_HEALTH_PORT=8086
+ # Log publisher — see the note on the backend service (UN-4046).
+ - LOG_TRANSPORT=redis
# Retention sweep (cadence-gated inside the leader's tick): drop expired
# pg_task_result rows + orphaned pg_batch_dedup markers so neither grows
# unbounded as the gate ramps. The dedup retention below (default 24h) should
@@ -958,8 +633,6 @@ services:
volumes:
- ./workflow_data:/data
- ${TOOL_REGISTRY_CONFIG_SRC_PATH}:/data/tool_registry_config
- profiles:
- - pg-queue
volumes:
prompt_studio_data:
diff --git a/docker/sample.compose.override.yaml b/docker/sample.compose.override.yaml
index b0e98d8b1..bdf379713 100644
--- a/docker/sample.compose.override.yaml
+++ b/docker/sample.compose.override.yaml
@@ -162,7 +162,7 @@ services:
# V2 Workers - Build configurations with memory-optimized settings
# All use --pool=threads and --concurrency=2 for lower memory footprint
- worker-api-deployment-v2:
+ worker-pg-orchestrator-api:
build:
dockerfile: docker/dockerfiles/worker-unified.Dockerfile
context: ..
@@ -179,7 +179,7 @@ services:
- action: rebuild
path: ../workers/uv.lock
- worker-callback-v2:
+ worker-pg-callback:
build:
dockerfile: docker/dockerfiles/worker-unified.Dockerfile
context: ..
@@ -196,7 +196,7 @@ services:
- action: rebuild
path: ../workers/uv.lock
- worker-file-processing-v2:
+ worker-pg-fileproc:
build:
dockerfile: docker/dockerfiles/worker-unified.Dockerfile
context: ..
@@ -213,7 +213,7 @@ services:
- action: rebuild
path: ../workers/uv.lock
- worker-general-v2:
+ worker-pg-orchestrator-general:
build:
dockerfile: docker/dockerfiles/worker-unified.Dockerfile
context: ..
@@ -230,7 +230,7 @@ services:
- action: rebuild
path: ../workers/uv.lock
- worker-notification-v2:
+ worker-pg-notification:
build:
dockerfile: docker/dockerfiles/worker-unified.Dockerfile
context: ..
@@ -247,7 +247,7 @@ services:
- action: rebuild
path: ../workers/uv.lock
- worker-log-consumer-v2:
+ worker-log-stream-consumer:
build:
dockerfile: docker/dockerfiles/worker-unified.Dockerfile
context: ..
@@ -270,7 +270,7 @@ services:
dockerfile: docker/dockerfiles/worker-unified.Dockerfile
context: ..
- worker-scheduler-v2:
+ worker-pg-scheduler:
build:
dockerfile: docker/dockerfiles/worker-unified.Dockerfile
context: ..
@@ -289,16 +289,20 @@ services:
#########################################################################################################
# IDE callback worker (workers image, processes ide_callback queue)
- worker-ide-callback:
+ worker-pg-ide-callback:
build:
dockerfile: docker/dockerfiles/worker-unified.Dockerfile
context: ..
- # Celery worker for dashboard metrics (memory optimized: reduced autoscale)
- worker-metrics:
+ # PG consumer for the dashboard_metrics.* periodics fired by the PG scheduler
+ # tick. Must build the WORKER image: the base file runs `pg-queue-consumer`,
+ # which the backend image's entrypoint rejects outright ("Unknown argument"),
+ # so building backend.Dockerfile here crash-loops the pod and leaves
+ # dashboard_metric_events with no consumer. No `command:` override either —
+ # the base file's is correct.
+ worker-pg-metrics:
build:
- dockerfile: docker/dockerfiles/backend.Dockerfile
+ dockerfile: docker/dockerfiles/worker-unified.Dockerfile
context: ..
- command: "-A backend worker --loglevel=info -Q dashboard_metric_events --autoscale=2,1"
develop:
watch:
- action: sync+restart
diff --git a/docker/sample.env b/docker/sample.env
index 6c96821a4..47905773d 100644
--- a/docker/sample.env
+++ b/docker/sample.env
@@ -144,7 +144,6 @@ WORKER_PG_REAPER_INTERVAL_SECONDS=5 # Reaper sweep interval (seconds)
# above the vt, which must stay above the task itself.
# above the vt, which must stay above the task itself.
#
-# NOTE: routing executions to PG is a SEPARATE, later step. Running these
-# services does NOT move any traffic — the single `pg_queue_enabled` Flipt flag
-# (default off, fail-closed) decides per-execution transport, and stays off until
-# the rollout ramp.
+# NOTE: since UN-4046 these services ARE the execution path — there is no flag
+# and no Celery fallback. Every dispatch goes to PG, so a queue whose consumer is
+# not running accepts work and never executes it, with no error at the producer.
diff --git a/tests/README.md b/tests/README.md
index bdd20fe47..0d2724e3f 100644
--- a/tests/README.md
+++ b/tests/README.md
@@ -221,7 +221,7 @@ Only `LLM` completions are mocked, not embeddings: `provisioned_workflow` pins `
### Fan-out (`MAX_PARALLEL_FILE_BATCHES`)
-Defaults to `1`, meaning every file of a multi-file run lands in one batch and is processed serially — so a fan-out test would pass without any fan-out happening. The overlay defaults it to `3` on `backend`, which is what normally takes effect: workers ask the backend for this value and fall back to their own env only when they can't reach it or it carries no value (the overlay sets it on `worker-api-deployment-v2` too, to keep the fallback in step). Batches are `min(MAX_PARALLEL_FILE_BATCHES, num_files)`, so N files with the same N gives one batch each.
+Defaults to `1`, meaning every file of a multi-file run lands in one batch and is processed serially — so a fan-out test would pass without any fan-out happening. The overlay defaults it to `3` on `backend`, which is what normally takes effect: workers ask the backend for this value and fall back to their own env only when they can't reach it or it carries no value (the overlay sets it on `worker-pg-orchestrator-api` too, to keep the fallback in step). Batches are `min(MAX_PARALLEL_FILE_BATCHES, num_files)`, so N files with the same N gives one batch each.
**The fan-out half is an open gap** (`workflow-execution-fan-out` in `critical_paths.yaml`), and closing it needs a product change, not a cleverer test. Nothing persists a batch or task id: the batch index is a discarded loop local, and the celery task id only ever reaches worker stdout. That leaves per-file timing as the only proxy, and timing does not work here — measured on CI, three files genuinely fanned out finished *further* apart (durations `6.6 / 18.1 / 22.9`) than the ~2s steps seen when they share a batch, because three concurrent tool containers on a loaded runner cost more in contention than they gain in overlap. No threshold separates those two distributions, and no delay value fixes it: the contention scales with the parallelism being measured.
diff --git a/tests/compose/docker-compose.test.yaml b/tests/compose/docker-compose.test.yaml
index 78713fefa..742d6b2b5 100644
--- a/tests/compose/docker-compose.test.yaml
+++ b/tests/compose/docker-compose.test.yaml
@@ -17,18 +17,25 @@ services:
runner:
image: unstract/runner:${UNSTRACT_TEST_VERSION:-latest}
+ # Retargeted onto the PG consumers (UN-4046). The Celery worker services these
+ # used to name — worker-executor-v2, worker-file-processing-v2,
+ # worker-api-deployment-v2 — are gone from docker/docker-compose.yaml. An
+ # override for a service that does not exist is NOT a no-op: compose creates it
+ # with neither image nor build context and rejects the whole project with
+ # "invalid compose project", so e2e could not bring the stack up at all.
+
# Execute-path e2e must never reach a real provider.
- worker-executor-v2:
+ worker-pg-executor:
environment:
- UNSTRACT_LLM_MOCK_RESPONSE=${UNSTRACT_LLM_MOCK_RESPONSE:-}
- worker-file-processing-v2:
+ worker-pg-fileproc:
environment:
- UNSTRACT_LLM_MOCK_RESPONSE=${UNSTRACT_LLM_MOCK_RESPONSE:-}
# Only the worker's fallback if it can't reach the backend; kept in step so it
# can't quietly serialise the fan-out test.
- worker-api-deployment-v2:
+ worker-pg-orchestrator-api:
environment:
- MAX_PARALLEL_FILE_BATCHES=${MAX_PARALLEL_FILE_BATCHES:-3}
diff --git a/unstract/core/src/unstract/core/data_models.py b/unstract/core/src/unstract/core/data_models.py
index bc2fd6dbc..00073ee2a 100644
--- a/unstract/core/src/unstract/core/data_models.py
+++ b/unstract/core/src/unstract/core/data_models.py
@@ -214,8 +214,32 @@ class WorkflowTransport(str, Enum):
PG_QUEUE = "pg_queue"
-# Default transport when none is resolved/carried — keeps every pre-existing
-# payload and caller on the legacy path until a transport is explicitly chosen.
+# Default transport when none is carried on the payload.
+#
+# ⚠️ KNOWN HAZARD, DELIBERATELY NOT FIXED HERE — the safe direction has inverted.
+#
+# CELERY was the safe fallback while Celery had consumers: a payload that lost the
+# field degraded onto a working substrate. With the Celery fleet at zero that is no
+# longer true. A payload defaulting to "celery" now selects CeleryChordBarrier in
+# `_barrier_for_transport`, skips the PG orchestration-claim and terminal-redelivery
+# guards, and publishes the fan-out to a broker nothing drains — and because no PG
+# rows are written, neither the reaper nor the undispatched sweep can see the
+# stranded execution. Reachable on a rolling deploy, where a not-yet-upgraded
+# backend still emits `transport: "celery"`.
+#
+# Flipping this constant to PG_QUEUE is the right fix and it is a one-line change,
+# but it is NOT a one-line consequence: it breaks 19 tests across 6 files
+# (test_pg_barrier, test_chord_sites_characterisation, test_barrier,
+# test_dispatch_with_callback, test_dispatch_sites_characterisation,
+# test_workflow_context_transport), every one of which relies on the implicit
+# Celery default — most of them characterising PgBarrier's Celery-link branch,
+# which is deferred-deletion code. Rewriting those to state their transport
+# explicitly is correct, but it is a behaviour change that deserves its own review
+# rather than riding along in a documentation sweep.
+#
+# Until then the mitigation is that every producer sets the field explicitly
+# (workflow_helper.py and internal_api_views.py both hardcode PG_QUEUE), so the
+# default is only reachable via version skew.
DEFAULT_WORKFLOW_TRANSPORT = WorkflowTransport.CELERY.value
@@ -227,14 +251,18 @@ def normalize_transport(value: object, *, logger: Any = None, context: str = "")
route an execution onto an unknown substrate. This **fails closed**: an
unrecognized value (``None``, ``"celary"``, ``""``) logs a warning (when a
``logger`` is given) and falls back to :data:`DEFAULT_WORKFLOW_TRANSPORT`
- (Celery). A recognized value passes through as its canonical string.
+ — see the hazard note on that constant. A recognized value passes through as
+ its canonical string.
Coerce-not-raise is deliberate, and differs from how
``WorkflowContextData.workflow_type`` is validated: ``transport`` has an
- explicit safe default by design (the whole point of the seam is reversible
- fallback to Celery), so a bad value should degrade to Celery, not crash the
- execution. ``context`` is an optional suffix for the log line (e.g. an
- ``exec:<id>`` tag) to make a version-skew warning traceable.
+ explicit safe default by design, so a bad value should degrade rather than
+ crash the execution. What changed with UN-4046 is WHICH default is safe — see
+ the note on the constant. A rolling deploy where an older backend still emits
+ ``"celery"`` is the live case: that value is recognized, so it passes through
+ here untouched and the guards keyed on it are skipped. This function only
+ protects the missing/garbage case. ``context`` is an optional suffix for the
+ log line (e.g. an ``exec:<id>`` tag) to make a version-skew warning traceable.
"""
try:
return WorkflowTransport(value).value
diff --git a/unstract/workflow-execution/src/unstract/workflow_execution/executor_rpc.py b/unstract/workflow-execution/src/unstract/workflow_execution/executor_rpc.py
index 14dcc6e04..3fc3a2017 100644
--- a/unstract/workflow-execution/src/unstract/workflow_execution/executor_rpc.py
+++ b/unstract/workflow-execution/src/unstract/workflow_execution/executor_rpc.py
@@ -1,13 +1,12 @@
-"""Shared executor-RPC dispatch for the PG path — the gate + reply_key orchestration.
+"""Shared executor-RPC dispatch — the reply_key/timeout orchestration.
The executor "RPC" is a synchronous request-reply: a caller sends an
``ExecutionContext`` to the executor worker and blocks for the ``ExecutionResult``.
-The legacy transport is Celery (the SDK ``ExecutionDispatcher``); the PG path adds a
-parallel Postgres transport. Backend (Django/prompt-studio) and workers
-(``structure_tool``) both need it, and used to carry **byte-for-byte mirrors** of
-this logic — the only thing that genuinely differs between them is the *transport
-primitive*: the backend enqueues via the Django ORM (``enqueue_task`` +
-``PgTaskResult``), the workers via psycopg2 (``PgQueueClient`` + ``PgResultBackend``).
+Backend (Django/prompt-studio) and workers (``structure_tool``) both need it, and
+used to carry **byte-for-byte mirrors** of this logic — the only thing that
+genuinely differs between them is the *transport primitive*: the backend enqueues
+via the Django ORM (``enqueue_task`` + ``PgTaskResult``), the workers via psycopg2
+(``PgQueueClient`` + ``PgResultBackend``).
So this module owns everything transport-agnostic exactly once, and the differing
primitive is **injected** (composition, not inheritance) via :class:`QueueTransport`:
@@ -16,12 +15,10 @@ primitive is **injected** (composition, not inheritance) via :class:`QueueTransp
``dispatch_with_callback`` + the reply_key/timeout orchestration and the
never-raises contract (timeout/failure → ``ExecutionResult.failure``). It calls
``transport.enqueue(...)`` and ``transport.wait_for_result(...)``.
-- :func:`resolve_pg_transport` — the gate: the single ``pg_queue_enabled`` Flipt
- flag (bucketed per org) is the sole control. Fails closed to Celery on a
- blind/unreachable Flipt or any error.
-- :class:`RoutingExecutionDispatcher` — picks PG-vs-Celery per call (instant
- rollout/rollback) for every mode; the Celery dispatcher, the PG dispatcher and the
- per-side ``resolve`` are all injected.
+
+PG is the only transport. This module used to also hold ``resolve_pg_transport``
+(the ``pg_queue_enabled`` Flipt gate) and ``RoutingExecutionDispatcher`` (which
+picked PG-vs-Celery per call); both went with the flag in UN-4046.
It lives in ``unstract-workflow-execution`` (which both backend and workers already
depend on) rather than ``unstract.core`` because it needs ``unstract.sdk1`` and
@@ -35,25 +32,19 @@ import logging
import os
import uuid
from dataclasses import dataclass
-from typing import TYPE_CHECKING, Any, Protocol
+from typing import TYPE_CHECKING, Protocol
from unstract.core.data_models import PgTaskStatus
from unstract.core.execution_dispatch import DispatchHandle, signature_to_continuation
-from unstract.flags.feature_flag import check_feature_flag_status
from unstract.sdk1.execution.result import ExecutionResult
if TYPE_CHECKING:
- from collections.abc import Callable
-
from unstract.core.data_models import ContinuationSpec
from unstract.core.execution_dispatch import CallbackSignature
from unstract.sdk1.execution.context import ExecutionContext
logger = logging.getLogger(__name__)
-# The single PG-queue rollout flag — the same key execution and the scheduler read,
-# so one flip turns the whole PG-queue feature on/off.
-PG_QUEUE_FLAG_KEY = "pg_queue_enabled"
EXECUTE_TASK = "execute_extraction"
# Mirror the SDK's queue-per-executor convention so the PG executor queue name
# matches the Celery one (the worker-pg-executor consumer subscribes to these).
@@ -64,32 +55,6 @@ DEFAULT_TIMEOUT_ENV = "EXECUTOR_RESULT_TIMEOUT"
DEFAULT_TIMEOUT = 3600
-class _CeleryDispatcher(Protocol):
- """The subset of the SDK ``ExecutionDispatcher`` the router delegates to on the
- Celery path — so ``RoutingExecutionDispatcher`` holds it typed, not as ``Any``.
- """
-
- def dispatch(
- self,
- context: ExecutionContext,
- timeout: int | None = ...,
- headers: dict[str, Any] | None = ...,
- ) -> ExecutionResult: ...
-
- def dispatch_async(
- self, context: ExecutionContext, headers: dict[str, Any] | None = ...
- ) -> str: ...
-
- def dispatch_with_callback(
- self,
- context: ExecutionContext,
- on_success: CallbackSignature | None = ...,
- on_error: CallbackSignature | None = ...,
- task_id: str | None = ...,
- headers: dict[str, Any] | None = ...,
- ) -> Any: ...
-
-
@dataclass
class ExecResultRow:
"""Normalised executor-RPC result row — the transport-agnostic shape
@@ -131,42 +96,6 @@ class QueueTransport(Protocol):
def wait_for_result(self, reply_key: str, timeout: float) -> ExecResultRow | None: ...
-def resolve_pg_transport(
- context: ExecutionContext,
- *,
- flag_key: str = PG_QUEUE_FLAG_KEY,
-) -> bool:
- """True → route this executor dispatch over PG; False → Celery (default).
-
- Gated by the single ``pg_queue_enabled`` Flipt flag, bucketed per org.
- **Fails closed to Celery** on a blind Flipt or any error — so the executor
- never silently loses its transport.
- """
- if os.environ.get("FLIPT_SERVICE_AVAILABLE", "false").lower() != "true":
- logger.warning(
- "resolve_pg_transport: FLIPT_SERVICE_AVAILABLE != true "
- "(Flipt blind); using Celery"
- )
- return False
- org = getattr(context, "organization_id", None)
- # %-bucket keyed on org; fall back to run_id so a context without an org still
- # resolves deterministically.
- entity_id = str(org or getattr(context, "run_id", "") or "default")
- flag_context = {"executor_name": str(context.executor_name)}
- if org:
- flag_context["organization_id"] = str(org)
- try:
- enabled = check_feature_flag_status(
- flag_key=flag_key, entity_id=entity_id, context=flag_context
- )
- except Exception:
- logger.warning(
- "resolve_pg_transport: Flipt check failed; using Celery", exc_info=True
- )
- return False
- return bool(enabled)
-
-
def _resolve_timeout(timeout: int | None) -> int:
"""Caller timeout, defaulting to ``EXECUTOR_RESULT_TIMEOUT`` env then 3600s.
@@ -215,8 +144,11 @@ class PgExecutionDispatcher:
callers branch on ``result.success`` identically on either transport.
No ``headers`` on any PG dispatch method: the PG path carries org/routing in
- the enqueue payload, not Celery headers, so the ``RoutingExecutionDispatcher``
- does not forward fairness headers to the PG path.
+ the enqueue payload (``transport.enqueue(..., org_id=...)``), not Celery
+ headers. This method takes no ``headers`` argument: callers must not pass
+ one. The routing dispatcher that used to absorb a ``headers=`` kwarg went
+ with the flag in UN-4046, and three call sites kept passing it — every
+ extraction raised ``TypeError`` until they were fixed.
"""
timeout = _resolve_timeout(timeout)
reply_key = str(uuid.uuid4())
@@ -393,69 +325,3 @@ class PgExecutionDispatcher:
error_spec["task_name"] if error_spec else None,
)
return DispatchHandle(task_id)
-
-
-class RoutingExecutionDispatcher:
- """Gate-routed executor dispatcher: every mode picks PG vs Celery per call
- (read at dispatch time → flipping the flag is an instant, no-redeploy
- rollout/rollback). Duck-typed against the SDK ``ExecutionDispatcher`` so call
- sites are unchanged.
-
- Composition-injected: the Celery dispatcher (``celery``), the PG dispatcher
- (``pg``) and the per-side gate (``resolve(context) -> bool``).
- """
-
- def __init__(
- self,
- *,
- celery: _CeleryDispatcher,
- pg: PgExecutionDispatcher,
- resolve: Callable[[ExecutionContext], bool],
- ) -> None:
- self._celery = celery
- self._pg = pg
- self._resolve = resolve
-
- def dispatch(
- self,
- context: ExecutionContext,
- timeout: int | None = None,
- headers: dict[str, Any] | None = None,
- ) -> ExecutionResult:
- if self._resolve(context):
- logger.info(
- "Executor RPC → PG transport (executor=%s run_id=%s)",
- context.executor_name,
- context.run_id,
- )
- # PG carries org/routing via the enqueue payload, not Celery headers, so
- # the fairness headers are intentionally not forwarded here.
- return self._pg.dispatch(context, timeout=timeout)
- return self._celery.dispatch(context, timeout=timeout, headers=headers)
-
- def dispatch_async(
- self, context: ExecutionContext, headers: dict[str, Any] | None = None
- ) -> str:
- if self._resolve(context):
- return self._pg.dispatch_async(context)
- return self._celery.dispatch_async(context, headers=headers)
-
- def dispatch_with_callback(
- self,
- context: ExecutionContext,
- on_success: CallbackSignature | None = None,
- on_error: CallbackSignature | None = None,
- task_id: str | None = None,
- headers: dict[str, Any] | None = None,
- ) -> Any:
- if self._resolve(context):
- return self._pg.dispatch_with_callback(
- context, on_success=on_success, on_error=on_error, task_id=task_id
- )
- return self._celery.dispatch_with_callback(
- context,
- on_success=on_success,
- on_error=on_error,
- task_id=task_id,
- headers=headers,
- )
diff --git a/workers/file_processing/structure_tool_task.py b/workers/file_processing/structure_tool_task.py
index 971a78398..9e6353e65 100644
--- a/workers/file_processing/structure_tool_task.py
+++ b/workers/file_processing/structure_tool_task.py
@@ -22,11 +22,9 @@ import time
from pathlib import Path
from typing import Any
-from file_processing.worker import app
-from queue_backend import FairnessKey, worker_task
-from queue_backend.fairness import WorkloadType
+from queue_backend import worker_task
from queue_backend.pg_queue.executor_rpc import (
- RoutingExecutionDispatcher,
+ PgExecutionDispatcher,
get_executor_dispatcher,
)
from shared.enums.task_enums import TaskName
@@ -43,25 +41,13 @@ logger = logging.getLogger(__name__)
EXECUTOR_TIMEOUT = int(os.environ.get("EXECUTOR_RESULT_TIMEOUT", 3600))
-def _fairness_headers(
- organization_id: str,
-) -> dict[str, dict[str, str | int | None]]:
- """Fairness header for executor dispatches.
-
- Structure-tool dispatches are workflow-execution work — both ETL
- and API workflows route through here. ``NON_API`` is the safe
- default (API traffic preempting is a strictly weaker mistake than
- the inverse).
-
- TODO(UN-3504): propagate the caller's WorkloadType (API vs ETL)
- instead of hard-coding ``NON_API``. Requires the chord-lift work
- so the workflow type flows from the chord caller down to here.
- """
- return FairnessKey(
- org_id=organization_id,
- workload_type=WorkloadType.NON_API,
- ).as_header()
-
+# NOTE: `_fairness_headers()` lived here and was passed as `headers=` to every
+# executor dispatch. It is gone with the routing dispatcher (UN-4046):
+# `PgExecutionDispatcher.dispatch()` takes no `headers`, and the router that used
+# to absorb them never forwarded them to the PG path anyway — PG carries org
+# routing in the enqueue payload (`transport.enqueue(..., org_id=...)`).
+# UN-3504 (propagating the caller's WorkloadType) is unaffected: it was always
+# about the PG payload, not these headers.
# -----------------------------------------------------------------------
# Constants mirrored from tools/structure/src/constants.py
@@ -279,9 +265,9 @@ def _execute_structure_tool_impl(params: dict) -> dict:
)
platform_helper = _create_platform_helper(shim, file_execution_id)
- # Gate-routed: PG executor RPC when pg_queue_enabled is on, else the unchanged
- # Celery ExecutionDispatcher (zero-regression by construction — see executor_rpc).
- dispatcher = get_executor_dispatcher(celery_app=app)
+ # PG executor RPC. ``celery_app`` is accepted and ignored by the factory —
+ # see executor_rpc.
+ dispatcher = get_executor_dispatcher()
fs = _get_file_storage()
# ---- Step 2: Fetch tool metadata ----
@@ -498,7 +484,6 @@ def _execute_structure_tool_impl(params: dict) -> dict:
at_result = dispatcher.dispatch(
at_ctx,
timeout=EXECUTOR_TIMEOUT,
- headers=_fairness_headers(organization_id),
)
if not at_result.success:
return at_result.to_dict()
@@ -541,7 +526,6 @@ def _execute_structure_tool_impl(params: dict) -> dict:
pipeline_result = dispatcher.dispatch(
pipeline_ctx,
timeout=EXECUTOR_TIMEOUT,
- headers=_fairness_headers(organization_id),
)
pipeline_elapsed = time.monotonic() - pipeline_start
@@ -684,7 +668,7 @@ def _run_agentic_extraction(
input_file_path: str,
output_dir_path: str,
tool_instance_metadata: dict,
- dispatcher: RoutingExecutionDispatcher,
+ dispatcher: PgExecutionDispatcher,
shim: Any,
file_execution_id: str,
execution_id: str,
@@ -758,7 +742,6 @@ def _run_agentic_extraction(
agentic_result = dispatcher.dispatch(
agentic_ctx,
timeout=EXECUTOR_TIMEOUT,
- headers=_fairness_headers(organization_id),
)
if not agentic_result.success:
diff --git a/workers/notification/providers/slack_webhook.py b/workers/notification/providers/slack_webhook.py
index 6c97cde02..6d8856412 100644
--- a/workers/notification/providers/slack_webhook.py
+++ b/workers/notification/providers/slack_webhook.py
@@ -55,9 +55,9 @@ class SlackWebhook(WebhookProvider):
Two input shapes are accepted:
- Backend-rendered ``{"text": "<mrkdwn>"}`` (any backend dispatch
through ``clubbed_renderer``) — passed through.
- - Flat per-event dict from the generic internal webhook-send endpoints —
- wrapped in a single-event envelope and rendered to the canonical
- Slack mrkdwn body.
+ - Flat per-event dict from a direct ``send_webhook_notification``
+ dispatch — wrapped in a single-event envelope and rendered to the
+ canonical Slack mrkdwn body.
"""
if "text" in payload and "events" not in payload:
return {"text": payload["text"]}
diff --git a/workers/queue_backend/__init__.py b/workers/queue_backend/__init__.py
index d1c9154aa..e148708a6 100644
--- a/workers/queue_backend/__init__.py
+++ b/workers/queue_backend/__init__.py
@@ -25,13 +25,14 @@ time rather than silently falling back, which would mask a typo'd
flag in a production env.
**Queue-transport routing (separate axis).** ``dispatch()`` consults
-:func:`~queue_backend.routing.select_backend`, which reads the
-PG-queue routing table (``WORKER_PG_QUEUE_ENABLED_TASKS``) to decide
-Celery-vs-PG per task. This is orthogonal to the barrier choice above:
-the barrier is *how a batch's fan-in fires the callback*; the transport
-is *how messages travel*. Both default to Celery. The routing table is
-a scaffold today — PG-selected tasks still ride Celery (no PG consumer
-yet) — so it is observable but inert.
+:func:`~queue_backend.routing.select_backend`, which returns PG
+unconditionally since UN-4046. This is orthogonal to the barrier choice
+above: the barrier is *how a batch's fan-in fires the callback*; the
+transport is *how messages travel*. It is emphatically NOT inert — every
+dispatch rides PG and needs its consumer running, or the message is
+durably enqueued and never executed, with no error at the producer.
+(It used to read a ``WORKER_PG_QUEUE_ENABLED_TASKS`` allow-list that was
+set nowhere, so everything defaulted to Celery; both are gone.)
"""
import os
diff --git a/workers/queue_backend/dispatch.py b/workers/queue_backend/dispatch.py
index 3af477dd2..88bfd9860 100644
--- a/workers/queue_backend/dispatch.py
+++ b/workers/queue_backend/dispatch.py
@@ -1,21 +1,25 @@
-"""Transport-agnostic task dispatch.
+"""Task dispatch onto the PG queue.
-Routes each task to its transport via :func:`resolve_backend` (which applies a
-per-call ``backend`` override, else defers to :func:`select_backend`):
+Routes each task via :func:`resolve_backend` (a per-call ``backend`` override,
+else :func:`select_backend`):
-- **Celery** (default) — a thin pass-through to ``current_app.send_task``.
-- **PG Queue** — when a task is opted into ``WORKER_PG_QUEUE_ENABLED_TASKS``
- (or pinned via a ``backend=`` override), the task is serialised and enqueued
- to ``pg_queue_message`` (9b); the PG consumer (9c) drains and runs it.
+- **PG Queue** (the only transport :func:`select_backend` returns since UN-4046)
+ — the task is serialised and enqueued to ``pg_queue_message``; the PG consumer
+ drains and runs it.
+- **Celery** — a thin pass-through to ``current_app.send_task``, reachable only
+ by an explicit ``backend=QueueBackend.CELERY`` override. Nothing passes one;
+ the branch goes with the rest of the Celery transport.
-The default (empty allow-list) routes everything to Celery, so dispatch is
-unchanged unless an operator explicitly opts a task in.
+This used to default to Celery and route to PG only for task names opted into
+``WORKER_PG_QUEUE_ENABLED_TASKS``. That allow-list was set nowhere, so the
+default sent everything to Celery; UN-4046 removed it along with the
+``pg_queue_enabled`` flag.
.. warning::
- A task opted into the PG queue **requires the PG consumer to be running**
- — otherwise the message is durably enqueued but never executed. Only opt
- in tasks once the consumer is deployed (and, per the migration-coherence
- decision, only *leaf* tasks until execution-level routing exists).
+ Every PG queue **requires its consumer to be running** — otherwise the
+ message is durably enqueued and never executed, with no error at the
+ producer. ``validate-pg-worker-fleet.yaml`` in the chart refuses a partial
+ fleet at render time for exactly this reason.
"""
from __future__ import annotations
@@ -88,16 +92,11 @@ def dispatch(
``backend`` is a per-call transport override (see
:func:`~queue_backend.routing.resolve_backend` for the precedence). When
- ``None`` (the default, and every call site today) the transport is the env
- allow-list decision via ``select_backend`` — behaviour is unchanged. When
- set, it wins over the allow-list: this is the seam the execution-level PG
- pipeline uses to route a whole execution's header/callback
- dispatches onto PG without opting their task *names* into
- ``WORKER_PG_QUEUE_ENABLED_TASKS``. (The allow-list is for *leaf* tasks; the
- coupled pipeline's migration unit is the whole execution — its transport is
- resolved once at creation and travels on the execution's task kwargs onto
- ``WorkflowContextData.transport``.) The override only forces the *transport*; it does not
- bypass ``_enqueue_pg``'s no-silent-fallback contract.
+ ``None`` — the default, and every call site today — ``select_backend`` decides,
+ and since UN-4046 it returns PG unconditionally. The override is therefore the
+ only remaining dial, and nothing in production passes one. It forces the
+ *transport* only; it does not bypass ``_enqueue_pg``'s no-silent-fallback
+ contract.
"""
if resolve_backend(task_name, backend) is QueueBackend.PG:
return _enqueue_pg(task_name, args, kwargs, queue, fairness)
@@ -130,12 +129,6 @@ def _enqueue_pg(
# true regardless of send outcome, so a first-dispatch failure
# (DB down / unmigrated) must not suppress the one announcement.
# INFO so it survives a default log config; once-per-task bounds it.
- # NOTE: keyed on task name only, not on *why* it routed to PG. If a name
- # is first PG-routed via a ``backend=`` override and the operator later
- # adds it to the allow-list, the allow-list cutover won't re-announce
- # (already in the set). Benign given the usage split (override = pipeline
- # headers, allow-list = leaf tasks, so no overlap expected); the
- # allow-list config itself is still announced by _log_allow_list_once.
_pg_routing_logged.add(task_name)
logger.info(
"PG-queue: routing task=%r to Postgres (queue=%r). Requires the "
diff --git a/workers/queue_backend/pg_barrier.py b/workers/queue_backend/pg_barrier.py
index b044b4438..d841ae0bc 100644
--- a/workers/queue_backend/pg_barrier.py
+++ b/workers/queue_backend/pg_barrier.py
@@ -472,8 +472,8 @@ def try_claim_orchestration(execution_id: str, organization_id: str) -> bool:
raise RuntimeError(
"pg_orchestration_claim schema is out of date — migrations "
"0012_pgorchestrationclaim / 0013_pgorchestrationclaim_organization_id "
- "have not been fully applied. Run backend migrations before enabling "
- "PG transport (pg_queue_enabled)."
+ "have not been fully applied. Run backend migrations — PG is the only "
+ "execution transport, so this blocks all orchestration."
) from exc
diff --git a/workers/queue_backend/pg_queue/README.md b/workers/queue_backend/pg_queue/README.md
index bd303c0f2..8ae3f8fea 100644
--- a/workers/queue_backend/pg_queue/README.md
+++ b/workers/queue_backend/pg_queue/README.md
@@ -3,8 +3,9 @@
A quick-reference dictionary for the bespoke Postgres-backed work queue that can
stand in for Celery/RabbitMQ. This file is the **terminology + config lookup**.
-The transport is chosen **per org** by the Flipt flag `pg_queue_enabled`
-(fail-closed to Celery), so the system runs **with and without** PG unchanged.
+PG is the **only** transport since UN-4046. There is no per-org flag and no Celery
+fallback: every dispatch is enqueued here, so a queue whose consumer is not running
+accepts work and never executes it — durably, and with no error at the producer.
---
@@ -52,11 +53,11 @@ integers.
| `WORKER_PG_DEDUP_RETENTION_SECONDS` | Retention for `pg_batch_dedup` rows |
| `WORKER_PG_QUEUE_CONNECT_RETRIES` / `_BACKOFF` | Reconnect attempts / backoff on a stale or broken DB connection |
-### Routing / flag
-| Env / flag | One-line |
-|---|---|
-| `pg_queue_enabled` (Flipt, `PG_QUEUE_FLAG_KEY`) | Per-org gate that routes dispatch to PG; **fail-closed to Celery** on any Flipt error |
-| `WORKER_PG_QUEUE_ENABLED_TASKS` | Comma-separated task names eligible for PG routing (empty → everything stays on Celery) |
+### Routing
+There are no routing knobs left. `select_backend()` returns PG unconditionally; the
+`pg_queue_enabled` Flipt flag and the `WORKER_PG_QUEUE_ENABLED_TASKS` allow-list were
+both removed in UN-4046. Which work reaches a given consumer is decided entirely by
+`WORKER_PG_QUEUE_CONSUMER_QUEUE` above.
---
diff --git a/workers/queue_backend/pg_queue/__init__.py b/workers/queue_backend/pg_queue/__init__.py
index 303fefd9d..01573a225 100644
--- a/workers/queue_backend/pg_queue/__init__.py
+++ b/workers/queue_backend/pg_queue/__init__.py
@@ -16,8 +16,8 @@ consumer poll loop that drains and runs claimed tasks, the leader-elected
reaper, and the result backend. :func:`queue_backend.dispatch.dispatch` enqueues
a PG-selected task here as a :class:`TaskPayload` instead of sending it to Celery.
-Default-empty ``WORKER_PG_QUEUE_ENABLED_TASKS`` → everything still routes to
-Celery, so this is inert unless a task is explicitly opted in.
+Every dispatch routes here since UN-4046 — this subpackage is on the critical
+path, not inert. A queue without its consumer running strands work silently.
"""
from .client import PgQueueClient, QueueMessage
diff --git a/workers/queue_backend/pg_queue/executor_rpc.py b/workers/queue_backend/pg_queue/executor_rpc.py
index 373a331f7..fadad7d04 100644
--- a/workers/queue_backend/pg_queue/executor_rpc.py
+++ b/workers/queue_backend/pg_queue/executor_rpc.py
@@ -1,29 +1,26 @@
"""Executor-RPC for the PG path — workers (psycopg2) transport adapter.
-The gate + reply_key/timeout orchestration + routing live ONCE in
+The reply_key/timeout orchestration lives ONCE in
``unstract.workflow_execution.executor_rpc`` (shared with the backend). This module
is the thin psycopg2 half: a :class:`PgClientQueueTransport` that enqueues via
:class:`~queue_backend.pg_queue.client.PgQueueClient` and polls via
:class:`~queue_backend.pg_queue.result_backend.PgResultBackend`, plus the
:func:`get_executor_dispatcher` factory.
-Zero-regression: with the ``pg_queue_enabled`` Flipt flag off the routing dispatcher
-delegates every mode to the unchanged Celery ``ExecutionDispatcher`` and no
-``pg_task_result`` row is created.
+Since UN-4046 the factory returns the PG dispatcher unconditionally: there is no
+gate, no routing dispatcher and no Celery fall-through, so a ``pg_task_result``
+row is written on every request-reply dispatch.
"""
from __future__ import annotations
from typing import TYPE_CHECKING
-from unstract.sdk1.execution.dispatcher import ExecutionDispatcher
from unstract.workflow_execution.executor_rpc import (
EXECUTE_TASK,
ExecResultRow,
PgExecutionDispatcher,
QueueTransport,
- RoutingExecutionDispatcher,
- resolve_pg_transport,
)
from .client import PgQueueClient
@@ -39,20 +36,10 @@ if TYPE_CHECKING:
__all__ = [
"PgClientQueueTransport",
"PgExecutionDispatcher",
- "RoutingExecutionDispatcher",
"get_executor_dispatcher",
- "resolve_executor_transport",
]
-def resolve_executor_transport(context: ExecutionContext) -> bool:
- """True → route this executor dispatch over PG; False → Celery (default).
-
- The single ``pg_queue_enabled`` Flipt flag (fail-closed).
- """
- return resolve_pg_transport(context)
-
-
class PgClientQueueTransport(QueueTransport):
""":class:`QueueTransport` over psycopg2 (the workers half).
@@ -107,12 +94,15 @@ class PgClientQueueTransport(QueueTransport):
)
-def get_executor_dispatcher(
- celery_app: object | None = None,
-) -> RoutingExecutionDispatcher:
- """Factory: the gate-routed executor dispatcher (PG when enabled, else Celery)."""
- return RoutingExecutionDispatcher(
- celery=ExecutionDispatcher(celery_app=celery_app),
- pg=PgExecutionDispatcher(PgClientQueueTransport()),
- resolve=resolve_executor_transport,
- )
+def get_executor_dispatcher() -> PgExecutionDispatcher:
+ """Factory: the executor dispatcher.
+
+ Takes no arguments. It used to accept a ``celery_app`` that fed the Celery
+ branch of the routing dispatcher; that branch went with the ``pg_queue_enabled``
+ flag (UN-4046), and the parameter was kept for a while so the call sites did not
+ all need editing at once. Keeping an ignored parameter made the signature
+ decoration rather than a contract — and it read as "this dispatcher may use
+ Celery", which is exactly the belief that left a ``headers=`` argument at three
+ call sites and broke every extraction. Removed.
+ """
+ return PgExecutionDispatcher(PgClientQueueTransport())
diff --git a/workers/queue_backend/routing.py b/workers/queue_backend/routing.py
index 0cf9d9d5d..b38c0e0b2 100644
--- a/workers/queue_backend/routing.py
+++ b/workers/queue_backend/routing.py
@@ -1,85 +1,42 @@
-"""Queue-transport routing gate (Strangler-Fig).
-
-Decides whether a given dispatch should ride the **Celery** transport
-(today's only real path) or the future **PG Queue** transport,
-based on a per-task-type opt-in allow-list read from env:
-
-- ``WORKER_PG_QUEUE_ENABLED_TASKS`` — comma-separated task names routed
- to PG. Empty / unset → everything routes to Celery.
-
-**Why an allow-list, not a global boolean.** A single
-``WORKER_QUEUE_BACKEND=pg`` switch would move *all* traffic at once —
-the big-bang migration the PG Queue rollout explicitly forbids. The
-allow-list lets an operator migrate one task type at a time
-(drain-and-cutover) and roll back instantly by removing an entry.
-
-(Per-org granularity is intentionally out of scope here. When tenant-level
-canarying is needed it slots in behind :func:`select_backend` as a small
-additive change — no call site touches the routing decision directly.)
-
-**Migration coherence — what the allow-list may and may not split.**
-Per-task routing is the right granularity for *independent / leaf* tasks
-(e.g. ``process_log_history``, ``send_webhook_notification``): no chain,
-no barrier, so one can ride PG while the rest stay on Celery — these are
-the safe first migration candidates. It is **not** valid to split the
-*coupled execution pipeline* (``async_execute_bin`` → file processing →
-callback, with the barrier fan-in) across substrates: a chain hands off
-stage→stage and the barrier coordinates a batch, so a single execution
-must run **entirely on one transport**. The migration unit for the
-pipeline is therefore the *execution*, not the task — the next phase will
-resolve the transport once at kickoff and carry it in ``ExecutionContext``
-(like the fairness key), and downstream dispatches will honour that
-carried choice over the per-task env. ``select_backend`` then becomes:
-"if the dispatch carries an execution-transport marker, use it; otherwise
-consult this allow-list." Until then, only enable *leaf* tasks here.
-
-**Routing posture.** This module only makes the routing *decision*.
-When the flag and the ``WORKER_PG_QUEUE_ENABLED_TASKS`` allow-list select
-PG, ``dispatch()`` really enqueues the task to Postgres (a durable INSERT
-via ``_enqueue_pg``) and the PG consumer runs it — this moves live traffic
-onto the PG transport, it is not a no-op. ``resolve_backend()`` wraps the
-``select_backend()`` allow-list with the per-call execution-transport
-override used to pin a whole coupled pipeline to one substrate.
-
-**Observability.** Routing is logged at INFO so the signals survive a
-default log config: the configured allow-list is logged once per process
-(:func:`_log_allow_list_once`, so a typo is visible even if it never
-matches a real task), and the first time each task is routed to PG
-(``dispatch()``, log-once per task name). A task not in the allow-list
-still routes to the safe ``CELERY`` default.
-
-**Fail-safe parsing.** Unlike ``get_barrier()`` (which raises on an
-unrecognised value, because a typo'd substrate must not silently fall
-back), the routing table is a membership set with no "invalid value"
-concept: blanks and stray whitespace are simply dropped. Malformed or
-empty config can only ever resolve to the safe ``CELERY`` default —
-``select_backend()`` never raises.
+"""Queue-transport routing.
+
+PG is the only transport. :func:`select_backend` returns ``PG``
+unconditionally and :func:`resolve_backend` applies the per-call override on
+top of it.
+
+**History (UN-4046).** This was the Strangler-Fig gate: a per-task opt-in
+allow-list, ``WORKER_PG_QUEUE_ENABLED_TASKS``, with everything not listed
+routed to Celery. That allow-list was never set in any environment, so every
+dispatch lacking an explicit ``backend=`` override went to Celery — harmless
+while Celery consumers existed, and a queue nothing drains once they were
+scaled to zero. The allow-list and the Celery default went with the
+``pg_queue_enabled`` flag.
+
+**The per-call override survives** (``resolve_backend(task_name, override)``).
+It is how the coupled execution pipeline (``async_execute_bin`` → file
+processing → callback, with the barrier fan-in) pins a whole execution to one
+substrate. That mattered when two substrates existed; it is kept because
+stating the transport at the dispatch site reads as intent rather than as
+reliance on a module default, and because the barrier's fan-in still threads it.
+
+**Observability.** The first dispatch of each task name is logged at INFO by
+``dispatch()``, so the signal survives a default log config.
"""
from __future__ import annotations
import logging
-import os
from enum import StrEnum
logger = logging.getLogger(__name__)
-# Env var name kept as a module literal (mirrors how the sibling
-# ``WORKER_BARRIER_BACKEND`` flag is read in ``queue_backend.__init__``)
-# so the queue-backend seam stays self-contained.
-_ENABLED_TASKS_ENV_VAR = "WORKER_PG_QUEUE_ENABLED_TASKS"
-
-# One-shot guard so the configured allow-list is logged once per process
-# (see ``_log_allow_list_once``). Module-level → per prefork child.
-_allow_list_logged = False
-
class QueueBackend(StrEnum):
"""Transport a dispatch is routed to.
``StrEnum`` (3.11+) is inherited for symmetry with ``BarrierBackend``,
but unlike that enum this one is never read from / written to env —
- only the task-name allow-list is. So callers MUST compare by identity
+ it is never read from env at all. So callers MUST compare by identity
(``backend is QueueBackend.PG``), never ``== "pg"``: ``StrEnum`` makes a
typo'd ``== "cellery"`` a silent ``False`` rather than an error.
"""
@@ -88,69 +45,34 @@ class QueueBackend(StrEnum):
PG = "pg"
-def _parse_allow_list() -> frozenset[str]:
- """Parse ``WORKER_PG_QUEUE_ENABLED_TASKS`` into a set of task names.
-
- Trims surrounding whitespace per entry and drops blanks, so
- ``"a, b ,, c"`` → ``{"a", "b", "c"}`` and ``""`` / unset → ``frozenset()``.
- Read at call time (not import) so test harnesses that
- ``monkeypatch.setenv`` flip the table per-test without a reload.
- """
- stripped = (
- entry.strip() for entry in os.getenv(_ENABLED_TASKS_ENV_VAR, "").split(",")
- )
- return frozenset(entry for entry in stripped if entry)
-
-
-def _log_allow_list_once(allow_list: frozenset[str]) -> None:
- """Log the configured allow-list once per process, at INFO.
-
- Makes a misconfiguration eyeballable at boot: a typo'd task name
- (``async_execute_bni``) silently routes everything to Celery, but it
- still shows up here so an operator can spot it. Only fires for a
- *non-empty* allow-list — the default (feature off) stays silent so
- the scaffold is truly inert when unused.
- """
- global _allow_list_logged
- if _allow_list_logged or not allow_list:
- return
- _allow_list_logged = True
- logger.info(
- "PG-queue routing enabled for tasks: %s (all others → Celery)",
- sorted(allow_list),
- )
-
-
-def select_backend(task_name: str) -> QueueBackend:
- """Return the transport a dispatch should ride.
+def select_backend() -> QueueBackend:
+ """Return the transport a dispatch should ride: always ``PG``.
- ``PG`` if ``task_name`` is in ``WORKER_PG_QUEUE_ENABLED_TASKS``;
- otherwise ``CELERY``. Empty / unset allow-list → always ``CELERY``.
+ This used to consult a ``WORKER_PG_QUEUE_ENABLED_TASKS`` allow-list and fall
+ back to ``CELERY`` for anything not opted in. The allow-list was set nowhere,
+ so every dispatch without an explicit ``backend=`` override published to
+ RabbitMQ — which, with no Celery consumers, is a queue nothing drains
+ (UN-4046). PG is the only transport now, so there is nothing to select.
- Never raises — the worst case is the safe ``CELERY`` default.
+ Takes no argument: the answer no longer depends on the task. It used to.
"""
- allow_list = _parse_allow_list()
- _log_allow_list_once(allow_list)
- if task_name in allow_list:
- return QueueBackend.PG
- return QueueBackend.CELERY
+ return QueueBackend.PG
def resolve_backend(task_name: str, override: QueueBackend | None) -> QueueBackend:
"""Resolve the transport for a dispatch, applying the per-call override.
- The single home for the override-wins-else-allow-list precedence so the
+ The single home for the override-wins-else-``select_backend`` precedence so the
rule reads in one place (and ``dispatch()`` plus the call sites
share it):
- - ``override`` is ``None`` → defer to :func:`select_backend` (the env
- allow-list) — the behaviour of every call site today.
- - ``override`` is a :class:`QueueBackend` → it wins. This is how the
- execution-level PG pipeline pins a whole execution's header/callback
- dispatches to one transport regardless of the per-task allow-list (the
- allow-list is for *leaf* tasks; the coupled pipeline's migration unit is
- the execution).
+ - ``override`` is ``None`` → defer to :func:`select_backend`, which is now
+ unconditionally ``PG``.
+ - ``override`` is a :class:`QueueBackend` → it wins. Retained because the
+ execution-level pipeline pins a whole execution's header/callback
+ dispatches explicitly, which reads at the call site as intent rather than
+ as reliance on a default.
Never raises — both branches resolve to a valid :class:`QueueBackend`.
"""
- return override if override is not None else select_backend(task_name)
+ return override if override is not None else select_backend()
diff --git a/workers/run-worker.sh b/workers/run-worker.sh
index 5c37466bf..5bd41dfbe 100755
--- a/workers/run-worker.sh
+++ b/workers/run-worker.sh
@@ -58,11 +58,13 @@ readonly PG_ROLE_CALLBACK="pg-callback"
readonly PG_ROLE_SCHEDULER="pg-scheduler"
readonly PG_ROLE_EXECUTOR="pg-executor"
readonly PG_ROLE_METRICS="pg-metrics"
+readonly PG_ROLE_NOTIFICATION="pg-notification"
+readonly PG_ROLE_IDE_CALLBACK="pg-ide-callback"
declare -rA PG_CONSUMER_ROLES=(
["$PG_ROLE_ORCH_API"]="api_deployment;celery_api_deployments"
["$PG_ROLE_ORCH_GENERAL"]="general;celery"
- ["$PG_ROLE_FILEPROC"]="file_processing;file_processing,api_file_processing"
- ["$PG_ROLE_CALLBACK"]="callback;file_processing_callback,api_file_processing_callback"
+ ["$PG_ROLE_FILEPROC"]="file_processing;file_processing,file_processing_priority,api_file_processing"
+ ["$PG_ROLE_CALLBACK"]="callback;file_processing_callback,celery_callback,api_file_processing_callback"
# Runs scheduler.tasks.execute_pipeline_task fired by the orchestrator's PG
# scheduler tick (Beat replacement). Distinct from the Celery 'scheduler'
# worker (which Beat fires onto RabbitMQ).
@@ -70,7 +72,7 @@ declare -rA PG_CONSUMER_ROLES=(
# Runs execute_extraction (the executor RPC) over PG — request-reply: writes
# the result to pg_task_result for the blocking caller. Queues mirror the
# Celery executor's CELERY_QUEUES_EXECUTOR.
- ["$PG_ROLE_EXECUTOR"]="executor;celery_executor_legacy,celery_executor_agentic,celery_executor_agentic_table"
+ ["$PG_ROLE_EXECUTOR"]="executor;celery_executor_legacy,celery_executor_agentic,celery_executor_table,celery_executor_smart_table,celery_executor_simple_prompt_studio,celery_executor_agentic_table,celery_executor_lookup_test"
# Runs the dashboard_metrics.* periodics fired by the PG scheduler tick,
# replacing the Celery 'workerMetrics' (-Q dashboard_metric_events). Its own
# role rather than a queue bolted onto pg-scheduler: the consumer's health
@@ -79,6 +81,13 @@ declare -rA PG_CONSUMER_ROLES=(
# pg-scheduler's tight 240s bound would restart that pod and take in-flight
# pipeline triggers down with it. Sized in docker-compose.yaml.
["$PG_ROLE_METRICS"]="scheduler;dashboard_metric_events"
+ # The backend notification seam enqueues send_webhook_notification onto
+ # these queues unconditionally (UN-4046). Without this role a host-run PG
+ # fleet accepts every buffered webhook and drains none — enqueue_task
+ # succeeds, so nothing errors at the producer.
+ ["$PG_ROLE_NOTIFICATION"]="notification;notifications,notifications_webhook,notifications_email,notifications_sms,notifications_priority"
+ # Prompt Studio IDE callbacks (ide_index_*/ide_prompt_*/extraction_*).
+ ["$PG_ROLE_IDE_CALLBACK"]="ide_callback;ide_callback"
)
declare -rA PG_QUEUE_MEMBERS=(
["$PG_QUEUE_CONSUMER_TYPE"]=1
@@ -90,6 +99,8 @@ declare -rA PG_QUEUE_MEMBERS=(
["$PG_ROLE_SCHEDULER"]=1
["$PG_ROLE_EXECUTOR"]=1
["$PG_ROLE_METRICS"]=1
+ ["$PG_ROLE_NOTIFICATION"]=1
+ ["$PG_ROLE_IDE_CALLBACK"]=1
)
# The Celery transport set: every worker EXCEPT the PG-queue members — the
# *complement* of the 'pg-queue' set, so the two transports' logs can be tailed
diff --git a/workers/sample.env b/workers/sample.env
index 6a80eb455..abb09255f 100644
--- a/workers/sample.env
+++ b/workers/sample.env
@@ -131,34 +131,6 @@ WORKER_BARRIER_BACKEND=chord
# workflows can plausibly span >6h.
WORKER_BARRIER_KEY_TTL_SECONDS=21600
-# =============================================================================
-# PG Queue Routing (Strangler-Fig)
-# =============================================================================
-# Selects the queue *transport* per task — Celery (default) or PG Queue
-# (PG Queue). Separate axis from the barrier above: the barrier is how a
-# batch's fan-in fires the callback; this is how messages travel.
-#
-# A comma-separated allow-list drives a gradual, per-task-type migration
-# (NOT a global on/off switch). A task routes to PG if its name is listed.
-# Unset / empty (the default) → everything routes to Celery, zero
-# behaviour change. No Flipt server required.
-#
-# ⚠️ A listed task is enqueued to Postgres instead of Celery. It will only
-# execute if the PG consumer is running — otherwise the message is durably
-# enqueued but never run. Do NOT opt a task in unless the consumer is
-# deployed.
-#
-# Roll back instantly by removing an entry; flips take effect on worker
-# restart (same posture as every other WORKER_* env).
-#
-# Only list *leaf* tasks dispatched via queue_backend.dispatch() — i.e. no
-# downstream chain or barrier fan-in (e.g. send_webhook_notification). Do
-# NOT list coupled pipeline tasks like async_execute_bin here: one execution
-# must run wholly on one transport, so the pipeline migrates per-execution
-# (via ExecutionContext) in a later phase, not per-task. See the routing.py
-# module docstring for the full constraint.
-# WORKER_PG_QUEUE_ENABLED_TASKS=send_webhook_notification
-
# Database URL (for fallback usage)
DATABASE_URL=postgresql://unstract_dev:unstract_pass@unstract-db:5432/unstract_db
diff --git a/workers/shared/api/internal_client.py b/workers/shared/api/internal_client.py
index bba4af529..ee9fbca7a 100644
--- a/workers/shared/api/internal_client.py
+++ b/workers/shared/api/internal_client.py
@@ -711,12 +711,6 @@ class InternalAPIClient(CachedAPIClientMixin):
workflow_id, execution_id, organization_id
)
- def submit_file_batch_for_processing(
- self, batch_data: dict[str, Any]
- ) -> dict[str, Any]:
- """Submit file batch for processing."""
- return self.execution_client.submit_file_batch_for_processing(batch_data)
-
def execute_workflow_async(self, execution_data: dict[str, Any]) -> dict[str, Any]:
"""Execute workflow asynchronously for scheduler.
diff --git a/workers/shared/clients/base_client.py b/workers/shared/clients/base_client.py
index 017a1d62e..585c0683c 100644
--- a/workers/shared/clients/base_client.py
+++ b/workers/shared/clients/base_client.py
@@ -19,8 +19,6 @@ from celery.exceptions import SoftTimeLimitExceeded
from requests.adapters import HTTPAdapter
from urllib3.util.retry import Retry
-from unstract.flags.feature_flag import check_feature_flag_status
-
from ..data.models import APIResponse
from ..enums import HTTPMethod
from ..infrastructure.config.worker_config import WorkerConfig
@@ -32,27 +30,8 @@ logger = WorkerLogger.get_logger(__name__)
APPLICATION_JSON = "application/json"
-# Single PG-queue rollout flag (same key as pg_queue.flags / executor_rpc).
-_PG_QUEUE_FLAG_KEY = "pg_queue_enabled"
-
-
-def _pg_queue_jitter_enabled() -> bool:
- """Whether retry jitter is on — gated by the single ``pg_queue_enabled`` Flipt
- flag (fail-closed to False, like ``check_feature_flag_status`` itself), so the
- Celery flow's retry cadence is unchanged and the jitter tracks the PG rollout.
-
- The retry path carries no per-execution/org context, so the flag is evaluated
- globally (entity ``"default"``) — a coarse "is PG being rolled out" gate, which
- is all a transport-agnostic HTTP-retry hardening needs.
- """
- try:
- return check_feature_flag_status(flag_key=_PG_QUEUE_FLAG_KEY, entity_id="default")
- except Exception:
- return False
-
-
def _backoff_with_jitter(backoff_factor: float, attempt: int) -> float:
- """Exponential backoff, with equal jitter when the PG-queue flag is on.
+ """Exponential backoff with equal jitter.
Without jitter, every concurrent worker retrying a saturated backend sleeps on
the identical 1s/2s/4s schedule and their retries arrive in synchronized waves
@@ -60,16 +39,14 @@ def _backoff_with_jitter(backoff_factor: float, attempt: int) -> float:
finalization-write failures under high concurrency. Equal jitter spreads each
retry across [base/2, base] so the waves de-correlate.
- Gated by the ``pg_queue_enabled`` Flipt flag (fail-closed): OFF keeps the exact
- exponential backoff for the Celery flow; evaluated at call time on the rare
- retry path (behind a sleep + network round-trip), so the Flipt cost is
- irrelevant and it stays runtime-togglable via the flag.
+ Previously gated on the ``pg_queue_enabled`` Flipt flag so the Celery flow kept
+ the exact exponential cadence. With that flag removed the jitter is
+ unconditional — it is transport-agnostic HTTP-retry hardening, and the jittered
+ cadence is the one integration has run under load.
"""
base = backoff_factor * (2**attempt)
if base <= 0:
return 0.0
- if not _pg_queue_jitter_enabled():
- return base # exact exponential backoff (unchanged for the Celery flow)
return base / 2 + random.uniform(0, base / 2)
diff --git a/workers/shared/clients/execution_client.py b/workers/shared/clients/execution_client.py
index 22bc9b355..ef98f8dae 100644
--- a/workers/shared/clients/execution_client.py
+++ b/workers/shared/clients/execution_client.py
@@ -712,23 +712,6 @@ class ExecutionAPIClient(BaseAPIClient):
organization_id=organization_id,
)
- def submit_file_batch_for_processing(
- self, batch_data: dict[str, Any]
- ) -> dict[str, Any]:
- """Submit file batch for processing.
-
- Args:
- batch_data: File batch data
-
- Returns:
- Submission result
- """
- return self.post(
- "v1/workflow-manager/file-batch/submit/",
- batch_data,
- organization_id=batch_data.get("organization_id"),
- )
-
# Pipeline Management Methods for Scheduler Worker
def get_pipeline_data(
diff --git a/workers/shared/constants/api_endpoints.py b/workers/shared/constants/api_endpoints.py
index 0737c202c..8d88ee845 100644
--- a/workers/shared/constants/api_endpoints.py
+++ b/workers/shared/constants/api_endpoints.py
@@ -57,7 +57,6 @@ class APIEndpoints:
)
# Notification endpoints
- WEBHOOK_SEND = build_internal_endpoint("webhook/send/")
NOTIFICATION_SEND = build_internal_endpoint("notification/send/")
# Health and monitoring endpoints
diff --git a/workers/shared/patterns/worker_patterns.py b/workers/shared/patterns/worker_patterns.py
index 77acb32dd..fbd915efe 100644
--- a/workers/shared/patterns/worker_patterns.py
+++ b/workers/shared/patterns/worker_patterns.py
@@ -152,7 +152,6 @@ class APIEndpoints:
"workflow-file-execution/create/"
)
PIPELINE_STATUS = build_internal_endpoint("pipeline/{pipeline_id}/status/")
- WEBHOOK_SEND = build_internal_endpoint("webhook/send/")
class WorkerConfig:
diff --git a/workers/tests/test_dispatch_pg.py b/workers/tests/test_dispatch_pg.py
index c97a2664f..65eb378c4 100644
--- a/workers/tests/test_dispatch_pg.py
+++ b/workers/tests/test_dispatch_pg.py
@@ -18,7 +18,6 @@ import pytest
from queue_backend import dispatch
from queue_backend.fairness import FairnessKey, WorkloadType
from queue_backend.pg_queue import to_payload
-from queue_backend.routing import _ENABLED_TASKS_ENV_VAR as ENABLED_TASKS_ENV
from queue_backend.routing import QueueBackend
# ``queue_backend.dispatch`` the attribute is the function (shadows the
@@ -28,7 +27,6 @@ dispatch_mod = importlib.import_module("queue_backend.dispatch")
@pytest.fixture(autouse=True)
def _reset(monkeypatch):
- monkeypatch.delenv(ENABLED_TASKS_ENV, raising=False)
dispatch_mod._pg_routing_logged.clear()
dispatch_mod._pg_local.client = None
@@ -83,7 +81,6 @@ class TestToPayload:
class TestDispatchEnqueueIntegration:
def test_dispatch_lands_decodable_row(self, monkeypatch, pg_client):
queue_name = f"test_dispatch_{os.getpid()}"
- monkeypatch.setenv(ENABLED_TASKS_ENV, "leaf_task")
monkeypatch.setattr(dispatch_mod, "_get_pg_client", lambda: pg_client)
fairness = FairnessKey(org_id="org-x", workload_type=WorkloadType.API)
try:
@@ -110,27 +107,17 @@ class TestDispatchEnqueueIntegration:
)
pg_client.conn.commit()
- def test_celery_dispatch_unaffected(self):
- # No opt-in → Celery path; never touches the PG client.
- with (
- patch("queue_backend.dispatch.current_app") as mock_app,
- patch("queue_backend.dispatch._get_pg_client") as mock_get,
- ):
- dispatch("some_task", args=["x"], queue="general")
- mock_app.send_task.assert_called_once()
- mock_get.assert_not_called()
-
class TestDispatchBackendOverride:
"""The per-call ``backend=`` override (9e PR 2a inert foundation).
- When set it wins over the ``WORKER_PG_QUEUE_ENABLED_TASKS`` allow-list, so
- the execution-level PG pipeline can route a whole execution's headers /
+ Retained after the allow-list went (UN-4046), so the execution-level PG
+ pipeline can state a whole execution's headers /
callback onto PG without opting their task *names* in. ``None`` (default)
preserves the allow-list decision exactly — every call site today.
"""
- def test_override_pg_forces_pg_without_allow_list(self, monkeypatch):
+ def test_override_pg_forces_pg(self, monkeypatch):
# Task name NOT in the (empty) allow-list → select_backend says Celery;
# the explicit override must still route it to PG.
captured: dict = {}
@@ -142,16 +129,13 @@ class TestDispatchBackendOverride:
monkeypatch.setattr(dispatch_mod, "_get_pg_client", lambda: _Client())
with patch("queue_backend.dispatch.current_app") as mock_app:
- handle = dispatch(
- "pipeline_header", queue="general", backend=QueueBackend.PG
- )
+ handle = dispatch("pipeline_header", queue="general", backend=QueueBackend.PG)
mock_app.send_task.assert_not_called()
assert handle.id == "11"
assert captured["payload"]["task_name"] == "pipeline_header"
- def test_override_celery_forces_celery_despite_allow_list(self, monkeypatch):
+ def test_override_celery_forces_celery(self, monkeypatch):
# Task name IS opted into PG, but the override pins it back to Celery.
- monkeypatch.setenv(ENABLED_TASKS_ENV, "pipeline_header")
with (
patch("queue_backend.dispatch.current_app") as mock_app,
patch("queue_backend.dispatch._get_pg_client") as mock_get,
@@ -160,7 +144,7 @@ class TestDispatchBackendOverride:
mock_app.send_task.assert_called_once()
mock_get.assert_not_called()
- def test_default_none_preserves_allow_list_decision(self, monkeypatch):
+ def test_default_none_resolves_to_pg(self, monkeypatch):
# No override → allow-list decides. Opted-in name → PG.
captured: dict = {}
@@ -169,7 +153,6 @@ class TestDispatchBackendOverride:
captured.update(queue=queue_name)
return 5
- monkeypatch.setenv(ENABLED_TASKS_ENV, "leaf_task")
monkeypatch.setattr(dispatch_mod, "_get_pg_client", lambda: _Client())
with patch("queue_backend.dispatch.current_app") as mock_app:
handle = dispatch("leaf_task", queue="general")
@@ -215,7 +198,6 @@ class TestDispatchPriorityWiring:
captured.update(queue=queue_name, **kwargs)
return 7
- monkeypatch.setenv(ENABLED_TASKS_ENV, "leaf_task")
monkeypatch.setattr(dispatch_mod, "_get_pg_client", lambda: _Client())
return captured
diff --git a/workers/tests/test_dispatch_with_callback.py b/workers/tests/test_dispatch_with_callback.py
index 1449e395b..dce8b73f9 100644
--- a/workers/tests/test_dispatch_with_callback.py
+++ b/workers/tests/test_dispatch_with_callback.py
@@ -928,17 +928,15 @@ class TestStructureToolSingleDispatch:
assert "answer_params" in ctx.executor_params
assert "pipeline_options" in ctx.executor_params
- # Fairness header lands on the dispatch call. A regression that
- # drops ``headers=`` or flips ``NON_API``→``API`` here would
- # otherwise stay green — keep this assertion paired with the
- # ``_fairness_headers`` unit test in test_structure_tool_task.py.
- assert dispatcher.dispatch.call_args.kwargs["headers"] == {
- "x-fairness-key": {
- "org_id": "org-1",
- "workload_type": "non_api",
- "pipeline_priority": 5,
- }
- }
+ # No ``headers`` on the dispatch call, and that is the assertion now
+ # (UN-4046). This used to pin the fairness header; the routing dispatcher
+ # that accepted ``headers=`` is gone, ``PgExecutionDispatcher.dispatch``
+ # takes no such argument, and passing one raises TypeError — which is
+ # exactly what shipped and broke every extraction until it was caught.
+ # Fairness is not lost: PG carries org routing in the enqueue payload
+ # (``transport.enqueue(..., org_id=...)``), which is queue_backend's to
+ # cover, not this call site's.
+ assert "headers" not in dispatcher.dispatch.call_args.kwargs
# ---------------------------------------------------------------------------
diff --git a/workers/tests/test_executor_rpc.py b/workers/tests/test_executor_rpc.py
index 97822e9ba..99fc1b261 100644
--- a/workers/tests/test_executor_rpc.py
+++ b/workers/tests/test_executor_rpc.py
@@ -14,15 +14,13 @@ from unittest.mock import MagicMock, patch
import pytest
from queue_backend.pg_queue.executor_rpc import (
PgClientQueueTransport,
- RoutingExecutionDispatcher,
get_executor_dispatcher,
- resolve_executor_transport,
)
+
from unstract.core.execution_dispatch import DispatchHandle, signature_to_continuation
from unstract.workflow_execution.executor_rpc import (
ExecResultRow,
PgExecutionDispatcher,
- resolve_pg_transport,
)
_WMOD = "queue_backend.pg_queue.executor_rpc"
@@ -96,20 +94,31 @@ class TestSharedDispatchContract:
def test_failed_row_returns_error(self):
row = ExecResultRow(status="failed", result=None, error="boom")
- res = PgExecutionDispatcher(_FakeTransport(wait_return=row)).dispatch(_ctx(), timeout=5)
+ res = PgExecutionDispatcher(_FakeTransport(wait_return=row)).dispatch(
+ _ctx(), timeout=5
+ )
assert res.success is False and res.error == "boom"
def test_failed_row_empty_error_falls_back(self):
row = ExecResultRow(status="failed", result=None, error="")
- res = PgExecutionDispatcher(_FakeTransport(wait_return=row)).dispatch(_ctx(), timeout=5)
+ res = PgExecutionDispatcher(_FakeTransport(wait_return=row)).dispatch(
+ _ctx(), timeout=5
+ )
assert res.success is False and "executor task failed" in res.error
def test_completed_but_result_none_is_failure(self):
row = ExecResultRow(status="completed", result=None, error="")
- assert PgExecutionDispatcher(_FakeTransport(wait_return=row)).dispatch(_ctx(), timeout=5).success is False
+ assert (
+ PgExecutionDispatcher(_FakeTransport(wait_return=row))
+ .dispatch(_ctx(), timeout=5)
+ .success
+ is False
+ )
def test_malformed_completed_row_is_failure_not_raise(self):
- d = PgExecutionDispatcher(_FakeTransport(wait_return=_completed({"bad": "shape"})))
+ d = PgExecutionDispatcher(
+ _FakeTransport(wait_return=_completed({"bad": "shape"}))
+ )
res = d.dispatch(_ctx(), timeout=5)
assert res.success is False and "Malformed" in res.error
@@ -152,15 +161,19 @@ class TestSharedDispatchContract:
@staticmethod
def _sig(task: str):
return MagicMock(
- task=task, args=(), kwargs={"callback_kwargs": {"room": "r1"}},
+ task=task,
+ args=(),
+ kwargs={"callback_kwargs": {"room": "r1"}},
options={"queue": "ide_callback"},
)
def test_dispatch_with_callback_translates_both_signatures(self):
t = _FakeTransport()
handle = PgExecutionDispatcher(t).dispatch_with_callback(
- _ctx(), on_success=self._sig("ide_prompt_complete"),
- on_error=self._sig("ide_prompt_error"), task_id="tid-7",
+ _ctx(),
+ on_success=self._sig("ide_prompt_complete"),
+ on_error=self._sig("ide_prompt_error"),
+ task_id="tid-7",
)
assert handle.id == "tid-7"
(kw,) = t.enqueue_calls
@@ -185,85 +198,9 @@ class TestSharedDispatchContract:
# --- Shared gate: resolve_pg_transport (single Flipt flag, fail-closed) ---
-class TestResolvePgTransport:
- def test_flipt_unavailable_is_celery(self, monkeypatch):
- monkeypatch.setenv("FLIPT_SERVICE_AVAILABLE", "false")
- with patch(f"{_SMOD}.check_feature_flag_status") as flag:
- assert resolve_pg_transport(_ctx()) is False
- flag.assert_not_called()
-
- def test_flag_true_is_pg_keyed_on_org(self, monkeypatch):
- monkeypatch.setenv("FLIPT_SERVICE_AVAILABLE", "true")
- with patch(f"{_SMOD}.check_feature_flag_status", return_value=True) as flag:
- assert resolve_pg_transport(_ctx("orgX")) is True
- assert flag.call_args.kwargs["entity_id"] == "orgX"
- assert flag.call_args.kwargs["flag_key"] == "pg_queue_enabled"
-
- def test_flag_false_is_celery(self, monkeypatch):
- monkeypatch.setenv("FLIPT_SERVICE_AVAILABLE", "true")
- with patch(f"{_SMOD}.check_feature_flag_status", return_value=False):
- assert resolve_pg_transport(_ctx()) is False
-
- def test_flipt_error_fails_closed(self, monkeypatch):
- monkeypatch.setenv("FLIPT_SERVICE_AVAILABLE", "true")
- with patch(f"{_SMOD}.check_feature_flag_status", side_effect=RuntimeError("x")):
- assert resolve_pg_transport(_ctx()) is False
-
- def test_org_less_context_buckets_on_run_id(self, monkeypatch):
- monkeypatch.setenv("FLIPT_SERVICE_AVAILABLE", "true")
- with patch(f"{_SMOD}.check_feature_flag_status", return_value=True) as flag:
- assert resolve_pg_transport(_ctx(org=None)) is True
- assert flag.call_args.kwargs["entity_id"] == "run-1"
- assert "organization_id" not in flag.call_args.kwargs["context"]
-
-
# --- Shared routing: RoutingExecutionDispatcher (zero-regression) ---
-class TestRoutingDispatch:
- @staticmethod
- def _build(route_to_pg: bool):
- celery, pg = MagicMock(), MagicMock()
- d = RoutingExecutionDispatcher(
- celery=celery, pg=pg, resolve=lambda _ctx: route_to_pg
- )
- return d, celery, pg
-
- def test_gate_off_forwards_timeout_and_headers_to_celery(self):
- d, celery, pg = self._build(route_to_pg=False)
- ctx = _ctx()
- hdrs = {"x-fairness-key": {"org_id": "o"}}
- d.dispatch(ctx, timeout=9, headers=hdrs)
- celery.dispatch.assert_called_once_with(ctx, timeout=9, headers=hdrs)
- pg.dispatch.assert_not_called()
-
- def test_gate_on_passes_timeout_to_pg_and_drops_headers(self):
- d, celery, pg = self._build(route_to_pg=True)
- ctx = _ctx()
- d.dispatch(ctx, timeout=7, headers={"x-fairness-key": {"o": 1}})
- pg.dispatch.assert_called_once_with(ctx, timeout=7) # headers dropped
- celery.dispatch.assert_not_called()
-
- def test_async_and_callback_stay_celery_when_gate_off(self):
- d, celery, pg = self._build(route_to_pg=False)
- d.dispatch_async(_ctx(), headers={"h": 1})
- d.dispatch_with_callback(_ctx(), on_success="s", on_error="e")
- celery.dispatch_async.assert_called_once()
- celery.dispatch_with_callback.assert_called_once()
- pg.dispatch_async.assert_not_called()
- pg.dispatch_with_callback.assert_not_called()
-
- def test_async_and_callback_route_to_pg_when_gated(self):
- d, celery, pg = self._build(route_to_pg=True)
- d.dispatch_async(_ctx())
- d.dispatch_with_callback(_ctx(), on_success="s", on_error="e", task_id="t")
- pg.dispatch_async.assert_called_once()
- pg.dispatch_with_callback.assert_called_once()
- assert "headers" not in pg.dispatch_with_callback.call_args.kwargs
- celery.dispatch_async.assert_not_called()
- celery.dispatch_with_callback.assert_not_called()
-
-
# --- Workers adapter: PgClientQueueTransport + env gate + factory wiring ---
@@ -286,8 +223,10 @@ class TestWorkersAdapter:
client = self._client()
with patch(f"{_WMOD}.PgQueueClient", return_value=client):
PgClientQueueTransport().enqueue(
- queue="celery_executor_legacy", context=self._ctx(),
- org_id="org9", reply_key="rk1",
+ queue="celery_executor_legacy",
+ context=self._ctx(),
+ org_id="org9",
+ reply_key="rk1",
)
client.send.assert_called_once()
queue_arg, payload_arg = client.send.call_args.args[:2]
@@ -302,8 +241,11 @@ class TestWorkersAdapter:
spec = {"task_name": "ide_prompt_complete", "kwargs": {}, "queue": "ide_callback"}
with patch(f"{_WMOD}.PgQueueClient", return_value=client):
PgClientQueueTransport().enqueue(
- queue="celery_executor_legacy", context=self._ctx(), org_id="o",
- on_success=spec, task_id="tid-7",
+ queue="celery_executor_legacy",
+ context=self._ctx(),
+ org_id="o",
+ on_success=spec,
+ task_id="tid-7",
)
payload = client.send.call_args.args[1]
assert payload["on_success"] == spec and payload["task_id"] == "tid-7"
@@ -312,7 +254,11 @@ class TestWorkersAdapter:
def test_wait_for_result_folds_dict_to_row(self):
rb = MagicMock()
rb.__enter__.return_value = rb
- rb.wait_for_result.return_value = {"status": "completed", "result": {"a": 1}, "error": ""}
+ rb.wait_for_result.return_value = {
+ "status": "completed",
+ "result": {"a": 1},
+ "error": "",
+ }
with patch(f"{_WMOD}.PgResultBackend", return_value=rb):
row = PgClientQueueTransport().wait_for_result("rk", 5)
assert isinstance(row, ExecResultRow)
@@ -354,24 +300,12 @@ class TestWorkersAdapter:
PgClientQueueTransport().wait_for_result("rk", 5)
rb.forget.assert_not_called()
- def test_resolve_delegates_to_shared_flipt_resolver_true(self):
- with patch(f"{_WMOD}.resolve_pg_transport", return_value=True) as r:
- assert resolve_executor_transport(self._ctx()) is True
- r.assert_called_once()
- # No env master-gate is threaded any more — Flipt is the sole gate.
- assert "master_gate_enabled" not in r.call_args.kwargs
-
- def test_resolve_delegates_to_shared_flipt_resolver_false(self):
- with patch(f"{_WMOD}.resolve_pg_transport", return_value=False):
- assert resolve_executor_transport(self._ctx()) is False
-
- def test_factory_wires_routing_with_workers_transport(self):
- d = get_executor_dispatcher(celery_app="app")
- assert isinstance(d, RoutingExecutionDispatcher)
- # The PG dispatcher is wired with the workers psycopg2 transport, and the gate
- # is the workers' Flipt resolver.
- assert isinstance(d._pg._transport, PgClientQueueTransport)
- assert d._resolve is resolve_executor_transport
+ def test_factory_wires_the_workers_transport(self):
+ # The factory takes no arguments (UN-4046) and returns the PG dispatcher
+ # wired with the workers psycopg2 transport.
+ d = get_executor_dispatcher()
+ assert isinstance(d, PgExecutionDispatcher)
+ assert isinstance(d._transport, PgClientQueueTransport)
# --- Core helpers (unchanged; the shared signature/handle primitives) ---
@@ -382,9 +316,12 @@ class TestSharedDispatchHelpers:
assert signature_to_continuation(None) is None
def test_signature_translates_task_kwargs_and_queue(self):
- sig = MagicMock(task="ide_prompt_complete", args=(),
- kwargs={"callback_kwargs": {"room": "r1"}},
- options={"queue": "ide_callback"})
+ sig = MagicMock(
+ task="ide_prompt_complete",
+ args=(),
+ kwargs={"callback_kwargs": {"room": "r1"}},
+ options={"queue": "ide_callback"},
+ )
assert signature_to_continuation(sig) == {
"task_name": "ide_prompt_complete",
"kwargs": {"callback_kwargs": {"room": "r1"}},
@@ -402,8 +339,12 @@ class TestSharedDispatchHelpers:
signature_to_continuation(sig)
def test_signature_with_positional_args_fails_fast(self):
- sig = MagicMock(task="ide_prompt_complete", args=("pos",), kwargs={},
- options={"queue": "ide_callback"})
+ sig = MagicMock(
+ task="ide_prompt_complete",
+ args=("pos",),
+ kwargs={},
+ options={"queue": "ide_callback"},
+ )
with pytest.raises(ValueError, match="positional args"):
signature_to_continuation(sig)
diff --git a/workers/tests/test_fairness_key.py b/workers/tests/test_fairness_key.py
index 0ef3e53a5..c720be4ed 100644
--- a/workers/tests/test_fairness_key.py
+++ b/workers/tests/test_fairness_key.py
@@ -11,6 +11,7 @@ import pytest
from celery import Celery
from queue_backend import FairnessKey, dispatch
+from queue_backend.routing import QueueBackend
from queue_backend.fairness import (
DEFAULT_PRIORITY,
FAIRNESS_HEADER_NAME,
@@ -153,25 +154,40 @@ class TestFairnessKey:
class TestDispatchAttachesFairness:
- def test_omitted_fairness_no_header_sent(self):
- with patch("queue_backend.dispatch.current_app") as mock_app:
+ """UN-4046: these assert the PG path, which is the only one dispatch takes.
+
+ They used to patch ``current_app`` and read ``send_task.call_args`` — the
+ Celery branch. ``select_backend`` returns PG unconditionally now, so that
+ branch is unreachable without an explicit ``backend=`` override and the
+ assertions failed on a mock that was never called. The property under test is
+ unchanged: fairness must reach the transport WITHOUT being folded into the
+ business kwargs, since tasks without ``**kwargs`` would break. On PG that
+ means the ``fairness`` payload slot plus the ``org_id``/``priority`` columns
+ the dequeue orders by.
+ """
+
+ def test_omitted_fairness_writes_neutral_defaults(self):
+ with patch("queue_backend.dispatch._get_pg_client") as mock_client:
dispatch("any_task", kwargs={"foo": "bar"})
- call_kwargs = mock_app.send_task.call_args.kwargs
- assert call_kwargs["headers"] is None
- assert call_kwargs["kwargs"] == {"foo": "bar"}
+ _, payload = mock_client.return_value.send.call_args.args
+ send_kwargs = mock_client.return_value.send.call_args.kwargs
+ assert payload["fairness"] is None
+ assert payload["kwargs"] == {"foo": "bar"}
+ assert send_kwargs["org_id"] is None
+ assert send_kwargs["priority"] == DEFAULT_PRIORITY
- def test_explicit_fairness_none_no_header_sent(self):
+ def test_explicit_fairness_none_writes_neutral_defaults(self):
# Documented opt-out for non-workflow dispatches.
- with patch("queue_backend.dispatch.current_app") as mock_app:
+ with patch("queue_backend.dispatch._get_pg_client") as mock_client:
dispatch("send_webhook_notification", kwargs={"x": 1}, fairness=None)
- call_kwargs = mock_app.send_task.call_args.kwargs
- assert call_kwargs["headers"] is None
- assert call_kwargs["kwargs"] == {"x": 1}
+ _, payload = mock_client.return_value.send.call_args.args
+ assert payload["fairness"] is None
+ assert payload["kwargs"] == {"x": 1}
- def test_provided_fairness_attached_as_message_header(self):
- with patch("queue_backend.dispatch.current_app") as mock_app:
+ def test_provided_fairness_rides_the_payload_and_the_columns(self):
+ with patch("queue_backend.dispatch._get_pg_client") as mock_client:
dispatch(
"any_task",
kwargs={"foo": "bar"},
@@ -180,40 +196,43 @@ class TestDispatchAttachesFairness:
),
)
- call_kwargs = mock_app.send_task.call_args.kwargs
- assert call_kwargs["headers"] == {
- FAIRNESS_HEADER_NAME: {
- "org_id": "org-1",
- "workload_type": "api",
- "pipeline_priority": 9,
- }
+ _, payload = mock_client.return_value.send.call_args.args
+ send_kwargs = mock_client.return_value.send.call_args.kwargs
+ assert payload["fairness"] == {
+ "org_id": "org-1",
+ "workload_type": "api",
+ "pipeline_priority": 9,
}
+ # org_id + priority are separate COLUMNS, not just payload keys: the
+ # dequeue orders by them, so a payload-only fairness would be inert.
+ assert send_kwargs["org_id"] == "org-1"
+ assert send_kwargs["priority"] == 9
# Business kwargs must NOT contain the fairness slot — tasks
# without **kwargs would break.
- sent_kwargs = call_kwargs["kwargs"]
- assert sent_kwargs == {"foo": "bar"}
- assert FAIRNESS_HEADER_NAME not in sent_kwargs
+ assert payload["kwargs"] == {"foo": "bar"}
+ assert FAIRNESS_HEADER_NAME not in payload["kwargs"]
def test_fairness_with_no_business_kwargs(self):
- with patch("queue_backend.dispatch.current_app") as mock_app:
+ with patch("queue_backend.dispatch._get_pg_client") as mock_client:
dispatch(
"any_task",
fairness=FairnessKey(org_id=None, workload_type=WorkloadType.NON_API),
)
- call_kwargs = mock_app.send_task.call_args.kwargs
- assert call_kwargs["kwargs"] is None
- assert call_kwargs["headers"] == {
- FAIRNESS_HEADER_NAME: {
- "org_id": None,
- "workload_type": "non_api",
- "pipeline_priority": DEFAULT_PRIORITY,
- }
+ _, payload = mock_client.return_value.send.call_args.args
+ send_kwargs = mock_client.return_value.send.call_args.kwargs
+ assert payload["kwargs"] == {}
+ assert payload["fairness"] == {
+ "org_id": None,
+ "workload_type": "non_api",
+ "pipeline_priority": DEFAULT_PRIORITY,
}
+ assert send_kwargs["org_id"] is None
+ assert send_kwargs["priority"] == DEFAULT_PRIORITY
def test_caller_kwargs_not_mutated_in_place(self):
caller_kwargs = {"foo": "bar"}
- with patch("queue_backend.dispatch.current_app"):
+ with patch("queue_backend.dispatch._get_pg_client"):
dispatch(
"any_task",
kwargs=caller_kwargs,
@@ -223,6 +242,33 @@ class TestDispatchAttachesFairness:
assert caller_kwargs == {"foo": "bar"}
assert FAIRNESS_HEADER_NAME not in caller_kwargs
+ def test_explicit_celery_override_still_attaches_the_header(self):
+ """The Celery branch survives only as a ``backend=`` override target.
+
+ Nothing selects it — ``select_backend`` returns PG unconditionally — and
+ it is slated for deletion with the rest of the Celery transport. Until
+ then it must not rot silently: an override is the one way to reach it.
+ """
+ with patch("queue_backend.dispatch.current_app") as mock_app:
+ dispatch(
+ "any_task",
+ kwargs={"foo": "bar"},
+ fairness=FairnessKey(
+ org_id="org-1", workload_type=WorkloadType.API, pipeline_priority=9
+ ),
+ backend=QueueBackend.CELERY,
+ )
+
+ call_kwargs = mock_app.send_task.call_args.kwargs
+ assert call_kwargs["headers"] == {
+ FAIRNESS_HEADER_NAME: {
+ "org_id": "org-1",
+ "workload_type": "api",
+ "pipeline_priority": 9,
+ }
+ }
+ assert call_kwargs["kwargs"] == {"foo": "bar"}
+
class TestDispatchCallSitesPassFairness:
"""AST audit: every production ``dispatch(...)`` declares fairness."""
@@ -279,7 +325,12 @@ class TestNoConsumerYet:
class TestHeaderSurvivesCeleryPipeline:
- """End-to-end: header survives Celery's real send_task code path."""
+ """End-to-end: header survives Celery's real send_task code path.
+
+ Reached via an explicit ``backend=`` override since UN-4046 — see
+ ``test_explicit_celery_override_still_attaches_the_header``. Kept as long as
+ the Celery branch exists, and deleted with it.
+ """
def test_header_present_on_outbound_message(self):
app = Celery(
@@ -296,6 +347,7 @@ class TestHeaderSurvivesCeleryPipeline:
workload_type=WorkloadType.NON_API,
pipeline_priority=9,
),
+ backend=QueueBackend.CELERY,
)
call_headers = wrapped_send.call_args.kwargs["headers"]
diff --git a/workers/tests/test_pg_finalization_fixes.py b/workers/tests/test_pg_finalization_fixes.py
index 064524459..b1daad5c2 100644
--- a/workers/tests/test_pg_finalization_fixes.py
+++ b/workers/tests/test_pg_finalization_fixes.py
@@ -3,51 +3,45 @@
- L2: ``_update_execution_status_unified`` re-raises on the PG path (so the failed
finalization vt-redelivers / poison-drops) but swallows on the Celery path.
- L3: ``_backoff_with_jitter`` spreads retries across [base/2, base] (equal jitter)
- so concurrent workers don't retry in synchronized waves.
+ so concurrent workers don't retry in synchronized waves. Unconditional since
+ UN-4046 — it was gated on ``pg_queue_enabled`` to keep the Celery flow on the
+ exact exponential cadence.
"""
-from unittest.mock import MagicMock, patch
+from unittest.mock import MagicMock
import pytest
-_JITTER_FLAG = "shared.clients.base_client.check_feature_flag_status"
-
class TestBackoffJitter:
- def test_flag_off_returns_exact_base(self):
- # Celery flow (pg_queue_enabled off): exact exponential backoff, no jitter.
- from shared.clients.base_client import _backoff_with_jitter
-
- with patch(_JITTER_FLAG, return_value=False):
- assert _backoff_with_jitter(1.0, 2) == pytest.approx(4.0)
-
- def test_flipt_error_fails_closed_to_no_jitter(self):
- from shared.clients.base_client import _backoff_with_jitter
-
- with patch(_JITTER_FLAG, side_effect=RuntimeError("flipt down")):
- assert _backoff_with_jitter(1.0, 2) == pytest.approx(4.0)
+ """UN-4046: jitter is unconditional, so there is no flag to patch.
+
+ The two flag-off cases (``flag off returns exact base``, ``Flipt error fails
+ closed to no jitter``) are GONE — they asserted the exact exponential cadence
+ the Celery flow kept, and there is no Celery flow and no Flipt call left. What
+ they were really protecting is that the RESULT stays a sane backoff, which the
+ band assertion below already pins from both sides. The remaining three drop
+ their patch and assert the same properties directly.
+ """
- def test_within_equal_jitter_band_when_flag_on(self):
+ def test_within_equal_jitter_band(self):
from shared.clients.base_client import _backoff_with_jitter
# base = backoff_factor(1.0) * 2**attempt(2) = 4.0 → equal jitter [2.0, 4.0]
- with patch(_JITTER_FLAG, return_value=True):
- for _ in range(500):
- assert 2.0 <= _backoff_with_jitter(1.0, 2) <= 4.0
+ for _ in range(500):
+ assert 2.0 <= _backoff_with_jitter(1.0, 2) <= 4.0
def test_zero_base_returns_zero(self):
from shared.clients.base_client import _backoff_with_jitter
- with patch(_JITTER_FLAG, return_value=True):
- assert _backoff_with_jitter(0.0, 3) == pytest.approx(0.0)
+ assert _backoff_with_jitter(0.0, 3) == pytest.approx(0.0)
- def test_is_not_constant_when_flag_on(self):
+ def test_is_not_constant(self):
from shared.clients.base_client import _backoff_with_jitter
# De-correlation: repeated calls must NOT all land on the same value
# (that synchronized schedule is exactly the thundering herd we fix).
- with patch(_JITTER_FLAG, return_value=True):
- assert len({_backoff_with_jitter(1.0, 3) for _ in range(64)}) > 1
+ assert len({_backoff_with_jitter(1.0, 3) for _ in range(64)}) > 1
class TestPGFinalizationReraise:
diff --git a/workers/tests/test_queue_backend_seam.py b/workers/tests/test_queue_backend_seam.py
index abc0ac154..12e85f9fd 100644
--- a/workers/tests/test_queue_backend_seam.py
+++ b/workers/tests/test_queue_backend_seam.py
@@ -1,163 +1,67 @@
-"""Equivalence tests for the queue-backend seam.
+"""Contract tests for the queue-backend seam.
-Today the seam is a transparent pass-through to Celery. These tests
-lock that in so future PRs adding per-task routing (PG Queue) can be
-proved to preserve the Celery default path.
+``dispatch()`` enqueues to Postgres — PG is the only transport (UN-4046). Two
+layers:
-Two layers:
+1. **No dispatch may reach Celery.** The canary below asserts that no call site
+ can resolve to ``CELERY``, because a producer without a consumer does not just
+ lose the work: the messages accumulate in RabbitMQ until it hits its memory
+ high-watermark and starts blocking publishers.
-1. **dispatch()** must produce the same ``current_app.send_task`` call
- as the raw Celery idiom — checked against representative
- ``send_webhook_notification`` and ``async_execute_bin``
- (``scheduler/tasks.py::_execute_scheduled_workflow``) shapes.
+2. **@worker_task** must still register a task with the Celery app. That is not
+ vestigial — the PG consumer resolves work via ``self._app.tasks.get(name)``, so
+ the Celery *registry* stays load-bearing even though the Celery *transport* is
+ gone.
-2. **@worker_task** must register a task with the Celery app so that
- functions decorated with it behave identically to ones decorated
- with ``@shared_task``.
+The old layer 1 was the mirror image of this: a suite proving ``dispatch()``
+produced exactly the same ``current_app.send_task`` call as the raw Celery idiom,
+so that adding PG routing could be shown to preserve the Celery default. That
+default is what we have now removed, so those cases are gone.
"""
from __future__ import annotations
-from unittest.mock import patch
+from unittest.mock import MagicMock, patch
import pytest
from celery import current_app
+from queue_backend import QueueBackend, dispatch, select_backend
+from queue_backend.routing import resolve_backend
-# --- dispatch() equivalence ---
+# --- no Celery producer ---
-class TestDispatchEquivalence:
- """dispatch() forwards to current_app.send_task with the same shape."""
+class TestNoCeleryProducer:
+ """The zero-producer canary (UN-4046).
- def test_dispatches_task_name(self):
- from queue_backend import dispatch
-
- with patch("queue_backend.dispatch.current_app") as mock_app:
- dispatch("send_webhook_notification")
-
- mock_app.send_task.assert_called_once()
- assert mock_app.send_task.call_args.args[0] == "send_webhook_notification"
-
- def test_forwards_args(self):
- from queue_backend import dispatch
-
- with patch("queue_backend.dispatch.current_app") as mock_app:
- dispatch("any_task", args=["a", "b", 42])
-
- assert mock_app.send_task.call_args.kwargs["args"] == ["a", "b", 42]
-
- def test_forwards_kwargs(self):
- from queue_backend import dispatch
-
- with patch("queue_backend.dispatch.current_app") as mock_app:
- dispatch("any_task", kwargs={"max_retries": 5, "platform": "SLACK"})
-
- assert mock_app.send_task.call_args.kwargs["kwargs"] == {
- "max_retries": 5,
- "platform": "SLACK",
- }
-
- def test_forwards_queue(self):
- from queue_backend import dispatch
-
- with patch("queue_backend.dispatch.current_app") as mock_app:
- dispatch("any_task", queue="notifications")
-
- assert mock_app.send_task.call_args.kwargs["queue"] == "notifications"
-
- def test_omitted_args_forwarded_as_none(self):
- """Omitted ``args`` is forwarded verbatim as ``None``.
-
- Celery's own ``send_task`` normalises ``None`` to its native default
- (a tuple) internally — the seam doesn't coerce, so any third-party
- router that checks ``isinstance(args, tuple)`` sees Celery's default.
- """
- from queue_backend import dispatch
-
- with patch("queue_backend.dispatch.current_app") as mock_app:
- dispatch("any_task")
-
- assert mock_app.send_task.call_args.kwargs["args"] is None
-
- def test_omitted_kwargs_forwarded_as_none(self):
- """Same as args — omitted kwargs reaches Celery as ``None``."""
- from queue_backend import dispatch
-
- with patch("queue_backend.dispatch.current_app") as mock_app:
- dispatch("any_task")
-
- assert mock_app.send_task.call_args.kwargs["kwargs"] is None
-
- def test_returns_underlying_handle(self):
- """Whatever send_task returns is what dispatch returns."""
- from queue_backend import dispatch
-
- sentinel = object()
- with patch("queue_backend.dispatch.current_app") as mock_app:
- mock_app.send_task.return_value = sentinel
- result = dispatch("any_task")
-
- assert result is sentinel
+ Any dispatch that resolves to CELERY publishes to RabbitMQ, where — with the
+ Celery fleet scaled to zero — nothing drains it. Left unnoticed the queue grows
+ until the broker blocks publishers, so this is asserted rather than assumed.
+ """
- def test_matches_notification_helper_shape(self):
- """Output mirrors the raw ``send_task`` shape for a notification task.
+ def test_select_backend_never_resolves_to_celery(self):
+ # `select_backend()` takes no argument since UN-4046: the answer cannot
+ # depend on the task name, so enumerating names proved nothing. (Sonar
+ # flagged the unused parameter; removing it makes that structural.)
+ assert select_backend() is QueueBackend.PG
- The callback notification path itself migrated to an HTTP
- buffer-enqueue in UN-3056; this keeps a representative
- ``send_webhook_notification`` call as a seam-equivalence reference.
+ def test_resolve_backend_defaults_to_pg_without_an_override(self):
+ assert resolve_backend("any_task", None) is QueueBackend.PG
- Reference call shape (paraphrased):
+ def test_an_explicit_pg_override_is_honoured(self):
+ assert resolve_backend("any_task", QueueBackend.PG) is QueueBackend.PG
- current_app.send_task(
- "send_webhook_notification",
- args=[url, payload_dict, headers, timeout],
- kwargs={"max_retries": ..., "retry_delay": ..., "platform": ...},
- queue="notifications",
- )
- """
- from queue_backend import dispatch
-
- with patch("queue_backend.dispatch.current_app") as mock_app:
- dispatch(
- "send_webhook_notification",
- args=["https://example.com/hook", {"event": "x"}, {}, 10],
- kwargs={"max_retries": 3, "retry_delay": 10, "platform": "API"},
- queue="notifications",
- )
-
- call = mock_app.send_task.call_args
- assert call.args[0] == "send_webhook_notification"
- assert call.kwargs["args"] == ["https://example.com/hook", {"event": "x"}, {}, 10]
- assert call.kwargs["kwargs"] == {
- "max_retries": 3,
- "retry_delay": 10,
- "platform": "API",
- }
- assert call.kwargs["queue"] == "notifications"
-
- def test_matches_scheduler_dispatch_shape(self):
- """Output mirrors the raw send_task at ``_execute_scheduled_workflow``."""
- from queue_backend import dispatch
-
- with patch("queue_backend.dispatch.current_app") as mock_app:
- dispatch(
- "async_execute_bin",
- args=["org-1", "wf-1", "exec-1", {}, True],
- kwargs={"use_file_history": False, "pipeline_id": "pipe-1"},
- queue="general",
- )
-
- call = mock_app.send_task.call_args
- assert call.args[0] == "async_execute_bin"
- assert call.kwargs["args"] == ["org-1", "wf-1", "exec-1", {}, True]
- assert call.kwargs["kwargs"] == {
- "use_file_history": False,
- "pipeline_id": "pipe-1",
- }
- assert call.kwargs["queue"] == "general"
-
-
-# --- @worker_task equivalence ---
+ def test_dispatch_never_calls_send_task(self):
+ client = MagicMock()
+ client.send.return_value = 1
+ with (
+ patch("queue_backend.dispatch.current_app") as mock_app,
+ patch("queue_backend.dispatch._get_pg_client", return_value=client),
+ ):
+ dispatch("send_webhook_notification", args=["a"], queue="notifications")
+ dispatch("async_execute_bin", args=["b"], queue="celery")
+ mock_app.send_task.assert_not_called()
+ assert client.send.call_count == 2
class TestWorkerTaskEquivalence:
diff --git a/workers/tests/test_routing.py b/workers/tests/test_routing.py
index 7c3e6447e..f9c46abcd 100644
--- a/workers/tests/test_routing.py
+++ b/workers/tests/test_routing.py
@@ -1,19 +1,13 @@
-"""Tests for the PG-queue routing gate (``queue_backend.routing``).
+"""Tests for queue-transport routing and ``dispatch()`` (UN-4046).
-Three layers:
+PG is the only transport: ``select_backend()`` returns ``PG`` unconditionally and
+``dispatch()`` enqueues to Postgres. What these pin is the *payload* the PG
+consumer receives, the no-silent-fallback contract, and the per-thread client
+reuse.
-1. **select_backend()** resolves the per-task allow-list correctly,
- defaults to ``CELERY`` when unset, and never raises on malformed
- input.
-
-2. **Observability** — the configured allow-list is logged once, and a
- PG cutover emits a log. These are the gate's only observable effect,
- so they're asserted explicitly (a future "the gate is inert, delete
- it" refactor must fail a test, not pass silently).
-
-3. **dispatch() is inert under routing** — the ``current_app.send_task``
- call is byte-identical whether the routing table selects ``CELERY``
- or ``PG``.
+The allow-list suites are gone with ``WORKER_PG_QUEUE_ENABLED_TASKS`` — the
+parsing, the tolerant-whitespace cases, the "logged once when configured"
+observability and every "celery path" case describe a gate that no longer exists.
"""
from __future__ import annotations
@@ -24,9 +18,7 @@ from unittest.mock import MagicMock, patch
import pytest
from queue_backend import QueueBackend, dispatch, select_backend
-from queue_backend import routing as routing_mod
from queue_backend.fairness import FairnessKey, WorkloadType
-from queue_backend.routing import _ENABLED_TASKS_ENV_VAR as ENABLED_TASKS_ENV
# ``queue_backend.__init__`` binds ``dispatch`` to the *function*, shadowing
# the submodule attribute — import the module explicitly to reach its globals.
@@ -34,62 +26,30 @@ dispatch_mod = importlib.import_module("queue_backend.dispatch")
@pytest.fixture(autouse=True)
-def _reset_routing_state(monkeypatch):
- """Empty allow-list + cleared log-once guards before each test.
+def _reset_routing_state():
+ """Clear the log-once guard and the per-thread PG client before each test.
- The allow-list-logged flag and the per-task routing-logged set are
- process-global one-shot guards; reset them so caplog assertions are
- deterministic regardless of test order.
+ The per-task routing-logged set is a process-global one-shot guard; reset it
+ so caplog assertions are deterministic regardless of test order.
"""
- monkeypatch.delenv(ENABLED_TASKS_ENV, raising=False)
- routing_mod._allow_list_logged = False
dispatch_mod._pg_routing_logged.clear()
- dispatch_mod._pg_local.client = None # drop the per-thread PG client
+ dispatch_mod._pg_local.client = None
-# --- select_backend() resolution ---
+def _mock_pg_client(monkeypatch, *, msg_id=99):
+ client = MagicMock()
+ client.send.return_value = msg_id
+ # Patch on the module object (string target would navigate the shadowing
+ # ``dispatch`` *function*, not the submodule).
+ monkeypatch.setattr(dispatch_mod, "_get_pg_client", lambda: client)
+ return client
class TestSelectBackend:
- def test_empty_table_routes_to_celery(self):
- assert select_backend("any_task") is QueueBackend.CELERY
-
- def test_task_opted_in_routes_to_pg(self, monkeypatch):
- monkeypatch.setenv(ENABLED_TASKS_ENV, "process_log_history")
- assert select_backend("process_log_history") is QueueBackend.PG
-
- def test_task_not_opted_in_routes_to_celery(self, monkeypatch):
- monkeypatch.setenv(ENABLED_TASKS_ENV, "process_log_history")
- assert select_backend("some_other_task") is QueueBackend.CELERY
-
- def test_multiple_entries_membership(self, monkeypatch):
- monkeypatch.setenv(ENABLED_TASKS_ENV, "a,b,c")
- assert select_backend("b") is QueueBackend.PG
- assert select_backend("d") is QueueBackend.CELERY
-
-
-class TestSelectBackendTolerantParsing:
- """Malformed / whitespace input resolves to the safe default, never raises."""
-
- def test_whitespace_around_entries_trimmed(self, monkeypatch):
- monkeypatch.setenv(ENABLED_TASKS_ENV, " a , b , c ")
- assert select_backend("b") is QueueBackend.PG
-
- def test_empty_and_blank_segments_ignored(self, monkeypatch):
- monkeypatch.setenv(ENABLED_TASKS_ENV, "a,, ,b")
- assert select_backend("a") is QueueBackend.PG
- # A blank segment must not become a matchable empty-string entry.
- assert select_backend("") is QueueBackend.CELERY
-
- def test_empty_string_env_routes_to_celery(self, monkeypatch):
- monkeypatch.setenv(ENABLED_TASKS_ENV, " ")
- assert select_backend("any_task") is QueueBackend.CELERY
-
- def test_does_not_raise_on_any_input(self, monkeypatch):
- for value in ("", " ", ",,,", "a,b", "\t\n", ",a,"):
- monkeypatch.setenv(ENABLED_TASKS_ENV, value)
- # Must resolve to an enum member, never raise.
- assert select_backend("a") in (QueueBackend.CELERY, QueueBackend.PG)
+ def test_always_pg(self):
+ # No argument since UN-4046 — the task name never mattered once the
+ # allow-list went, and Sonar flagged it as an unused parameter.
+ assert select_backend() is QueueBackend.PG
class TestQueueBackendEnum:
@@ -101,52 +61,10 @@ class TestQueueBackendEnum:
assert {b.value for b in QueueBackend} == {"celery", "pg"}
-# --- Observability (the gate's only observable effect) ---
-
-
-class TestObservability:
- def test_allow_list_logged_once_when_configured(self, monkeypatch, caplog):
- monkeypatch.setenv(ENABLED_TASKS_ENV, "async_execute_bin")
- with caplog.at_level(logging.INFO, logger="queue_backend.routing"):
- select_backend("x")
- select_backend("y") # second call must NOT re-log
- hits = [r for r in caplog.records if "PG-queue routing enabled" in r.getMessage()]
- assert len(hits) == 1
- assert "async_execute_bin" in hits[0].getMessage()
-
- def test_empty_allow_list_logs_nothing(self, caplog):
- """Default (feature off) stays silent — truly inert."""
- with caplog.at_level(logging.INFO, logger="queue_backend.routing"):
- select_backend("x")
- assert "PG-queue routing enabled" not in caplog.text
-
-
-# --- dispatch() routing behaviour (9b: PG-selected enqueues to Postgres) ---
-
-
-def _mock_pg_client(monkeypatch, *, msg_id=99):
- client = MagicMock()
- client.send.return_value = msg_id
- # Patch on the module object (string target would navigate the shadowing
- # ``dispatch`` *function*, not the submodule).
- monkeypatch.setattr(dispatch_mod, "_get_pg_client", lambda: client)
- return client
-
-
class TestDispatchRouting:
- """Celery path unchanged; a PG-selected task enqueues to Postgres, not Celery."""
-
- def test_celery_path_sends_to_celery_and_never_touches_pg(self):
- with (
- patch("queue_backend.dispatch.current_app") as mock_app,
- patch("queue_backend.dispatch._get_pg_client") as mock_get,
- ):
- dispatch("t1", args=["a"], kwargs={"k": "v"}, queue="general")
- mock_app.send_task.assert_called_once()
- mock_get.assert_not_called()
+ """A dispatch enqueues to Postgres and never touches Celery."""
- def test_pg_selected_enqueues_to_pg_not_celery(self, monkeypatch):
- monkeypatch.setenv(ENABLED_TASKS_ENV, "t1")
+ def test_enqueues_to_pg_not_celery(self, monkeypatch):
client = _mock_pg_client(monkeypatch, msg_id=99)
fairness = FairnessKey(org_id="org-1", workload_type=WorkloadType.API)
with patch("queue_backend.dispatch.current_app") as mock_app:
@@ -166,17 +84,15 @@ class TestDispatchRouting:
# Handle satisfies TaskHandle (.id) and carries the msg_id.
assert handle.id == "99"
- def test_pg_default_queue_name_when_unset(self, monkeypatch):
- monkeypatch.setenv(ENABLED_TASKS_ENV, "t1")
+ def test_default_queue_name_when_unset(self, monkeypatch):
client = _mock_pg_client(monkeypatch)
dispatch("t1")
assert client.send.call_args.args[0] == "default"
# No fairness → org_id None (client coerces to "").
assert client.send.call_args.kwargs["org_id"] is None
- def test_pg_enqueue_failure_propagates_and_does_not_fall_back(self, monkeypatch):
+ def test_enqueue_failure_propagates_and_does_not_fall_back(self, monkeypatch):
"""A PG enqueue failure raises — no silent Celery fallback."""
- monkeypatch.setenv(ENABLED_TASKS_ENV, "t1")
client = MagicMock()
client.send.side_effect = RuntimeError("db down")
monkeypatch.setattr(dispatch_mod, "_get_pg_client", lambda: client)
@@ -202,28 +118,17 @@ class TestDispatchRouting:
class TestCutoverLog:
- """The PG cutover log: visible (INFO), once per task name."""
+ """The PG routing log: visible (INFO), once per task name."""
def test_pg_enqueue_logs_once_per_task(self, monkeypatch, caplog):
- monkeypatch.setenv(ENABLED_TASKS_ENV, "t1")
_mock_pg_client(monkeypatch)
with caplog.at_level(logging.INFO, logger="queue_backend.dispatch"):
dispatch("t1")
dispatch("t1")
dispatch("t1")
- hits = [
- r for r in caplog.records if "routing task=" in r.getMessage()
- ]
+ hits = [r for r in caplog.records if "routing task=" in r.getMessage()]
assert len(hits) == 1
- def test_celery_path_emits_no_pg_log(self, caplog):
- with (
- patch("queue_backend.dispatch.current_app"),
- caplog.at_level(logging.INFO, logger="queue_backend.dispatch"),
- ):
- dispatch("t1") # empty allow-list → Celery
- assert "routing task=" not in caplog.text
-
if __name__ == "__main__":
pytest.main([__file__, "-v"])
diff --git a/workers/tests/test_structure_tool_pipeline.py b/workers/tests/test_structure_tool_pipeline.py
index c565621da..4c2812bf6 100644
--- a/workers/tests/test_structure_tool_pipeline.py
+++ b/workers/tests/test_structure_tool_pipeline.py
@@ -7,7 +7,15 @@ single ``structure_pipeline`` operation to the executor worker instead of
are assembled and the result is written to filesystem.
"""
-from unittest.mock import MagicMock, patch
+from unittest.mock import MagicMock, create_autospec, patch
+
+# create_autospec, NOT MagicMock(spec=...): only autospec enforces the method
+# SIGNATURE. A bare MagicMock accepts any keyword, which is how a `headers=`
+# argument the real dispatcher does not take (UN-4046) reached production and
+# made every extraction raise TypeError while this suite stayed green.
+# `spec=` alone does not close that hole — it restricts attribute names only,
+# and was verified to still accept `headers=`.
+from unstract.workflow_execution.executor_rpc import PgExecutionDispatcher
import pytest
from shared.enums.task_enums import TaskName
@@ -174,7 +182,7 @@ class TestStructureToolPipeline:
"tool_metadata": tool_metadata_regular,
}
- dispatcher_instance = MagicMock()
+ dispatcher_instance = create_autospec(PgExecutionDispatcher, instance=True)
mock_get_executor_dispatcher.return_value = dispatcher_instance
pipeline_result = _make_pipeline_result(
@@ -225,7 +233,7 @@ class TestStructureToolPipeline:
"tool_metadata": tool_metadata_regular,
}
- dispatcher_instance = MagicMock()
+ dispatcher_instance = create_autospec(PgExecutionDispatcher, instance=True)
mock_get_executor_dispatcher.return_value = dispatcher_instance
dispatcher_instance.dispatch.return_value = _make_pipeline_result()
@@ -289,7 +297,7 @@ class TestStructureToolSinglePass:
base_params["tool_instance_metadata"]["single_pass_extraction_mode"] = True
- dispatcher_instance = MagicMock()
+ dispatcher_instance = create_autospec(PgExecutionDispatcher, instance=True)
mock_get_executor_dispatcher.return_value = dispatcher_instance
dispatcher_instance.dispatch.return_value = _make_pipeline_result(
output={"field_a": "answer"},
@@ -337,7 +345,7 @@ class TestStructureToolSummarize:
tool_metadata_regular["tool_settings"]["summarize_prompt"] = "Summarize this doc"
base_params["tool_instance_metadata"]["summarize_as_source"] = True
- dispatcher_instance = MagicMock()
+ dispatcher_instance = create_autospec(PgExecutionDispatcher, instance=True)
mock_get_executor_dispatcher.return_value = dispatcher_instance
dispatcher_instance.dispatch.return_value = _make_pipeline_result(
output={"field_a": "answer"},
@@ -396,7 +404,7 @@ class TestStructureToolSmartTable:
"tool_metadata": tool_metadata_regular,
}
- dispatcher_instance = MagicMock()
+ dispatcher_instance = create_autospec(PgExecutionDispatcher, instance=True)
mock_get_executor_dispatcher.return_value = dispatcher_instance
dispatcher_instance.dispatch.return_value = _make_pipeline_result(
output={"field_a": "table_answer"},
@@ -451,7 +459,7 @@ class TestStructureToolAgentic:
"tool_metadata": agentic_metadata,
}
- dispatcher_instance = MagicMock()
+ dispatcher_instance = create_autospec(PgExecutionDispatcher, instance=True)
mock_get_executor_dispatcher.return_value = dispatcher_instance
# Mock X2Text extraction
@@ -511,7 +519,7 @@ class TestStructureToolProfileOverrides:
"llm_id": "llm-override",
}
- dispatcher_instance = MagicMock()
+ dispatcher_instance = create_autospec(PgExecutionDispatcher, instance=True)
mock_get_executor_dispatcher.return_value = dispatcher_instance
dispatcher_instance.dispatch.return_value = _make_pipeline_result(
output={"field_a": "answer"},
@@ -552,7 +560,7 @@ class TestStructureToolPipelineFailure:
"tool_metadata": tool_metadata_regular,
}
- dispatcher_instance = MagicMock()
+ dispatcher_instance = create_autospec(PgExecutionDispatcher, instance=True)
mock_get_executor_dispatcher.return_value = dispatcher_instance
pipeline_failure = ExecutionResult.failure(
@@ -601,7 +609,7 @@ class TestStructureToolMultipleOutputs:
"tool_metadata": tool_metadata_regular,
}
- dispatcher_instance = MagicMock()
+ dispatcher_instance = create_autospec(PgExecutionDispatcher, instance=True)
mock_get_executor_dispatcher.return_value = dispatcher_instance
dispatcher_instance.dispatch.return_value = _make_pipeline_result(
output={"field_a": "a", "field_b": "b"},
@@ -647,7 +655,7 @@ class TestStructureToolOutputWritten:
"tool_metadata": tool_metadata_regular,
}
- dispatcher_instance = MagicMock()
+ dispatcher_instance = create_autospec(PgExecutionDispatcher, instance=True)
mock_get_executor_dispatcher.return_value = dispatcher_instance
dispatcher_instance.dispatch.return_value = _make_pipeline_result(
output={"field_a": "answer"},
@@ -717,7 +725,7 @@ class TestStructureToolMetadataFileName:
"tool_metadata": tool_metadata_regular,
}
- dispatcher_instance = MagicMock()
+ dispatcher_instance = create_autospec(PgExecutionDispatcher, instance=True)
mock_get_executor_dispatcher.return_value = dispatcher_instance
dispatcher_instance.dispatch.return_value = _make_pipeline_result(
output={"field_a": "answer"},
@@ -758,7 +766,7 @@ class TestStructureToolNoSummarize:
"tool_metadata": tool_metadata_regular,
}
- dispatcher_instance = MagicMock()
+ dispatcher_instance = create_autospec(PgExecutionDispatcher, instance=True)
mock_get_executor_dispatcher.return_value = dispatcher_instance
dispatcher_instance.dispatch.return_value = _make_pipeline_result()
diff --git a/workers/tests/test_structure_tool_task.py b/workers/tests/test_structure_tool_task.py
index dcef6f0b1..969ef9eed 100644
--- a/workers/tests/test_structure_tool_task.py
+++ b/workers/tests/test_structure_tool_task.py
@@ -1,18 +1,20 @@
"""Direct unit tests for ``file_processing.structure_tool_task`` helpers.
-Locks contracts that the integration test in ``test_sanity_phase5.py``
-exercises indirectly. Specifically guards ``_fairness_headers``:
-
-* The hard-coded ``WorkloadType.NON_API`` default. Per the docstring,
- ``API`` is "strictly worse" here — flipping it would silently
- preempt ETL work under API traffic.
-* The wire shape consumers see at the dispatcher. A regression in
- ``FairnessKey.as_header()``'s serialisation surfaces here too.
-
-Paired with the ``headers=`` assertion in
-``test_sanity_phase5.TestStructureToolSingleDispatch`` — together
-they cover both "the helper returns the right thing" and "the call
-site forwards it".
+UN-4046 removed ``TestFairnessHeaders``. It pinned the wire shape of
+``_fairness_headers()``, which was passed as ``headers=`` to every executor
+dispatch — but the routing dispatcher never forwarded headers to the PG path
+(see ``PgExecutionDispatcher.dispatch``'s own docstring), so the value was
+inert on the only transport that runs. With the router gone the helper is
+deleted and the three tests describe a function that no longer exists.
+
+Fairness on PG is not lost and is not tested here: it rides the enqueue payload
+(``transport.enqueue(..., org_id=...)``), and ``queue_backend`` owns its
+coverage. The paired "call site forwards it" assertion referenced by the old
+docstring lived in a ``test_sanity_phase5.py`` that no longer exists either.
+
+What remains is ``TestDispatcherFactory``, which guards the UN-3779 regression:
+the impl must build its dispatcher via the factory, not a raw
+``ExecutionDispatcher``.
"""
from __future__ import annotations
@@ -20,43 +22,13 @@ from __future__ import annotations
from unittest.mock import patch
import pytest
-
from file_processing import structure_tool_task as st
-from file_processing.structure_tool_task import _fairness_headers
-from queue_backend.fairness import WorkloadType
-
-
-class TestFairnessHeaders:
- def test_returns_non_api_wire_shape(self):
- """``_fairness_headers`` emits the canonical ``x-fairness-key``
- envelope with ``workload_type='non_api'`` and the default
- ``pipeline_priority=5``."""
- assert _fairness_headers("org-1") == {
- "x-fairness-key": {
- "org_id": "org-1",
- "workload_type": "non_api",
- "pipeline_priority": 5,
- }
- }
-
- def test_org_id_is_propagated_verbatim(self):
- wire = _fairness_headers("another-org")
- assert wire["x-fairness-key"]["org_id"] == "another-org"
-
- def test_workload_type_is_non_api_not_api(self):
- """Pinned because the docstring calls ``NON_API`` "the safe
- default" and ``API`` "strictly worse" — a regression that
- flips this would silently let API traffic preempt ETL work.
- """
- wire = _fairness_headers("org-1")
- assert wire["x-fairness-key"]["workload_type"] == WorkloadType.NON_API.value
- assert wire["x-fairness-key"]["workload_type"] != WorkloadType.API.value
class TestDispatcherFactory:
"""Pin the call-site swap this PR makes: the impl builds its dispatcher via
- ``get_executor_dispatcher(celery_app=app)`` (the gate-routed dispatcher), not
- the raw SDK ``ExecutionDispatcher``. A mis-import or wrong arg would otherwise
+ ``get_executor_dispatcher()`` (the shared factory), not
+ the raw SDK ``ExecutionDispatcher``. A mis-import would otherwise
silently bypass the PG routing with nothing failing.
"""
@@ -73,7 +45,7 @@ class TestDispatcherFactory:
"execution_data_dir": "/data",
}
- def test_impl_builds_dispatcher_via_factory_with_app(self):
+ def test_impl_builds_dispatcher_via_factory(self):
# Stub everything up to (and just past) the dispatcher construction, then
# raise to stop before the heavy tool-metadata work runs.
with (
@@ -86,7 +58,9 @@ class TestDispatcherFactory:
params = self._params()
with pytest.raises(RuntimeError, match="stop"):
st._execute_structure_tool_impl(params)
- factory.assert_called_once_with(celery_app=st.app)
+ # No arguments since UN-4046 — the factory's accepted-and-ignored
+ # `celery_app` was removed once Sonar flagged it as an unused parameter.
+ factory.assert_called_once_with()
if __name__ == "__main__":